From f36b5a5d0c363d8d3d44161edad6b394664ca376 Mon Sep 17 00:00:00 2001 From: stumpylog <797416+stumpylog@users.noreply.github.com> Date: Tue, 25 Aug 2026 10:28:55 -0700 Subject: [PATCH] feat(search): parse advanced search with whoosh-compat and delete the hand-written translator --- src/documents/search/__init__.py | 12 +- src/documents/search/_backend.py | 68 +- src/documents/search/_dates.py | 171 ---- src/documents/search/_errors.py | 71 ++ src/documents/search/_query.py | 595 ++++++++++--- src/documents/search/_translate.py | 610 ------------- src/documents/tests/search/conftest.py | 12 - src/documents/tests/search/test_acceptance.py | 411 +++++++++ .../search/test_checksum_prefix_queries.py | 57 ++ src/documents/tests/search/test_cjk_clause.py | 133 +++ .../test_date_keyword_phrase_removal.py | 72 ++ .../search/test_date_keyword_timezone.py | 83 ++ .../test_default_search_fields_guard.py | 20 + .../tests/search/test_error_routing.py | 247 ++++++ .../search/test_exists_on_json_fields.py | 92 ++ .../tests/search/test_fuzzy_tokenization.py | 174 ++++ .../search/test_highlight_query_guard.py | 192 ++++ .../tests/search/test_json_field_prefixes.py | 148 ++++ .../search/test_permission_field_isolation.py | 148 ++++ src/documents/tests/search/test_query.py | 836 ++++-------------- .../tests/search/test_query_negation.py | 153 ++++ src/documents/tests/search/test_translate.py | 810 ----------------- src/documents/tests/test_api_bulk_download.py | 18 + src/documents/tests/test_api_bulk_edit.py | 19 + src/documents/tests/test_api_search.py | 89 ++ src/documents/tests/test_api_search_errors.py | 275 ++++++ src/documents/views.py | 85 +- 27 files changed, 3130 insertions(+), 2471 deletions(-) delete mode 100644 src/documents/search/_dates.py create mode 100644 src/documents/search/_errors.py delete mode 100644 src/documents/search/_translate.py create mode 100644 src/documents/tests/search/test_acceptance.py create mode 100644 src/documents/tests/search/test_checksum_prefix_queries.py create mode 100644 src/documents/tests/search/test_cjk_clause.py create mode 100644 src/documents/tests/search/test_date_keyword_phrase_removal.py create mode 100644 src/documents/tests/search/test_date_keyword_timezone.py create mode 100644 src/documents/tests/search/test_default_search_fields_guard.py create mode 100644 src/documents/tests/search/test_error_routing.py create mode 100644 src/documents/tests/search/test_exists_on_json_fields.py create mode 100644 src/documents/tests/search/test_fuzzy_tokenization.py create mode 100644 src/documents/tests/search/test_highlight_query_guard.py create mode 100644 src/documents/tests/search/test_json_field_prefixes.py create mode 100644 src/documents/tests/search/test_permission_field_isolation.py create mode 100644 src/documents/tests/search/test_query_negation.py delete mode 100644 src/documents/tests/search/test_translate.py create mode 100644 src/documents/tests/test_api_search_errors.py diff --git a/src/documents/search/__init__.py b/src/documents/search/__init__.py index f0e65e5cd..0e89caa6f 100644 --- a/src/documents/search/__init__.py +++ b/src/documents/search/__init__.py @@ -6,13 +6,20 @@ from documents.search._backend import TantivyRelevanceList from documents.search._backend import WriteBatch from documents.search._backend import get_backend from documents.search._backend import reset_backend +from documents.search._errors import InvalidDateQuery +from documents.search._errors import InvalidNumberQuery +from documents.search._errors import MultipleSearchQueryErrors +from documents.search._errors import QueryTooLongError +from documents.search._errors import SearchQueryError +from documents.search._errors import search_query_error_messages from documents.search._schema import needs_rebuild from documents.search._schema import wipe_index -from documents.search._translate import InvalidDateQuery -from documents.search._translate import SearchQueryError __all__ = [ "InvalidDateQuery", + "InvalidNumberQuery", + "MultipleSearchQueryErrors", + "QueryTooLongError", "SearchHit", "SearchIndexLockError", "SearchMode", @@ -23,5 +30,6 @@ __all__ = [ "get_backend", "needs_rebuild", "reset_backend", + "search_query_error_messages", "wipe_index", ] diff --git a/src/documents/search/_backend.py b/src/documents/search/_backend.py index e5bae979e..6fe1e6da2 100644 --- a/src/documents/search/_backend.py +++ b/src/documents/search/_backend.py @@ -22,7 +22,6 @@ import tantivy from django.conf import settings from django.utils.timezone import get_current_timezone -from documents.search._query import build_permission_filter from documents.search._query import extract_cjk_text from documents.search._query import parse_simple_text_highlight_query from documents.search._query import parse_simple_text_query @@ -40,6 +39,7 @@ from documents.utils import QuerySetStream from documents.utils import identity if TYPE_CHECKING: + from collections.abc import Iterable from collections.abc import Iterator from collections.abc import Sequence from pathlib import Path @@ -325,6 +325,47 @@ class WriteBatch: self._writer.add_document(doc) +def build_permission_filter( + schema: tantivy.Schema, + user: AbstractUser, + viewer_group_ids: Iterable[int] = (), +) -> tantivy.Query: + """ + Build a query filter for user document permissions. + + Creates a query that matches only documents visible to the specified user + according to paperless-ngx permission rules: + - Public documents (no owner) are visible to all users + - Private documents are visible to their owner + - Documents explicitly shared with the user are visible + - Documents shared with one of the user's current groups are visible + + Args: + schema: Tantivy schema for field validation + user: User to check permissions for + viewer_group_ids: Current group memberships for the user + + Returns: + Tantivy query that filters results to visible documents + """ + owner_any = tantivy.Query.exists_query("owner_id") + no_owner = tantivy.Query.boolean_query( + [ + (tantivy.Occur.Must, tantivy.Query.all_query()), + (tantivy.Occur.MustNot, owner_any), + ], + ) + owned = tantivy.Query.term_query(schema, "owner_id", user.pk) + shared = tantivy.Query.term_query(schema, "viewer_id", user.pk) + group_shared = [ + tantivy.Query.term_query(schema, "viewer_group_id", group_id) + for group_id in viewer_group_ids + ] + return tantivy.Query.disjunction_max_query( + [no_owner, owned, shared, *group_shared], + ) + + class TantivyBackend: """ Tantivy search backend with explicit lifecycle management. @@ -498,7 +539,6 @@ class TantivyBackend: doc.add_text("correspondent_sort", document.correspondent.name) if cjk_corr := extract_cjk_text(document.correspondent.name): doc.add_text("bigram_correspondent", cjk_corr) - doc.add_unsigned("correspondent_id", document.correspondent_id) # Document type if document.document_type: @@ -506,12 +546,10 @@ class TantivyBackend: doc.add_text("type_sort", document.document_type.name) if cjk_type := extract_cjk_text(document.document_type.name): doc.add_text("bigram_document_type", cjk_type) - doc.add_unsigned("document_type_id", document.document_type_id) # Storage path if document.storage_path: doc.add_text("storage_path", document.storage_path.name) - doc.add_unsigned("storage_path_id", document.storage_path_id) # Tags — collect names for autocomplete in the same pass tag_names: list[str] = [] @@ -519,12 +557,13 @@ class TantivyBackend: doc.add_text("tag", tag.name) if cjk_tag := extract_cjk_text(tag.name): doc.add_text("bigram_tag", cjk_tag) - doc.add_unsigned("tag_id", tag.pk) tag_names.append(tag.name) # Notes — JSON for structured queries (notes.user:alice, notes.note:text). # notes_text is a plain-text companion for snippet/highlight generation; - # tantivy's SnippetGenerator does not support JSON fields. + # tantivy's SnippetGenerator does not support JSON fields. It is not in + # _DEFAULT_SEARCH_FIELDS, so an unqualified query never searches it: a + # note matches through the JSON field or not at all. num_notes = 0 note_texts: list[str] = [] for note in document.notes.all(): @@ -540,8 +579,9 @@ class TantivyBackend: if note_texts: doc.add_text("notes_text", " ".join(note_texts)) - # Custom fields — JSON for structured queries (custom_fields.name:x, custom_fields.value:y), - # companion text field for default full-text search. + # Custom fields — JSON for structured queries (custom_fields.name:x, + # custom_fields.value:y). There is no companion text field here, unlike + # notes: custom field values are reachable only through the JSON field. for cfi in document.custom_fields.all(): search_value = cfi.value_for_search # Skip fields where there is no value yet @@ -708,7 +748,17 @@ class TantivyBackend: user_query = self._parse_query(query, search_mode) highlight_query = user_query if search_mode is SearchMode.TEXT: - highlight_query = parse_simple_text_highlight_query(self._index, query) + try: + highlight_query = parse_simple_text_highlight_query( + self._index, + query, + ) + except ValueError: + logger.debug( + "Skipping simple text highlight query: token string is not " + "valid tantivy query syntax: %r", + query, + ) # For notes_text snippet generation, we need a query that targets the # notes_text field directly. user_query may contain JSON-field terms diff --git a/src/documents/search/_dates.py b/src/documents/search/_dates.py deleted file mode 100644 index dbdf6b42b..000000000 --- a/src/documents/search/_dates.py +++ /dev/null @@ -1,171 +0,0 @@ -from __future__ import annotations - -from datetime import UTC -from datetime import date -from datetime import datetime -from datetime import timedelta -from typing import TYPE_CHECKING -from typing import Final - -from dateutil.relativedelta import relativedelta - -if TYPE_CHECKING: - from datetime import tzinfo - -_DATE_ONLY_FIELDS = frozenset({"created"}) - -_TODAY: Final[str] = "today" -_YESTERDAY: Final[str] = "yesterday" -_PREVIOUS_WEEK: Final[str] = "previous week" -_THIS_MONTH: Final[str] = "this month" -_PREVIOUS_MONTH: Final[str] = "previous month" -_THIS_YEAR: Final[str] = "this year" -_PREVIOUS_YEAR: Final[str] = "previous year" -_PREVIOUS_QUARTER: Final[str] = "previous quarter" - -_DATE_KEYWORDS = frozenset( - { - _TODAY, - _YESTERDAY, - _PREVIOUS_WEEK, - _THIS_MONTH, - _PREVIOUS_MONTH, - _THIS_YEAR, - _PREVIOUS_YEAR, - _PREVIOUS_QUARTER, - }, -) - - -def _fmt(dt: datetime) -> str: - """Format a datetime as an ISO 8601 UTC string for use in Tantivy range queries.""" - return dt.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") - - -def _iso_range(lo: datetime, hi: datetime) -> str: - """ - Format a half-open ``[lo TO hi)`` range in ISO 8601 for Tantivy query syntax. - - ``hi`` is always the exclusive ceiling of a computed period (the start of - the *next* day/week/month/quarter/year), so the closing bracket must be - the Tantivy exclusive-range brace ``}`` rather than ``]`` — otherwise the - first instant of the following period (e.g. the 1st of next month) is - incorrectly included in the match. - """ - return f"[{_fmt(lo)} TO {_fmt(hi)}}}" - - -def _quarter_start(d: date) -> date: - """Return the first day of the calendar quarter containing ``d``.""" - return date(d.year, ((d.month - 1) // 3) * 3 + 1, 1) - - -def _midnight(d: date, tz: tzinfo) -> datetime: - """Convert a calendar date at local-timezone midnight to a UTC datetime.""" - return datetime(d.year, d.month, d.day, tzinfo=tz).astimezone(UTC) - - -def _keyword_bounds(keyword: str, tz: tzinfo) -> tuple[date, date]: - """ - Map a relative date keyword to ``(start, exclusive_end)`` calendar dates. - - ``tz`` only determines what "today" is; the caller decides how the returned - dates become UTC datetime boundaries (date-only vs. local-midnight offset). - """ - today = datetime.now(tz).date() - if keyword == _TODAY: - return today, today + timedelta(days=1) - if keyword == _YESTERDAY: - return today - timedelta(days=1), today - if keyword == _PREVIOUS_WEEK: - this_monday = today - timedelta(days=today.weekday()) - return this_monday - timedelta(weeks=1), this_monday - if keyword == _THIS_MONTH: - first = today.replace(day=1) - return first, first + relativedelta(months=1) - if keyword == _PREVIOUS_MONTH: - this_first = today.replace(day=1) - return this_first - relativedelta(months=1), this_first - if keyword == _THIS_YEAR: - return date(today.year, 1, 1), date(today.year + 1, 1, 1) - if keyword == _PREVIOUS_YEAR: - return date(today.year - 1, 1, 1), date(today.year, 1, 1) - if keyword == _PREVIOUS_QUARTER: - this_quarter = _quarter_start(today) - return this_quarter - relativedelta(months=3), this_quarter - raise ValueError(f"Unknown keyword: {keyword}") - - -def _date_only_range(keyword: str, tz: tzinfo) -> str: - """ - For `created` (DateField): use the local calendar date, converted to - midnight UTC boundaries. No offset arithmetic — date only. - """ - start, end = _keyword_bounds(keyword, tz) - lo = datetime(start.year, start.month, start.day, tzinfo=UTC) - hi = datetime(end.year, end.month, end.day, tzinfo=UTC) - return _iso_range(lo, hi) - - -def _datetime_range(keyword: str, tz: tzinfo) -> str: - """ - For `added` / `modified` (DateTimeField, stored as UTC): convert local day - boundaries to UTC — full offset arithmetic required. - """ - start, end = _keyword_bounds(keyword, tz) - return _iso_range(_midnight(start, tz), _midnight(end, tz)) - - -def _precision_bounds(digits: str) -> tuple[date, date] | None: - """ - Map a 4/6/8-digit date token to (start, exclusive_end) calendar dates. - - YYYY -> whole year, YYYYMM -> whole month, YYYYMMDD -> single day. - Returns None for any unparsable or out-of-range value (e.g. month 23), - so callers can emit a no-match clause instead of erroring (Whoosh parity). - """ - try: - if len(digits) == 4: - year = int(digits) - return date(year, 1, 1), date(year + 1, 1, 1) - if len(digits) == 6: - year, month = int(digits[:4]), int(digits[4:6]) - start = date(year, month, 1) - end = date(year + 1, 1, 1) if month == 12 else date(year, month + 1, 1) - return start, end - if len(digits) == 8: - start = date(int(digits[:4]), int(digits[4:6]), int(digits[6:8])) - return start, start + timedelta(days=1) - except ValueError: - return None - return None - - -def _utc_bounds_for_field( - field: str, - start: date, - end: date, - tz: tzinfo, -) -> tuple[datetime, datetime]: - """ - Convert calendar-date bounds to UTC datetimes per the field's storage type. - - For DateField (``created``) the bounds are UTC midnight (no offset). For - DateTimeField (``added``/``modified``) the bounds are local-tz midnight - converted to UTC, matching how each field is indexed. - """ - if field in _DATE_ONLY_FIELDS: - return ( - datetime(start.year, start.month, start.day, tzinfo=UTC), - datetime(end.year, end.month, end.day, tzinfo=UTC), - ) - return ( - datetime(start.year, start.month, start.day, tzinfo=tz).astimezone(UTC), - datetime(end.year, end.month, end.day, tzinfo=tz).astimezone(UTC), - ) - - -def _field_range_from_dates(field: str, start: date, end: date, tz: tzinfo) -> str: - """Build a Tantivy ``field:[lo TO hi]`` ISO range from calendar-date bounds.""" - lo, hi = _utc_bounds_for_field(field, start, end, tz) - return f"{field}:{_iso_range(lo, hi)}" diff --git a/src/documents/search/_errors.py b/src/documents/search/_errors.py new file mode 100644 index 000000000..a7bf4be04 --- /dev/null +++ b/src/documents/search/_errors.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Sequence + + +class SearchQueryError(ValueError): + """ + Base for user-fixable search query errors. + + Carries a message safe to surface to the user (no internal details). The + view layer catches this and returns an HTTP 400, so any future subclass + gets the same treatment. + """ + + +class InvalidDateQuery(SearchQueryError): + """Raised when a date field value or range bound cannot be parsed.""" + + def __init__(self, field: str | None, value: str | None) -> None: + self.field = field + self.value = value + super().__init__(f"Invalid date value {value!r} for field {field!r}.") + + +class InvalidNumberQuery(SearchQueryError): + """Raised when a numeric field value or range bound cannot be parsed.""" + + def __init__(self, field: str | None, value: str | None) -> None: + self.field = field + self.value = value + super().__init__(f"Invalid numeric value {value!r} for field {field!r}.") + + +class QueryTooLongError(SearchQueryError): + """Raised when a query string exceeds the maximum allowed length. + + whoosh-compat's fieldname tagger is O(n^2) in plain word characters, so an + unbounded query is a CPU-exhaustion vector against a single request + handler. This is a hard boundary, not a validation nicety. + """ + + def __init__(self, length: int, limit: int) -> None: + self.length = length + self.limit = limit + super().__init__( + f"The search query is too long ({length} characters). " + f"The maximum allowed length is {limit} characters.", + ) + + +class MultipleSearchQueryErrors(SearchQueryError): + """Aggregates every user-fixable error from one parse, not just the first.""" + + def __init__(self, errors: Sequence[SearchQueryError]) -> None: + self.errors = tuple(errors) + super().__init__("; ".join(str(e) for e in self.errors)) + + +def search_query_error_messages(e: SearchQueryError) -> list[str]: + """The user-facing message list for a SearchQueryError. + + Every offending value's message, not just the first, so the user can + fix them all in one round-trip. Shared by every view that maps + SearchQueryError to an HTTP 400. + """ + if isinstance(e, MultipleSearchQueryErrors): + return [str(sub) for sub in e.errors] + return [str(e)] diff --git a/src/documents/search/_query.py b/src/documents/search/_query.py index 3584767bc..96623a56c 100644 --- a/src/documents/search/_query.py +++ b/src/documents/search/_query.py @@ -6,22 +6,30 @@ from typing import Final import regex import tantivy +import whoosh_compat as wc from django.conf import settings +from whoosh_compat.emitters.tantivy_ import emit as tantivy_emit +from whoosh_compat.errors import Cause +from whoosh_compat.errors import Diagnostic +from whoosh_compat.errors import DiagnosticKind +from whoosh_compat.errors import QueryError +from documents.search._errors import InvalidDateQuery +from documents.search._errors import InvalidNumberQuery +from documents.search._errors import MultipleSearchQueryErrors +from documents.search._errors import SearchQueryError +from documents.search._registry import get_field_registry from documents.search._tokenizer import simple_search_tokens -from documents.search._translate import SearchQueryError -from documents.search._translate import translate_query if TYPE_CHECKING: - from collections.abc import Iterable from datetime import tzinfo - from django.contrib.auth.base_user import AbstractBaseUser - logger = logging.getLogger("paperless.search") -# Maximum seconds any single regex substitution may run. -# Prevents ReDoS on adversarial user-supplied query strings. +# Maximum seconds any single regex substitution over user-supplied query text +# may run. The one remaining use is a character class, which cannot backtrack, +# so the bound is an upper limit on that substitution's cost, not the ReDoS +# guard it was originally written as. _REGEX_TIMEOUT: Final[float] = 1.0 # Matches CJK/Hangul characters so queries can be routed to bigram fields. @@ -29,6 +37,64 @@ _REGEX_TIMEOUT: Final[float] = 1.0 _CJK_RE: Final = regex.compile(r"[\p{Han}\p{Hiragana}\p{Katakana}\p{Hangul}]+") +def _user_facing_emit_message(d: Diagnostic) -> str: + """A user-safe message for an emit-time QueryError's Diagnostic. + + Built from the Diagnostic's structured fields (kind, field), never from + d.message: whoosh-compat documents that as developer/log output with no + stability guarantee, and PATTERN_TOO_COMPLEX embeds the raw backend + error text in it. + """ + field = str(d.field) if d.field is not None else None + if d.kind is DiagnosticKind.EXISTS_REQUIRES_FAST: + return f"Existence searches (field:*) are not supported for field {field!r}." + if d.kind is DiagnosticKind.TEXT_RANGE: + return f"Range searches are not supported for field {field!r}." + if d.kind is DiagnosticKind.PATTERN_TOO_COMPLEX: + return f"The wildcard pattern for field {field!r} is too complex." + if d.kind is DiagnosticKind.SCHEMA_FIELD_MISSING: + return f"Field {field!r} is not available in the search index." + logger.warning("Unmapped emit diagnostic %s: %s", d.kind, d.message) + return "The search query could not be executed." + + +def _map_emit_error(e: QueryError) -> SearchQueryError: + """Route an emit-time QueryError by its Diagnostic's Cause. + + INVALID_INPUT/UNSUPPORTED are user-input errors, exactly like a parse + diagnostic, and map to a 400. INTERNAL means a defect in whoosh-compat + or in our own AST handling, never the user's query, so the QueryError is + re-raised rather than converted, reaching the generic 500 handler instead + of blaming the query. MISCONFIGURED is deliberately both: the registry and + the index schema disagree, which only an operator can fix, so it is logged + as an error, but a request is still waiting and the query cannot run + either way, so it also returns a 400. + + EXISTS_REQUIRES_FAST is the one MISCONFIGURED kind that is not a + disagreement. whoosh-compat derives it from the registry's own FieldSpec + (kind plus fast) without ever consulting the index schema, so it fires + whenever a non-fast field of a kind that cannot answer "exists" is asked + to: for us that is only the JSON fields, which field_descriptors() builds + non-fast on purpose. "notes:*" and the five other spellings of it are + ordinary user error that no operator action can clear, so they get the + 400 without the alert. + """ + d = e.diagnostic + if d.cause is Cause.INTERNAL: + raise e + if ( + d.cause is Cause.MISCONFIGURED + and d.kind is not DiagnosticKind.EXISTS_REQUIRES_FAST + ): + logger.error( + "Search index misconfiguration for field %s (%s): %s", + d.field, + d.kind.name, + d.message, + ) + return SearchQueryError(_user_facing_emit_message(d)) + + def _has_cjk(text: str) -> bool: """Return True if text contains any CJK characters.""" return bool(_CJK_RE.search(text)) @@ -37,14 +103,36 @@ def _has_cjk(text: str) -> bool: def extract_cjk_text(text: str) -> str: """Join the CJK runs in ``text`` for indexing into bigram (char-ngram) fields. - Mirrors the query side (``_build_cjk_query``): only CJK runs are ever searched - against the bigram fields, so only CJK runs are worth indexing there. Latin - text fed to a character-bigram field is never matched and only bloats the + Mirrors the query side, which extracts the CJK runs of whatever it is + about to search for (the raw string in simple modes, the parsed query's + free-text tokens in query mode): only CJK runs are ever searched against + the bigram fields, so only CJK runs are worth indexing there. Latin text + fed to a character-bigram field is never matched and only bloats the index and slows indexing/merge. Returns "" when there is no CJK text. """ return " ".join(_CJK_RE.findall(text)) +def _parse_cjk_text( + index: tantivy.Index, + cjk_text: str, + fields: list[str], +) -> tantivy.Query | None: + """Parse a plain CJK run string against ``fields``, or None if it won't parse.""" + try: + return index.parse_query(cjk_text, fields) + except Exception: + # Broad on purpose, unlike _try_parse_fuzzy_query's narrower + # ValueError: cjk_text isn't filtered to a guaranteed-safe token + # set the way the fuzzy blend's word string is, so the exact + # failure mode tantivy could raise here isn't pinned down. + logger.debug( + "Skipping CJK search clause: could not parse CJK text: %r", + cjk_text, + ) + return None + + def _build_cjk_query( index: tantivy.Index, raw_query: str, @@ -52,91 +140,259 @@ def _build_cjk_query( ) -> tantivy.Query | None: """Build a bigram-field query from the CJK runs in ``raw_query``. - Only the CJK character runs are extracted and parsed; ASCII field prefixes, - boolean operators and date keywords are discarded. This keeps the CJK clause - plain-text and consistent across query/simple modes (no leaked ``field:`` - semantics, no parse failures from spaced ``-``/``+``), and avoids feeding - Latin tokens into the character-bigram matcher (which would produce spurious - matches against unrelated Latin text). Returns None when there is no CJK - text or the parse fails. + For the simple (TEXT/TITLE) modes, whose input is plain text and carries + no query grammar to respect. Only the CJK character runs are extracted, so + a stray ``field:`` prefix or ``-``/``+`` in the input can neither leak + field semantics nor fail the parse, and no Latin token reaches the + character-bigram matcher (where it would produce spurious matches against + unrelated Latin text). Returns None when there is no CJK text or the parse + fails. """ - cjk_text = " ".join(_CJK_RE.findall(raw_query)) + cjk_text = extract_cjk_text(raw_query) if not cjk_text: return None + return _parse_cjk_text(index, cjk_text, fields) + + +def _build_ast_cjk_query( + index: tantivy.Index, + ast: wc.ast.Node, + registry: wc.FieldRegistry, +) -> tantivy.Query | None: + """Build the bigram clause of a QUERY-mode search from the parsed AST. + + Same discipline as the fuzzy clause (see _try_parse_fuzzy_query): the CJK + runs come from whoosh_compat's ``free_text_tokens`` over the parsed tree, + never from the raw query string, so a term the user negated or restricted + to a field outside the default search fields contributes nothing, instead + of resurfacing as a top-level clause matching every bigram field. + + ``free_text_tokens`` reports no field of its own, so the tokens are + collected one default field at a time: a bare term, which the parser has + already copied onto every default field, is therefore searched across + every bigram field, while ``title:東京`` reaches ``bigram_title`` alone. + Fields whose CJK text is identical (the bare-term case) share a single + parse over all of their bigram fields at once. + + Raw (``analyzed=False``) tokens are used because the bigram fields have + their own character-ngram analyzer: the default fields' word analyzers + have no useful say over a CJK run, and running them first would only + risk dropping it (remove_long) before the run is ever extracted. + Returns None when the query has no CJK free text. + """ + fields_by_text: dict[str, list[str]] = {} + for field, bigram_field in _CJK_BIGRAM_FIELDS.items(): + tokens = wc.free_text_tokens( + ast, + registry=registry, + fields=[field], + analyzed=False, + ) + cjk_text = extract_cjk_text(" ".join(tokens)) + if cjk_text: + fields_by_text.setdefault(cjk_text, []).append(bigram_field) + + clauses: list[tuple[tantivy.Occur, tantivy.Query]] = [ + (tantivy.Occur.Should, query) + for cjk_text, bigram_fields in fields_by_text.items() + if (query := _parse_cjk_text(index, cjk_text, bigram_fields)) is not None + ] + return _any_of(clauses) if clauses else None + + +# A joined fuzzy word string must stay plain words: it goes back through +# tantivy's own query parser, and the raw query text the clause collects +# routinely carries characters that parser reads as grammar (a colon, a +# bracket, a quote, a leading -). Each token is cut into its word runs and +# only those are kept, so no field syntax, pattern, range or grouping can +# reach the parser. Cutting rather than dropping the whole token is what +# keeps ordinary hyphenated, dotted and quoted input ("COVID-19", +# "hello@example.com", "tax reports") contributing to the clause at all. +_WORD_RUN_RE = regex.compile(r"\w+") + +# The one piece of tantivy grammar that survives the cut: its boolean +# keywords are themselves word runs. Only these exact spellings are +# grammar there ("And"/"and" are ordinary terms), so lowercasing exactly +# these turns them back into the ordinary terms the field analyzer used to +# make of them, before the clause switched to raw text. Left alone, a +# quoted phrase would silently restructure the clause ("tax AND reports" +# becoming a conjunction) or fail to parse and drop it entirely +# ("tax AND", or "IN" anywhere). +# +# Only these words are touched: tantivy lowercases query terms with the +# field's own analyzer, and doing it ourselves first is not always the +# same operation (Python folds a final sigma to a different letter than +# tantivy does, and turns Turkish 'İ' into a sequence tantivy then splits +# in two), which would search for terms the index does not contain. +_TANTIVY_KEYWORDS: Final[frozenset[str]] = frozenset({"AND", "OR", "NOT", "IN"}) + + +def _try_parse_fuzzy_query( + index: tantivy.Index, + ast: wc.ast.Node, + registry: wc.FieldRegistry, +) -> tantivy.Query | None: + """Build the fuzzy blend clause from the parsed query's free-text + words, or None if it has none. + + The clause is built by handing tantivy's own query parser a plain + word string (there's no clean AST-level fuzzy equivalent to + whoosh-compat's parse tree, and fuzzy matching was always an + approximate, secondary, 0.1-boosted clause). The words come from + whoosh_compat's ``free_text_tokens`` over the already-parsed AST, + never from the raw query string: raw whoosh grammar (date keywords, + ``[2005 to 2009]`` ranges, bracket-class wildcards) is not tantivy + syntax, and feeding it here used to knock the fuzzy clause out for + the whole query the moment any such construct appeared alongside a + typo'd word. The helper also keeps excluded terms out: a ``NOT``'d + word must not resurface through the fuzzy clause. + + Chosen trade-off: a term explicitly fielded on one of the default + search fields (``correspondent:acme``) contributes its text to the + word string UNFIELDED, so the fuzzy clause searches it across all + default fields rather than just the one the user named. That is + recall-only widening on a secondary 0.1-boosted clause the score + threshold already disciplines, accepted in exchange for never feeding + field syntax to tantivy's parser. What the word string guarantees is + exactly that: no field prefix, pattern, range, grouping or quoting + survives, and the boolean keywords that do survive (they are word + runs) are lowercased into ordinary terms; see _TANTIVY_KEYWORDS. + + The words are the query's RAW text, not the analyzer's output + (``analyzed=False``), because ``index.parse_query`` analyzes whatever + it is given and analysis is not idempotent: ``universities`` stems to + ``univers``, and handing that back stems it again to ``univ``, a term + the index does not contain. ``prefix=True`` hid this as over-broad + matching (``univ`` also prefixes ``unicycle``) rather than as no + matches at all. Raw text is untokenized, which is why it is cut into + word runs above rather than taken whole. + + The ValueError guard stays as insurance (the word string is plain + tokens, so tantivy accepting it is expected, not assumed): on a parse + failure the fuzzy clause is skipped and the exact/CJK clauses stand, + rather than the whole query failing. + """ + tokens = wc.free_text_tokens( + ast, + registry=registry, + fields=_DEFAULT_SEARCH_FIELDS, + analyzed=False, + ) + words = list( + dict.fromkeys( + word.lower() if word in _TANTIVY_KEYWORDS else word + for token in tokens + for word in _WORD_RUN_RE.findall(token) + ), + ) + if not words: + return None + fuzzy_text = " ".join(words) try: - return index.parse_query(cjk_text, fields) - except Exception: + return index.parse_query( + fuzzy_text, + _DEFAULT_SEARCH_FIELDS, + field_boosts=_FIELD_BOOSTS, + fuzzy_fields={f: (True, 1, True) for f in _DEFAULT_SEARCH_FIELDS}, + ) + except ValueError: + logger.debug( + "Skipping fuzzy search clause: token string is not valid " + "tantivy query syntax: %r", + fuzzy_text, + ) return None -def build_permission_filter( - schema: tantivy.Schema, - user: AbstractBaseUser, - viewer_group_ids: Iterable[int] = (), -) -> tantivy.Query: - """ - Build a query filter for user document permissions. - - Creates a query that matches only documents visible to the specified user - according to paperless-ngx permission rules: - - Public documents (no owner) are visible to all users - - Private documents are visible to their owner - - Documents explicitly shared with the user are visible - - Documents shared with one of the user's current groups are visible - - Args: - schema: Tantivy schema for field validation - user: User to check permissions for - viewer_group_ids: Current group memberships for the user - - Returns: - Tantivy query that filters results to visible documents - """ - owner_any = tantivy.Query.exists_query("owner_id") - no_owner = tantivy.Query.boolean_query( - [ - (tantivy.Occur.Must, tantivy.Query.all_query()), - (tantivy.Occur.MustNot, owner_any), - ], - ) - owned = tantivy.Query.term_query(schema, "owner_id", user.pk) - shared = tantivy.Query.term_query(schema, "viewer_id", user.pk) - group_shared = [ - tantivy.Query.term_query(schema, "viewer_group_id", group_id) - for group_id in viewer_group_ids - ] - return tantivy.Query.disjunction_max_query( - [no_owner, owned, shared, *group_shared], - ) - - -DEFAULT_SEARCH_FIELDS = [ +_DEFAULT_SEARCH_FIELDS: Final[list[str]] = [ "title", "content", "correspondent", "document_type", "tag", ] -SIMPLE_SEARCH_FIELDS = ["simple_title", "simple_content"] -TITLE_SEARCH_FIELDS = ["simple_title"] -_CJK_ALL_FIELDS: Final[list[str]] = [ - "bigram_content", - "bigram_title", - "bigram_correspondent", - "bigram_document_type", - "bigram_tag", -] +_SIMPLE_SEARCH_FIELDS: Final[list[str]] = ["simple_title", "simple_content"] +_TITLE_SEARCH_FIELDS: Final[list[str]] = ["simple_title"] +# The bigram (character-ngram) companion of each default search field. +_CJK_BIGRAM_FIELDS: Final[dict[str, str]] = { + field: f"bigram_{field}" for field in _DEFAULT_SEARCH_FIELDS +} _CJK_CONTENT_FIELDS: Final[list[str]] = ["bigram_content"] _CJK_TITLE_FIELDS: Final[list[str]] = ["bigram_title"] _FIELD_BOOSTS = {"title": 2.0} _SIMPLE_FIELD_BOOSTS = {"simple_title": 2.0} -def _simple_query_tokens(raw_query: str) -> list[str]: - # Tokenize and fold via the same analyzer used to index simple_title / - # simple_content, so query terms fold identically to the indexed terms - # (single source of truth for ASCII folding). - return simple_search_tokens(raw_query) +class _ConjunctiveNegations(wc.ast.Visitor[tuple["wc.ast.Node", ...]]): + """Collect the subtrees an AST excludes from every document it matches. + + A negation reached through ``And``/``AndNot``/``Require`` (and through + the required half of an ``AndMaybe``) constrains the whole query, so it + can be re-stated above the blend. ``Or`` is deliberately not descended + into: in ``invoice OR NOT secret`` the negation is one branch's own + condition, and hoisting it would throw away documents the other branch + matches. Nor is a collected subtree descended into, since a negation + inside a negation is not an exclusion. + + Node types with no negation to contribute (every leaf, ``Or``) fall + through to ``generic_visit``. + """ + + def generic_visit(self, node: wc.ast.Node) -> tuple[wc.ast.Node, ...]: + return () + + def visit_not(self, node: wc.ast.Not) -> tuple[wc.ast.Node, ...]: + return (node.child,) + + def visit_andnot(self, node: wc.ast.AndNot) -> tuple[wc.ast.Node, ...]: + return (*self.visit(node.positive), node.negative) + + def visit_and(self, node: wc.ast.And) -> tuple[wc.ast.Node, ...]: + return tuple( + negation for child in node.children for negation in self.visit(child) + ) + + def visit_boosted(self, node: wc.ast.Boosted) -> tuple[wc.ast.Node, ...]: + return self.visit(node.child) + + def visit_andmaybe(self, node: wc.ast.AndMaybe) -> tuple[wc.ast.Node, ...]: + return self.visit(node.required) + + def visit_require(self, node: wc.ast.Require) -> tuple[wc.ast.Node, ...]: + return (*self.visit(node.scored), *self.visit(node.filter_only)) + + +def _negation_clauses( + index: tantivy.Index, + ast: wc.ast.Node, + registry: wc.FieldRegistry, +) -> list[tuple[tantivy.Occur, tantivy.Query]]: + """MustNot clauses for everything ``ast`` excludes conjunctively. + + Each excluded subtree is emitted as its own positive query and attached + with ``MustNot``, rather than emitting a negative query and hoping + tantivy accepts a bare one. + """ + try: + return [ + ( + tantivy.Occur.MustNot, + tantivy_emit(negation, index=index, registry=registry), + ) + for negation in _ConjunctiveNegations().visit(ast) + ] + except QueryError as e: + raise _map_emit_error(e) from e + + +def _any_of(clauses: list[tuple[tantivy.Occur, tantivy.Query]]) -> tantivy.Query: + """Collapse a clause list: none -> empty, one -> itself (no wasted + single-clause boolean_query wrapping), many -> boolean_query(clauses).""" + if not clauses: + return tantivy.Query.empty_query() + if len(clauses) == 1: + return clauses[0][1] + return tantivy.Query.boolean_query(clauses) def _build_simple_token_query( @@ -168,9 +424,7 @@ def _build_simple_token_query( query = tantivy.Query.boost_query(query, boost) field_queries.append((tantivy.Occur.Should, query)) - if len(field_queries) == 1: - return field_queries[0][1] - return tantivy.Query.boolean_query(field_queries) + return _any_of(field_queries) def parse_user_query( @@ -179,52 +433,53 @@ def parse_user_query( tz: tzinfo, ) -> tantivy.Query: """ - Parse user query through the complete preprocessing pipeline. + Parse user query through whoosh-compat, then blend in fuzzy/CJK clauses. - Transforms the raw user query through multiple stages: - 1. Date keyword rewriting (today → ISO 8601 ranges) - 2. Query normalization (comma expansion, whitespace cleanup) - 3. Tantivy parsing with field boosts - 4. Optional fuzzy query blending (if ADVANCED_FUZZY_SEARCH_THRESHOLD set) - - Args: - index: Tantivy index with registered tokenizers - raw_query: Original user query string - tz: Timezone for date boundary calculations - - Returns: - Parsed Tantivy query ready for execution - - Note: - When ADVANCED_FUZZY_SEARCH_THRESHOLD is configured, adds a low-priority - fuzzy query as a Should clause (0.1 boost) to catch approximate matches - while keeping exact matches ranked higher. The threshold value is applied - as a post-search score filter, not during query construction. + 1. wc.parse() against the shared FieldRegistry (whoosh grammar -> AST). + Bare notes:/custom_fields: prefixes resolve to their default subpath + (notes.note:/custom_fields.value:) directly in the registry, via + each JSON field's SubpathSpec(default=True). + 2. Any diagnostics (bad dates/numbers) map to SearchQueryError subclasses + and raise — the view returns HTTP 400 with every offending field + listed, not just the first. + 3. emit() turns the AST into a tantivy.Query directly (no string + round-trip). A QueryError is routed by its Diagnostic's Cause + (_map_emit_error): a construct that parses but can't execute against + tantivy (e.g. a text-field range) is a 400, a registry/schema + mismatch is logged and a 400, and an INTERNAL defect is re-raised. + 4. Optional fuzzy blend (ADVANCED_FUZZY_SEARCH_THRESHOLD) builds a + plain word string from the parsed AST's free-text tokens + (whoosh_compat.free_text_tokens) and feeds THAT to + index.parse_query — never raw_query, whose whoosh grammar (date + keywords, bracket-class wildcards, etc.) tantivy's parser rejects, + which used to silently knock the fuzzy clause out of any mixed + query (see _try_parse_fuzzy_query). + 5. Optional CJK bigram clause, built from the same parsed AST for the + same reason (see _build_ast_cjk_query): a CJK term the query negated + or fielded must not resurface through it. + 6. When any optional clause was added, the query's conjunctive + exclusions are restated as MustNot above the blend + (_negation_clauses): a clause built from positive terms cannot + express them, and as a bare Should it would undo them. """ + registry = get_field_registry(settings.SEARCH_LANGUAGE) + result = wc.parse( + raw_query, + registry=registry, + default_fields=_DEFAULT_SEARCH_FIELDS, + field_boosts=_FIELD_BOOSTS, + tz=tz, + ) + if result.diagnostics: + raise _diagnostics_to_error(result.diagnostics) try: - query_str = translate_query(raw_query, tz) - except SearchQueryError: - # Intentional, user-fixable error (e.g. an unparsable date). Propagate so - # the view can return a 400 with a helpful message rather than falling - # back to the raw (still-invalid) query. - raise - except Exception: # pragma: no cover - defensive - logger.warning("Query translation failed; using raw query", exc_info=True) - query_str = raw_query + exact = tantivy_emit(result.ast, index=index, registry=registry) + except QueryError as e: + raise _map_emit_error(e) from e - exact = index.parse_query( - query_str, - DEFAULT_SEARCH_FIELDS, - field_boosts=_FIELD_BOOSTS, - ) - - # The standard analyzer keeps a whitespace-free CJK run as a single token, - # so substring queries can't match content/title (and long runs are dropped - # by remove_long). Route CJK queries to the bigram fields, whose ngram - # tokenizer indexes overlapping 2-grams for substring matching. cjk_query = ( - _build_cjk_query(index, raw_query, _CJK_ALL_FIELDS) + _build_ast_cjk_query(index, result.ast, registry) if _has_cjk(raw_query) else None ) @@ -235,22 +490,73 @@ def parse_user_query( threshold = settings.ADVANCED_FUZZY_SEARCH_THRESHOLD if threshold is not None: - fuzzy = index.parse_query( - query_str, - DEFAULT_SEARCH_FIELDS, - field_boosts=_FIELD_BOOSTS, - # (prefix=True, distance=1, transposition_cost_one=True) — edit-distance fuzziness - fuzzy_fields={f: (True, 1, True) for f in DEFAULT_SEARCH_FIELDS}, - ) - # 0.1 boost keeps fuzzy hits ranked below exact matches (intentional) - clauses.append((tantivy.Occur.Should, tantivy.Query.boost_query(fuzzy, 0.1))) + fuzzy = _try_parse_fuzzy_query(index, result.ast, registry) + if fuzzy is not None: + clauses.append( + (tantivy.Occur.Should, tantivy.Query.boost_query(fuzzy, 0.1)), + ) if cjk_query is not None: clauses.append((tantivy.Occur.Should, cjk_query)) if len(clauses) == 1: return exact - return tantivy.Query.boolean_query(clauses) + # The fuzzy and CJK clauses are built from positive terms only, so as + # plain Shoulds beside the exact clause they re-admit exactly the + # documents the query excluded. Restate the exclusions once, above the + # whole blend. Redundant against the exact clause, which already + # carries them, but idempotently so. + negations = _negation_clauses(index, result.ast, registry) + if not negations: + return _any_of(clauses) + return tantivy.Query.boolean_query( + [(tantivy.Occur.Must, _any_of(clauses)), *negations], + ) + + +# The three whoosh-compat kinds for a wildcard on a field that cannot +# carry one. d.field_kind supplies the discriminator, so naming the field's +# type needs no second trip through the registry. +_PATTERN_ON_KINDS: Final = frozenset( + { + DiagnosticKind.PATTERN_ON_NUMERIC, + DiagnosticKind.PATTERN_ON_BOOLEAN_EXISTS, + DiagnosticKind.PATTERN_ON_SUBPATH, + }, +) + + +def _diagnostics_to_error(diagnostics: tuple[Diagnostic, ...]) -> SearchQueryError: + errors = [_single_diagnostic_to_error(d) for d in diagnostics] + return errors[0] if len(errors) == 1 else MultipleSearchQueryErrors(errors) + + +def _single_diagnostic_to_error(d: Diagnostic) -> SearchQueryError: + # d.field is a FieldRef, not a str: str(d.field) gives the canonical + # dotted name (an aliased query, e.g. type:, reports document_type). + field_name = str(d.field) if d.field is not None else None + if d.kind is DiagnosticKind.BAD_DATE: + return InvalidDateQuery(field_name, d.raw_value) + if d.kind is DiagnosticKind.BAD_NUMBER: + return InvalidNumberQuery(field_name, d.raw_value) + if d.kind is DiagnosticKind.TOO_DEEP: + return SearchQueryError("The search query is nested too deeply.") + if d.kind in _PATTERN_ON_KINDS: + kind_label = f" ({d.field_kind.name.lower()})" if d.field_kind else "" + return SearchQueryError( + f"Wildcard patterns are not supported for field " + f"{field_name!r}{kind_label}.", + ) + if d.kind is DiagnosticKind.SINGLE_CHAR_BRACKET_RANGE: + field_label = f" for field {field_name!r}" if field_name else "" + return SearchQueryError( + f"{d.raw_value!r} looks like a bracket range{field_label}, but " + "'[' is not a wildcard character on its own. Combine it with a " + "wildcard, e.g. a trailing '*', or double-quote the value to " + "search it as literal text.", + ) + logger.warning("Unmapped parse diagnostic %s: %s", d.kind, d.message) + return SearchQueryError("The search query could not be executed.") def parse_simple_query( @@ -268,7 +574,7 @@ def parse_simple_query( CJK substrings the simple analyzer can't (long whitespace-free runs are dropped by remove_long). """ - tokens = _simple_query_tokens(raw_query) + tokens = simple_search_tokens(raw_query) clauses: list[tuple[tantivy.Occur, tantivy.Query]] = [] if tokens: @@ -291,23 +597,14 @@ def parse_simple_query( ) for token in tokens ] - simple_query = ( - token_queries[0][1] - if len(token_queries) == 1 - else tantivy.Query.boolean_query(token_queries) - ) - clauses.append((tantivy.Occur.Should, simple_query)) + clauses.append((tantivy.Occur.Should, _any_of(token_queries))) if cjk_fields and _has_cjk(raw_query): cjk_q = _build_cjk_query(index, raw_query, cjk_fields) if cjk_q is not None: clauses.append((tantivy.Occur.Should, cjk_q)) - if not clauses: - return tantivy.Query.empty_query() - if len(clauses) == 1: - return clauses[0][1] - return tantivy.Query.boolean_query(clauses) + return _any_of(clauses) def parse_simple_text_highlight_query( @@ -322,13 +619,21 @@ def parse_simple_text_highlight_query( # Strip Tantivy operator chars before tokenizing: this is a plain-text # highlight query, not a structured boolean query, so +/- are separators. - tokens = _simple_query_tokens( + tokens = simple_search_tokens( regex.sub(r"[-+]", " ", raw_query, timeout=_REGEX_TIMEOUT), ) if not tokens: return tantivy.Query.empty_query() - return index.parse_query(" ".join(tokens), ["content"]) + # Quote each token as its own phrase, escaping backslashes and embedded + # quotes. simple search tokens can carry arbitrary Tantivy syntax + # characters (`"`, `:`, `(`, `[`, `/`, ...) that the query-string parser + # would otherwise interpret as query grammar rather than literal text. + quoted_tokens = [ + '"' + token.replace("\\", "\\\\").replace('"', '\\"') + '"' for token in tokens + ] + + return index.parse_query(" ".join(quoted_tokens), ["content"]) def parse_simple_text_query( @@ -342,7 +647,7 @@ def parse_simple_text_query( return parse_simple_query( index, raw_query, - SIMPLE_SEARCH_FIELDS, + _SIMPLE_SEARCH_FIELDS, cjk_fields=_CJK_CONTENT_FIELDS, ) @@ -358,6 +663,6 @@ def parse_simple_title_query( return parse_simple_query( index, raw_query, - TITLE_SEARCH_FIELDS, + _TITLE_SEARCH_FIELDS, cjk_fields=_CJK_TITLE_FIELDS, ) diff --git a/src/documents/search/_translate.py b/src/documents/search/_translate.py deleted file mode 100644 index f020f1c89..000000000 --- a/src/documents/search/_translate.py +++ /dev/null @@ -1,610 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from datetime import UTC -from datetime import datetime -from datetime import timedelta -from typing import TYPE_CHECKING -from typing import TypeAlias - -import regex -from dateutil.relativedelta import relativedelta - -from documents.search._dates import _DATE_KEYWORDS -from documents.search._dates import _DATE_ONLY_FIELDS -from documents.search._dates import _date_only_range -from documents.search._dates import _datetime_range -from documents.search._dates import _field_range_from_dates -from documents.search._dates import _fmt -from documents.search._dates import _precision_bounds -from documents.search._dates import _utc_bounds_for_field - -# Compiled regex that matches any known multi-word (or single-word) date keyword -# at the start of a match position, longest alternatives first so "previous week" -# wins over a hypothetical shorter "previous". -_KEYWORD_VALUE_RE = regex.compile( - "|".join(sorted((regex.escape(k) for k in _DATE_KEYWORDS), key=len, reverse=True)), - regex.IGNORECASE, -) - -if TYPE_CHECKING: - from datetime import tzinfo - -# TODO: this module translates date queries into Tantivy *string* syntax, which -# forces a workaround for something Tantivy's string parser cannot express on -# date fields: open-ended ranges use far-past/far-future string sentinels -# (OPEN_LO/OPEN_HI). These can be replaced with a real tantivy.Query object -# (Query.range_query(..., None) for open bounds) once tantivy-py accepts Python -# datetimes in range_query/term_query on Date fields. That support exists on -# tantivy-py master (PRs #655 + #666) but postdates the pinned 0.26.0 wheel, so -# it is blocked only on a published release > 0.26.0 and a dependency bump. -# (Unparsable dates now raise InvalidDateQuery -> HTTP 400 rather than using a -# no-match string sentinel.) - -# Fields that store exact, non-analyzed comma-joined tokens in the index and so -# need explicit comma->AND expansion (Whoosh KEYWORD(commas=True) set). -MULTI_VALUE_FIELDS = frozenset({"tag", "tag_id", "viewer_id"}) - -# Date fields whose values/ranges get rewritten to RFC3339 Tantivy ranges. -DATE_FIELDS = frozenset({"created", "modified", "added"}) - -# Field aliases: Whoosh (v2) field names that were renamed in the Tantivy schema. -# Preserved here so v2 queries using the old names continue to work without 400 -# errors instead of silently failing. Applied by _render to non-date field tokens. -FIELD_ALIASES: dict[str, str] = { - "type": "document_type", - "type_id": "document_type_id", - "path": "storage_path", - "path_id": "storage_path_id", -} - -# Known schema fields: a comma immediately followed by ``:`` is a clause -# separator. Restricting to known fields prevents URL-like ``http:`` misfires. -KNOWN_FIELDS = frozenset( - { - "title", - "content", - "correspondent", - "document_type", - "type", # v2 alias -> document_type - "storage_path", - "path", # v2 alias -> storage_path - "tag", - "tag_id", - "correspondent_id", - "document_type_id", - "type_id", # v2 alias -> document_type_id - "storage_path_id", - "path_id", # v2 alias -> storage_path_id - "owner_id", - "viewer_id", - "asn", - "page_count", - "num_notes", - "created", - "modified", - "added", - "original_filename", - "checksum", - "notes", - "custom_fields", - }, -) - -_FIELD_RE = regex.compile(r"(?P\w+):") - -# Matches the TO separator inside a range bracket. Handles three forms: -# middle: "lo TO hi" (either lo or hi may be empty) -# trailing: "lo TO" (open upper bound) -# leading: "TO hi" (open lower bound) -# Bounds MAY contain internal spaces (e.g. "-7 days"), so we use .*? / .+? -# and split on the whitespace-delimited " TO " / " to " separator. -_RANGE_RE = regex.compile( - r"^\s*(?P.*?)\s+[Tt][Oo]\s+(?P.+?)\s*$" - r"|" - r"^\s*(?P.+?)\s+[Tt][Oo]\s*$" - r"|" - r"^\s*[Tt][Oo]\s+(?P.+?)\s*$", -) - - -@dataclass(frozen=True, slots=True) -class FieldValue: - field: str - value: str - - -# Produced by the comma-resolution pass (not by scan()). -@dataclass(frozen=True, slots=True) -class FieldValueList: - field: str - values: tuple[str, ...] - - -@dataclass(frozen=True, slots=True) -class FieldRange: - field: str - open: str - lo: str - hi: str - close: str - - -# Produced by the comma-resolution pass (not by scan()). -@dataclass(frozen=True, slots=True) -class Comma: - pass - - -@dataclass(frozen=True, slots=True) -class Passthrough: - raw: str - - -Token: TypeAlias = FieldValue | FieldValueList | FieldRange | Comma | Passthrough - -_CLOSE: dict[str, str] = {"[": "]", "{": "}"} - - -def scan(query: str) -> list[Token]: - """ - Tokenize a raw query into date/comma-aware tokens, leaving everything else - as verbatim ``Passthrough`` runs. Non-recursive: finds the first matching - close bracket/quote. Nested brackets are not valid Tantivy range syntax and - pass through verbatim on mismatch. - """ - tokens: list[Token] = [] - buf: list[str] = [] # accumulates passthrough chars - i, n = 0, len(query) - while i < n: - matched = _match_field_token(query, i) - if matched is None: - buf.append(query[i]) - i += 1 - continue - token, i = matched - if buf and buf[-1] == ",": - buf.pop() - _flush(buf, tokens) - tokens.append(Comma()) - else: - _flush(buf, tokens) - tokens.append(token) - i = _maybe_comma(query, i, tokens) - _flush(buf, tokens) - return tokens - - -def _flush(buf: list[str], tokens: list[Token]) -> None: - """Emit any accumulated passthrough characters as a single token.""" - if buf: - tokens.append(Passthrough("".join(buf))) - buf.clear() - - -def _at_word_boundary(query: str, i: int) -> bool: - """A field token may begin only at the start or after a non-word character.""" - return i == 0 or not (query[i - 1].isalnum() or query[i - 1] == "_") - - -def _match_field_token(query: str, i: int) -> tuple[Token, int] | None: - """ - If a known ``field:`` token starts at ``i``, consume it and return - ``(token, end_index)``; otherwise return None so the caller treats the - character as passthrough. Handles both ``field:[range]`` and ``field:value``, - and returns None when the range/value cannot be consumed. - """ - m = _FIELD_RE.match(query, i) - if m is None or m.group("field") not in KNOWN_FIELDS: - return None - if not _at_word_boundary(query, i): - return None - field = m.group("field") - j = m.end() - if j < len(query) and query[j] in "[{": - return _consume_range(query, j, field) - consumed = _consume_field_value(query, field, j) - if consumed is None: - return None - value, end = consumed - return FieldValue(field, value), end - - -def _consume_field_value(query: str, field: str, start: int) -> tuple[str, int] | None: - """ - Consume a field value starting at ``start``: a multi-word date keyword phrase - (date fields only), or a bare/quoted value, then absorb any comma-joined - continuation that is not a clause separator. ``resolve_commas`` later splits a - multi-value field's joined value into a ``FieldValueList``; for other fields - the comma stays literal. - """ - n = len(query) - consumed = None - if field in DATE_FIELDS: - km = _KEYWORD_VALUE_RE.match(query, start) - if km is not None and (km.end() >= n or query[km.end()] in " \t),"): - consumed = (km.group(0), km.end()) - if consumed is None: - consumed = _consume_value(query, start) - if consumed is None: - return None - value, k = consumed - while k < n and query[k] == ",": - if _looks_like_known_field(query, k + 1): - break # clause separator: left for _maybe_comma to emit a Comma() - more = _consume_value(query, k + 1) - if more is None: - break - value = f"{value},{more[0]}" - k = more[1] - return value, k - - -def _consume_range( - query: str, - start: int, - field: str, -) -> tuple[FieldRange, int] | None: - """Consume ``[lo TO hi]`` / ``{lo TO hi}`` from ``start`` (the bracket).""" - open_br = query[start] - close_br = _CLOSE[open_br] - end = query.find(close_br, start + 1) - if end == -1: - return None - inner = query[start + 1 : end] - m = _RANGE_RE.match(inner) - if m is not None: - if m.group("lo") is not None or m.group("hi") is not None: - # Middle form: "lo TO hi" (either may be empty string) - lo = (m.group("lo") or "").strip() - hi = (m.group("hi") or "").strip() - elif m.group("lo2") is not None: - # Trailing form: "lo TO" - lo = m.group("lo2").strip() - hi = "" - else: - # Leading form: "TO hi" - lo = "" - hi = (m.group("hi2") or "").strip() - else: - lo, hi = inner.strip(), "" - return FieldRange(field, open_br, lo, hi, close_br), end + 1 - - -def _consume_value(query: str, start: int) -> tuple[str, int] | None: - """Consume a bare or quoted field value from ``start``, stopping at comma.""" - n = len(query) - if start >= n or query[start] in " \t": - return None - if query[start] in "\"'": - quote = query[start] - end = query.find(quote, start + 1) - if end == -1: - return None - return query[start : end + 1], end + 1 - j = start - while j < n and query[j] not in " \t),": - j += 1 - return query[start:j], j - - -def _looks_like_known_field(query: str, pos: int) -> bool: - """True if a known ``field:`` token starts at ``pos``.""" - m = _FIELD_RE.match(query, pos) - return bool(m and m.group("field") in KNOWN_FIELDS) - - -def _maybe_comma(query: str, i: int, tokens: list) -> int: - """If a clause-separator comma follows at ``i``, emit ``Comma()`` and advance.""" - if i < len(query) and query[i] == "," and _looks_like_known_field(query, i + 1): - tokens.append(Comma()) - return i + 1 - return i - - -def resolve_commas(tokens: list) -> list: - """ - Collapse value-list commas into ``FieldValueList`` and keep clause-separator - commas as ``Comma``. (Clause-sep commas are already emitted by ``scan`` via - the value-stop logic; this pass folds value-lists.) - """ - out: list = [] - for tok in tokens: - if ( - isinstance(tok, FieldValue) - and tok.field in MULTI_VALUE_FIELDS - and "," in tok.value - ): - values = tuple(v for v in tok.value.split(",") if v) - out.append(FieldValueList(tok.field, values)) - else: - out.append(tok) - return out - - -class SearchQueryError(ValueError): - """ - Base for user-fixable search query errors. - - Carries a message safe to surface to the user (no internal details). The view - layer catches this and returns an HTTP 400, so any future subclass (unknown - field, malformed range, wrapped parser errors) gets the same treatment. - """ - - -class InvalidDateQuery(SearchQueryError): - """Raised when a date field value or range bound cannot be parsed.""" - - def __init__(self, field: str, value: str) -> None: - self.field = field - self.value = value - super().__init__(f"Invalid date value {value!r} for field {field!r}.") - - -_DIGITS_RE = regex.compile(r"^\d{4}(?:\d{2}){0,2}$") -_ISO_RE = regex.compile(r"^\d{4}(?:-\d{2}(?:-\d{2})?)?$") - - -def translate_scalar(field: str, value: str, tz: tzinfo) -> str: - """Translate a bare date-field value to a Tantivy range string.""" - bare = value.strip("\"'").lower() - if bare in _DATE_KEYWORDS: - if field in _DATE_ONLY_FIELDS: - return f"{field}:{_date_only_range(bare, tz)}" - return f"{field}:{_datetime_range(bare, tz)}" - digits = value.replace("-", "") - if _DIGITS_RE.match(value) or _ISO_RE.match(value): - bounds = _precision_bounds(digits) - if bounds is None: - raise InvalidDateQuery(field, value) - return _field_range_from_dates(field, bounds[0], bounds[1], tz) - if regex.fullmatch(r"\d{14}", value): - try: - dt = datetime( - int(value[0:4]), - int(value[4:6]), - int(value[6:8]), - int(value[8:10]), - int(value[10:12]), - int(value[12:14]), - tzinfo=UTC, - ) - except ValueError: - raise InvalidDateQuery(field, value) from None - iso = _fmt(dt) - return f"{field}:[{iso} TO {iso}]" - # Unrecognized shape -> tell the user their date is malformed rather than - # silently matching nothing or emitting invalid Tantivy syntax. - raise InvalidDateQuery(field, value) - - -# Open-bound sentinels for date ranges. These far-past/far-future strings allow -# open-ended ranges to be expressed as Tantivy string queries until tantivy-py -# exposes Query.range_query(..., None) on Date fields (see module TODO). -OPEN_LO = "0001-01-01T00:00:00Z" -OPEN_HI = "9999-12-31T23:59:59Z" - - -# Matches compact now-offset tokens like now-7d, now+1h, now-30m. -_NOW_COMPACT_RE = regex.compile( - r"^now(?P[+-])(?P\d+)(?P[dhm])$", - regex.IGNORECASE, -) - -# Matches "±N " Whoosh-style offsets (e.g. -7 days, -1 week, +3 hours). -# Whoosh's own date parser (qparser.dateparse.PlusMinus) additionally accepted -# abbreviated unit spellings (e.g. "yrs", "yr", "y", "mos", "wks", "hrs", "mins", -# "secs"); saved views/searches created under the old Whoosh backend can still -# contain those tokens (e.g. "-999yrs"), so they are accepted here too and -# normalized to a canonical unit via _UNIT_ALIASES below. -_NOW_SPACED_RE = regex.compile( - r"^(?P[+-])(?P\d+)\s*" - r"(?Pyears|year|yrs|yr|ys|y" - r"|months|month|mons|mon|mos|mo" - r"|weeks|week|wks|wk|ws|w" - r"|days|day|dys|dy|ds|d" - r"|hours|hour|hrs|hr|hs|h" - r"|minutes|minute|mins|min|ms|m" - r"|seconds|second|secs|sec|s)$", - regex.IGNORECASE, -) - -# Maps every accepted unit spelling (including Whoosh-era abbreviations) to the -# canonical unit name used as a key into the delta map in _resolve_relative_bound. -_UNIT_ALIASES: dict[str, str] = { - alias: canonical - for canonical, aliases in { - "year": ("years", "year", "yrs", "yr", "ys", "y"), - "month": ("months", "month", "mons", "mon", "mos", "mo"), - "week": ("weeks", "week", "wks", "wk", "ws", "w"), - "day": ("days", "day", "dys", "dy", "ds", "d"), - "hour": ("hours", "hour", "hrs", "hr", "hs", "h"), - "minute": ("minutes", "minute", "mins", "min", "ms", "m"), - "second": ("seconds", "second", "secs", "sec", "s"), - }.items() - for alias in aliases -} - - -def _resolve_relative_bound(token: str) -> datetime | None: - """ - Resolve a relative bound token to an exact UTC instant, or return None. - - Supported forms: - - ``now`` -> current UTC instant - - ``now+/-d/h/m`` -> now +/- timedelta (d=days, h=hours, m=minutes) - - ``±N `` -> now +/- delta; month/year use relativedelta; - unit also accepts Whoosh-era abbreviations - (e.g. "yrs", "mos", "wks", "hrs", "mins", "secs") - """ - stripped = token.strip() - low = stripped.lower() - now = datetime.now(UTC) - - if low == "now": - return now - - m = _NOW_COMPACT_RE.match(stripped) - if m: - sign = 1 if m.group("sign") == "+" else -1 - n = int(m.group("n")) - unit = m.group("unit").lower() - delta = ( - sign - * { - "d": timedelta(days=n), - "h": timedelta(hours=n), - "m": timedelta(minutes=n), - }[unit] - ) - return now + delta - - m = _NOW_SPACED_RE.match(stripped) - if m: - sign = 1 if m.group("sign") == "+" else -1 - n = int(m.group("n")) - unit = _UNIT_ALIASES[m.group("unit").lower()] - delta_map: dict[str, timedelta | relativedelta] = { - "second": timedelta(seconds=n), - "minute": timedelta(minutes=n), - "hour": timedelta(hours=n), - "day": timedelta(days=n), - "week": timedelta(weeks=n), - "month": relativedelta(months=n), - "year": relativedelta(years=n), - } - return now - delta_map[unit] if sign == -1 else now + delta_map[unit] - - return None - - -def _bound_datetimes( - field: str, - token: str, - tz: tzinfo, -) -> tuple[datetime, datetime] | None: - """ - Return (floor_dt, ceil_dt) UTC datetimes for a single range bound token, or - None if the token is unparsable. ``now`` and relative offsets resolve to the - current instant (floor == ceil == that instant; no day-flooring). - """ - token = token.strip() - - # Try relative/now forms first (before stripping hyphens which would mangle them). - rel = _resolve_relative_bound(token) - if rel is not None: - return rel, rel - - # Full ISO datetime token (contains "T"): parse directly and return an exact - # instant (floor == ceil). Python 3.11+ datetime.fromisoformat accepts trailing Z. - if "T" in token: - try: - dt = datetime.fromisoformat(token) - # Ensure timezone-aware UTC result. - dt = dt.replace(tzinfo=UTC) if dt.tzinfo is None else dt.astimezone(UTC) - return dt, dt - except ValueError: - return None - - digits = token.replace("-", "") - bounds = _precision_bounds(digits) - if bounds is None: - return None - start, end = bounds - return _utc_bounds_for_field(field, start, end, tz) - - -def _render(tok: Token, tz: tzinfo) -> str: - """Render a single token back to a Tantivy query string fragment.""" - if isinstance(tok, Passthrough): - return tok.raw - if isinstance(tok, Comma): - return " AND " - if isinstance(tok, FieldValueList): - field = FIELD_ALIASES.get(tok.field, tok.field) - return " AND ".join(f"{field}:{v}" for v in tok.values) - if isinstance(tok, FieldValue): - field = FIELD_ALIASES.get(tok.field, tok.field) - if field in DATE_FIELDS: - return translate_scalar(field, tok.value, tz) - return f"{field}:{tok.value}" - if isinstance(tok, FieldRange): - field = FIELD_ALIASES.get(tok.field, tok.field) - if field in DATE_FIELDS: - return translate_range(field, tok.lo, tok.hi, tz) - return f"{field}:{tok.open}{tok.lo} TO {tok.hi}{tok.close}" - return "" # pragma: no cover - - -# Post-render operator normalization patterns: collapse repeated whitespace and -# strip spaced/trailing Tantivy boolean operators that would otherwise be invalid. -_MULTI_SPACE_RE = regex.compile(r" {2,}") -_TRAILING_OP_RE = regex.compile(r"\s+[-+]+\s*$") -_SPACED_OP_RE = regex.compile(r"\s+[-+]\s+") - - -def _normalize_operators(text: str) -> str: - """ - Collapse multiple spaces, strip trailing dangling operators, and replace - spaced operators (`` - `` / `` + ``) with a single space. - - Applied only to Passthrough fragments (the rendered output is scanned for - operator artifacts outside bracketed ranges) via a post-render pass on the - full rendered string. This preserves date ranges (``[... TO ...]``) verbatim - while cleaning natural-language separators in the surrounding text. - """ - text = _MULTI_SPACE_RE.sub(" ", text) - text = _TRAILING_OP_RE.sub("", text).strip() - text = _SPACED_OP_RE.sub(" ", text).strip() - return text - - -def translate_query(raw: str, tz: tzinfo) -> str: - """Translate a raw Whoosh-style query into Tantivy-compatible syntax.""" - tokens = resolve_commas(scan(raw)) - rendered = "".join(_render(t, tz) for t in tokens) - return _normalize_operators(rendered) - - -def translate_range(field: str, lo: str, hi: str, tz: tzinfo) -> str: - """Translate a date-field ``[lo TO hi]`` range to a Tantivy ISO range string. - - Handles partial-date bounds (YYYY, YYYYMM, YYYYMMDD, ISO dash variants), - open bounds (empty string -> OPEN_LO/OPEN_HI), ``now``, and reversed ranges - (swaps tokens before computing floor/ceil so the span is always correct). - """ - lo_s = lo.strip() - hi_s = hi.strip() - - # Parse both bounds to (floor, ceil) pairs when present. - lo_pair: tuple[datetime, datetime] | None = None - hi_pair: tuple[datetime, datetime] | None = None - - if lo_s: - lo_pair = _bound_datetimes(field, lo_s, tz) - if lo_pair is None: - raise InvalidDateQuery(field, lo_s) - if hi_s: - hi_pair = _bound_datetimes(field, hi_s, tz) - if hi_pair is None: - raise InvalidDateQuery(field, hi_s) - - # Detect a reversed range: only swap when BOTH bounds are present. - if lo_pair is not None and hi_pair is not None and lo_pair[0] > hi_pair[0]: - lo_pair, hi_pair = hi_pair, lo_pair - - lo_iso = _fmt(lo_pair[0]) if lo_pair is not None else OPEN_LO - - # A bound resolves to (floor, ceil) where floor == ceil for an exact instant - # (a full ISO datetime, "now", or a "+/-N unit" offset) and floor != ceil for - # a coarser period token (year/month/day precision). Only the latter needs a - # half-open close: its ceil is the start of the *next* period and must be - # excluded, or that instant (e.g. the 1st of next month) wrongly matches. - if hi_pair is not None: - hi_iso = _fmt(hi_pair[1]) - hi_close = "]" if hi_pair[0] == hi_pair[1] else "}" - else: - hi_iso = OPEN_HI - hi_close = "]" - - return f"{field}:[{lo_iso} TO {hi_iso}{hi_close}" diff --git a/src/documents/tests/search/conftest.py b/src/documents/tests/search/conftest.py index 4f9216df7..b65ce8899 100644 --- a/src/documents/tests/search/conftest.py +++ b/src/documents/tests/search/conftest.py @@ -1,15 +1,11 @@ from __future__ import annotations -import tempfile from typing import TYPE_CHECKING import pytest -import tantivy from documents.search._backend import TantivyBackend from documents.search._backend import reset_backend -from documents.search._schema import build_schema -from documents.search._tokenizer import register_tokenizers if TYPE_CHECKING: from collections.abc import Generator @@ -35,11 +31,3 @@ def backend() -> Generator[TantivyBackend, None, None]: finally: b.close() reset_backend() - - -@pytest.fixture(scope="module") -def index() -> tantivy.Index: - """A real Tantivy index for parse-acceptance tests (module scope for speed).""" - idx = tantivy.Index(build_schema(), path=tempfile.mkdtemp()) - register_tokenizers(idx, "english") - return idx diff --git a/src/documents/tests/search/test_acceptance.py b/src/documents/tests/search/test_acceptance.py new file mode 100644 index 000000000..1a1557c71 --- /dev/null +++ b/src/documents/tests/search/test_acceptance.py @@ -0,0 +1,411 @@ +"""Result-level acceptance corpus: real documents indexed via build_schema(), +real queries run through parse_user_query(), matched-document-ID sets +asserted — not intermediate ASTs or query strings. This is paperless-ngx's +analogue of whoosh-compat's own tests/emitter/test_acceptance_e2e.py. + +Supersedes test_query.py's TestParseUserQuery result-level cases. +""" + +from __future__ import annotations + +from datetime import UTC +from datetime import datetime +from typing import TYPE_CHECKING + +import pytest +import time_machine +from django.contrib.auth.models import User + +from documents.models import CustomField +from documents.models import CustomFieldInstance +from documents.models import Document +from documents.models import DocumentType +from documents.models import Note +from documents.models import StoragePath +from documents.search._query import parse_user_query + +if TYPE_CHECKING: + from documents.search._backend import TantivyBackend + +pytestmark = [pytest.mark.search, pytest.mark.django_db] + +FROZEN_NOW = datetime(2026, 6, 15, 12, 0, tzinfo=UTC) + + +def _matched_ids(backend: TantivyBackend, query: str) -> set[int]: + return set(backend.search_ids(query, user=None)) + + +def _index(backend: TantivyBackend, **kwargs: object) -> Document: + """Create a Document and index it in one step, for the common case + where nothing needs to happen between the two (no related Note/ + CustomFieldInstance to attach first).""" + doc = Document.objects.create(**kwargs) + backend.add_or_update(doc) + return doc + + +@pytest.fixture +def indexed_documents(backend: TantivyBackend) -> dict[str, int]: + """Index a small fixture set, return {label: doc_id} for corpus queries.""" + docs = { + "invoice_2020": _index( + backend, + title="Invoice 2020", + content="invoice total due", + checksum="acc-invoice-2020", + archive_serial_number=100, + ), + "invoice_2021": _index( + backend, + title="Invoice 2021", + content="invoice total due", + checksum="acc-invoice-2021", + archive_serial_number=101, + ), + "invoice_2023": _index( + backend, + title="Invoice 2023", + content="invoice total due", + checksum="acc-invoice-2023", + archive_serial_number=102, + ), + "receipt_2022": _index( + backend, + title="Receipt 2022", + content="receipt total due", + checksum="acc-receipt-2022", + archive_serial_number=103, + ), + } + return {label: doc.pk for label, doc in docs.items()} + + +class TestIssue13568BracketWildcard: + """paperless-ngx#13568: title:202[0-3]* must keep its character class, + not fold to a prefix query that silently drops it.""" + + def test_bracket_class_wildcard_matches_only_in_range_years( + self, + backend: TantivyBackend, + indexed_documents: dict[str, int], + ) -> None: + # [0-1] (not [0-3]) is deliberate: the fixture's four years are + # 2020/2021/2022/2023, i.e. their trailing digit is 0/1/2/3 + # respectively - a [0-3] class would match all four and the test + # would pass even if the character class were silently dropped and + # folded to an unconstrained "202*" prefix. [0-1] partitions the + # fixture into a genuine in-range/out-of-range split. + matched = _matched_ids(backend, "title:202[0-1]*") + expected = { + indexed_documents["invoice_2020"], + indexed_documents["invoice_2021"], + } + assert matched == expected, ( + "title:202[0-1]* must match 2020/2021 titles and exclude 2022/2023 " + "- if this matches everything, the wildcard's character class was " + "silently dropped (issue #13568's original bug)" + ) + + +class TestFieldBoosts: + def test_title_boost_ranks_title_match_above_content_only_match( + self, + backend: TantivyBackend, + ) -> None: + title_match = _index( + backend, + title="urgent", + content="nothing else relevant", + checksum="acc-boost-title", + ) + _index( + backend, + title="nothing", + content="urgent matter here", + checksum="acc-boost-content", + ) + query = parse_user_query(backend._index, "urgent", UTC) + searcher = backend._index.searcher() + results = searcher.search(query, limit=10) + ranked_ids = [ + searcher.doc(addr).to_dict()["id"][0] for _score, addr in results.hits + ] + assert ranked_ids[0] == title_match.pk + + +class TestJsonSubpaths: + def test_notes_user_matches_document_with_that_note_author( + self, + backend: TantivyBackend, + ) -> None: + alice = User.objects.create_user(username="alice") + doc_with_note = Document.objects.create( + title="Has note", + content="x", + checksum="acc-note-with", + ) + Note.objects.create(document=doc_with_note, user=alice, note="reminder") + backend.add_or_update(doc_with_note) + _index(backend, title="No note", content="x", checksum="acc-note-without") + matched = _matched_ids(backend, "notes.user:alice") + assert matched == {doc_with_note.pk} + + def test_custom_fields_name_and_value_combine( + self, + backend: TantivyBackend, + ) -> None: + field = CustomField.objects.create( + name="Contract Number", + data_type=CustomField.FieldDataType.STRING, + ) + other_field = CustomField.objects.create( + name="Other Field", + data_type=CustomField.FieldDataType.STRING, + ) + matching = Document.objects.create( + title="Matching", + content="x", + checksum="acc-cf-matching", + ) + CustomFieldInstance.objects.create( + document=matching, + field=field, + value_text="policy", + ) + backend.add_or_update(matching) + non_matching = Document.objects.create( + title="Non-matching", + content="x", + checksum="acc-cf-nonmatching", + ) + CustomFieldInstance.objects.create( + document=non_matching, + field=other_field, + value_text="policy", + ) + backend.add_or_update(non_matching) + matched = _matched_ids( + backend, + 'custom_fields.name:"Contract Number" custom_fields.value:policy', + ) + assert matched == {matching.pk} + + +class TestUnregisteredIdFieldFoldsToLiteralText: + """tag_id, owner_id, etc. are intentionally excluded from the + FieldRegistry - always internal index columns, never meant to be + query-addressable. Prove an unregistered field folds to a literal + text search that matches nothing, rather than erroring.""" + + def test_tag_id_query_matches_nothing( + self, + backend: TantivyBackend, + indexed_documents: dict[str, int], + ) -> None: + matched = _matched_ids(backend, "tag_id:5") + assert matched == set() + + +class TestFuzzyBlendSurvivesWhooshGrammar: + """A query mixing whoosh-only grammar (a date keyword) with a typo'd + free-text word must still fuzzy-match the intended document when + ADVANCED_FUZZY_SEARCH_THRESHOLD is enabled. The fuzzy clause is built + from the parsed query's free-text tokens (whoosh_compat's + free_text_tokens), never from the raw query string, so whoosh grammar + that tantivy's own parser rejects cannot knock the fuzzy clause out.""" + + def test_typo_fuzzy_matches_alongside_date_keyword( + self, + backend: TantivyBackend, + settings, + ) -> None: + settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5 + with time_machine.travel(FROZEN_NOW, tick=False): + doc = _index( + backend, + title="Receipt March", + content="receipt total due", + checksum="fuzzy-blend-1", + archive_serial_number=900, + ) + # Sanity: the exact spelling matches through the exact clause. + assert doc.pk in _matched_ids(backend, "added:today receipt") + # The regression: the misspelling (one transposition) only + # matches via the fuzzy clause, and "added:today" is + # whoosh-only grammar tantivy's parser rejects, so raw-string + # fuzzy parsing skips the clause entirely and this returns + # nothing. The typo is deliberate; keep codespell away from it. + typo_query = "added:today reciept" # codespell:ignore reciept + assert doc.pk in _matched_ids(backend, typo_query) + + def test_negated_words_do_not_fuzzy_match( + self, + backend: TantivyBackend, + settings, + ) -> None: + # A term the user excluded must not resurface through the fuzzy + # clause. The shape is chosen so this genuinely discriminates: the + # indexed document contains the NOT'd word but NOT the positive + # word, so nothing matches the exact clause, and a fuzzy string + # naively built from ALL words (including the NOT'd one) would + # make this document the sole hit, normalize its score to 1.0, + # and survive any threshold. (A shape with an exact-matching + # sibling document does NOT discriminate: normalization ranks the + # resurfaced doc far below the exact match and the threshold cuts + # it even for a naive implementation.) + settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5 + with time_machine.travel(FROZEN_NOW, tick=False): + _index( + backend, + title="Receipt Archive", + content="receipt archived stack", + checksum="fuzzy-blend-2", + archive_serial_number=901, + ) + assert _matched_ids(backend, "added:today total NOT receipt") == set() + + +class TestUnquotedDateKeywordPhrases: + """The unquoted spelling (added:previous month) is honored natively by + whoosh-compat's own grammar for this closed phrase vocabulary — no + app-level rewrite is involved. Pins that the historically supported + spelling keeps working now that paperless no longer pre-quotes it.""" + + @pytest.fixture + def period_documents(self, backend: TantivyBackend) -> dict[str, int]: + with time_machine.travel(FROZEN_NOW, tick=False): + in_may = _index( + backend, + title="May Doc", + content="statement", + checksum="kw-may", + archive_serial_number=910, + added=datetime(2026, 5, 20, 12, 0, tzinfo=UTC), + ) + in_june = _index( + backend, + title="June Doc", + content="statement", + checksum="kw-june", + archive_serial_number=911, + added=datetime(2026, 6, 10, 12, 0, tzinfo=UTC), + ) + return {"in_may": in_may.pk, "in_june": in_june.pk} + + @pytest.mark.parametrize( + "query", + [ + pytest.param("added:previous month", id="unquoted"), + pytest.param('added:"previous month"', id="quoted"), + pytest.param("added:Previous Month", id="unquoted-mixed-case"), + ], + ) + def test_unquoted_matches_the_same_documents_as_quoted( + self, + backend: TantivyBackend, + period_documents: dict[str, int], + query: str, + ) -> None: + with time_machine.travel(FROZEN_NOW, tick=False): + assert _matched_ids(backend, query) == {period_documents["in_may"]} + + @pytest.mark.parametrize( + "query", + [ + pytest.param("added:this month", id="this-month"), + pytest.param("added:this year", id="this-year"), + pytest.param("added:previous week", id="previous-week"), + pytest.param("added:previous quarter", id="previous-quarter"), + pytest.param("added:previous year", id="previous-year"), + pytest.param("created:previous month", id="created-field"), + pytest.param("modified:previous month", id="modified-field"), + ], + ) + def test_every_phrase_and_date_field_parses_without_error( + self, + backend: TantivyBackend, + period_documents: dict[str, int], + query: str, + ) -> None: + # The whole vocabulary times every date field must at least parse + # and search cleanly (no SearchQueryError -> no HTTP 400); exact + # window semantics are whoosh-compat's, pinned in its own suite. + with time_machine.travel(FROZEN_NOW, tick=False): + _matched_ids(backend, query) + + def test_text_field_keyword_words_are_ordinary_text( + self, + backend: TantivyBackend, + period_documents: dict[str, int], + ) -> None: + # "previous month" after a TEXT field (or unfielded) is ordinary + # text, not a date phrase: a title actually containing the words + # matches, and the date-window documents do not. + with time_machine.travel(FROZEN_NOW, tick=False): + wordy = _index( + backend, + title="Notes from the previous month", + content="meeting notes", + checksum="kw-text", + archive_serial_number=912, + ) + assert _matched_ids(backend, "title:previous month") == {wordy.pk} + + +class TestFieldAliases: + """type:/path: are registry aliases for document_type:/storage_path:. + The only other alias coverage is parse-shape; these prove resolution + end-to-end against a real index.""" + + def test_type_alias_and_canonical_name_match_the_same_document( + self, + backend: TantivyBackend, + ) -> None: + invoice_type = DocumentType.objects.create(name="invoice") + # Discriminating shape: document_type is itself a default search + # field, so if alias resolution ever broke and "type:invoice" + # demoted to unfielded text, the token would STILL match the typed + # document through the field value. The decoy carries the query + # word in content, so a demoted search matches BOTH documents and + # the exact-set assertions fail. (The title avoids stemming to + # "type": english stems Typed -> type.) + typed = _index( + backend, + title="First", + content="quarterly statement", + checksum="alias-type-1", + document_type=invoice_type, + ) + _index( + backend, + title="Second", + content="invoice mentioned in body", + checksum="alias-type-2", + ) + assert _matched_ids(backend, "type:invoice") == {typed.pk} + assert _matched_ids(backend, "document_type:invoice") == {typed.pk} + + def test_path_alias_and_canonical_name_match_the_same_document( + self, + backend: TantivyBackend, + ) -> None: + archive = StoragePath.objects.create(name="archive", path="archive/{title}") + stored = _index( + backend, + title="Stored", + content="quarterly statement", + checksum="alias-path-1", + storage_path=archive, + ) + # storage_path is NOT a default search field today, so a demoted + # "path:archive" already matches nothing; the content decoy keeps + # this test discriminating even if it ever joins the defaults. + _index( + backend, + title="Loose", + content="archive mentioned in body", + checksum="alias-path-2", + ) + assert _matched_ids(backend, "path:archive") == {stored.pk} + assert _matched_ids(backend, "storage_path:archive") == {stored.pk} diff --git a/src/documents/tests/search/test_checksum_prefix_queries.py b/src/documents/tests/search/test_checksum_prefix_queries.py new file mode 100644 index 000000000..c7dd66076 --- /dev/null +++ b/src/documents/tests/search/test_checksum_prefix_queries.py @@ -0,0 +1,57 @@ +"""``checksum`` wildcard patterns stay literal end to end, once user queries +route through whoosh-compat. + +The registry-level fact (the pattern normalizer folds a KEYWORD pattern +rather than stemming it) is pinned on its own in +``test_keyword_pattern_literal.py``. This proves it actually reaches a real +query: ``checksum:ceded*`` must match only the document whose checksum +starts with "ceded", not the one whose checksum stems to the same run. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from documents.models import Document + +if TYPE_CHECKING: + from documents.search._backend import TantivyBackend + +pytestmark = [pytest.mark.search, pytest.mark.django_db] + +CEDEF00D = "cedef00ddeadbeef0123456789abcdef01234567" +CEDEDEAD = "cededeadbeef567801234567" + "89abcdef01234567" + + +class TestChecksumPrefixQueries: + @pytest.fixture + def indexed(self, backend: TantivyBackend) -> None: + for i, checksum in enumerate((CEDEF00D, CEDEDEAD)): + doc = Document.objects.create( + title=f"Checksum doc {i}", + content="invoices for the quarter", + checksum=checksum, + archive_serial_number=940 + i, + ) + backend.add_or_update(doc) + + def _ids(self, backend: TantivyBackend, query: str) -> set[int]: + return set(backend.search_ids(query, user=None)) + + def test_prefix_matches_only_the_document_that_starts_with_it( + self, + backend: TantivyBackend, + indexed: None, + ) -> None: + matched = self._ids(backend, "checksum:ceded*") + expected = Document.objects.get(checksum=CEDEDEAD).pk + assert matched == {expected} + + def test_text_prefix_still_reaches_the_stemmed_index( + self, + backend: TantivyBackend, + indexed: None, + ) -> None: + assert len(self._ids(backend, "invoice*")) == 2 diff --git a/src/documents/tests/search/test_cjk_clause.py b/src/documents/tests/search/test_cjk_clause.py new file mode 100644 index 000000000..ebd907fe9 --- /dev/null +++ b/src/documents/tests/search/test_cjk_clause.py @@ -0,0 +1,133 @@ +"""The CJK bigram clause blended into QUERY-mode searches. + +The clause exists so CJK runs are matchable at all (the default analyzers +keep a whitespace-free CJK run as one indivisible token), but it must not +widen the query beyond what the user asked for: a CJK term the query +excludes, or restricts to one field, must not come back through it. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from documents.models import Document + +if TYPE_CHECKING: + from pytest_django.fixtures import SettingsWrapper + + from documents.search._backend import TantivyBackend + +pytestmark = [pytest.mark.search, pytest.mark.django_db] + + +def _matched_ids(backend: TantivyBackend, query: str) -> set[int]: + return set(backend.search_ids(query, user=None)) + + +def _index(backend: TantivyBackend, **kwargs: object) -> Document: + doc = Document.objects.create(**kwargs) + backend.add_or_update(doc) + return doc + + +class TestCjkClauseFollowsTheParsedQuery: + def test_negated_cjk_term_is_excluded(self, backend: TantivyBackend) -> None: + """'invoice NOT 漢字' must not return the document containing 漢字.""" + with_cjk = _index( + backend, + title="Invoice A", + content="invoice total 漢字", + checksum="cjk-neg-1", + ) + without_cjk = _index( + backend, + title="Invoice B", + content="invoice total only", + checksum="cjk-neg-2", + ) + + assert _matched_ids(backend, "invoice") == {with_cjk.pk, without_cjk.pk} + assert _matched_ids(backend, "invoice NOT 漢字") == {without_cjk.pk} + + @pytest.mark.parametrize( + ("threshold", "expected"), + [ + pytest.param(None, {"titled"}, id="fuzzy_off"), + pytest.param(0.0, {"titled", "content_only"}, id="fuzzy_on"), + ], + ) + def test_fielded_cjk_term_searches_only_that_field( + self, + backend: TantivyBackend, + settings: SettingsWrapper, + threshold: float | None, + expected: set[str], + ) -> None: + """'title:東京' must not match a document whose 東京 is in the content. + + The CJK clause honours the field. The fuzzy clause, when enabled, + does not: it contributes every free-text term UNFIELDED by design + (see _try_parse_fuzzy_query), so it brings the content-only + document back on its own 0.1-boosted terms. That is the documented + trade-off, pinned here so it stays deliberate. + """ + settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = threshold + content_only = _index( + backend, + title="Tokyo report", + content="東京都の人口は約1400万人です", + checksum="cjk-field-1", + ) + titled = _index( + backend, + title="東京都の報告書", + content="an english summary", + checksum="cjk-field-2", + ) + pks = {"titled": titled.pk, "content_only": content_only.pk} + + assert _matched_ids(backend, "東京") == set(pks.values()) + assert _matched_ids(backend, "title:東京") == {pks[label] for label in expected} + + def test_cjk_on_a_non_default_field_builds_no_clause( + self, + backend: TantivyBackend, + ) -> None: + """A CJK term restricted to a field outside the default search fields + has nothing to contribute to the bigram clause: 'notes:東京' must not + fall back to matching 東京 in the content.""" + _index( + backend, + title="Tokyo report", + content="東京都の人口は約1400万人です", + checksum="cjk-notes-1", + ) + + assert _matched_ids(backend, "notes:東京") == set() + + def test_bare_cjk_term_still_matches_every_default_field( + self, + backend: TantivyBackend, + ) -> None: + """The clause's reason for existing: an unfielded CJK run matches + wherever it is indexed, and does so alongside a latin term.""" + in_content = _index( + backend, + title="report", + content="本文に重要な情報", + checksum="cjk-bare-1", + ) + in_title = _index( + backend, + title="重要な報告書", + content="english only", + checksum="cjk-bare-2", + ) + + assert _matched_ids(backend, "重要") == {in_content.pk, in_title.pk} + assert _matched_ids(backend, "重要 OR report") == { + in_content.pk, + in_title.pk, + } diff --git a/src/documents/tests/search/test_date_keyword_phrase_removal.py b/src/documents/tests/search/test_date_keyword_phrase_removal.py new file mode 100644 index 000000000..06ddb3f3c --- /dev/null +++ b/src/documents/tests/search/test_date_keyword_phrase_removal.py @@ -0,0 +1,72 @@ +"""Pins the correctness gained by deleting the pre-parse +_quote_date_keyword_phrases rewrite. + +That rewrite matched date-keyword phrases (e.g. "previous month" after a +date field) anywhere in the raw query string, including inside an +unrelated quoted string, and inserted quotes mid-phrase there too — its +own docstring gave ``title:"see added:previous month notes"`` as the +example of what it corrupted. whoosh-compat's grammar accepts the same +phrase vocabulary unquoted natively (see TestUnquotedDateKeywordPhrases +in test_acceptance.py), so the rewrite was redundant everywhere it was +safe and actively wrong everywhere it was not. This is the one case that +tells the two apart: a literal title phrase that happens to contain +"added:previous month" as running text. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from documents.models import Document + +if TYPE_CHECKING: + from documents.search._backend import TantivyBackend + +pytestmark = [pytest.mark.search, pytest.mark.django_db] + + +def _matched_ids(backend: TantivyBackend, query: str) -> set[int]: + return set(backend.search_ids(query, user=None)) + + +def _index(backend: TantivyBackend, **kwargs: object) -> Document: + doc = Document.objects.create(**kwargs) + backend.add_or_update(doc) + return doc + + +class TestQuotedStringContainingDateKeywordText: + """A quoted title phrase containing the literal text + "added:previous month" as running words must match on that literal + text alone, never spill into an unfielded search for "previous" and + "month" across the default search fields the way the deleted rewrite + would have decomposed it into.""" + + def test_matches_only_the_literal_phrase( + self, + backend: TantivyBackend, + ) -> None: + literal = _index( + backend, + title="see added:previous month notes", + content="quarterly filing", + checksum="dkp-literal", + archive_serial_number=920, + ) + # Under the deleted rewrite, this decoy would incorrectly match: + # its title contains the "see added:" and " notes" fragments the + # corrupted parse required as title phrases, and its content + # supplies "previous" and "month" as the decomposed word-match + # clauses the rewrite turned the middle of the phrase into. + decoy = _index( + backend, + title="see added: quarterly report notes", + content="we reviewed the previous statement about month end", + checksum="dkp-decoy", + archive_serial_number=921, + ) + query = 'title:"see added:previous month notes"' + assert _matched_ids(backend, query) == {literal.pk} + assert decoy.pk not in _matched_ids(backend, query) diff --git a/src/documents/tests/search/test_date_keyword_timezone.py b/src/documents/tests/search/test_date_keyword_timezone.py new file mode 100644 index 000000000..243fce260 --- /dev/null +++ b/src/documents/tests/search/test_date_keyword_timezone.py @@ -0,0 +1,83 @@ +"""Date keyword phrases (``today``, etc.) resolved in a non-UTC timezone, +end to end. + +paperless's own ``tz=get_current_timezone()`` plumbing +(``TantivyBackend._parse_query``) is exercised elsewhere only for +relative *ranges* (``added:[-1 week to now]``, in +documents/tests/test_api_search.py). This covers a date *keyword* +(``today``), whose day boundary depends on the active timezone the same +way but goes through whoosh-compat's DateParserPlugin resolution instead +of an explicit range. + +Discriminating shape: frozen at 2026-06-15T02:00 UTC, which is +2026-06-14T22:00 in America/New_York -- still "today" (06-14) there, but +already "today" (06-15) in UTC. Two documents pin both directions of the +mistake a hardcoded-UTC bug would make: + +- ``in_ny_today`` (added 2026-06-14T20:00 UTC = 2026-06-14T16:00 NY) is + inside New York's "today" window and outside a naive UTC-calendar-day + window. A ``tz``-ignoring bug would miss it. +- ``in_utc_calendar_day_only`` (added 2026-06-15T10:00 UTC = + 2026-06-15T06:00 NY) is inside a naive UTC-calendar-day window but + outside New York's actual "today" window. A ``tz``-ignoring bug would + wrongly match it. +""" + +from __future__ import annotations + +from datetime import UTC +from datetime import datetime +from typing import TYPE_CHECKING + +import pytest +import time_machine + +from documents.models import Document + +if TYPE_CHECKING: + from pytest_django.fixtures import SettingsWrapper + + from documents.search._backend import TantivyBackend + +pytestmark = [pytest.mark.search, pytest.mark.django_db] + +FROZEN_NOW = datetime(2026, 6, 15, 2, 0, tzinfo=UTC) + + +def _matched_ids(backend: TantivyBackend, query: str) -> set[int]: + return set(backend.search_ids(query, user=None)) + + +def _index(backend: TantivyBackend, **kwargs: object) -> Document: + doc = Document.objects.create(**kwargs) + backend.add_or_update(doc) + return doc + + +class TestDateKeywordUsesTheActiveTimezone: + def test_today_matches_the_new_york_calendar_day_not_the_utc_one( + self, + backend: TantivyBackend, + settings: SettingsWrapper, + ) -> None: + settings.TIME_ZONE = "America/New_York" + with time_machine.travel(FROZEN_NOW, tick=False): + in_ny_today = _index( + backend, + title="NY today", + content="x", + checksum="tz-keyword-ny-today", + added=datetime(2026, 6, 14, 20, 0, tzinfo=UTC), + ) + # Not captured: the exact-set assertion below already proves + # this document (inside a naive UTC-calendar-day window, but + # outside New York's actual "today") does not match. + _index( + backend, + title="UTC calendar day only", + content="x", + checksum="tz-keyword-utc-calendar-day-only", + added=datetime(2026, 6, 15, 10, 0, tzinfo=UTC), + ) + + assert _matched_ids(backend, "added:today") == {in_ny_today.pk} diff --git a/src/documents/tests/search/test_default_search_fields_guard.py b/src/documents/tests/search/test_default_search_fields_guard.py new file mode 100644 index 000000000..ae97018e9 --- /dev/null +++ b/src/documents/tests/search/test_default_search_fields_guard.py @@ -0,0 +1,20 @@ +"""``_DEFAULT_SEARCH_FIELDS`` must stay a subset of the registered public +field names. + +Nothing enforced this before: a rename in PUBLIC_FIELDS not mirrored in +``_DEFAULT_SEARCH_FIELDS`` (documents/search/_query.py) would 400 every +unfielded search at request time, since ``index.parse_query`` and the +fuzzy/CJK clause builders are handed a field name the schema no longer +has. +""" + +from __future__ import annotations + +from documents.search._fields import PUBLIC_FIELDS +from documents.search._query import _DEFAULT_SEARCH_FIELDS + + +class TestDefaultSearchFieldsAreRegistered: + def test_every_default_search_field_is_a_public_field(self) -> None: + public_field_names = {f.name for f in PUBLIC_FIELDS} + assert set(_DEFAULT_SEARCH_FIELDS) <= public_field_names diff --git a/src/documents/tests/search/test_error_routing.py b/src/documents/tests/search/test_error_routing.py new file mode 100644 index 000000000..e882c7e64 --- /dev/null +++ b/src/documents/tests/search/test_error_routing.py @@ -0,0 +1,247 @@ +"""Diagnostics route by Cause, and user-facing messages are host-owned. + +whoosh-compat documents ``Diagnostic.message`` as developer output with no +stability guarantee, so it must never reach an HTTP response body. +""" + +from __future__ import annotations + +import logging +from datetime import UTC + +import pytest +import tantivy +from whoosh_compat.errors import Diagnostic +from whoosh_compat.errors import DiagnosticKind +from whoosh_compat.errors import QueryError +from whoosh_compat.errors import cause_for +from whoosh_compat.fields import FieldKind +from whoosh_compat.fields import FieldRef + +from documents.search._errors import SearchQueryError +from documents.search._query import _map_emit_error +from documents.search._query import _single_diagnostic_to_error +from documents.search._query import parse_user_query +from documents.search._schema import build_schema +from documents.search._tokenizer import register_tokenizers + +pytestmark = pytest.mark.search + +_LIBRARY_PROSE = "INTERNAL LIBRARY WORDING WITH raw tantivy detail" + + +@pytest.fixture(scope="module") +def query_index() -> tantivy.Index: + """An in-memory, unstemmed index; these tests only parse, never index.""" + idx = tantivy.Index(build_schema(), path=None) + register_tokenizers(idx, "") + return idx + + +def _diagnostic( + kind: DiagnosticKind, + *, + field: FieldRef | None = FieldRef("title"), + field_kind: FieldKind | None = FieldKind.TEXT, +) -> Diagnostic: + """A Diagnostic shaped like the emitter's, with the library's own + kind -> cause mapping rather than a hand-picked cause.""" + return Diagnostic( + kind=kind, + cause=cause_for(kind), + message=_LIBRARY_PROSE, + field=field, + field_kind=field_kind, + ) + + +class TestEmitErrorRouting: + """Every Cause gets a distinguishable treatment, not just "a 400".""" + + @pytest.mark.parametrize( + "kind", + [ + DiagnosticKind.BACKEND_REJECTED, + DiagnosticKind.AST_INVALID_SHAPE, + DiagnosticKind.AST_UNKNOWN_FIELD, + ], + ) + def test_internal_cause_is_not_converted(self, kind: DiagnosticKind) -> None: + """A library defect must surface as a 500 monitoring can see, not a + 400 blaming the user.""" + error = QueryError(_diagnostic(kind)) + with pytest.raises(QueryError) as excinfo: + _map_emit_error(error) + assert excinfo.value is error + + def test_misconfigured_cause_is_logged_and_becomes_a_400( + self, + caplog: pytest.LogCaptureFixture, + ) -> None: + kind = DiagnosticKind.SCHEMA_FIELD_MISSING + with caplog.at_level(logging.ERROR, logger="paperless.search"): + error = _map_emit_error( + QueryError(_diagnostic(kind, field=FieldRef("asn"))), + ) + assert isinstance(error, SearchQueryError) + errors = [r for r in caplog.records if r.levelno == logging.ERROR] + assert len(errors) == 1 + assert "asn" in errors[0].getMessage() + assert kind.name in errors[0].getMessage() + + @pytest.mark.parametrize( + "kind", + [ + DiagnosticKind.TEXT_RANGE, + DiagnosticKind.PATTERN_TOO_COMPLEX, + DiagnosticKind.EXISTS_REQUIRES_FAST, + ], + ) + def test_unsupported_cause_is_a_400_with_no_operator_log( + self, + kind: DiagnosticKind, + caplog: pytest.LogCaptureFixture, + ) -> None: + """A query tantivy cannot run is the user's to fix; it must not page + an operator the way a registry/schema mismatch does. + + EXISTS_REQUIRES_FAST is nominally MISCONFIGURED but belongs here: it + is decided from the registry's own FieldSpec, so it never reports a + disagreement anyone could resolve.""" + with caplog.at_level(logging.WARNING, logger="paperless.search"): + error = _map_emit_error(QueryError(_diagnostic(kind))) + assert isinstance(error, SearchQueryError) + assert caplog.records == [] + + @pytest.mark.parametrize( + "kind", + [ + DiagnosticKind.TEXT_RANGE, + DiagnosticKind.PATTERN_TOO_COMPLEX, + DiagnosticKind.EXISTS_REQUIRES_FAST, + DiagnosticKind.SCHEMA_FIELD_MISSING, + ], + ) + def test_user_facing_message_never_echoes_library_prose( + self, + kind: DiagnosticKind, + ) -> None: + error = _map_emit_error(QueryError(_diagnostic(kind))) + assert _LIBRARY_PROSE not in str(error) + + @pytest.mark.parametrize( + "kind", + [ + DiagnosticKind.TEXT_RANGE, + DiagnosticKind.PATTERN_TOO_COMPLEX, + DiagnosticKind.EXISTS_REQUIRES_FAST, + DiagnosticKind.SCHEMA_FIELD_MISSING, + ], + ) + def test_user_facing_message_names_the_field( + self, + kind: DiagnosticKind, + ) -> None: + """FieldRef.__str__ yields the canonical dotted name, including a + JSON subpath, so every user-reachable emit kind can name it.""" + diagnostic = _diagnostic( + kind, + field=FieldRef("custom_fields", "value"), + field_kind=FieldKind.JSON, + ) + error = _map_emit_error(QueryError(diagnostic)) + assert "custom_fields.value" in str(error) + + +class TestParseDiagnosticMessages: + """Parse-time diagnostics are host-worded too, off field_kind.""" + + def test_too_deep_is_a_400_without_library_prose(self) -> None: + error = _single_diagnostic_to_error( + _diagnostic(DiagnosticKind.TOO_DEEP, field=None, field_kind=None), + ) + assert isinstance(error, SearchQueryError) + assert _LIBRARY_PROSE not in str(error) + + @pytest.mark.parametrize( + ("kind", "field_kind"), + [ + (DiagnosticKind.PATTERN_ON_NUMERIC, FieldKind.U64), + (DiagnosticKind.PATTERN_ON_BOOLEAN_EXISTS, FieldKind.BOOLEAN_EXISTS), + (DiagnosticKind.PATTERN_ON_SUBPATH, FieldKind.JSON), + ], + ) + def test_pattern_on_kinds_name_the_field_and_its_kind( + self, + kind: DiagnosticKind, + field_kind: FieldKind, + ) -> None: + error = _single_diagnostic_to_error( + _diagnostic(kind, field=FieldRef("asn"), field_kind=field_kind), + ) + message = str(error) + assert _LIBRARY_PROSE not in message + assert "asn" in message + assert field_kind.name.lower() in message + + def test_single_char_bracket_range_names_the_field_and_the_value(self) -> None: + diagnostic = Diagnostic( + kind=DiagnosticKind.SINGLE_CHAR_BRACKET_RANGE, + cause=cause_for(DiagnosticKind.SINGLE_CHAR_BRACKET_RANGE), + message=_LIBRARY_PROSE, + field=FieldRef("title"), + field_kind=FieldKind.TEXT, + raw_value="200[1-9]", + ) + error = _single_diagnostic_to_error(diagnostic) + message = str(error) + assert isinstance(error, SearchQueryError) + assert _LIBRARY_PROSE not in message + assert "title" in message + assert "200[1-9]" in message + + +class TestRealQueriesRouteCorrectly: + """The routing table against diagnostics emit() really produces.""" + + def test_text_range_is_a_400_naming_the_field( + self, + query_index: tantivy.Index, + ) -> None: + with pytest.raises(SearchQueryError) as excinfo: + parse_user_query(query_index, "title:[a to b]", UTC) + assert "title" in str(excinfo.value) + + def test_wildcard_on_a_numeric_field_is_a_400_naming_the_field( + self, + query_index: tantivy.Index, + ) -> None: + with pytest.raises(SearchQueryError) as excinfo: + parse_user_query(query_index, "asn:12*", UTC) + assert "asn" in str(excinfo.value) + + def test_single_char_bracket_range_is_a_400_naming_field_and_value( + self, + query_index: tantivy.Index, + ) -> None: + with pytest.raises(SearchQueryError) as excinfo: + parse_user_query(query_index, "title:200[1-9]", UTC) + message = str(excinfo.value) + assert "title" in message + assert "200[1-9]" in message + + def test_internal_diagnostic_escapes_as_a_query_error( + self, + query_index: tantivy.Index, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """The one case with no query text that reaches it: emit() reporting + a defect in itself must not be converted to a user-facing 400.""" + import documents.search._query as query_mod + + def raise_internal(*args: object, **kwargs: object) -> None: + raise QueryError(_diagnostic(DiagnosticKind.BACKEND_REJECTED)) + + monkeypatch.setattr(query_mod, "tantivy_emit", raise_internal) + with pytest.raises(QueryError): + parse_user_query(query_index, "invoice", UTC) diff --git a/src/documents/tests/search/test_exists_on_json_fields.py b/src/documents/tests/search/test_exists_on_json_fields.py new file mode 100644 index 000000000..89290ac55 --- /dev/null +++ b/src/documents/tests/search/test_exists_on_json_fields.py @@ -0,0 +1,92 @@ +"""``field:*`` on a JSON field is user error, not an operator alert. + +whoosh-compat classifies EXISTS_REQUIRES_FAST as MISCONFIGURED, and +_map_emit_error used to route every MISCONFIGURED diagnostic to an ERROR log. +But the kind is decided from the registry's own FieldSpec (kind plus fast) +without consulting the index schema, and field_descriptors() builds the JSON +fields non-fast deliberately, so nothing is misconfigured and no operator +action can clear the condition. Any authenticated user could otherwise emit +ERROR lines in a loop by repeating ``notes:*``. + +SCHEMA_FIELD_MISSING, the other MISCONFIGURED kind, does compare the registry +against the live schema, so it stays an ERROR. +""" + +from __future__ import annotations + +import logging +from datetime import UTC + +import pytest +import tantivy +from whoosh_compat.errors import Diagnostic +from whoosh_compat.errors import DiagnosticKind +from whoosh_compat.errors import QueryError +from whoosh_compat.errors import cause_for +from whoosh_compat.fields import FieldKind +from whoosh_compat.fields import FieldRef + +from documents.search._errors import SearchQueryError +from documents.search._query import _map_emit_error +from documents.search._query import parse_user_query +from documents.search._schema import build_schema +from documents.search._tokenizer import register_tokenizers + +pytestmark = pytest.mark.search + +# Every spelling of "does this JSON field have a value" a user can type. +EXISTS_QUERIES = [ + "notes:*", + "notes.note:*", + "notes.user:*", + "custom_fields:*", + "custom_fields.name:*", + "custom_fields.value:*", +] + + +@pytest.fixture(scope="module") +def query_index() -> tantivy.Index: + idx = tantivy.Index(build_schema(), path=None) + register_tokenizers(idx, "") + return idx + + +class TestJsonExistsIsUserError: + @pytest.mark.parametrize("query", EXISTS_QUERIES) + def test_query_is_a_400_that_emits_no_error_log( + self, + query_index: tantivy.Index, + caplog: pytest.LogCaptureFixture, + query: str, + ) -> None: + with caplog.at_level(logging.WARNING, logger="paperless.search"): + with pytest.raises(SearchQueryError) as excinfo: + parse_user_query(query_index, query, UTC) + assert query.split(":", maxsplit=1)[0] in str(excinfo.value) + assert [r for r in caplog.records if r.levelno >= logging.ERROR] == [] + + +class TestGenuineMisconfigurationStillLogs: + def test_schema_field_missing_is_an_error_log( + self, + caplog: pytest.LogCaptureFixture, + ) -> None: + """The registry naming a field the index schema does not have is a + real mismatch an operator can fix, so it keeps the alert.""" + kind = DiagnosticKind.SCHEMA_FIELD_MISSING + error = QueryError( + Diagnostic( + kind=kind, + cause=cause_for(kind), + message="field 'asn' is not defined in the index schema", + field=FieldRef("asn"), + field_kind=FieldKind.U64, + ), + ) + with caplog.at_level(logging.ERROR, logger="paperless.search"): + mapped = _map_emit_error(error) + assert isinstance(mapped, SearchQueryError) + records = [r for r in caplog.records if r.levelno == logging.ERROR] + assert len(records) == 1 + assert kind.name in records[0].getMessage() diff --git a/src/documents/tests/search/test_fuzzy_tokenization.py b/src/documents/tests/search/test_fuzzy_tokenization.py new file mode 100644 index 000000000..3048228a2 --- /dev/null +++ b/src/documents/tests/search/test_fuzzy_tokenization.py @@ -0,0 +1,174 @@ +"""The words the fuzzy blend clause hands back to tantivy's parser. + +The clause re-parses a word string through tantivy, which analyzes it +again, so the words must be the query's raw text rather than the analyzed +text (analysis is not idempotent), and must still be split into plain +words so that hyphenated, dotted and quoted terms keep contributing. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from documents.models import Document + +if TYPE_CHECKING: + from pytest_django.fixtures import SettingsWrapper + + from documents.search._backend import TantivyBackend + +pytestmark = [pytest.mark.search, pytest.mark.django_db] + + +def _matched_ids(backend: TantivyBackend, query: str) -> set[int]: + return set(backend.search_ids(query, user=None)) + + +def _index(backend: TantivyBackend, **kwargs: object) -> Document: + doc = Document.objects.create(**kwargs) + backend.add_or_update(doc) + return doc + + +@pytest.fixture(autouse=True) +def fuzzy_enabled(settings: SettingsWrapper) -> None: + """Enable the fuzzy blend clause. The threshold doubles as a minimum + score filter, so it is set to 0.0: every hit passes and the test sees + the clause's matching behaviour, not the filter's.""" + settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.0 + + +class TestFuzzyClauseWords: + def test_a_stemmed_word_is_not_stemmed_a_second_time( + self, + backend: TantivyBackend, + ) -> None: + """'universities' stems to 'univers'; feeding that back to tantivy + stems it again to 'univ', whose fuzzy prefix reaches unrelated + words. The clause must stay wide enough for a typo and no wider.""" + wanted = _index( + backend, + title="A", + content="universities of europe", + checksum="fuzz-stem-1", + ) + typo = _index( + backend, + title="B", + content="universties of europe", + checksum="fuzz-stem-2", + ) + _index( + backend, + title="C", + content="univalent chemical bonds", + checksum="fuzz-stem-3", + ) + _index( + backend, + title="D", + content="unicycle repair manual", + checksum="fuzz-stem-4", + ) + + assert _matched_ids(backend, "universities") == {wanted.pk, typo.pk} + + def test_a_hyphenated_term_still_reaches_the_clause( + self, + backend: TantivyBackend, + ) -> None: + """'COVID-19' is one raw token: unless it is split into words, it + carries characters the re-parse would read as grammar, is dropped, + and the whole query loses its fuzzy clause.""" + misspelled = _index( + backend, + title="A", + content="covidx testing results", + checksum="fuzz-hyphen-1", + ) + + assert _matched_ids(backend, "COVID-19") == {misspelled.pk} + + def test_a_phrase_still_reaches_the_clause( + self, + backend: TantivyBackend, + ) -> None: + """A phrase is one raw token carrying a space, and is the whole + query's only free text here.""" + near_miss = _index( + backend, + title="A", + content="taxation reportage weekly", + checksum="fuzz-phrase-1", + ) + + assert _matched_ids(backend, '"tax reports"') == {near_miss.pk} + + +class TestBooleanKeywordsInRawText: + """Tantivy's boolean keywords are word runs, so they survive the cut + into words and its own parser reads them as grammar. Raw query text + reaches that parser with its case intact, so a quoted phrase can carry + them in.""" + + @pytest.fixture + def corpus(self, backend: TantivyBackend) -> dict[str, int]: + both = _index( + backend, + title="A", + content="taxation reportage weekly", + checksum="fuzz-kw-1", + ) + tax_only = _index( + backend, + title="B", + content="taxation only here", + checksum="fuzz-kw-2", + ) + report_only = _index( + backend, + title="C", + content="reportage only here", + checksum="fuzz-kw-3", + ) + return { + "both": both.pk, + "tax_only": tax_only.pk, + "report_only": report_only.pk, + } + + @pytest.mark.parametrize( + "query", + [ + pytest.param('"tax AND reports"', id="and"), + pytest.param('"tax OR reports"', id="or"), + pytest.param('"tax NOT reports"', id="not"), + pytest.param('"tax IN reports"', id="in"), + ], + ) + def test_a_keyword_inside_a_phrase_stays_an_ordinary_word( + self, + backend: TantivyBackend, + corpus: dict[str, int], + query: str, + ) -> None: + """The phrase asks for three words, so the clause must stay the + disjunction it is for '"tax reports"': AND must not turn it into a + conjunction, NOT must not give it its own exclusion, IN must not + fail the parse.""" + assert _matched_ids(backend, '"tax reports"') == set(corpus.values()) + assert _matched_ids(backend, query) == set(corpus.values()) + + def test_a_trailing_keyword_does_not_drop_the_clause( + self, + backend: TantivyBackend, + corpus: dict[str, int], + ) -> None: + """'tax AND' is a syntax error to tantivy's parser, which would + cost the whole query its fuzzy clause.""" + assert _matched_ids(backend, '"tax AND"') == { + corpus["both"], + corpus["tax_only"], + } diff --git a/src/documents/tests/search/test_highlight_query_guard.py b/src/documents/tests/search/test_highlight_query_guard.py new file mode 100644 index 000000000..f70f37e06 --- /dev/null +++ b/src/documents/tests/search/test_highlight_query_guard.py @@ -0,0 +1,192 @@ +"""Regression coverage for the unguarded TEXT-mode highlight query. + +parse_simple_text_highlight_query re-parses simple-search tokens through +Tantivy's query-string parser to build a SnippetGenerator-compatible query. +Simple-search tokens keep arbitrary punctuation (quotes, colons, brackets, +slashes), so any token carrying Tantivy query grammar raised an unguarded +ValueError. The search itself had already succeeded by the time this ran: +only the highlight step failed, and with the DocumentViewSet.list +exception handler narrowed elsewhere on this branch, that ValueError now +reaches the client as a bare 500 rather than a 400. + +Covers three angles: + - the query builder itself: quoting each token as its own escaped phrase + should let it parse instead of raising, for every failure mode a plain- + text query can trigger (syntax error, unknown field, unsupported regex). + - highlight_hits: even when a token still can't be expressed as a + highlight query, the guard must fall back to a query that still + produces usable highlight HTML, not silently empty ones. + - the real API endpoint: pinning the previously-500 status to 200. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest +import tantivy +from rest_framework import status + +from documents.search._backend import SearchMode +from documents.search._query import parse_simple_text_highlight_query +from documents.search._schema import build_schema +from documents.search._tokenizer import register_tokenizers +from documents.tests.factories import DocumentFactory + +if TYPE_CHECKING: + from rest_framework.test import APIClient + + from documents.search._backend import TantivyBackend + +pytestmark = [pytest.mark.search, pytest.mark.django_db] + +# Each spelling below trips a different Tantivy parser failure mode: +# 'a"b' -> Syntax Error (unterminated quote) +# foo:bar -> unknown field +# (a -> Syntax Error (unbalanced group) +# [a -> Syntax Error (unbalanced range) +# /a/ -> Unsupported query (regex queries disallowed) +_MALFORMED_QUERIES = [ + pytest.param('a"b', id="unterminated_quote"), + pytest.param("foo:bar", id="unknown_field"), + pytest.param("(a", id="unbalanced_group"), + pytest.param("[a", id="unbalanced_range"), + pytest.param("/a/", id="unsupported_regex"), +] + + +@pytest.fixture(scope="module") +def query_index() -> tantivy.Index: + """An in-memory, unstemmed index for parse-only tests.""" + schema = build_schema() + idx = tantivy.Index(schema, path=None) + register_tokenizers(idx, "") + return idx + + +class TestParseSimpleTextHighlightQueryDoesNotRaise: + """The query builder itself must tolerate Tantivy syntax in its tokens.""" + + @pytest.mark.parametrize("raw_query", _MALFORMED_QUERIES) + def test_malformed_token_does_not_raise( + self, + query_index: tantivy.Index, + raw_query: str, + ) -> None: + assert isinstance( + parse_simple_text_highlight_query(query_index, raw_query), + tantivy.Query, + ) + + +class TestHighlightHitsProducesUsableHighlights: + """highlight_hits must keep producing real -wrapped snippet HTML for + these queries, not merely avoid raising.""" + + @pytest.mark.parametrize( + "raw_query", + [*_MALFORMED_QUERIES, pytest.param("plain text", id="plain_text_sanity")], + ) + def test_highlight_still_contains_matched_text( + self, + backend: TantivyBackend, + raw_query: str, + ) -> None: + doc = DocumentFactory.create( + title="probe", + content=f"needle content containing {raw_query} literally here", + ) + backend.add_or_update(doc) + + hits = backend.highlight_hits( + raw_query, + [doc.pk], + search_mode=SearchMode.TEXT, + ) + + assert len(hits) == 1 + highlights = hits[0]["highlights"] + assert "content" in highlights, ( + f"Expected a content highlight for {raw_query!r}, got: {highlights!r}" + ) + assert "" in highlights["content"], ( + f"Highlight for {raw_query!r} carries no matched-term markup: " + f"{highlights['content']!r}" + ) + + +class TestHighlightGuardDiscriminatesOnValueError: + """The guard added to highlight_hits must catch exactly ValueError, the + same shape as the sibling notes_text guard, and let anything else + through -- so a real library defect is never mistaken for a harmless + syntax error.""" + + def test_non_value_error_is_not_swallowed( + self, + backend: TantivyBackend, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + import documents.search._backend as backend_mod + + def raise_runtime_error(*args: object, **kwargs: object) -> object: + raise RuntimeError("synthetic bug, unrelated to query syntax") + + monkeypatch.setattr( + backend_mod, + "parse_simple_text_highlight_query", + raise_runtime_error, + ) + + doc = DocumentFactory.create(title="probe", content="anything here") + backend.add_or_update(doc) + + with pytest.raises(RuntimeError): + backend.highlight_hits( + "anything", + [doc.pk], + search_mode=SearchMode.TEXT, + ) + + +@pytest.mark.usefixtures("_search_index") +class TestApiNoLongerReturns500: + """Pins the actual regression: a matching TEXT-mode search whose query + string carries Tantivy syntax must return results, not a server error.""" + + @pytest.mark.parametrize("raw_query", _MALFORMED_QUERIES) + def test_malformed_text_query_returns_200( + self, + admin_client: APIClient, + raw_query: str, + ) -> None: + from documents.search import get_backend + + doc = DocumentFactory.create( + title="probe", + content=f"needle content containing {raw_query} literally here", + ) + get_backend().add_or_update(doc) + + response = admin_client.get(f"/api/documents/?text={raw_query}") + + assert response.status_code == status.HTTP_200_OK + assert response.data["count"] == 1 + + def test_plain_text_query_still_returns_200( + self, + admin_client: APIClient, + ) -> None: + """Sanity check: the guard must not mask a total failure of the + ordinary highlight path.""" + from documents.search import get_backend + + doc = DocumentFactory.create( + title="probe", + content="needle content containing plain text literally here", + ) + get_backend().add_or_update(doc) + + response = admin_client.get("/api/documents/?text=plain text") + + assert response.status_code == status.HTTP_200_OK + assert response.data["count"] == 1 diff --git a/src/documents/tests/search/test_json_field_prefixes.py b/src/documents/tests/search/test_json_field_prefixes.py new file mode 100644 index 000000000..e8a62cf6c --- /dev/null +++ b/src/documents/tests/search/test_json_field_prefixes.py @@ -0,0 +1,148 @@ +"""Bare notes:/custom_fields: prefix resolution. + +"notes:foo"/"custom_fields:foo" were valid fielded searches before the +whoosh-compat migration. The registry only exposes them as JSON subpaths, so +each JSON FieldSpec declares a default subpath (SubpathSpec(default=True)): +notes: resolves to notes.note:, custom_fields: resolves to +custom_fields.value:. This replaced an earlier regex-based rewrite +(_rewrite_bare_json_field_prefixes) that ran on the raw query string before +parsing and was blind to quoting, so a phrase like +content:"payment notes: none" was silently corrupted into a notes-field +search and matched nothing. Resolving the default subpath inside the parser +instead means quoting is already understood by the time it happens. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest +from django.contrib.auth.models import User + +from documents.models import CustomField +from documents.models import CustomFieldInstance +from documents.models import Document +from documents.models import Note + +if TYPE_CHECKING: + from documents.search._backend import TantivyBackend + +pytestmark = [pytest.mark.search, pytest.mark.django_db] + + +def _matched_ids(backend: TantivyBackend, query: str) -> set[int]: + return set(backend.search_ids(query, user=None)) + + +def _index(backend: TantivyBackend, **kwargs: object) -> Document: + doc = Document.objects.create(**kwargs) + backend.add_or_update(doc) + return doc + + +class TestBareJsonFieldPrefixes: + def test_bare_notes_prefix_searches_note_text( + self, + backend: TantivyBackend, + ) -> None: + alice = User.objects.create_user(username="alice") + with_note = Document.objects.create( + title="Has note", + content="x", + checksum="bare-notes-with", + ) + Note.objects.create(document=with_note, user=alice, note="crocodile") + backend.add_or_update(with_note) + # This document's CONTENT contains the words a demoted text search + # would match; it must NOT match once the prefix addresses notes. + _index( + backend, + title="Notes about things", + content="notes crocodile mention", + checksum="bare-notes-decoy", + ) + assert _matched_ids(backend, "notes:crocodile") == {with_note.pk} + + def test_bare_custom_fields_prefix_searches_values( + self, + backend: TantivyBackend, + ) -> None: + field = CustomField.objects.create( + name="Policy Number", + data_type=CustomField.FieldDataType.STRING, + ) + with_value = Document.objects.create( + title="Has field", + content="x", + checksum="bare-cf-with", + ) + CustomFieldInstance.objects.create( + document=with_value, + field=field, + value_text="crocodile", + ) + backend.add_or_update(with_value) + _index( + backend, + title="Custom things", + content="custom fields crocodile", + checksum="bare-cf-decoy", + ) + assert _matched_ids(backend, "custom_fields:crocodile") == {with_value.pk} + + def test_subpath_spellings_are_untouched( + self, + backend: TantivyBackend, + ) -> None: + bob = User.objects.create_user(username="bob") + doc = Document.objects.create( + title="Bob note", + content="x", + checksum="bare-subpath", + ) + Note.objects.create(document=doc, user=bob, note="remark") + backend.add_or_update(doc) + assert _matched_ids(backend, "notes.user:bob") == {doc.pk} + assert _matched_ids(backend, "notes.note:remark") == {doc.pk} + + +class TestQuotedPhraseContainingNotesColonIsNotCorrupted: + """The regex rewrite this migration removes was blind to quoting: it + matched "notes:" anywhere in the raw query string, including inside an + already-quoted phrase on an unrelated field, silently turning + content:"payment notes: none" into a notes-field search that matched + nothing. Resolving the default subpath during parsing (which is + quote-aware) fixes this.""" + + def test_quoted_phrase_with_notes_colon_matches_by_content( + self, + backend: TantivyBackend, + ) -> None: + target = _index( + backend, + title="Statement", + content="payment notes: none", + checksum="quoted-phrase-notes-colon", + ) + assert _matched_ids( + backend, + 'content:"payment notes: none"', + ) == {target.pk} + + def test_quoted_phrase_matches_the_same_document_unquoted( + self, + backend: TantivyBackend, + ) -> None: + # Same document, phrasing without the colon: this proves the fix is + # about quote-awareness, not about the words themselves being + # unsearchable. + target = _index( + backend, + title="Statement", + content="payment notes none", + checksum="quoted-phrase-no-colon", + ) + assert _matched_ids( + backend, + 'content:"payment notes none"', + ) == {target.pk} diff --git a/src/documents/tests/search/test_permission_field_isolation.py b/src/documents/tests/search/test_permission_field_isolation.py new file mode 100644 index 000000000..dd196ef44 --- /dev/null +++ b/src/documents/tests/search/test_permission_field_isolation.py @@ -0,0 +1,148 @@ +"""Permission filtering must hold against the real indexed document shape. + +Only three of the index's unsigned ``*_id`` columns are load-bearing: +``owner_id``, ``viewer_id`` and ``viewer_group_id``, all read by +build_permission_filter. The rest (correspondent/document_type/storage_path/tag +ids) were written on every document and read by nothing, and were dropped. + +These tests index real Documents through the backend's own document builder and +assert result-level visibility per user, so a mistake about which columns are +load-bearing shows up as documents leaking across users rather than as a passing +unit test over a hand-built index. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest +from django.contrib.auth.models import Group +from django.contrib.auth.models import User +from guardian.shortcuts import assign_perm + +from documents.models import Correspondent +from documents.models import Document +from documents.models import DocumentType +from documents.models import StoragePath +from documents.models import Tag + +if TYPE_CHECKING: + from documents.search._backend import TantivyBackend + +pytestmark = [pytest.mark.search, pytest.mark.django_db] + + +@pytest.fixture +def owner() -> User: + return User.objects.create_user(username="owner") + + +@pytest.fixture +def stranger() -> User: + return User.objects.create_user(username="stranger") + + +@pytest.fixture +def viewer() -> User: + return User.objects.create_user(username="viewer") + + +@pytest.fixture +def group_member() -> User: + user = User.objects.create_user(username="group_member") + user.groups.add(Group.objects.create(name="accounting")) + return user + + +class TestPermissionFilteringOnIndexedDocuments: + def test_unowned_document_is_visible_to_everyone( + self, + backend: TantivyBackend, + stranger: User, + ) -> None: + doc = Document.objects.create( + title="Public Invoice", + content="invoice total due", + checksum="perm-unowned", + ) + backend.add_or_update(doc) + + assert backend.search_ids("invoice", user=stranger) == [doc.pk] + + def test_owned_document_is_visible_only_to_its_owner( + self, + backend: TantivyBackend, + owner: User, + stranger: User, + ) -> None: + doc = Document.objects.create( + title="Private Invoice", + content="invoice total due", + checksum="perm-owned", + owner=owner, + ) + backend.add_or_update(doc) + + assert backend.search_ids("invoice", user=owner) == [doc.pk] + assert backend.search_ids("invoice", user=stranger) == [] + + def test_explicitly_shared_document_is_visible_to_the_viewer( + self, + backend: TantivyBackend, + owner: User, + viewer: User, + stranger: User, + ) -> None: + doc = Document.objects.create( + title="Shared Invoice", + content="invoice total due", + checksum="perm-shared-user", + owner=owner, + ) + assign_perm("view_document", viewer, doc) + backend.add_or_update(doc) + + assert backend.search_ids("invoice", user=viewer) == [doc.pk] + assert backend.search_ids("invoice", user=stranger) == [] + + def test_group_shared_document_is_visible_to_group_members( + self, + backend: TantivyBackend, + owner: User, + group_member: User, + stranger: User, + ) -> None: + doc = Document.objects.create( + title="Group Invoice", + content="invoice total due", + checksum="perm-shared-group", + owner=owner, + ) + assign_perm("view_document", group_member.groups.first(), doc) + backend.add_or_update(doc) + + assert backend.search_ids("invoice", user=group_member) == [doc.pk] + assert backend.search_ids("invoice", user=stranger) == [] + + def test_metadata_does_not_widen_visibility( + self, + backend: TantivyBackend, + owner: User, + stranger: User, + ) -> None: + """A document carrying correspondent/type/storage-path/tag metadata is + still filtered by owner alone.""" + doc = Document.objects.create( + title="Tagged Invoice", + content="invoice total due", + checksum="perm-metadata", + owner=owner, + correspondent=Correspondent.objects.create(name="ACME"), + document_type=DocumentType.objects.create(name="Bill"), + storage_path=StoragePath.objects.create(name="Archive", path="archive/"), + ) + doc.tags.add(Tag.objects.create(name="paid")) + backend.add_or_update(doc) + + assert backend.search_ids("invoice", user=owner) == [doc.pk] + assert backend.search_ids("invoice", user=stranger) == [] diff --git a/src/documents/tests/search/test_query.py b/src/documents/tests/search/test_query.py index 6260b67af..8b2962542 100644 --- a/src/documents/tests/search/test_query.py +++ b/src/documents/tests/search/test_query.py @@ -1,448 +1,96 @@ from __future__ import annotations -import re from datetime import UTC from datetime import datetime -from datetime import tzinfo from typing import TYPE_CHECKING -from zoneinfo import ZoneInfo import pytest import tantivy import time_machine -from documents.search._dates import _date_only_range -from documents.search._dates import _datetime_range -from documents.search._query import build_permission_filter +from documents.search._backend import build_permission_filter +from documents.search._errors import InvalidDateQuery +from documents.search._errors import InvalidNumberQuery +from documents.search._errors import MultipleSearchQueryErrors +from documents.search._errors import SearchQueryError from documents.search._query import parse_simple_text_highlight_query from documents.search._query import parse_user_query from documents.search._schema import build_schema from documents.search._tokenizer import register_tokenizers -from documents.search._translate import InvalidDateQuery -from documents.search._translate import translate_query if TYPE_CHECKING: from django.contrib.auth.base_user import AbstractBaseUser pytestmark = pytest.mark.search -EASTERN = ZoneInfo("America/New_York") # UTC-5 / UTC-4 (DST) -AUCKLAND = ZoneInfo("Pacific/Auckland") # UTC+13 in southern-hemisphere summer + +@pytest.fixture(scope="module") +def query_index() -> tantivy.Index: + """An in-memory, unstemmed index shared read-only across this module's + parse-only tests (none of them index documents).""" + schema = build_schema() + idx = tantivy.Index(schema, path=None) + register_tokenizers(idx, "") + return idx -def _range(result: str, field: str) -> tuple[str, str]: - # Half-open period ranges close with "}" (exclusive); exact-instant ranges - # (full ISO datetimes, "now", relative offsets) close with "]" (inclusive). - m = re.search(rf"{field}:\[(.+?) TO (.+?)[\]}}]", result) - assert m, f"No range for {field!r} in: {result!r}" - return m.group(1), m.group(2) +@pytest.fixture(scope="module") +def populated_index() -> tantivy.Index: + """An index holding one document, so a query matching nothing is + distinguishable from one matching everything.""" + idx = tantivy.Index(build_schema(), path=None) + register_tokenizers(idx, "") + writer = idx.writer() + doc = tantivy.Document() + doc.add_unsigned("id", 1) + doc.add_text("content", "needle in indexed content") + writer.add_document(doc) + writer.commit() + idx.reload() + return idx -class TestCreatedDateField: - """ - created is a Django DateField: indexed as midnight UTC of the local calendar - date. No offset arithmetic needed - the local calendar date is what matters. - """ - - @pytest.mark.parametrize( - ("tz", "expected_lo", "expected_hi"), - [ - pytest.param(UTC, "2026-03-28T00:00:00Z", "2026-03-29T00:00:00Z", id="utc"), - pytest.param( - EASTERN, - "2026-03-28T00:00:00Z", - "2026-03-29T00:00:00Z", - id="eastern_same_calendar_date", - ), - ], - ) - @time_machine.travel(datetime(2026, 3, 28, 15, 30, tzinfo=UTC), tick=False) - def test_today(self, tz: tzinfo, expected_lo: str, expected_hi: str) -> None: - lo, hi = _range(translate_query("created:today", tz), "created") - assert lo == expected_lo - assert hi == expected_hi - - @time_machine.travel(datetime(2026, 3, 28, 3, 0, tzinfo=UTC), tick=False) - def test_today_auckland_ahead_of_utc(self) -> None: - # UTC 03:00 -> Auckland (UTC+13) = 16:00 same date; local date = 2026-03-28 - lo, _ = _range( - translate_query("created:today", AUCKLAND), - "created", - ) - assert lo == "2026-03-28T00:00:00Z" - - @pytest.mark.parametrize( - ("field", "keyword", "expected_lo", "expected_hi"), - [ - pytest.param( - "created", - "yesterday", - "2026-03-27T00:00:00Z", - "2026-03-28T00:00:00Z", - id="yesterday", - ), - pytest.param( - "created", - "previous week", - "2026-03-16T00:00:00Z", - "2026-03-23T00:00:00Z", - id="previous_week", - ), - pytest.param( - "created", - "this month", - "2026-03-01T00:00:00Z", - "2026-04-01T00:00:00Z", - id="this_month", - ), - pytest.param( - "created", - "previous month", - "2026-02-01T00:00:00Z", - "2026-03-01T00:00:00Z", - id="previous_month", - ), - pytest.param( - "created", - "this year", - "2026-01-01T00:00:00Z", - "2027-01-01T00:00:00Z", - id="this_year", - ), - pytest.param( - "created", - "previous year", - "2025-01-01T00:00:00Z", - "2026-01-01T00:00:00Z", - id="previous_year", - ), - ], - ) - @time_machine.travel(datetime(2026, 3, 28, 15, 0, tzinfo=UTC), tick=False) - def test_date_keywords( - self, - field: str, - keyword: str, - expected_lo: str, - expected_hi: str, - ) -> None: - # 2026-03-28 is Saturday; Mon-Sun week calculation built into expectations - query = f"{field}:{keyword}" - lo, hi = _range(translate_query(query, UTC), field) - assert lo == expected_lo - assert hi == expected_hi - - @time_machine.travel(datetime(2026, 12, 15, 12, 0, tzinfo=UTC), tick=False) - def test_this_month_december_wraps_to_next_year(self) -> None: - # December: next month must roll over to January 1 of next year - lo, hi = _range( - translate_query("created:this month", UTC), - "created", - ) - assert lo == "2026-12-01T00:00:00Z" - assert hi == "2027-01-01T00:00:00Z" - - @time_machine.travel(datetime(2026, 1, 15, 12, 0, tzinfo=UTC), tick=False) - def test_last_month_january_wraps_to_previous_year(self) -> None: - # January: last month must roll back to December 1 of previous year - lo, hi = _range( - translate_query("created:previous month", UTC), - "created", - ) - assert lo == "2025-12-01T00:00:00Z" - assert hi == "2026-01-01T00:00:00Z" - - @time_machine.travel(datetime(2026, 7, 15, 12, 0, tzinfo=UTC), tick=False) - def test_previous_quarter(self) -> None: - lo, hi = _range( - translate_query('created:"previous quarter"', UTC), - "created", - ) - assert lo == "2026-04-01T00:00:00Z" - assert hi == "2026-07-01T00:00:00Z" - - def test_unknown_keyword_raises(self) -> None: - with pytest.raises(ValueError, match="Unknown keyword"): - _date_only_range("bogus_keyword", UTC) - - -class TestDateTimeFields: - """ - added/modified store full UTC datetimes. Natural keywords must convert - the local day boundaries to UTC - timezone offset arithmetic IS required. - """ - - @time_machine.travel(datetime(2026, 3, 28, 15, 30, tzinfo=UTC), tick=False) - def test_added_today_eastern(self) -> None: - # EDT = UTC-4; local midnight 2026-03-28 00:00 EDT = 2026-03-28 04:00 UTC - lo, hi = _range(translate_query("added:today", EASTERN), "added") - assert lo == "2026-03-28T04:00:00Z" - assert hi == "2026-03-29T04:00:00Z" - - @time_machine.travel(datetime(2026, 3, 29, 2, 0, tzinfo=UTC), tick=False) - def test_added_today_auckland_midnight_crossing(self) -> None: - # UTC 02:00 on 2026-03-29 -> Auckland (UTC+13) = 2026-03-29 15:00 local - # Auckland midnight = UTC 2026-03-28 11:00 - lo, hi = _range(translate_query("added:today", AUCKLAND), "added") - assert lo == "2026-03-28T11:00:00Z" - assert hi == "2026-03-29T11:00:00Z" - - @time_machine.travel(datetime(2026, 3, 28, 15, 0, tzinfo=UTC), tick=False) - def test_modified_today_utc(self) -> None: - lo, hi = _range( - translate_query("modified:today", UTC), - "modified", - ) - assert lo == "2026-03-28T00:00:00Z" - assert hi == "2026-03-29T00:00:00Z" - - @pytest.mark.parametrize( - ("keyword", "expected_lo", "expected_hi"), - [ - pytest.param( - "yesterday", - "2026-03-27T00:00:00Z", - "2026-03-28T00:00:00Z", - id="yesterday", - ), - pytest.param( - "previous week", - "2026-03-16T00:00:00Z", - "2026-03-23T00:00:00Z", - id="previous_week", - ), - pytest.param( - "this month", - "2026-03-01T00:00:00Z", - "2026-04-01T00:00:00Z", - id="this_month", - ), - pytest.param( - "previous month", - "2026-02-01T00:00:00Z", - "2026-03-01T00:00:00Z", - id="previous_month", - ), - pytest.param( - "this year", - "2026-01-01T00:00:00Z", - "2027-01-01T00:00:00Z", - id="this_year", - ), - pytest.param( - "previous year", - "2025-01-01T00:00:00Z", - "2026-01-01T00:00:00Z", - id="previous_year", - ), - ], - ) - @time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False) - def test_datetime_keywords_utc( - self, - keyword: str, - expected_lo: str, - expected_hi: str, - ) -> None: - # 2026-03-28 is Saturday; weekday()==5 so Monday=2026-03-23 - lo, hi = _range(translate_query(f"added:{keyword}", UTC), "added") - assert lo == expected_lo - assert hi == expected_hi - - @time_machine.travel(datetime(2026, 12, 15, 12, 0, tzinfo=UTC), tick=False) - def test_this_month_december_wraps_to_next_year(self) -> None: - # December: next month wraps to January of next year - lo, hi = _range(translate_query("added:this month", UTC), "added") - assert lo == "2026-12-01T00:00:00Z" - assert hi == "2027-01-01T00:00:00Z" - - @time_machine.travel(datetime(2026, 1, 15, 12, 0, tzinfo=UTC), tick=False) - def test_last_month_january_wraps_to_previous_year(self) -> None: - # January: last month wraps back to December of previous year - lo, hi = _range( - translate_query("added:previous month", UTC), - "added", - ) - assert lo == "2025-12-01T00:00:00Z" - assert hi == "2026-01-01T00:00:00Z" - - @pytest.mark.parametrize( - ("query", "expected_lo", "expected_hi"), - [ - pytest.param( - 'added:"previous quarter"', - "2026-04-01T00:00:00Z", - "2026-07-01T00:00:00Z", - id="quoted_previous_quarter", - ), - pytest.param( - "added:previous month", - "2026-06-01T00:00:00Z", - "2026-07-01T00:00:00Z", - id="bare_previous_month", - ), - pytest.param( - "added:this month", - "2026-07-01T00:00:00Z", - "2026-08-01T00:00:00Z", - id="bare_this_month", - ), - ], - ) - @time_machine.travel(datetime(2026, 7, 15, 12, 0, tzinfo=UTC), tick=False) - def test_legacy_natural_language_aliases( - self, - query: str, - expected_lo: str, - expected_hi: str, - ) -> None: - lo, hi = _range(translate_query(query, UTC), "added") - assert lo == expected_lo - assert hi == expected_hi - - def test_unknown_keyword_raises(self) -> None: - with pytest.raises(ValueError, match="Unknown keyword"): - _datetime_range("bogus_keyword", UTC) - - -class TestWhooshQueryRewriting: - """All Whoosh query syntax variants must be rewritten to ISO 8601 before Tantivy parses them.""" - - @time_machine.travel(datetime(2026, 3, 28, 15, 0, tzinfo=UTC), tick=False) - def test_compact_date_shim_rewrites_to_iso(self) -> None: - result = translate_query("created:20240115120000", UTC) - assert "2024-01-15" in result - assert "20240115120000" not in result - - @time_machine.travel(datetime(2026, 3, 28, 15, 0, tzinfo=UTC), tick=False) - def test_relative_range_shim_removes_now(self) -> None: - result = translate_query("added:[now-7d TO now]", UTC) - assert "now" not in result - assert "2026-03-" in result - - @time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False) - def test_bracket_minus_7_days(self) -> None: - lo, hi = _range( - translate_query("added:[-7 days to now]", UTC), - "added", - ) - assert lo == "2026-03-21T12:00:00Z" - assert hi == "2026-03-28T12:00:00Z" - - @time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False) - def test_bracket_minus_1_week(self) -> None: - lo, hi = _range( - translate_query("added:[-1 week to now]", UTC), - "added", - ) - assert lo == "2026-03-21T12:00:00Z" - assert hi == "2026-03-28T12:00:00Z" - - @time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False) - def test_bracket_minus_1_month_uses_relativedelta(self) -> None: - # relativedelta(months=1) from 2026-03-28 = 2026-02-28 (not 29) - lo, hi = _range( - translate_query("created:[-1 month to now]", UTC), - "created", - ) - assert lo == "2026-02-28T12:00:00Z" - assert hi == "2026-03-28T12:00:00Z" - - @time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False) - def test_bracket_minus_1_year(self) -> None: - lo, hi = _range( - translate_query("modified:[-1 year to now]", UTC), - "modified", - ) - assert lo == "2025-03-28T12:00:00Z" - assert hi == "2026-03-28T12:00:00Z" - - @time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False) - def test_bracket_plural_unit_hours(self) -> None: - lo, hi = _range( - translate_query("added:[-3 hours to now]", UTC), - "added", - ) - assert lo == "2026-03-28T09:00:00Z" - assert hi == "2026-03-28T12:00:00Z" - - @time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False) - def test_bracket_case_insensitive(self) -> None: - result = translate_query("added:[-1 WEEK TO NOW]", UTC) - assert "now" not in result.lower() - lo, hi = _range(result, "added") - assert lo == "2026-03-21T12:00:00Z" - assert hi == "2026-03-28T12:00:00Z" - - @time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False) - def test_relative_range_swaps_bounds_when_lo_exceeds_hi(self) -> None: - # [now+1h TO now-1h] has lo > hi before substitution; they must be swapped - lo, hi = _range( - translate_query("added:[now+1h TO now-1h]", UTC), - "added", - ) - assert lo == "2026-03-28T11:00:00Z" - assert hi == "2026-03-28T13:00:00Z" - - def test_8digit_created_date_field_always_uses_utc_midnight(self) -> None: - # created is a DateField: boundaries are always UTC midnight, no TZ offset - result = translate_query("created:20231201", EASTERN) - lo, hi = _range(result, "created") - assert lo == "2023-12-01T00:00:00Z" - assert hi == "2023-12-02T00:00:00Z" - - def test_8digit_added_datetime_field_converts_local_midnight_to_utc(self) -> None: - # added is DateTimeField: midnight Dec 1 Eastern (EST = UTC-5) = 05:00 UTC - result = translate_query("added:20231201", EASTERN) - lo, hi = _range(result, "added") - assert lo == "2023-12-01T05:00:00Z" - assert hi == "2023-12-02T05:00:00Z" - - def test_8digit_modified_datetime_field_converts_local_midnight_to_utc( - self, - ) -> None: - result = translate_query("modified:20231201", EASTERN) - lo, hi = _range(result, "modified") - assert lo == "2023-12-01T05:00:00Z" - assert hi == "2023-12-02T05:00:00Z" - - def test_8digit_invalid_date_raises(self) -> None: - # The translation pipeline raises InvalidDateQuery for unparsable dates - # (e.g. month=13) so the API can surface a 400 telling the user the date - # is malformed instead of silently returning zero results. - with pytest.raises(InvalidDateQuery) as exc_info: - translate_query("added:20231340", UTC) - assert exc_info.value.field == "added" - assert exc_info.value.value == "20231340" +def _highlight_hit_count(index: tantivy.Index, raw_query: str) -> int: + query = parse_simple_text_highlight_query(index, raw_query) + return index.searcher().search(query, limit=1).count class TestParseUserQuery: """parse_user_query runs the full preprocessing pipeline.""" - @pytest.fixture - def query_index(self) -> tantivy.Index: - schema = build_schema() - idx = tantivy.Index(schema, path=None) - register_tokenizers(idx, "") - return idx - def test_returns_tantivy_query(self, query_index: tantivy.Index) -> None: assert isinstance(parse_user_query(query_index, "invoice", UTC), tantivy.Query) + @pytest.mark.parametrize( + "raw_query", + [ + pytest.param("invoice", id="plain_text"), + pytest.param("created:today", id="date_keyword"), + pytest.param("created:[2005 to 2009]", id="whoosh_date_range"), + pytest.param('added:"previous month"', id="quoted_date_phrase"), + pytest.param("title:202[0-1]*", id="bracket_class_wildcard"), + ], + ) def test_fuzzy_mode_does_not_raise( self, query_index: tantivy.Index, settings, + raw_query: str, ) -> None: + # These are all valid whoosh grammar that tantivy's own query parser + # (used only by the fuzzy blend clause) cannot parse; the fuzzy + # clause must degrade gracefully instead of raising and failing the + # whole query. See _try_parse_fuzzy_query. settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5 - assert isinstance(parse_user_query(query_index, "invoice", UTC), tantivy.Query) + assert isinstance(parse_user_query(query_index, raw_query, UTC), tantivy.Query) - def test_date_rewriting_applied_before_tantivy_parse( + def test_date_keyword_resolves_without_raising( self, query_index: tantivy.Index, ) -> None: - # created:today must be rewritten to an ISO range before Tantivy parses it; - # if passed raw, Tantivy would reject "today" as an invalid date value + # whoosh-compat's DateParserPlugin resolves "today" against the AST + # directly (no string rewrite to an ISO range happens anywhere in + # this pipeline); the emitted tantivy query must still build cleanly. with time_machine.travel(datetime(2026, 3, 28, 12, 0, tzinfo=UTC), tick=False): q = parse_user_query(query_index, "created:today", UTC) assert isinstance(q, tantivy.Query) @@ -466,302 +114,58 @@ class TestParseUserQuery: ) -> None: assert isinstance(parse_user_query(query_index, raw_query, UTC), tantivy.Query) - @pytest.mark.parametrize( - "raw_query", - [ - # Partial date scalar (year only) - pytest.param("created:2020", id="created_year_scalar"), - # 8-digit compact date range in brackets - pytest.param( - "created:[20200101 TO 20201231]", - id="created_8digit_bracket_range", - ), - # Comma-separated field + date range (Whoosh v2 multi-clause syntax) - pytest.param( - "title:x,created:[2020 TO 2021]", - id="title_comma_created_range", - ), - # Field alias: type -> document_type - pytest.param("type:invoice", id="type_alias"), - # Multi-word date keyword - pytest.param("created:previous week", id="created_previous_week"), - # Full ISO datetime range - pytest.param( - "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]", - id="created_iso_range", - ), - # Comma-separated ISO ranges (Whoosh v2 syntax) - pytest.param( - "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]," - "added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]", - id="comma_iso_ranges", - ), - ], - ) - def test_advanced_search_queries_do_not_raise( - self, - query_index: tantivy.Index, - raw_query: str, - ) -> None: - """ - End-to-end: queries that the frontend sends must parse without raising. - - This tests the full pipeline: translate_query -> tantivy parse_query. - Equivalent to asserting HTTP 200 (not 400) for each query form. - """ - with time_machine.travel(datetime(2026, 6, 15, 12, 0, tzinfo=UTC), tick=False): - assert isinstance( - parse_user_query(query_index, raw_query, UTC), - tantivy.Query, - ) - def test_invalid_date_propagates_not_swallowed( self, query_index: tantivy.Index, ) -> None: - # parse_user_query falls back to the raw query on unexpected translation - # errors, but an InvalidDateQuery is intentional and must propagate so the - # view can return a 400 instead of silently parsing the raw (invalid) date. + # parse_user_query never falls back to the raw query string on a parse + # error — a bad date diagnostic from whoosh-compat always maps to an + # InvalidDateQuery and must propagate, so the view can return a 400 + # instead of silently parsing the raw (invalid) date. with pytest.raises(InvalidDateQuery) as exc_info: parse_user_query(query_index, "created:202023", UTC) assert exc_info.value.field == "created" assert exc_info.value.value == "202023" - -class TestYearRangeRewriting: - """Whoosh-style year-only date ranges must be rewritten to ISO 8601.""" - - @pytest.mark.parametrize( - ("query", "field", "expected_lo", "expected_hi"), - [ - pytest.param( - "created:[2020 TO 2020]", - "created", - "2020-01-01T00:00:00Z", - "2021-01-01T00:00:00Z", - id="single_year_created", - ), - pytest.param( - "created:[2018 TO 2021]", - "created", - "2018-01-01T00:00:00Z", - "2022-01-01T00:00:00Z", - id="multi_year_range_created", - ), - pytest.param( - "added:[2022 TO 2023]", - "added", - "2022-01-01T00:00:00Z", - "2024-01-01T00:00:00Z", - id="added_field", - ), - pytest.param( - "modified:[2021 TO 2021]", - "modified", - "2021-01-01T00:00:00Z", - "2022-01-01T00:00:00Z", - id="modified_field", - ), - pytest.param( - "created:[2020 to 2020]", - "created", - "2020-01-01T00:00:00Z", - "2021-01-01T00:00:00Z", - id="lowercase_to_keyword", - ), - ], - ) - def test_year_range_rewritten( + def test_invalid_number_raises_invalid_number_query( self, - query: str, - field: str, - expected_lo: str, - expected_hi: str, + query_index: tantivy.Index, ) -> None: - result = translate_query(query, UTC) - lo, hi = _range(result, field) - assert lo == expected_lo - assert hi == expected_hi + with pytest.raises(InvalidNumberQuery) as exc_info: + parse_user_query(query_index, "asn:notanumber", UTC) + assert exc_info.value.field == "asn" + assert exc_info.value.value == "notanumber" - def test_reversed_year_range_is_swapped(self) -> None: - # A reversed range must not yield lo > hi, which Tantivy treats as an - # empty range (silently zero results). The bounds are swapped instead. - result = translate_query("created:[2025 TO 2020]", UTC) - lo, hi = _range(result, "created") - assert lo == "2020-01-01T00:00:00Z" - assert hi == "2026-01-01T00:00:00Z" - - def test_year_range_in_complex_boolean_query(self) -> None: - query = "tag:steuer AND (title:2020 OR (NOT title:2019 AND NOT title:2018 AND created:[2020 TO 2020]))" - result = translate_query(query, UTC) - lo, hi = _range(result, "created") - assert lo == "2020-01-01T00:00:00Z" - assert hi == "2021-01-01T00:00:00Z" - assert "title:2020" in result - assert "title:2019" in result - assert "title:2018" in result - - def test_already_iso_date_range_passes_through_unchanged(self) -> None: - original = "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z]" - assert translate_query(original, UTC) == original - - def test_8digit_in_brackets_not_matched_as_year_range(self) -> None: - # [YYYYMMDD TO YYYYMMDD]: the translation layer converts 8-digit bounds to - # ISO day ranges. 20200101 -> 2020-01-01T00:00:00Z (lo of that day); - # 20201231 -> the ceil of Dec 31 = 2021-01-01T00:00:00Z (exclusive end). - # This is the correct and accepted behavior: old compact form becomes a - # proper Tantivy-parseable ISO range. - original = "created:[20200101 TO 20201231]" - result = translate_query(original, UTC) - lo, hi = _range(result, "created") - assert lo == "2020-01-01T00:00:00Z" - assert hi == "2021-01-01T00:00:00Z" - - -class TestNonDateFieldsNotRewritten: - """Date rewriters must only fire on the date fields (created/modified/added). - - Integer fields like asn/id/page_count and unknown fields would otherwise be - rewritten into date ranges and rejected by Tantivy as type mismatches. - """ - - @pytest.mark.parametrize( - "query", - [ - pytest.param("asn:20240101", id="asn_8digit"), - pytest.param("id:20240101", id="id_8digit"), - pytest.param("page_count:12345678", id="page_count_8digit"), - pytest.param("num_notes:20231201", id="num_notes_8digit"), - ], - ) - def test_8digit_on_integer_field_passes_through_unchanged(self, query: str) -> None: - assert translate_query(query, EASTERN) == query - - @pytest.mark.parametrize( - "query", - [ - pytest.param("asn:[2000 TO 2024]", id="asn_year_range"), - pytest.param("id:[2000 TO 2024]", id="id_year_range"), - pytest.param("page_count:[2000 TO 2024]", id="page_count_year_range"), - ], - ) - def test_year_range_on_integer_field_passes_through_unchanged( + def test_multiple_bad_fields_raise_multiple_search_query_errors( self, - query: str, + query_index: tantivy.Index, ) -> None: - assert translate_query(query, UTC) == query + with pytest.raises(MultipleSearchQueryErrors) as exc_info: + parse_user_query( + query_index, + "created:notadate AND asn:notanumber", + UTC, + ) + assert len(exc_info.value.errors) == 2 + kinds = {type(e) for e in exc_info.value.errors} + assert kinds == {InvalidDateQuery, InvalidNumberQuery} - def test_unknown_field_keyword_passes_through_unchanged(self) -> None: - # foobar is not a date field: 'foobar:today' must not become a date range, - # which Tantivy would otherwise reject as an unknown/typed field. - assert translate_query("foobar:today", UTC) == "foobar:today" - - -class TestPassthrough: - """Queries without field prefixes or unrelated content pass through unchanged.""" - - def test_bare_keyword_no_field_prefix_unchanged(self) -> None: - # Bare 'today' with no field: prefix passes through unchanged - result = translate_query("bank statement today", UTC) - assert "today" in result - - def test_unrelated_query_unchanged(self) -> None: - assert translate_query("title:invoice", UTC) == "title:invoice" - - -class TestNormalizeQuery: - """translate_query expands comma-separated values and collapses whitespace.""" - - def test_normalize_expands_comma_separated_tags(self) -> None: - assert translate_query("tag:foo,bar", UTC) == "tag:foo AND tag:bar" - - def test_normalize_comma_between_range_expressions(self) -> None: - # Comma-separated field range expressions (Whoosh v2 syntax) must be - # converted to AND so Tantivy does not receive an invalid comma. - q = "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z],added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]" - assert translate_query(q, UTC) == ( - "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]" - " AND " - "added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]" - ) - - def test_normalize_expands_three_values(self) -> None: - assert ( - translate_query("tag:foo,bar,baz", UTC) == "tag:foo AND tag:bar AND tag:baz" - ) - - def test_normalize_collapses_whitespace(self) -> None: - assert translate_query("bank statement", UTC) == "bank statement" - - def test_normalize_no_commas_unchanged(self) -> None: - assert translate_query("bank statement", UTC) == "bank statement" - - @pytest.mark.parametrize( - ("raw", "expected"), - [ - pytest.param( - "h52.1 - kurzsichtigkeit", - "h52.1 kurzsichtigkeit", - id="icd_code_dash_description", - ), - pytest.param( - "H52.1 - asd", - "H52.1 asd", - id="icd_code_uppercase_dash", - ), - pytest.param( - "h52.1 -", - "h52.1", - id="trailing_minus", - ), - pytest.param( - ". -", - ".", - id="dot_trailing_minus", - ), - pytest.param( - "h52. -", - "h52.", - id="partial_code_trailing_minus", - ), - pytest.param( - "foo - bar - baz", - "foo bar baz", - id="multiple_dashes", - ), - pytest.param( - "foo + bar", - "foo bar", - id="spaced_plus_operator", - ), - ], - ) - def test_normalize_strips_dangling_operators(self, raw: str, expected: str) -> None: - assert translate_query(raw, UTC) == expected - - @pytest.mark.parametrize( - "query", - [ - pytest.param("term -other", id="adjacent_not_operator"), - pytest.param("-term", id="leading_not_operator"), - pytest.param("+term", id="leading_must_operator"), - pytest.param("foo -bar +baz", id="mixed_adjacent_operators"), - ], - ) - def test_normalize_preserves_valid_operators(self, query: str) -> None: - assert translate_query(query, UTC) == query + def test_unregistered_id_field_folds_to_literal_text_not_error( + self, + query_index: tantivy.Index, + ) -> None: + # tag_id is intentionally excluded from the FieldRegistry — whoosh-compat + # parity leniency folds it into literal text, not a diagnostic/400. + # A result-level assertion that this fold actually matches nothing + # against real documents lives in + # test_acceptance.py::TestUnregisteredIdFieldFoldsToLiteralText. + q = parse_user_query(query_index, "tag_id:5", UTC) + assert isinstance(q, tantivy.Query) class TestParseSimpleTextHighlightQuery: """parse_simple_text_highlight_query must not raise on natural-language queries.""" - @pytest.fixture - def query_index(self) -> tantivy.Index: - schema = build_schema() - idx = tantivy.Index(schema, path=None) - register_tokenizers(idx, "") - return idx - @pytest.mark.parametrize( "raw_query", [ @@ -783,16 +187,25 @@ class TestParseSimpleTextHighlightQuery: tantivy.Query, ) - def test_empty_query_returns_empty_query(self, query_index: tantivy.Index) -> None: - result = parse_simple_text_highlight_query(query_index, "") - assert isinstance(result, tantivy.Query) - - def test_all_operators_returns_empty_query( + def test_a_real_token_matches_the_corpus( self, - query_index: tantivy.Index, + populated_index: tantivy.Index, ) -> None: - result = parse_simple_text_highlight_query(query_index, "- +") - assert isinstance(result, tantivy.Query) + """Without this, an empty corpus would make the two assertions below + pass for a query that matches every document.""" + assert _highlight_hit_count(populated_index, "needle") == 1 + + def test_empty_query_matches_no_document( + self, + populated_index: tantivy.Index, + ) -> None: + assert _highlight_hit_count(populated_index, "") == 0 + + def test_all_operators_query_matches_no_document( + self, + populated_index: tantivy.Index, + ) -> None: + assert _highlight_hit_count(populated_index, "- +") == 0 class TestPermissionFilter: @@ -884,3 +297,52 @@ class TestPermissionFilter: user = django_user_model(pk=20) perm = build_permission_filter(perm_index.schema, user) assert perm_index.searcher().search(perm, limit=10).count == 1 # only unowned + + +class TestSearchQueryErrors: + def test_invalid_date_query_is_a_search_query_error(self) -> None: + err = InvalidDateQuery("created", "notadate") + assert isinstance(err, SearchQueryError) + assert err.field == "created" + assert err.value == "notadate" + assert "created" in str(err) + assert "notadate" in str(err) + + def test_invalid_number_query_is_a_search_query_error(self) -> None: + err = InvalidNumberQuery("asn", "notanumber") + assert isinstance(err, SearchQueryError) + assert err.field == "asn" + assert err.value == "notanumber" + assert "asn" in str(err) + assert "notanumber" in str(err) + + def test_multiple_search_query_errors_aggregates(self) -> None: + sub_errors = [ + InvalidDateQuery("created", "notadate"), + InvalidNumberQuery("asn", "notanumber"), + ] + err = MultipleSearchQueryErrors(sub_errors) + assert isinstance(err, SearchQueryError) + assert err.errors == tuple(sub_errors) + assert "created" in str(err) + assert "asn" in str(err) + + +class TestEmitErrorContract: + """A QueryError from emit() surfaces as a SearchQueryError (HTTP 400). + + The Cause-based routing table itself is covered in test_error_routing.py. + """ + + def test_exists_requires_fast_gets_the_user_facing_rewrite( + self, + query_index: tantivy.Index, + ) -> None: + # whoosh-compat's own message advises a host-side fast=True config + # change the user can't act on, so this checks OUR wording, not + # whoosh-compat's (that's its own test suite's job now). + with pytest.raises(SearchQueryError) as exc_info: + parse_user_query(query_index, "notes.user:*", UTC) + assert str(exc_info.value) == ( + "Existence searches (field:*) are not supported for field 'notes.user'." + ) diff --git a/src/documents/tests/search/test_query_negation.py b/src/documents/tests/search/test_query_negation.py new file mode 100644 index 000000000..8b3cd0b4c --- /dev/null +++ b/src/documents/tests/search/test_query_negation.py @@ -0,0 +1,153 @@ +"""Negation must survive the blended query. + +parse_user_query ORs an exact clause with optional fuzzy and CJK clauses. +Each of those is built from positive terms only, so unless the query's +exclusions are applied to the blend as a whole, a document the exact +clause excluded is re-admitted by whichever other clause is enabled. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from documents.models import Document + +if TYPE_CHECKING: + from pytest_django.fixtures import SettingsWrapper + + from documents.search._backend import TantivyBackend + +pytestmark = [pytest.mark.search, pytest.mark.django_db] + + +def _matched_ids(backend: TantivyBackend, query: str) -> set[int]: + return set(backend.search_ids(query, user=None)) + + +def _index(backend: TantivyBackend, **kwargs: object) -> Document: + doc = Document.objects.create(**kwargs) + backend.add_or_update(doc) + return doc + + +@pytest.fixture +def fuzzy_enabled(settings: SettingsWrapper) -> None: + """Enable the fuzzy blend clause. The threshold doubles as a minimum + score filter, so it is set to 0.0: every hit passes and the test sees + the clause's matching behaviour, not the filter's.""" + settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.0 + + +class TestNegationConstrainsEveryClause: + @pytest.mark.usefixtures("fuzzy_enabled") + def test_fuzzy_clause_does_not_readmit_an_excluded_document( + self, + backend: TantivyBackend, + ) -> None: + secret = _index( + backend, + title="Invoice A", + content="invoice total secret", + checksum="neg-fuzzy-1", + ) + public = _index( + backend, + title="Invoice B", + content="invoice total public", + checksum="neg-fuzzy-2", + ) + + assert _matched_ids(backend, "invoice") == {secret.pk, public.pk} + assert _matched_ids(backend, "invoice NOT secret") == {public.pk} + + def test_cjk_clause_does_not_readmit_an_excluded_document( + self, + backend: TantivyBackend, + ) -> None: + """The CJK clause legitimately carries 東京 here, so rebuilding it + from the AST cannot help: only applying the exclusion above the + blend keeps the secret document out.""" + secret = _index( + backend, + title="Tokyo A", + content="東京都の秘密です secret", + checksum="neg-cjk-1", + ) + public = _index( + backend, + title="Tokyo B", + content="東京都の報告書です public", + checksum="neg-cjk-2", + ) + + assert _matched_ids(backend, "東京") == {secret.pk, public.pk} + assert _matched_ids(backend, "東京 NOT secret") == {public.pk} + + @pytest.mark.usefixtures("fuzzy_enabled") + def test_disjunctive_negation_still_admits_the_other_branch( + self, + backend: TantivyBackend, + ) -> None: + """'invoice OR NOT secret' excludes nothing on its own: a document + matching the left branch stays in even though it contains secret.""" + secret_invoice = _index( + backend, + title="Invoice A", + content="invoice total secret", + checksum="neg-or-1", + ) + unrelated = _index( + backend, + title="Recipe", + content="flour and water", + checksum="neg-or-2", + ) + + assert _matched_ids(backend, "invoice OR NOT secret") == { + secret_invoice.pk, + unrelated.pk, + } + + def test_a_negation_under_or_does_not_constrain_the_cjk_clause( + self, + backend: TantivyBackend, + ) -> None: + """The limit of the hoist, pinned deliberately. + + An exclusion that is one branch's own condition cannot be restated + above the blend without dropping documents the other branch + matches, so it is left where it is and the CJK clause stays + unconstrained by it. That shows through here in a way it does not + for latin text: the exact clause cannot match a CJK run at all, so + the CJK clause is the only thing matching the tokyo documents, and + the secret one comes with it. + """ + secret = _index( + backend, + title="Tokyo A", + content="東京都の秘密です secret", + checksum="neg-or-cjk-1", + ) + public = _index( + backend, + title="Tokyo B", + content="東京都の報告書です public", + checksum="neg-or-cjk-2", + ) + bill = _index( + backend, + title="Bill", + content="bill payment received", + checksum="neg-or-cjk-3", + ) + + assert _matched_ids(backend, "(東京 AND NOT secret) OR bill") == { + bill.pk, + public.pk, + secret.pk, + } + # The same exclusion in conjunctive position is hoisted, and does + # constrain the CJK clause. + assert _matched_ids(backend, "東京 AND NOT secret") == {public.pk} diff --git a/src/documents/tests/search/test_translate.py b/src/documents/tests/search/test_translate.py deleted file mode 100644 index 25e5bc774..000000000 --- a/src/documents/tests/search/test_translate.py +++ /dev/null @@ -1,810 +0,0 @@ -from __future__ import annotations - -from datetime import UTC -from datetime import datetime -from typing import TYPE_CHECKING -from zoneinfo import ZoneInfo - -import pytest -import time_machine - -from documents.search._dates import _precision_bounds - -if TYPE_CHECKING: - import tantivy -from documents.search._query import _FIELD_BOOSTS -from documents.search._query import DEFAULT_SEARCH_FIELDS -from documents.search._translate import OPEN_HI -from documents.search._translate import OPEN_LO -from documents.search._translate import Comma -from documents.search._translate import FieldRange -from documents.search._translate import FieldValue -from documents.search._translate import FieldValueList -from documents.search._translate import InvalidDateQuery -from documents.search._translate import Passthrough -from documents.search._translate import resolve_commas -from documents.search._translate import scan -from documents.search._translate import translate_query -from documents.search._translate import translate_range -from documents.search._translate import translate_scalar - - -@pytest.mark.search -class TestPrecisionBounds: - @pytest.mark.parametrize( - ("digits", "expected"), - [ - ("2020", ((2020, 1, 1), (2021, 1, 1))), - ("202003", ((2020, 3, 1), (2020, 4, 1))), - ("202012", ((2020, 12, 1), (2021, 1, 1))), - ("20200115", ((2020, 1, 15), (2020, 1, 16))), - ("20201231", ((2020, 12, 31), (2021, 1, 1))), - ], - ) - def test_valid(self, digits, expected): - lo, hi = _precision_bounds(digits) - assert (lo.year, lo.month, lo.day) == expected[0] - assert (hi.year, hi.month, hi.day) == expected[1] - - @pytest.mark.parametrize("digits", ["202023", "20200230", "20201301", "20", "abcd"]) - def test_invalid_returns_none(self, digits): - assert _precision_bounds(digits) is None - - -@pytest.mark.search -class TestScan: - def test_plain_words_are_passthrough(self): - assert scan("bank statement") == [Passthrough("bank statement")] - - def test_field_value(self): - assert scan("created:2020") == [FieldValue("created", "2020")] - - def test_field_value_in_boolean(self): - toks = scan("created:2020 OR foo") - assert toks == [ - FieldValue("created", "2020"), - Passthrough(" OR foo"), - ] - - def test_field_value_in_parens(self): - toks = scan("(created:2020 OR foo)") - assert toks == [ - Passthrough("("), - FieldValue("created", "2020"), - Passthrough(" OR foo)"), - ] - - def test_quoted_value(self): - assert scan('correspondent:"A B"') == [FieldValue("correspondent", '"A B"')] - - def test_field_range(self): - assert scan("created:[2020 TO 2021]") == [ - FieldRange("created", "[", "2020", "2021", "]"), - ] - - @pytest.mark.parametrize( - ("query", "expected"), - [ - pytest.param( - "created:[2020 to]", - FieldRange("created", "[", "2020", "", "]"), - id="open_upper", - ), - pytest.param( - "created:[to 2020]", - FieldRange("created", "[", "", "2020", "]"), - id="open_lower", - ), - ], - ) - def test_open_range(self, query, expected): - assert scan(query) == [expected] - - def test_comma_inside_range_not_split(self): - # No depth-0 comma here; the whole thing is one range token. - toks = scan("created:[2020 TO 2021]") - assert len(toks) == 1 - - # --- Edge-case / regression tests (scan must never raise) --- - - def test_url_is_passthrough(self): - # "http" is not a known field; the whole URL must pass through verbatim. - assert scan("http://example.com") == [Passthrough("http://example.com")] - - def test_unterminated_quote_is_passthrough(self): - # title is a known field but the quoted value has no closing quote; - # _consume_value returns None so the whole string falls into passthrough. - assert scan('title:"abc') == [Passthrough('title:"abc')] - - def test_unterminated_bracket_is_passthrough(self): - # created is a known field but the range bracket is never closed; - # _consume_range returns None so the whole string falls into passthrough. - assert scan("created:[2020") == [Passthrough("created:[2020")] - - def test_empty_value_at_end_is_passthrough(self): - # created is a known field but there is no value after the colon - # (_consume_value returns None for start >= n), so passthrough. - assert scan("created:") == [Passthrough("created:")] - - def test_value_containing_colon(self): - # The bare-word value reader stops at whitespace/paren, not at colon, - # so "2020:30" is consumed as a single value token. - assert scan("created:2020:30") == [FieldValue("created", "2020:30")] - - def test_comma_followed_by_unconsumable_value_stops(self): - # A comma followed by whitespace is neither a value-list continuation nor a - # clause separator: the value stops and the comma stays as passthrough. - assert scan("tag:foo, bar") == [ - FieldValue("tag", "foo"), - Passthrough(", bar"), - ] - - def test_bracket_without_to_is_open_upper_bound(self): - # A bracketed value with no TO falls back to (value, "") -> open upper bound. - assert scan("created:[2020]") == [ - FieldRange("created", "[", "2020", "", "]"), - ] - - def test_known_field_name_midword_is_passthrough(self): - # A known field name embedded mid-word is not a field token (the - # word-boundary guard); the whole run stays passthrough. - assert scan("xtag:foo") == [Passthrough("xtag:foo")] - - -@pytest.mark.search -class TestCommaResolution: - def test_value_list_multi_value_field(self): - toks = resolve_commas(scan("tag:foo,bar")) - assert toks == [FieldValueList("tag", ("foo", "bar"))] - - def test_value_list_three(self): - toks = resolve_commas(scan("tag_id:1,2,3")) - assert toks == [FieldValueList("tag_id", ("1", "2", "3"))] - - def test_text_field_comma_is_literal(self): - # correspondent is not multi-value: comma stays inside the value. - toks = resolve_commas(scan("correspondent:foo,bar")) - assert toks == [FieldValue("correspondent", "foo,bar")] - - def test_clause_separator_before_known_field(self): - toks = resolve_commas(scan("tag:foo,type:bar")) - assert toks == [FieldValue("tag", "foo"), Comma(), FieldValue("type", "bar")] - - def test_clause_separator_after_range(self): - toks = resolve_commas(scan("created:[2020 TO 2021],added:[2022 TO 2023]")) - assert toks == [ - FieldRange("created", "[", "2020", "2021", "]"), - Comma(), - FieldRange("added", "[", "2022", "2023", "]"), - ] - - def test_clause_separator_after_quote(self): - toks = resolve_commas(scan('correspondent:"A B",created:[2020 TO 2021]')) - assert toks == [ - FieldValue("correspondent", '"A B"'), - Comma(), - FieldRange("created", "[", "2020", "2021", "]"), - ] - - def test_url_comma_is_literal_passthrough(self): - toks = resolve_commas(scan("http://example.com/a,b")) - assert toks == [Passthrough("http://example.com/a,b")] - - def test_non_multi_value_comma_is_literal(self): - # title is not in MULTI_VALUE_FIELDS: comma stays inside the value. - toks = resolve_commas(scan("title:10,20")) - assert toks == [FieldValue("title", "10,20")] - - def test_clause_separator_before_known_date_field(self): - # The comma between a bare value and a known date field acts as a - # clause separator; both sides survive as distinct tokens. - toks = resolve_commas(scan("correspondent:foo,created:[2020 TO 2021]")) - assert toks == [ - FieldValue("correspondent", "foo"), - Comma(), - FieldRange("created", "[", "2020", "2021", "]"), - ] - - -@pytest.mark.search -class TestTranslateScalar: - @pytest.mark.parametrize( - ("field", "value", "expected"), - [ - ( - "created", - "2020", - "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}", - ), - ( - "created", - "202003", - "created:[2020-03-01T00:00:00Z TO 2020-04-01T00:00:00Z}", - ), - ( - "created", - "20200115", - "created:[2020-01-15T00:00:00Z TO 2020-01-16T00:00:00Z}", - ), - ( - "created", - "2020-01-15", - "created:[2020-01-15T00:00:00Z TO 2020-01-16T00:00:00Z}", - ), - ( - "created", - "2020-03", - "created:[2020-03-01T00:00:00Z TO 2020-04-01T00:00:00Z}", - ), - ], - ) - def test_partial_and_iso_dates(self, field: str, value: str, expected: str) -> None: - assert translate_scalar(field, value, UTC) == expected - - def test_invalid_date_raises(self) -> None: - with pytest.raises(InvalidDateQuery) as exc_info: - translate_scalar("created", "202023", UTC) - assert exc_info.value.field == "created" - assert exc_info.value.value == "202023" - - def test_keyword_delegates(self) -> None: - # keyword path produces a half-open range; just assert it is a created range - out = translate_scalar("created", "today", UTC) - assert out.startswith("created:[") and out.endswith("}") - - def test_14digit_compact_datetime(self) -> None: - out = translate_scalar("created", "20240115120000", UTC) - assert "20240115120000" not in out - assert out.startswith("created:") - assert out == "created:[2024-01-15T12:00:00Z TO 2024-01-15T12:00:00Z]" - - def test_14digit_invalid_month_raises(self) -> None: - with pytest.raises(InvalidDateQuery) as exc_info: - translate_scalar("created", "20231300120000", UTC) - assert exc_info.value.field == "created" - assert exc_info.value.value == "20231300120000" - - def test_unrecognized_value_raises(self) -> None: - # A value that is not a keyword, digits, ISO date, or compact timestamp - # raises rather than producing invalid Tantivy syntax or silently matching - # nothing. - with pytest.raises(InvalidDateQuery) as exc_info: - translate_scalar("created", "garbage", UTC) - assert exc_info.value.field == "created" - assert exc_info.value.value == "garbage" - - -@pytest.mark.search -class TestTranslateRange: - @pytest.mark.parametrize( - ("lo", "hi", "expected"), - [ - ("2005", "2009", "created:[2005-01-01T00:00:00Z TO 2010-01-01T00:00:00Z}"), - ( - "202001", - "202006", - "created:[2020-01-01T00:00:00Z TO 2020-07-01T00:00:00Z}", - ), - ( - "20200101", - "20201231", - "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}", - ), - ( - "2020-01-01", - "2020-12-31", - "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}", - ), - ], - ) - def test_absolute_ranges(self, lo, hi, expected): - assert translate_range("created", lo, hi, UTC) == expected - - def test_reversed_swaps(self): - assert translate_range("created", "2009", "2005", UTC) == ( - "created:[2005-01-01T00:00:00Z TO 2010-01-01T00:00:00Z}" - ) - - def test_open_upper(self): - out = translate_range("created", "2020", "", UTC) - assert out == f"created:[2020-01-01T00:00:00Z TO {OPEN_HI}]" - - def test_open_lower(self): - out = translate_range("created", "", "2020", UTC) - assert out == f"created:[{OPEN_LO} TO 2021-01-01T00:00:00Z}}" - - def test_invalid_bound_raises(self): - with pytest.raises(InvalidDateQuery) as exc_info: - translate_range("created", "202023", "2025", UTC) - assert exc_info.value.field == "created" - assert exc_info.value.value == "202023" - - def test_invalid_high_bound_raises(self): - # Low bound parses, high bound does not -> raise on the high bound. - with pytest.raises(InvalidDateQuery) as exc_info: - translate_range("created", "2020", "garbage", UTC) - assert exc_info.value.field == "created" - assert exc_info.value.value == "garbage" - - -@pytest.mark.search -class TestTranslateQuery: - @pytest.mark.parametrize( - ("raw", "expected"), - [ - ( - "created:2020", - "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}", - ), - ("tag:foo,bar", "tag:foo AND tag:bar"), - # 'type' is a user-facing alias rewritten to 'document_type' (the real schema field) - ("tag:foo,type:bar", "tag:foo AND document_type:bar"), - ( - "created:[2020 TO 2021],added:[2022 TO 2023]", - ( - "created:[2020-01-01T00:00:00Z TO 2022-01-01T00:00:00Z}" - " AND " - "added:[2022-01-01T00:00:00Z TO 2024-01-01T00:00:00Z}" - ), - ), - # correspondent is not multi-value: comma stays literal inside the value - ("correspondent:foo,bar", "correspondent:foo,bar"), - ], - ) - def test_golden(self, raw: str, expected: str) -> None: - assert translate_query(raw, UTC) == expected - - @pytest.mark.parametrize( - "raw", - [ - "created:2020", - "created:202003", - "created:[20200101 TO 20201231]", - "created:[2020-01-01 TO 2020-12-31]", - "created:[2020 to]", - "created:[to 2020]", - "title:x,created:[2020 TO 2021]", - "created:2020 OR foo", - "(created:2020 OR invoice)", - "tag:foo,type:bar", - "bank statement", - ], - ) - def test_parse_acceptance(self, index: tantivy.Index, raw: str) -> None: - translated = translate_query(raw, UTC) - # Must not raise: - index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS) - - -@pytest.mark.search -class TestFieldAliasing: - """Whoosh->Tantivy field-name aliasing (type/path -> document_type/storage_path).""" - - def test_type_alias(self) -> None: - assert translate_query("type:invoice", UTC) == "document_type:invoice" - - def test_path_alias(self) -> None: - assert translate_query("path:/foo/bar", UTC) == "storage_path:/foo/bar" - - def test_type_id_alias(self) -> None: - assert translate_query("type_id:5", UTC) == "document_type_id:5" - - def test_path_id_alias(self) -> None: - assert translate_query("path_id:7", UTC) == "storage_path_id:7" - - def test_clause_separator_plus_alias(self) -> None: - # Comma between known fields acts as AND separator; alias still applied. - assert ( - translate_query("tag:foo,type:bar", UTC) == "tag:foo AND document_type:bar" - ) - - def test_type_range_alias(self) -> None: - # type is not a date field; range passes through verbatim with alias applied. - assert ( - translate_query("type:[2020 TO 2021]", UTC) - == "document_type:[2020 TO 2021]" - ) - - def test_parse_acceptance_type(self, index: tantivy.Index) -> None: - # Translated output must be accepted by the real Tantivy parser. - translated = translate_query("type:invoice", UTC) - index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS) - - def test_parse_acceptance_path(self, index: tantivy.Index) -> None: - translated = translate_query("path:foo", UTC) - index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS) - - -# Freeze time so relative-date tests are deterministic. -_FROZEN_NOW = datetime(2026, 3, 28, 12, 0, 0, tzinfo=UTC) - - -@pytest.mark.search -class TestRelativeRanges: - """Relative date-range tokens resolved against a frozen clock.""" - - @time_machine.travel(_FROZEN_NOW, tick=False) - def test_minus_7_days_to_now(self) -> None: - assert translate_query("added:[-7 days to now]", UTC) == ( - "added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]" - ) - - @time_machine.travel(_FROZEN_NOW, tick=False) - def test_minus_1_week_to_now(self) -> None: - assert translate_query("added:[-1 week to now]", UTC) == ( - "added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]" - ) - - @time_machine.travel(_FROZEN_NOW, tick=False) - def test_minus_1_month_to_now(self) -> None: - assert translate_query("created:[-1 month to now]", UTC) == ( - "created:[2026-02-28T12:00:00Z TO 2026-03-28T12:00:00Z]" - ) - - @time_machine.travel(_FROZEN_NOW, tick=False) - def test_minus_1_year_to_now(self) -> None: - assert translate_query("modified:[-1 year to now]", UTC) == ( - "modified:[2025-03-28T12:00:00Z TO 2026-03-28T12:00:00Z]" - ) - - @time_machine.travel(_FROZEN_NOW, tick=False) - def test_minus_3_hours_to_now(self) -> None: - assert translate_query("added:[-3 hours to now]", UTC) == ( - "added:[2026-03-28T09:00:00Z TO 2026-03-28T12:00:00Z]" - ) - - @time_machine.travel(_FROZEN_NOW, tick=False) - def test_uppercase_units(self) -> None: - assert translate_query("added:[-1 WEEK TO NOW]", UTC) == ( - "added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]" - ) - - @time_machine.travel(_FROZEN_NOW, tick=False) - def test_now_minus_7d_compact(self) -> None: - assert translate_query("added:[now-7d TO now]", UTC) == ( - "added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]" - ) - - @time_machine.travel(_FROZEN_NOW, tick=False) - def test_reversed_range_swapped(self) -> None: - # now+1h TO now-1h is reversed; translate_range swaps -> lo=now-1h, hi=now+1h - assert translate_query("added:[now+1h TO now-1h]", UTC) == ( - "added:[2026-03-28T11:00:00Z TO 2026-03-28T13:00:00Z]" - ) - - @pytest.mark.parametrize( - "raw", - [ - "added:[-7 days to now]", - "added:[-1 week to now]", - "created:[-1 month to now]", - "modified:[-1 year to now]", - "added:[-3 hours to now]", - "added:[now-7d TO now]", - "added:[now+1h TO now-1h]", - ], - ) - @time_machine.travel(_FROZEN_NOW, tick=False) - def test_parse_acceptance(self, index: tantivy.Index, raw: str) -> None: - translated = translate_query(raw, UTC) - index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS) - - -@pytest.mark.search -class TestWhooshUnitAbbreviations: - """ - Whoosh's PlusMinus date grammar accepted abbreviated unit spellings - (e.g. "yrs", "mos", "wks", "hrs", "mins", "secs"); saved views/searches - created under the old Whoosh backend can contain those tokens (see - https://github.com/paperless-ngx/paperless-ngx/issues/13482), so the - Tantivy translator must still accept them. - """ - - @time_machine.travel(_FROZEN_NOW, tick=False) - def test_minus_999_yrs(self) -> None: - assert translate_query("created:[-999yrs to now]", UTC) == ( - "created:[1027-03-28T12:00:00Z TO 2026-03-28T12:00:00Z]" - ) - - @pytest.mark.parametrize( - ("token", "expected_lo"), - [ - ("-1y", "2025-03-28T12:00:00Z"), - ("-1yr", "2025-03-28T12:00:00Z"), - ("-3mos", "2025-12-28T12:00:00Z"), - ("-3mo", "2025-12-28T12:00:00Z"), - ("-2wks", "2026-03-14T12:00:00Z"), - ("-2wk", "2026-03-14T12:00:00Z"), - ("-5dys", "2026-03-23T12:00:00Z"), - ("-5dy", "2026-03-23T12:00:00Z"), - ("-1hrs", "2026-03-28T11:00:00Z"), - ("-1hr", "2026-03-28T11:00:00Z"), - ("-10mins", "2026-03-28T11:50:00Z"), - ("-10min", "2026-03-28T11:50:00Z"), - ("-30secs", "2026-03-28T11:59:30Z"), - ("-30sec", "2026-03-28T11:59:30Z"), - ], - ) - @time_machine.travel(_FROZEN_NOW, tick=False) - def test_abbreviated_units(self, token: str, expected_lo: str) -> None: - assert translate_query(f"added:[{token} to now]", UTC) == ( - f"added:[{expected_lo} TO 2026-03-28T12:00:00Z]" - ) - - @pytest.mark.parametrize( - "raw", - [ - "created:[-999yrs to now]", - "added:[-1y to now]", - "created:[-3mos to now]", - "added:[-2wks to now]", - "added:[-5dys to now]", - "added:[-1hrs to now]", - "added:[-10mins to now]", - "added:[-30secs to now]", - ], - ) - @time_machine.travel(_FROZEN_NOW, tick=False) - def test_parse_acceptance(self, index: tantivy.Index, raw: str) -> None: - translated = translate_query(raw, UTC) - index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS) - - -@pytest.mark.search -class TestOperatorNormalization: - """Post-render operator normalization in translate_query.""" - - def test_spaced_dash_removed(self) -> None: - assert ( - translate_query("H52.1 - Kurzsichtigkeit", UTC) == "H52.1 Kurzsichtigkeit" - ) - - def test_spaced_dash_simple(self) -> None: - assert translate_query("bar - baz", UTC) == "bar baz" - - def test_trailing_operator_stripped(self) -> None: - assert translate_query("foo -", UTC) == "foo" - - def test_date_range_preserved(self) -> None: - out = translate_query("created:[2020 TO 2021]", UTC) - # Must not corrupt the ISO range - assert out == "created:[2020-01-01T00:00:00Z TO 2022-01-01T00:00:00Z}" - - def test_date_scalar_with_or(self) -> None: - out = translate_query("created:2020 OR foo", UTC) - # The created scalar becomes a range; " OR foo" passes through verbatim. - assert out.startswith("created:[") - assert "OR foo" in out - - def test_parse_acceptance_spaced_dash(self, index: tantivy.Index) -> None: - translated = translate_query("H52.1 - Kurzsichtigkeit", UTC) - index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS) - - def test_parse_acceptance_trailing_op(self, index: tantivy.Index) -> None: - translated = translate_query("foo -", UTC) - index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS) - - -@pytest.mark.search -class TestMultiWordDateKeywords: - """scan() must consume multi-word date keywords as a single value.""" - - def test_scan_previous_week_as_single_token(self) -> None: - # "created:previous week" must produce one FieldValue with value "previous week", - # not FieldValue("created","previous") + Passthrough(" week"). - toks = scan("created:previous week") - assert toks == [FieldValue("created", "previous week")] - - def test_scan_this_month_as_single_token(self) -> None: - toks = scan("added:this month") - assert toks == [FieldValue("added", "this month")] - - def test_scan_previous_month_as_single_token(self) -> None: - toks = scan("created:previous month") - assert toks == [FieldValue("created", "previous month")] - - def test_scan_this_year_as_single_token(self) -> None: - toks = scan("added:this year") - assert toks == [FieldValue("added", "this year")] - - def test_scan_previous_year_as_single_token(self) -> None: - toks = scan("created:previous year") - assert toks == [FieldValue("created", "previous year")] - - def test_scan_previous_quarter_as_single_token(self) -> None: - toks = scan("created:previous quarter") - assert toks == [FieldValue("created", "previous quarter")] - - def test_quoted_multi_word_keyword_still_works(self) -> None: - # The quoted form must continue to work as before. - toks = scan('created:"previous week"') - assert toks == [FieldValue("created", '"previous week"')] - - def test_non_date_field_not_affected(self) -> None: - # "previous" stops at the space for non-date fields; " week" passes through. - toks = scan("correspondent:previous week") - assert toks == [ - FieldValue("correspondent", "previous"), - Passthrough(" week"), - ] - - -@pytest.mark.search -class TestKeywordDateResolution: - """Relative date keywords resolve to exact ISO ranges against a frozen clock. - - Frozen at 2026-03-28 12:00 UTC (a Saturday in Q1) so the week, month, - quarter and year rollovers are all exercised by a single anchor. - """ - - # created is a DateField: bounds are UTC midnight, no timezone offset. - @pytest.mark.parametrize( - ("keyword", "expected"), - [ - pytest.param( - "today", - "created:[2026-03-28T00:00:00Z TO 2026-03-29T00:00:00Z}", - id="today", - ), - pytest.param( - "yesterday", - "created:[2026-03-27T00:00:00Z TO 2026-03-28T00:00:00Z}", - id="yesterday", - ), - pytest.param( - "previous week", - "created:[2026-03-16T00:00:00Z TO 2026-03-23T00:00:00Z}", - id="previous-week", - ), - pytest.param( - "this month", - "created:[2026-03-01T00:00:00Z TO 2026-04-01T00:00:00Z}", - id="this-month", - ), - pytest.param( - "previous month", - "created:[2026-02-01T00:00:00Z TO 2026-03-01T00:00:00Z}", - id="previous-month", - ), - pytest.param( - "this year", - "created:[2026-01-01T00:00:00Z TO 2027-01-01T00:00:00Z}", - id="this-year", - ), - pytest.param( - "previous year", - "created:[2025-01-01T00:00:00Z TO 2026-01-01T00:00:00Z}", - id="previous-year", - ), - pytest.param( - "previous quarter", - "created:[2025-10-01T00:00:00Z TO 2026-01-01T00:00:00Z}", - id="previous-quarter", - ), - ], - ) - @time_machine.travel(_FROZEN_NOW, tick=False) - def test_date_only_field_keyword_ranges( - self, - keyword: str, - expected: str, - ) -> None: - assert translate_query(f"created:{keyword}", UTC) == expected - - # added is a DateTimeField: local-tz midnight converted to UTC. Tokyo - # (+09:00, no DST) shifts each midnight boundary back to 15:00Z the day - # before, so this also exercises the local-midnight offset path. - @pytest.mark.parametrize( - ("keyword", "expected"), - [ - pytest.param( - "today", - "added:[2026-03-27T15:00:00Z TO 2026-03-28T15:00:00Z}", - id="today", - ), - pytest.param( - "yesterday", - "added:[2026-03-26T15:00:00Z TO 2026-03-27T15:00:00Z}", - id="yesterday", - ), - pytest.param( - "previous week", - "added:[2026-03-15T15:00:00Z TO 2026-03-22T15:00:00Z}", - id="previous-week", - ), - pytest.param( - "this month", - "added:[2026-02-28T15:00:00Z TO 2026-03-31T15:00:00Z}", - id="this-month", - ), - pytest.param( - "previous month", - "added:[2026-01-31T15:00:00Z TO 2026-02-28T15:00:00Z}", - id="previous-month", - ), - pytest.param( - "this year", - "added:[2025-12-31T15:00:00Z TO 2026-12-31T15:00:00Z}", - id="this-year", - ), - pytest.param( - "previous year", - "added:[2024-12-31T15:00:00Z TO 2025-12-31T15:00:00Z}", - id="previous-year", - ), - pytest.param( - "previous quarter", - "added:[2025-09-30T15:00:00Z TO 2025-12-31T15:00:00Z}", - id="previous-quarter", - ), - ], - ) - @time_machine.travel(_FROZEN_NOW, tick=False) - def test_datetime_field_keyword_ranges_local_tz( - self, - keyword: str, - expected: str, - ) -> None: - assert translate_query(f"added:{keyword}", ZoneInfo("Asia/Tokyo")) == expected - - -@pytest.mark.search -class TestISODatetimeBounds: - """Full ISO datetime tokens in range bounds must be parsed directly.""" - - def test_translate_range_iso_bounds_passthrough(self) -> None: - # Already-ISO datetime bounds must pass through as-is (exact instant). - result = translate_range( - "created", - "2020-01-01T00:00:00Z", - "2021-01-01T00:00:00Z", - UTC, - ) - assert result == "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z]" - - def test_translate_query_iso_range_preserved(self) -> None: - q = "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]" - assert translate_query(q, UTC) == q - - def test_translate_query_comma_separated_iso_ranges(self) -> None: - q = ( - "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]," - "added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]" - ) - result = translate_query(q, UTC) - assert result == ( - "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]" - " AND " - "added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]" - ) - - def test_translate_query_text_before_comma_separated_date_clause(self) -> None: - result = translate_query("schäfersee,created:previous year", UTC) - assert result == ( - "schäfersee AND created:[2025-01-01T00:00:00Z TO 2026-01-01T00:00:00Z}" - ) - - def test_invalid_iso_datetime_raises(self) -> None: - # A token with "T" that is not valid ISO datetime -> raise. - with pytest.raises(InvalidDateQuery) as exc_info: - translate_range( - "created", - "2020-01-01T99:00:00Z", - "2021-01-01T00:00:00Z", - UTC, - ) - assert exc_info.value.field == "created" - assert exc_info.value.value == "2020-01-01T99:00:00Z" - - def test_parse_acceptance_iso_bounds(self, index: tantivy.Index) -> None: - q = "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]" - translated = translate_query(q, UTC) - index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS) - - def test_parse_acceptance_comma_iso_ranges(self, index: tantivy.Index) -> None: - q = ( - "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]," - "added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]" - ) - translated = translate_query(q, UTC) - index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS) diff --git a/src/documents/tests/test_api_bulk_download.py b/src/documents/tests/test_api_bulk_download.py index eae03a3ed..9a8531244 100644 --- a/src/documents/tests/test_api_bulk_download.py +++ b/src/documents/tests/test_api_bulk_download.py @@ -339,3 +339,21 @@ class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase): self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) self.assertEqual(response.content, b"Insufficient permissions") + + def test_bad_search_query_returns_400(self) -> None: + response = self.client.post( + self.ENDPOINT, + json.dumps( + { + "all": True, + "filters": {"query": "added:notadate"}, + "content": "originals", + }, + ), + content_type="application/json", + ) + + # A user-fixable query error must surface as a 400 naming the bad + # value, exactly like the search list endpoint, never a 500. + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn(b"notadate", response.content) diff --git a/src/documents/tests/test_api_bulk_edit.py b/src/documents/tests/test_api_bulk_edit.py index 2f34b2425..0160371b8 100644 --- a/src/documents/tests/test_api_bulk_edit.py +++ b/src/documents/tests/test_api_bulk_edit.py @@ -2059,3 +2059,22 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(LogEntry.objects.filter(object_pk=self.doc1.id).count(), 2) + + def test_api_bulk_edit_with_bad_search_query_returns_400(self) -> None: + response = self.client.post( + "/api/documents/bulk_edit/", + json.dumps( + { + "all": True, + "filters": {"query": "added:notadate"}, + "method": "set_storage_path", + "parameters": {"storage_path": self.sp1.id}, + }, + ), + content_type="application/json", + ) + + # A user-fixable query error must surface as a 400 naming the bad + # value, exactly like the search list endpoint, never a 500. + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn(b"notadate", response.content) diff --git a/src/documents/tests/test_api_search.py b/src/documents/tests/test_api_search.py index f4ebf6d97..3bfc7f7ae 100644 --- a/src/documents/tests/test_api_search.py +++ b/src/documents/tests/test_api_search.py @@ -786,6 +786,10 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): tick=False, ): response = self.client.get("/api/documents/?query=added:previous month") + assert response.status_code == 200, ( + f"expected a successful search response, got {response.status_code}: " + f"{response.data!r}" + ) results = response.data["results"] self.assertEqual(len(results), 1) @@ -818,6 +822,26 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) self.assertIn("invalid-date", str(response.data["query"])) + def test_search_multiple_bad_fields_returns_all_messages(self) -> None: + """ + GIVEN: + - One document added + WHEN: + - Query with multiple bad fields (e.g. invalid date and invalid number) + THEN: + - 400 Bad Request with error messages for every bad field, + so the user can fix them all in one round-trip + """ + response = self.client.get( + "/api/documents/", + {"query": "created:notadate AND asn:notanumber"}, + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + messages = response.data["query"] + self.assertEqual(len(messages), 2) + self.assertTrue(any("created" in m for m in messages)) + self.assertTrue(any("asn" in m for m in messages)) + @override_settings( TIME_ZONE="UTC", ) @@ -861,6 +885,29 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): results = response.data["results"] self.assertEqual({r["id"] for r in results}, {1, 2}) + @mock.patch("documents.search._backend.parse_user_query") + def test_search_parser_bug_surfaces_as_500_not_400(self, m) -> None: + """ + GIVEN: + - The query parser itself fails (a whoosh-compat bug, per + QueryParserError's own contract: not user-fixable input) + WHEN: + - Any search request runs + THEN: + - The error surfaces as a 500 monitoring can see, never a 400 + blaming the user for a library defect + """ + from whoosh_compat.errors import QueryParserError + + m.side_effect = QueryParserError("synthetic parser bug") + + self.client.raise_request_exception = False + response = self.client.get("/api/documents/?query=anything") + self.assertEqual( + response.status_code, + status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + @mock.patch("documents.search._backend.TantivyBackend.autocomplete") def test_search_autocomplete_limits(self, m) -> None: """ @@ -2035,3 +2082,45 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) response = self.client.get("/api/search/?query=no") self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + + def _assert_query_finds(self, doc: Document, query: str) -> None: + get_backend().add_or_update(doc) + response = self.client.get("/api/documents/", {"query": query}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + ids = [r["id"] for r in response.data["results"]] + self.assertIn(doc.id, ids) + + def test_search_by_asn(self) -> None: + doc = Document.objects.create( + title="Has ASN", + content="content", + checksum="asn-checksum", + archive_serial_number=555, + ) + self._assert_query_finds(doc, "asn:555") + + def test_search_by_page_count(self) -> None: + doc = Document.objects.create( + title="Multi-page", + content="content", + checksum="page-count-checksum", + page_count=42, + ) + self._assert_query_finds(doc, "page_count:42") + + def test_search_by_original_filename(self) -> None: + doc = Document.objects.create( + title="Named file", + content="content", + checksum="filename-checksum", + original_filename="quarterly-report.pdf", + ) + self._assert_query_finds(doc, "original_filename:quarterly-report.pdf") + + def test_search_by_checksum(self) -> None: + doc = Document.objects.create( + title="Checksum doc", + content="content", + checksum="deadbeef1234", + ) + self._assert_query_finds(doc, "checksum:deadbeef1234") diff --git a/src/documents/tests/test_api_search_errors.py b/src/documents/tests/test_api_search_errors.py new file mode 100644 index 000000000..4a08051d1 --- /dev/null +++ b/src/documents/tests/test_api_search_errors.py @@ -0,0 +1,275 @@ +"""The search list endpoint's exception handling: what becomes a 400 and +what a library defect surfaces as instead. + +Companion to documents/tests/search/test_error_routing.py, which pins the +Cause -> SearchQueryError/QueryError routing inside documents/search/_query.py. +These tests pin the layer above it: DocumentViewSet.list's own except clauses, +which decide what an already-routed error becomes on the wire. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest +from rest_framework import status +from whoosh_compat.errors import Cause +from whoosh_compat.errors import Diagnostic +from whoosh_compat.errors import DiagnosticKind +from whoosh_compat.errors import QueryError + +from documents.search import SearchQueryError +from documents.tests.factories import DocumentFactory + +if TYPE_CHECKING: + from rest_framework.test import APIClient + + from documents.models import Document + +pytestmark = [pytest.mark.django_db, pytest.mark.usefixtures("_search_index")] + + +@pytest.fixture +def indexed_document() -> Document: + from documents.search import get_backend + + doc = DocumentFactory.create(title="quarterly invoice", content="acme corp") + get_backend().add_or_update(doc) + return doc + + +class TestSearchQueryErrorStillBecomesA400: + def test_search_query_error_becomes_a_400_naming_the_field( + self, + admin_client: APIClient, + monkeypatch: pytest.MonkeyPatch, + indexed_document: Document, + ) -> None: + import documents.search._backend as backend_mod + + def raise_search_query_error(*args: object, **kwargs: object) -> object: + raise SearchQueryError("bad value for field 'added'") + + monkeypatch.setattr( + backend_mod, + "parse_user_query", + raise_search_query_error, + ) + + response = admin_client.get("/api/documents/?query=anything") + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "added" in str(response.data["query"]) + + +class TestLibraryDefectsPropagate: + """The exact regression this task exists to fix: an unexpected or + INTERNAL-cause library error must not be relabeled a 400.""" + + def test_unexpected_exception_is_not_converted_to_a_400( + self, + admin_client: APIClient, + monkeypatch: pytest.MonkeyPatch, + indexed_document: Document, + ) -> None: + import documents.search._backend as backend_mod + + def raise_zero_division(*args: object, **kwargs: object) -> object: + raise ZeroDivisionError("synthetic bug, unrelated to search grammar") + + monkeypatch.setattr( + backend_mod, + "parse_user_query", + raise_zero_division, + ) + + with pytest.raises(ZeroDivisionError): + admin_client.get("/api/documents/?query=anything") + + def test_internal_cause_query_error_is_not_converted_to_a_400( + self, + admin_client: APIClient, + monkeypatch: pytest.MonkeyPatch, + indexed_document: Document, + ) -> None: + """Forces the one library-internal failure mode reachable from a real + query: emit() reporting a defect in itself (Cause.INTERNAL) after a + real query string went through the real parse and routing pipeline. + + ``tantivy_emit`` (the whoosh-compat emitter) is monkeypatched rather + than ``parse_user_query`` itself, so everything upstream of it -- + the pre-parse rewrites, ``wc.parse()``, and ``_map_emit_error``'s own + Cause routing in documents/search/_query.py -- runs for real; only + the final emit call is forced to report the defect. + """ + import documents.search._query as query_mod + + def raise_internal(*args: object, **kwargs: object) -> object: + raise QueryError( + Diagnostic( + kind=DiagnosticKind.BACKEND_REJECTED, + cause=Cause.INTERNAL, + message="synthetic whoosh-compat emitter defect", + ), + ) + + monkeypatch.setattr(query_mod, "tantivy_emit", raise_internal) + + with pytest.raises(QueryError): + admin_client.get("/api/documents/?query=invoice") + + +class TestSelectionPathsAgreeWithSearch: + """DocumentSelectionMixin backs bulk edit, bulk download, and a + more_like_id selection filter. It catches only SearchQueryError -- the + same contract the search list endpoint enforces above -- so all three + must map SearchQueryError to a 400 and let anything else surface.""" + + def test_bulk_edit_maps_search_query_error_to_a_400( + self, + admin_client: APIClient, + monkeypatch: pytest.MonkeyPatch, + indexed_document: Document, + ) -> None: + import documents.search._backend as backend_mod + + def raise_search_query_error(*args: object, **kwargs: object) -> object: + raise SearchQueryError("bad value for field 'added'") + + monkeypatch.setattr( + backend_mod, + "parse_user_query", + raise_search_query_error, + ) + + response = admin_client.post( + "/api/documents/bulk_edit/", + { + "documents": [], + "all": True, + "filters": {"query": "anything"}, + "method": "set_document_type", + "parameters": {"document_type": None}, + }, + format="json", + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "added" in str(response.data["query"]) + + def test_bulk_edit_lets_an_unexpected_exception_surface( + self, + admin_client: APIClient, + monkeypatch: pytest.MonkeyPatch, + indexed_document: Document, + ) -> None: + import documents.search._backend as backend_mod + + def raise_zero_division(*args: object, **kwargs: object) -> object: + raise ZeroDivisionError("synthetic bug, unrelated to search grammar") + + monkeypatch.setattr( + backend_mod, + "parse_user_query", + raise_zero_division, + ) + + with pytest.raises(ZeroDivisionError): + admin_client.post( + "/api/documents/bulk_edit/", + { + "documents": [], + "all": True, + "filters": {"query": "anything"}, + "method": "set_document_type", + "parameters": {"document_type": None}, + }, + format="json", + ) + + def test_bulk_download_maps_search_query_error_to_a_400( + self, + admin_client: APIClient, + monkeypatch: pytest.MonkeyPatch, + indexed_document: Document, + ) -> None: + import documents.search._backend as backend_mod + + def raise_search_query_error(*args: object, **kwargs: object) -> object: + raise SearchQueryError("bad value for field 'added'") + + monkeypatch.setattr( + backend_mod, + "parse_user_query", + raise_search_query_error, + ) + + response = admin_client.post( + "/api/documents/bulk_download/", + { + "documents": [], + "all": True, + "filters": {"query": "anything"}, + }, + format="json", + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "added" in str(response.data["query"]) + + def test_more_like_id_selection_filter_maps_search_query_error_to_a_400( + self, + admin_client: APIClient, + monkeypatch: pytest.MonkeyPatch, + indexed_document: Document, + ) -> None: + import documents.search._backend as backend_mod + + def raise_search_query_error(*args: object, **kwargs: object) -> object: + raise SearchQueryError("similar-document lookup is unavailable") + + monkeypatch.setattr( + backend_mod.TantivyBackend, + "more_like_this_ids", + raise_search_query_error, + ) + + response = admin_client.post( + "/api/documents/bulk_download/", + { + "documents": [], + "all": True, + "filters": {"more_like_id": indexed_document.pk}, + }, + format="json", + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_more_like_id_selection_filter_lets_an_unexpected_exception_surface( + self, + admin_client: APIClient, + monkeypatch: pytest.MonkeyPatch, + indexed_document: Document, + ) -> None: + import documents.search._backend as backend_mod + + def raise_zero_division(*args: object, **kwargs: object) -> object: + raise ZeroDivisionError("synthetic bug, unrelated to similarity lookup") + + monkeypatch.setattr( + backend_mod.TantivyBackend, + "more_like_this_ids", + raise_zero_division, + ) + + with pytest.raises(ZeroDivisionError): + admin_client.post( + "/api/documents/bulk_download/", + { + "documents": [], + "all": True, + "filters": {"more_like_id": indexed_document.pk}, + }, + format="json", + ) diff --git a/src/documents/views.py b/src/documents/views.py index 4cf775bd6..bfbf654f0 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -16,6 +16,7 @@ from time import mktime from time import sleep from typing import TYPE_CHECKING from typing import Any +from typing import Final from typing import Literal from typing import NamedTuple from unicodedata import normalize @@ -284,17 +285,40 @@ logger = logging.getLogger("paperless.api") _TANTIVY_INTERSECT_THRESHOLD = 5_000 _TANTIVY_SEARCH_PARAM_NAMES = ("text", "title_search", "query", "more_like_id") +# whoosh-compat's fieldname tagger (used only for SearchMode.QUERY, via the +# whoosh grammar in parse_user_query) is O(n^2) in plain word characters: +# measured at ~0.96s/10k chars, ~3.67s/20k, ~14.4s/40k against the real field +# registry. Django's DATA_UPLOAD_MAX_MEMORY_SIZE default (2.5 MB) does not +# bound this on the POST-body selection-filter path, so an unbounded query +# is a single-request CPU exhaustion vector. 4096 chars caps the worst case +# at roughly 0.16s (quadratic extrapolation from the measurements above), +# far beyond any plausible hand-typed advanced query, while still being fast +# enough to absorb inside a request handler. Applied to all three modes at +# this shared choke point: TEXT and TITLE route through simple_search_tokens +# instead and measure linear even at 20k chars, so the cap is hygiene for +# them, not a fix, but a single limit here is simpler than one exemption. +# Not exposed as a PAPERLESS_* setting: this is a hard security boundary, +# not a tunable, and a raisable ceiling would let a misconfiguration +# reintroduce the exact hazard this exists to close. +_MAX_QUERY_LENGTH: Final[int] = 4096 + def _get_tantivy_query_and_mode(params): + from documents.search import QueryTooLongError from documents.search import SearchMode if "text" in params: - return str(params["text"]), SearchMode.TEXT - if "title_search" in params: - return str(params["title_search"]), SearchMode.TITLE - if "query" in params: - return str(params["query"]), SearchMode.QUERY - return None # pragma: no cover + raw, mode = str(params["text"]), SearchMode.TEXT + elif "title_search" in params: + raw, mode = str(params["title_search"]), SearchMode.TITLE + elif "query" in params: + raw, mode = str(params["query"]), SearchMode.QUERY + else: + return None # pragma: no cover + + if len(raw) > _MAX_QUERY_LENGTH: + raise QueryTooLongError(len(raw), _MAX_QUERY_LENGTH) + return raw, mode def _get_more_like_id(query_params: dict[str, Any], user: User | None) -> int: @@ -2435,6 +2459,7 @@ class UnifiedSearchViewSet(DocumentViewSet): from documents.search import TantivyBackend from documents.search import TantivyRelevanceList from documents.search import get_backend + from documents.search import search_query_error_messages def parse_search_params() -> SearchParams: """Extract query string, search mode, and ordering from request.""" @@ -2625,15 +2650,10 @@ class UnifiedSearchViewSet(DocumentViewSet): except ValidationError: raise except SearchQueryError as e: - # User-fixable query error (e.g. an unparsable date): surface the - # specific message so the user can correct it, rather than a generic - # 400 or silently empty results. - raise ValidationError({"query": [str(e)]}) from e - except Exception as e: - logger.warning(f"An error occurred listing search results: {e!s}") - return HttpResponseBadRequest( - "Error listing search results, check logs for more detail.", - ) + # User-fixable query error(s) (e.g. unparsable dates/numbers): + # surface every offending field's message, not just the first, + # so the user can fix them all in one round-trip. + raise ValidationError({"query": search_query_error_messages(e)}) from e @action(detail=False, methods=["GET"], name="Get Next ASN") def next_asn(self, request, *args, **kwargs): @@ -2769,23 +2789,34 @@ class DocumentSelectionMixin: }, ) + from documents.search import SearchQueryError from documents.search import get_backend + from documents.search import search_query_error_messages filter_name = search_filters[0] backend = get_backend() search_user = None if user.is_superuser else user - if filter_name == "more_like_id": - more_like_doc_id = _get_more_like_id(filters, user) + try: + if filter_name == "more_like_id": + more_like_doc_id = _get_more_like_id(filters, user) - search_ids = backend.more_like_this_ids(more_like_doc_id, user=search_user) - else: - query_str, search_mode = _get_tantivy_query_and_mode(filters) - search_ids = backend.search_ids( - query_str, - user=search_user, - search_mode=search_mode, - ) + search_ids = backend.more_like_this_ids( + more_like_doc_id, + user=search_user, + ) + else: + query_str, search_mode = _get_tantivy_query_and_mode(filters) + search_ids = backend.search_ids( + query_str, + user=search_user, + search_mode=search_mode, + ) + except SearchQueryError as e: + # Same user-fixable-query mapping as the search list endpoint: + # a bad date/number in a bulk selection filter is a 400 naming + # the value, never a 500. + raise ValidationError({"query": search_query_error_messages(e)}) from e return search_ids @@ -3626,6 +3657,10 @@ class GlobalSearchView(PassUserMixin): return HttpResponseBadRequest("Query required") if len(query) < 3: return HttpResponseBadRequest("Query must be at least 3 characters") + if len(query) > _MAX_QUERY_LENGTH: + return HttpResponseBadRequest( + f"Query must be at most {_MAX_QUERY_LENGTH} characters", + ) db_only = request.query_params.get("db_only", False)