From 171b0a6f77d3c15602118e63ea39b28f51718bad Mon Sep 17 00:00:00 2001 From: Trenton Holmes <797416+stumpylog@users.noreply.github.com> Date: Tue, 18 Aug 2026 09:16:46 -0700 Subject: [PATCH] fix(search): rewrite bare notes:/custom_fields: prefixes to their subpaths The v2 whoosh schema had plural notes/custom_fields TEXT fields (notes indexed the joined note texts, custom_fields indexed joined "name : value" strings), so "notes:foo" and "custom_fields:foo" were valid fielded searches in released paperless and through the deleted translation layer. On the whoosh-compat registry those names are JSON fields addressable only via subpaths, and the bare spelling silently demoted to an unfielded text search of the words themselves, matching unrelated documents that merely contain "notes" or "custom". parse_user_query now rewrites the bare prefixes live to the same targets migration 0017 chose for the singular whoosh-era spellings: notes: becomes notes.note: and custom_fields: becomes custom_fields.value:, with 0017's lookbehind guard so subpath spellings and words merely ending in the prefix are untouched. Prefix substitution only; values ride through unchanged, and every value shape lands in a documented outcome downstream (ranges, wildcards and exists on JSON subpaths are typed errors, not crashes). The inherited trade-off stands: custom_fields.value: drops the name-matching half of v2's combined indexing, with custom_fields.name: available for it. Acceptance tests pin the rewrite with decoy documents whose content contains the literal prefix words, which the old demotion matched and the fielded search must not, plus untouched-subpath controls. docs/usage.md documents the bare prefixes as subpath shorthand. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WMsn6DgzbvSqh1pwy66VVF --- docs/usage.md | 3 + src/documents/search/_query.py | 39 ++++++++- src/documents/tests/search/test_acceptance.py | 80 +++++++++++++++++++ 3 files changed, 118 insertions(+), 4 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index d1cd2b24c..8fff4d78e 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -954,6 +954,7 @@ custom_fields.name:Insurance custom_fields.value:policy - `custom_fields.value` matches against the value of any custom field. - `custom_fields.name` matches the name of the field (use quotes for multi-word names). - Combine both to find documents where a specific named field contains a specific value. +- The bare `custom_fields:` prefix is shorthand for `custom_fields.value:`. Because separators are stripped during indexing, individual parts of formatted codes are searchable on their own. A value stored as `A-1312/99.50` produces the @@ -981,6 +982,8 @@ notes.note:reminder notes.user:alice notes.note:insurance ``` +The bare `notes:` prefix is shorthand for `notes.note:`. + All of these constructs can be combined as you see fit. If you want to learn more about the query language used by paperless, see the [Tantivy query language documentation](https://docs.rs/tantivy/latest/tantivy/query/struct.QueryParser.html). diff --git a/src/documents/search/_query.py b/src/documents/search/_query.py index bb12dde79..37137ce91 100644 --- a/src/documents/search/_query.py +++ b/src/documents/search/_query.py @@ -125,6 +125,34 @@ def _quote_date_keyword_phrases(raw_query: str) -> str: ) +# The v2 whoosh schema had plural notes/custom_fields TEXT fields (notes +# indexed the joined note texts; custom_fields indexed joined +# "name : value" strings), so the bare plural prefixes were valid fielded +# searches in released paperless and at the deleted translation layer. On +# the whoosh-compat registry they are JSON fields addressable only via +# subpaths, and the bare spelling would demote to an unfielded text search +# of the words themselves. Rewrite the prefixes live to the same targets +# migration 0017 chose for the singular whoosh-era spellings (note: -> +# notes.note:, custom_field: -> custom_fields.value:), values untouched. +# Trade-off inherited from that migration: custom_fields.value: drops the +# name-matching half of v2's "name : value" indexing (custom_fields.name: +# remains available for it). Same lookbehind guard as 0017: not preceded +# by a word character or dot, so subpath spellings and words that merely +# end in the prefix are untouched. +_BARE_JSON_PREFIX_RES: Final = ( + (regex.compile(r"(? str: + """Rewrite bare ``notes:``/``custom_fields:`` prefixes to their + subpath equivalents. Prefix substitution only, values untouched.""" + for pattern, replacement in _BARE_JSON_PREFIX_RES: + raw_query = pattern.sub(replacement, raw_query, timeout=_REGEX_TIMEOUT) + return raw_query + + def _has_cjk(text: str) -> bool: """Return True if text contains any CJK characters.""" return bool(_CJK_RE.search(text)) @@ -339,10 +367,12 @@ def parse_user_query( """ Parse user query through whoosh-compat, then blend in fuzzy/CJK clauses. - 1. Unquoted multi-word date keyword phrases on date fields are quoted - (_quote_date_keyword_phrases) so the historically honored - "added:previous month" spelling keeps working; then wc.parse() - against the shared FieldRegistry (whoosh grammar -> AST). + 1. Two small pre-parse rewrites keep historically honored spellings + working: unquoted multi-word date keyword phrases on date fields + are quoted (_quote_date_keyword_phrases), and bare + notes:/custom_fields: prefixes become their subpath equivalents + (_rewrite_bare_json_field_prefixes). Then wc.parse() against the + shared FieldRegistry (whoosh grammar -> AST). 2. Any diagnostics (bad dates/numbers) map to SearchQueryError subclasses and raise — the view returns HTTP 400 with every offending field listed, not just the first. @@ -361,6 +391,7 @@ def parse_user_query( """ registry = get_field_registry(settings.SEARCH_LANGUAGE) raw_query = _quote_date_keyword_phrases(raw_query) + raw_query = _rewrite_bare_json_field_prefixes(raw_query) result = wc.parse( raw_query, registry=registry, diff --git a/src/documents/tests/search/test_acceptance.py b/src/documents/tests/search/test_acceptance.py index 807ca8a17..923a021ab 100644 --- a/src/documents/tests/search/test_acceptance.py +++ b/src/documents/tests/search/test_acceptance.py @@ -500,3 +500,83 @@ class TestUnquotedDateKeywordPhrases: ) backend.add_or_update(wordy) assert _matched_ids(backend, "title:previous month") == {wordy.pk} + + +class TestBareJsonFieldPrefixes: + """The v2 whoosh schema had plural notes/custom_fields TEXT fields, so + "notes:foo" and "custom_fields:foo" were valid fielded searches in + released paperless (and at the tantivy translation layer). On the + whoosh-compat registry they are JSON fields addressable only via + subpaths, and the bare spelling would demote to a nonsense unfielded + text search. parse_user_query rewrites the bare prefixes live to the + same targets migration 0017 chose for the singular whoosh-era + spellings: notes: -> notes.note:, custom_fields: -> + custom_fields.value:.""" + + def test_bare_notes_prefix_searches_note_text( + self, + backend: TantivyBackend, + ) -> None: + from django.contrib.auth.models import User + + alice = User.objects.create_user(username="alice") + with_note = Document.objects.create( + title="Has note", + content="x", + checksum="bare-notes-with", + ) + Note.objects.create(document=with_note, user=alice, note="crocodile") + # This document's CONTENT contains the words a demoted text search + # would match; it must NOT match once the prefix addresses notes. + decoy = Document.objects.create( + title="Notes about things", + content="notes crocodile mention", + checksum="bare-notes-decoy", + ) + backend.add_or_update(with_note) + backend.add_or_update(decoy) + assert _matched_ids(backend, "notes:crocodile") == {with_note.pk} + + def test_bare_custom_fields_prefix_searches_values( + self, + backend: TantivyBackend, + ) -> None: + field = CustomField.objects.create( + name="Policy Number", + data_type=CustomField.FieldDataType.STRING, + ) + with_value = Document.objects.create( + title="Has field", + content="x", + checksum="bare-cf-with", + ) + CustomFieldInstance.objects.create( + document=with_value, + field=field, + value_text="crocodile", + ) + decoy = Document.objects.create( + title="Custom things", + content="custom fields crocodile", + checksum="bare-cf-decoy", + ) + backend.add_or_update(with_value) + backend.add_or_update(decoy) + assert _matched_ids(backend, "custom_fields:crocodile") == {with_value.pk} + + def test_subpath_spellings_are_untouched( + self, + backend: TantivyBackend, + ) -> None: + from django.contrib.auth.models import User + + bob = User.objects.create_user(username="bob") + doc = Document.objects.create( + title="Bob note", + content="x", + checksum="bare-subpath", + ) + Note.objects.create(document=doc, user=bob, note="remark") + backend.add_or_update(doc) + assert _matched_ids(backend, "notes.user:bob") == {doc.pk} + assert _matched_ids(backend, "notes.note:remark") == {doc.pk}