diff --git a/docs/usage.md b/docs/usage.md index d1cd2b24c..8fff4d78e 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -954,6 +954,7 @@ custom_fields.name:Insurance custom_fields.value:policy - `custom_fields.value` matches against the value of any custom field. - `custom_fields.name` matches the name of the field (use quotes for multi-word names). - Combine both to find documents where a specific named field contains a specific value. +- The bare `custom_fields:` prefix is shorthand for `custom_fields.value:`. Because separators are stripped during indexing, individual parts of formatted codes are searchable on their own. A value stored as `A-1312/99.50` produces the @@ -981,6 +982,8 @@ notes.note:reminder notes.user:alice notes.note:insurance ``` +The bare `notes:` prefix is shorthand for `notes.note:`. + All of these constructs can be combined as you see fit. If you want to learn more about the query language used by paperless, see the [Tantivy query language documentation](https://docs.rs/tantivy/latest/tantivy/query/struct.QueryParser.html). diff --git a/src/documents/search/_query.py b/src/documents/search/_query.py index bb12dde79..37137ce91 100644 --- a/src/documents/search/_query.py +++ b/src/documents/search/_query.py @@ -125,6 +125,34 @@ def _quote_date_keyword_phrases(raw_query: str) -> str: ) +# The v2 whoosh schema had plural notes/custom_fields TEXT fields (notes +# indexed the joined note texts; custom_fields indexed joined +# "name : value" strings), so the bare plural prefixes were valid fielded +# searches in released paperless and at the deleted translation layer. On +# the whoosh-compat registry they are JSON fields addressable only via +# subpaths, and the bare spelling would demote to an unfielded text search +# of the words themselves. Rewrite the prefixes live to the same targets +# migration 0017 chose for the singular whoosh-era spellings (note: -> +# notes.note:, custom_field: -> custom_fields.value:), values untouched. +# Trade-off inherited from that migration: custom_fields.value: drops the +# name-matching half of v2's "name : value" indexing (custom_fields.name: +# remains available for it). Same lookbehind guard as 0017: not preceded +# by a word character or dot, so subpath spellings and words that merely +# end in the prefix are untouched. +_BARE_JSON_PREFIX_RES: Final = ( + (regex.compile(r"(? str: + """Rewrite bare ``notes:``/``custom_fields:`` prefixes to their + subpath equivalents. Prefix substitution only, values untouched.""" + for pattern, replacement in _BARE_JSON_PREFIX_RES: + raw_query = pattern.sub(replacement, raw_query, timeout=_REGEX_TIMEOUT) + return raw_query + + def _has_cjk(text: str) -> bool: """Return True if text contains any CJK characters.""" return bool(_CJK_RE.search(text)) @@ -339,10 +367,12 @@ def parse_user_query( """ Parse user query through whoosh-compat, then blend in fuzzy/CJK clauses. - 1. Unquoted multi-word date keyword phrases on date fields are quoted - (_quote_date_keyword_phrases) so the historically honored - "added:previous month" spelling keeps working; then wc.parse() - against the shared FieldRegistry (whoosh grammar -> AST). + 1. Two small pre-parse rewrites keep historically honored spellings + working: unquoted multi-word date keyword phrases on date fields + are quoted (_quote_date_keyword_phrases), and bare + notes:/custom_fields: prefixes become their subpath equivalents + (_rewrite_bare_json_field_prefixes). Then wc.parse() against the + shared FieldRegistry (whoosh grammar -> AST). 2. Any diagnostics (bad dates/numbers) map to SearchQueryError subclasses and raise — the view returns HTTP 400 with every offending field listed, not just the first. @@ -361,6 +391,7 @@ def parse_user_query( """ registry = get_field_registry(settings.SEARCH_LANGUAGE) raw_query = _quote_date_keyword_phrases(raw_query) + raw_query = _rewrite_bare_json_field_prefixes(raw_query) result = wc.parse( raw_query, registry=registry, diff --git a/src/documents/tests/search/test_acceptance.py b/src/documents/tests/search/test_acceptance.py index 807ca8a17..923a021ab 100644 --- a/src/documents/tests/search/test_acceptance.py +++ b/src/documents/tests/search/test_acceptance.py @@ -500,3 +500,83 @@ class TestUnquotedDateKeywordPhrases: ) backend.add_or_update(wordy) assert _matched_ids(backend, "title:previous month") == {wordy.pk} + + +class TestBareJsonFieldPrefixes: + """The v2 whoosh schema had plural notes/custom_fields TEXT fields, so + "notes:foo" and "custom_fields:foo" were valid fielded searches in + released paperless (and at the tantivy translation layer). On the + whoosh-compat registry they are JSON fields addressable only via + subpaths, and the bare spelling would demote to a nonsense unfielded + text search. parse_user_query rewrites the bare prefixes live to the + same targets migration 0017 chose for the singular whoosh-era + spellings: notes: -> notes.note:, custom_fields: -> + custom_fields.value:.""" + + def test_bare_notes_prefix_searches_note_text( + self, + backend: TantivyBackend, + ) -> None: + from django.contrib.auth.models import User + + alice = User.objects.create_user(username="alice") + with_note = Document.objects.create( + title="Has note", + content="x", + checksum="bare-notes-with", + ) + Note.objects.create(document=with_note, user=alice, note="crocodile") + # This document's CONTENT contains the words a demoted text search + # would match; it must NOT match once the prefix addresses notes. + decoy = Document.objects.create( + title="Notes about things", + content="notes crocodile mention", + checksum="bare-notes-decoy", + ) + backend.add_or_update(with_note) + backend.add_or_update(decoy) + assert _matched_ids(backend, "notes:crocodile") == {with_note.pk} + + def test_bare_custom_fields_prefix_searches_values( + self, + backend: TantivyBackend, + ) -> None: + field = CustomField.objects.create( + name="Policy Number", + data_type=CustomField.FieldDataType.STRING, + ) + with_value = Document.objects.create( + title="Has field", + content="x", + checksum="bare-cf-with", + ) + CustomFieldInstance.objects.create( + document=with_value, + field=field, + value_text="crocodile", + ) + decoy = Document.objects.create( + title="Custom things", + content="custom fields crocodile", + checksum="bare-cf-decoy", + ) + backend.add_or_update(with_value) + backend.add_or_update(decoy) + assert _matched_ids(backend, "custom_fields:crocodile") == {with_value.pk} + + def test_subpath_spellings_are_untouched( + self, + backend: TantivyBackend, + ) -> None: + from django.contrib.auth.models import User + + bob = User.objects.create_user(username="bob") + doc = Document.objects.create( + title="Bob note", + content="x", + checksum="bare-subpath", + ) + Note.objects.create(document=doc, user=bob, note="remark") + backend.add_or_update(doc) + assert _matched_ids(backend, "notes.user:bob") == {doc.pk} + assert _matched_ids(backend, "notes.note:remark") == {doc.pk}