fix(search): rewrite bare notes:/custom_fields: prefixes to their subpaths

The v2 whoosh schema had plural notes/custom_fields TEXT fields (notes
indexed the joined note texts, custom_fields indexed joined
"name : value" strings), so "notes:foo" and "custom_fields:foo" were
valid fielded searches in released paperless and through the deleted
translation layer. On the whoosh-compat registry those names are JSON
fields addressable only via subpaths, and the bare spelling silently
demoted to an unfielded text search of the words themselves, matching
unrelated documents that merely contain "notes" or "custom".

parse_user_query now rewrites the bare prefixes live to the same
targets migration 0017 chose for the singular whoosh-era spellings:
notes: becomes notes.note: and custom_fields: becomes
custom_fields.value:, with 0017's lookbehind guard so subpath spellings
and words merely ending in the prefix are untouched. Prefix
substitution only; values ride through unchanged, and every value shape
lands in a documented outcome downstream (ranges, wildcards and exists
on JSON subpaths are typed errors, not crashes). The inherited
trade-off stands: custom_fields.value: drops the name-matching half of
v2's combined indexing, with custom_fields.name: available for it.

Acceptance tests pin the rewrite with decoy documents whose content
contains the literal prefix words, which the old demotion matched and
the fielded search must not, plus untouched-subpath controls.
docs/usage.md documents the bare prefixes as subpath shorthand.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WMsn6DgzbvSqh1pwy66VVF
This commit is contained in:
Trenton HolmesandClaude Fable 5 authored and stumpylog committed 2026-08-19 13:36:53 -07:00
1 parent ff540041b5
commit bd87624b29
3 files changed
+118 -4

No files matched your search

@@ -500,3 +500,83 @@ class TestUnquotedDateKeywordPhrases:
)
backend.add_or_update(wordy)
assert _matched_ids(backend, "title:previous month") == {wordy.pk}
class TestBareJsonFieldPrefixes:
"""The v2 whoosh schema had plural notes/custom_fields TEXT fields, so
"notes:foo" and "custom_fields:foo" were valid fielded searches in
released paperless (and at the tantivy translation layer). On the
whoosh-compat registry they are JSON fields addressable only via
subpaths, and the bare spelling would demote to a nonsense unfielded
text search. parse_user_query rewrites the bare prefixes live to the
same targets migration 0017 chose for the singular whoosh-era
spellings: notes: -> notes.note:, custom_fields: ->
custom_fields.value:."""
def test_bare_notes_prefix_searches_note_text(
self,
backend: TantivyBackend,
) -> None:
from django.contrib.auth.models import User
alice = User.objects.create_user(username="alice")
with_note = Document.objects.create(
title="Has note",
content="x",
checksum="bare-notes-with",
)
Note.objects.create(document=with_note, user=alice, note="crocodile")
# This document's CONTENT contains the words a demoted text search
# would match; it must NOT match once the prefix addresses notes.
decoy = Document.objects.create(
title="Notes about things",
content="notes crocodile mention",
checksum="bare-notes-decoy",
)
backend.add_or_update(with_note)
backend.add_or_update(decoy)
assert _matched_ids(backend, "notes:crocodile") == {with_note.pk}
def test_bare_custom_fields_prefix_searches_values(
self,
backend: TantivyBackend,
) -> None:
field = CustomField.objects.create(
name="Policy Number",
data_type=CustomField.FieldDataType.STRING,
)
with_value = Document.objects.create(
title="Has field",
content="x",
checksum="bare-cf-with",
)
CustomFieldInstance.objects.create(
document=with_value,
field=field,
value_text="crocodile",
)
decoy = Document.objects.create(
title="Custom things",
content="custom fields crocodile",
checksum="bare-cf-decoy",
)
backend.add_or_update(with_value)
backend.add_or_update(decoy)
assert _matched_ids(backend, "custom_fields:crocodile") == {with_value.pk}
def test_subpath_spellings_are_untouched(
self,
backend: TantivyBackend,
) -> None:
from django.contrib.auth.models import User
bob = User.objects.create_user(username="bob")
doc = Document.objects.create(
title="Bob note",
content="x",
checksum="bare-subpath",
)
Note.objects.create(document=doc, user=bob, note="remark")
backend.add_or_update(doc)
assert _matched_ids(backend, "notes.user:bob") == {doc.pk}
assert _matched_ids(backend, "notes.note:remark") == {doc.pk}