test(search): add pattern normalizer stem-alternates unit tests

docs/configuration.md

@@ -1209,14 +1209,6 @@ left unassigned, preventing low-confidence guesses from being applied.
 
     Defaults to 0.6.
 
-#### [`PAPERLESS_MATCH_REGEX_TIMEOUT_SECONDS=<float>`](#PAPERLESS_MATCH_REGEX_TIMEOUT_SECONDS) {#PAPERLESS_MATCH_REGEX_TIMEOUT_SECONDS}
-
-: Sets the timeout, in seconds, for regular expression matching. Increase this
-value if date parsing or user-defined matching rules time out when processing
-long documents, especially on slower hardware.
-
-    Defaults to 0.1 seconds.
-
 #### [`PAPERLESS_DATE_PARSER_LANGUAGES=<lang>`](#PAPERLESS_DATE_PARSER_LANGUAGES) {#PAPERLESS_DATE_PARSER_LANGUAGES}
 
 : Specifies which language Paperless should use when parsing dates from documents.

pyproject.toml

@@ -77,6 +77,7 @@ dependencies = [
   "torch~=2.13.0",
   "watchfiles>=1.2",
   "whitenoise~=6.11",
+  "whoosh-compat[tantivy]==0.1",
   "zxing-cpp~=3.1.0",
 ]
 [project.optional-dependencies]

src/documents/search/_fields.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from whoosh_compat import FieldKind
+from whoosh_compat import FieldSpec
+from whoosh_compat import SubpathSpec
+
+# Internal-only schema fields with no query-syntax meaning of their own
+# (sort shadow fields, bigram CJK fields, simple_title/simple_content,
+# autocomplete_word, notes_text) are NOT represented here, they are
+# declared in _schema.py's field_descriptors().
+#
+# analyzer/pattern_normalizer are deliberately left at FieldSpec's default
+# (None): they're language-specific and only meaningful to whoosh-compat's
+# parser, so _registry.py attaches them per-language via dataclasses.replace()
+# rather than PUBLIC_FIELDS declaring them itself. _schema.py only reads
+# name/kind/fast and never sees the analyzer at all.
+PUBLIC_FIELDS: tuple[FieldSpec, ...] = (
+    FieldSpec("title", FieldKind.TEXT),
+    FieldSpec("content", FieldKind.TEXT),
+    FieldSpec("correspondent", FieldKind.TEXT),
+    FieldSpec("document_type", FieldKind.TEXT, aliases=("type",)),
+    FieldSpec("storage_path", FieldKind.TEXT, aliases=("path",)),
+    FieldSpec("original_filename", FieldKind.TEXT),
+    FieldSpec("tag", FieldKind.TEXT, comma_values=True),
+    FieldSpec("checksum", FieldKind.KEYWORD),
+    FieldSpec("asn", FieldKind.U64, fast=True),
+    FieldSpec("page_count", FieldKind.U64, fast=True),
+    FieldSpec("num_notes", FieldKind.U64, fast=True),
+    FieldSpec("created", FieldKind.DATE, date_only=True, fast=True),
+    FieldSpec("modified", FieldKind.DATETIME, fast=True),
+    FieldSpec("added", FieldKind.DATETIME, fast=True),
+    FieldSpec(
+        "notes",
+        FieldKind.JSON,
+        subpaths={"user": SubpathSpec(), "note": SubpathSpec(default=True)},
+    ),
+    FieldSpec(
+        "custom_fields",
+        FieldKind.JSON,
+        subpaths={"name": SubpathSpec(), "value": SubpathSpec(default=True)},
+    ),
+)

src/documents/search/_registry.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+import dataclasses
+from typing import TYPE_CHECKING
+
+from whoosh_compat import FieldKind
+from whoosh_compat import FieldRegistry
+
+from documents.search._fields import PUBLIC_FIELDS
+from documents.search._tokenizer import ascii_fold
+from documents.search._tokenizer import paperless_text_analyzer
+from documents.search._tokenizer import stem_pattern_text
+
+if TYPE_CHECKING:
+    from whoosh_compat import PatternNormalizer
+
+_registry_cache: dict[str | None, FieldRegistry] = {}
+
+
+def _identity_analyzer(text: str) -> list[str]:
+    """Analyzer for KEYWORD fields indexed with the raw tokenizer (no splitting)."""
+    return [text]
+
+
+def _fold_normalizer(text: str) -> str:
+    """Wildcard/regex literal-run normalizer for fields indexed without stemming."""
+    return ascii_fold(text.lower())
+
+
+def _make_pattern_normalizer(language: str | None) -> PatternNormalizer:
+    """Build the wildcard/regex literal-run normalizer for a search language."""
+
+    def _pattern_normalizer(text: str) -> tuple[str, ...]:
+        """Normalize a literal run into the forms a term may match.
+
+        TEXT index terms go through lowercase -> ascii_fold -> stem, so a
+        pattern that skips stemming can never match one: "invoice*" would look
+        for a term starting with "invoice" while the index holds "invoic". The
+        run is therefore offered stemmed as well. KEYWORD fields are indexed
+        raw and get _fold_normalizer instead, so their patterns stay literal.
+
+        Both forms are returned, as alternatives, because neither is a prefix
+        of the other in general: English stemming substitutes as well as
+        truncates ("copy" -> "copi"), so the stem alone loses the compounds
+        the typed run reaches ("copyright") while the typed run alone loses
+        the inflections the stem reaches ("copies"). whoosh-compat ORs the
+        alternatives per literal run and deduplicates them, so a run the
+        stemmer leaves alone costs exactly the one branch it did before.
+
+        Inside a bracket class the emitter calls this once per character and
+        uses the answer only if it is a single one-character form; two forms
+        there leave the character as typed. A stemmer does not change a lone
+        character, so the two forms deduplicate to one and the class body is
+        folded as before.
+        """
+        folded = ascii_fold(text.lower())
+        stemmed = stem_pattern_text(folded, language)
+        return (folded, stemmed)
+
+    return _pattern_normalizer
+
+
+def get_field_registry(language: str | None) -> FieldRegistry:
+    """Build (or return the cached) FieldRegistry for the given search language.
+
+    Cached keyed by language, rebuilt on the same trigger register_tokenizers()
+    uses (settings.SEARCH_LANGUAGE change). A fresh call with a new language
+    builds and caches a new registry rather than mutating the old one.
+    """
+    if language in _registry_cache:
+        return _registry_cache[language]
+
+    text_analyzer = paperless_text_analyzer(language).analyze
+    pattern_normalizer = _make_pattern_normalizer(language)
+
+    specs = [
+        dataclasses.replace(
+            field,
+            analyzer=_identity_analyzer
+            if field.kind is FieldKind.KEYWORD
+            else text_analyzer,
+            pattern_normalizer=_fold_normalizer
+            if field.kind is FieldKind.KEYWORD
+            else pattern_normalizer,
+        )
+        for field in PUBLIC_FIELDS
+    ]
+
+    registry = FieldRegistry(specs)
+    _registry_cache[language] = registry
+    return registry

src/documents/search/_schema.py

@@ -1,14 +1,19 @@
 from __future__ import annotations
 
+import hashlib
 import json
 import logging
 import shutil
 from typing import TYPE_CHECKING
 from typing import Final
+from typing import NamedTuple
 from typing import cast
 
 import tantivy
 from django.conf import settings
+from whoosh_compat import FieldKind
+
+from documents.search._fields import PUBLIC_FIELDS
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -16,7 +21,185 @@ if TYPE_CHECKING:
 logger = logging.getLogger("paperless.search")
 
 # v1 - Initial tantivy schema format
-SCHEMA_VERSION: Final[int] = 1
+# v2 - build_schema() derived from PUBLIC_FIELDS, changing the field declaration
+#      order, and the write-only correspondent/document_type/storage_path/tag id
+#      columns dropped. tantivy compares schemas by ordered field list, so an
+#      index built by v1 rejects every write against the v2 schema.
+SCHEMA_VERSION: Final[int] = 2
+
+
+class FieldDescriptor(NamedTuple):
+    """One tantivy field, in declaration order.
+
+    The descriptor vocabulary is paperless', not tantivy-py's: it is both the
+    input to the SchemaBuilder and the input to schema_fingerprint(), so the
+    persisted fingerprint cannot move under a tantivy-py upgrade.
+    """
+
+    name: str
+    kind: str
+    stored: bool
+    indexed: bool
+    fast: bool
+    tokenizer: str | None
+
+
+# (schema kind, tokenizer) for the FieldKind -> FieldDescriptor mapping that
+# doesn't need special-casing. JSON is handled separately below since it can
+# emit a second, synthetic descriptor.
+_KIND_TABLE: Final[dict[FieldKind, tuple[str, str | None]]] = {
+    FieldKind.TEXT: ("text", "paperless_text"),
+    FieldKind.KEYWORD: ("text", "raw"),
+    FieldKind.U64: ("u64", None),
+    FieldKind.DATE: ("date", None),
+    FieldKind.DATETIME: ("date", None),
+}
+# Kinds whose fast-field flag follows FieldSpec.fast rather than always False.
+_FAST_FROM_FIELD: Final[frozenset[FieldKind]] = frozenset(
+    {FieldKind.U64, FieldKind.DATE, FieldKind.DATETIME},
+)
+
+
+def _public_field_descriptors() -> list[FieldDescriptor]:
+    """Descriptors for the query-visible fields declared in PUBLIC_FIELDS."""
+    descriptors: list[FieldDescriptor] = []
+    for field in PUBLIC_FIELDS:
+        if field.kind is FieldKind.JSON:
+            descriptors.append(
+                FieldDescriptor(
+                    field.name,
+                    "json",
+                    stored=True,
+                    indexed=True,
+                    fast=False,
+                    tokenizer="paperless_text",
+                ),
+            )
+            if field.name == "notes":
+                # Plain-text companion for snippet generation: tantivy's
+                # SnippetGenerator does not support JSON fields. Schema-only,
+                # no query-syntax meaning, not in PUBLIC_FIELDS.
+                descriptors.append(
+                    FieldDescriptor(
+                        "notes_text",
+                        "text",
+                        stored=True,
+                        indexed=True,
+                        fast=False,
+                        tokenizer="paperless_text",
+                    ),
+                )
+            continue
+        schema_kind, tokenizer = _KIND_TABLE[field.kind]
+        descriptors.append(
+            FieldDescriptor(
+                field.name,
+                schema_kind,
+                stored=True,
+                indexed=True,
+                fast=field.fast if field.kind in _FAST_FROM_FIELD else False,
+                tokenizer=tokenizer,
+            ),
+        )
+    return descriptors
+
+
+def field_descriptors() -> list[FieldDescriptor]:
+    """Every field of the document index, in the order tantivy declares them.
+
+    tantivy compares schemas by *ordered* field list, so the order here is
+    part of the on-disk contract: schema_fingerprint() hashes it and
+    needs_rebuild() acts on the result.
+    """
+    return [
+        FieldDescriptor(
+            "id",
+            "u64",
+            stored=True,
+            indexed=True,
+            fast=True,
+            tokenizer=None,
+        ),
+        *_public_field_descriptors(),
+        # Shadow sort fields - fast, not stored
+        *(
+            FieldDescriptor(
+                name,
+                "text",
+                stored=False,
+                indexed=True,
+                fast=True,
+                tokenizer="simple_analyzer",
+            )
+            for name in ("title_sort", "correspondent_sort", "type_sort")
+        ),
+        # CJK support - not stored, indexed only
+        *(
+            FieldDescriptor(
+                name,
+                "text",
+                stored=False,
+                indexed=True,
+                fast=False,
+                tokenizer="bigram_analyzer",
+            )
+            for name in (
+                "bigram_content",
+                "bigram_title",
+                "bigram_correspondent",
+                "bigram_document_type",
+                "bigram_tag",
+            )
+        ),
+        # Simple substring search support for title/content - not stored,
+        # indexed only
+        *(
+            FieldDescriptor(
+                name,
+                "text",
+                stored=False,
+                indexed=True,
+                fast=False,
+                tokenizer="simple_search_analyzer",
+            )
+            for name in ("simple_title", "simple_content")
+        ),
+        # Autocomplete prefix scan via terms_with_prefix, which walks the
+        # field's term dictionary - so the field must be indexed (term dict),
+        # not stored. The stored value is never read back, so storing it only
+        # wastes space.
+        FieldDescriptor(
+            "autocomplete_word",
+            "text",
+            stored=False,
+            indexed=True,
+            fast=False,
+            tokenizer="raw",
+        ),
+        # Permission filter columns, read by build_permission_filter.
+        *(
+            FieldDescriptor(
+                name,
+                "u64",
+                stored=False,
+                indexed=True,
+                fast=True,
+                tokenizer=None,
+            )
+            for name in ("owner_id", "viewer_id", "viewer_group_id")
+        ),
+    ]
+
+
+def schema_fingerprint() -> str:
+    """Hash of the field descriptors, stamped into .index_settings.json.
+
+    Changes whenever a field is added, removed, retyped, re-optioned or
+    reordered, so an index built from a different schema shape is detected
+    even when SCHEMA_VERSION was not bumped.
+    """
+    payload = json.dumps([list(descriptor) for descriptor in field_descriptors()])
+    return hashlib.blake2b(payload.encode()).hexdigest()
 
 
 def build_schema() -> tantivy.Schema:
@@ -32,85 +215,37 @@ def build_schema() -> tantivy.Schema:
     """
     sb = tantivy.SchemaBuilder()
 
-    sb.add_unsigned_field("id", stored=True, indexed=True, fast=True)
-    sb.add_text_field("checksum", stored=True, tokenizer_name="raw")
-
-    for field in (
-        "title",
-        "correspondent",
-        "document_type",
-        "storage_path",
-        "original_filename",
-        "content",
-    ):
-        sb.add_text_field(field, stored=True, tokenizer_name="paperless_text")
-
-    # Shadow sort fields - fast, not stored/indexed
-    for field in ("title_sort", "correspondent_sort", "type_sort"):
-        sb.add_text_field(
-            field,
-            stored=False,
-            tokenizer_name="simple_analyzer",
-            fast=True,
-        )
-
-    # CJK support - not stored, indexed only
-    sb.add_text_field("bigram_content", stored=False, tokenizer_name="bigram_analyzer")
-    sb.add_text_field("bigram_title", stored=False, tokenizer_name="bigram_analyzer")
-    sb.add_text_field(
-        "bigram_correspondent",
-        stored=False,
-        tokenizer_name="bigram_analyzer",
-    )
-    sb.add_text_field(
-        "bigram_document_type",
-        stored=False,
-        tokenizer_name="bigram_analyzer",
-    )
-    sb.add_text_field("bigram_tag", stored=False, tokenizer_name="bigram_analyzer")
-
-    # Simple substring search support for title/content - not stored, indexed only
-    sb.add_text_field(
-        "simple_title",
-        stored=False,
-        tokenizer_name="simple_search_analyzer",
-    )
-    sb.add_text_field(
-        "simple_content",
-        stored=False,
-        tokenizer_name="simple_search_analyzer",
-    )
-
-    # Autocomplete prefix scan via terms_with_prefix, which walks the field's
-    # term dictionary - so the field must be indexed (term dict), not stored.
-    # The stored value is never read back, so storing it only wastes space.
-    sb.add_text_field("autocomplete_word", stored=False, tokenizer_name="raw")
-
-    sb.add_text_field("tag", stored=True, tokenizer_name="paperless_text")
-
-    # JSON fields — structured queries: notes.user:alice, custom_fields.name:invoice
-    sb.add_json_field("notes", stored=True, tokenizer_name="paperless_text")
-    # Plain-text companion for notes — tantivy's SnippetGenerator does not support
-    # JSON fields, so highlights require a text field with the same content.
-    sb.add_text_field("notes_text", stored=True, tokenizer_name="paperless_text")
-    sb.add_json_field("custom_fields", stored=True, tokenizer_name="paperless_text")
-
-    for field in (
-        "correspondent_id",
-        "document_type_id",
-        "storage_path_id",
-        "tag_id",
-        "owner_id",
-        "viewer_id",
-        "viewer_group_id",
-    ):
-        sb.add_unsigned_field(field, stored=False, indexed=True, fast=True)
-
-    for field in ("created", "modified", "added"):
-        sb.add_date_field(field, stored=True, indexed=True, fast=True)
-
-    for field in ("asn", "page_count", "num_notes"):
-        sb.add_unsigned_field(field, stored=True, indexed=True, fast=True)
+    for descriptor in field_descriptors():
+        if descriptor.kind == "text":
+            sb.add_text_field(
+                descriptor.name,
+                stored=descriptor.stored,
+                fast=descriptor.fast,
+                tokenizer_name=cast("str", descriptor.tokenizer),
+            )
+        elif descriptor.kind == "json":
+            sb.add_json_field(
+                descriptor.name,
+                stored=descriptor.stored,
+                fast=descriptor.fast,
+                tokenizer_name=cast("str", descriptor.tokenizer),
+            )
+        elif descriptor.kind == "u64":
+            sb.add_unsigned_field(
+                descriptor.name,
+                stored=descriptor.stored,
+                indexed=descriptor.indexed,
+                fast=descriptor.fast,
+            )
+        elif descriptor.kind == "date":
+            sb.add_date_field(
+                descriptor.name,
+                stored=descriptor.stored,
+                indexed=descriptor.indexed,
+                fast=descriptor.fast,
+            )
+        else:
+            raise ValueError(f"Unknown schema field kind: {descriptor.kind}")
 
     return sb.build()
 
@@ -119,9 +254,9 @@ def needs_rebuild(index_dir: Path) -> bool:
     """
     Check if the search index needs rebuilding.
 
-    Reads .index_settings.json to compare the stored schema version and
-    search language against the current configuration. Returns True if the
-    file is missing, unparsable, or either value mismatches.
+    Reads .index_settings.json to compare the stored schema version, search
+    language and schema fingerprint against the current configuration. Returns
+    True if the file is missing, unparsable, or any value mismatches.
 
     Args:
         index_dir: Path to the search index directory
@@ -140,6 +275,9 @@ def needs_rebuild(index_dir: Path) -> bool:
         if "language" not in data or data["language"] != settings.SEARCH_LANGUAGE:
             logger.info("Search index language changed - rebuilding.")
             return True
+        if data.get("schema_fingerprint") != schema_fingerprint():
+            logger.info("Search index schema fingerprint mismatch - rebuilding.")
+            return True
     except ValueError:
         return True
     return False
@@ -170,6 +308,7 @@ def _write_sentinels(index_dir: Path) -> None:
             {
                 "schema_version": SCHEMA_VERSION,
                 "language": settings.SEARCH_LANGUAGE,
+                "schema_fingerprint": schema_fingerprint(),
             },
         ),
     )

src/documents/search/_tokenizer.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+from functools import cache
 from typing import Final
 
 import tantivy
@@ -71,7 +72,7 @@ def register_tokenizers(index: tantivy.Index, language: str | None) -> None:
         use fast=True and Tantivy requires fast-field tokenizers to exist
         even for documents that omit those fields.
     """
-    index.register_tokenizer("paperless_text", _paperless_text(language))
+    index.register_tokenizer("paperless_text", paperless_text_analyzer(language))
     index.register_tokenizer("simple_analyzer", _simple_analyzer())
     index.register_tokenizer("bigram_analyzer", _bigram_analyzer())
     index.register_tokenizer("simple_search_analyzer", _simple_search_analyzer())
@@ -79,7 +80,7 @@ def register_tokenizers(index: tantivy.Index, language: str | None) -> None:
     index.register_fast_field_tokenizer("simple_analyzer", _simple_analyzer())
 
 
-def _paperless_text(language: str | None) -> tantivy.TextAnalyzer:
+def paperless_text_analyzer(language: str | None) -> tantivy.TextAnalyzer:
     """Main full-text tokenizer for content, title, etc: simple -> remove_long(129) -> lowercase -> ascii_fold [-> stemmer]"""
     builder = (
         tantivy.TextAnalyzerBuilder(tantivy.Tokenizer.simple())
@@ -100,6 +101,54 @@ def _paperless_text(language: str | None) -> tantivy.TextAnalyzer:
     return builder.build()
 
 
+@cache
+def _pattern_stemmer(language: str | None) -> tantivy.TextAnalyzer | None:
+    """The stemming tail of paperless_text_analyzer, over a whole literal run.
+
+    Same language gate and same Snowball stemmer paperless_text_analyzer
+    applies at index time, so query patterns follow SEARCH_LANGUAGE. Returns
+    None when that gate disables stemming; paperless_text_analyzer already
+    warns about an unsupported language, so this stays quiet.
+
+    The raw tokenizer keeps the run whole (a wildcard literal is a fragment,
+    not necessarily a word), and remove_long is kept so an over-long run is
+    treated the same way the index treats it.
+    """
+    if not language:
+        return None
+    tantivy_lang = _LANGUAGE_MAP.get(language.lower())
+    if tantivy_lang is None:
+        return None
+    return (
+        tantivy.TextAnalyzerBuilder(tantivy.Tokenizer.raw())
+        .filter(tantivy.Filter.remove_long(_TOKEN_REMOVE_LONG_LIMIT))
+        .filter(tantivy.Filter.stemmer(tantivy_lang))
+        .build()
+    )
+
+
+def stem_pattern_text(text: str, language: str | None) -> str:
+    """Stem an already lowercased/ascii-folded run the way index terms are.
+
+    Returns text unchanged when stemming is disabled for language, and also
+    when the stem step does not yield exactly one token: remove_long drops a run
+    past the length limit, leaving no stem to substitute. Falling back to the
+    text as typed is the safe direction for a pattern prefix, since it can only
+    be as narrow as it was before stemming was considered.
+
+    The raw tokenizer emits one token whatever the input and the stemmer is
+    1-to-1, so only the zero-token case can fire today; the guard covers both
+    counts so a tokenizer change cannot turn this into an IndexError.
+    """
+    analyzer = _pattern_stemmer(language)
+    if analyzer is None:
+        return text
+    tokens = analyzer.analyze(text)
+    if len(tokens) != 1:
+        return text
+    return tokens[0]
+
+
 def _simple_analyzer() -> tantivy.TextAnalyzer:
     """Tokenizer for shadow sort fields (title_sort, correspondent_sort, type_sort): simple -> lowercase -> ascii_fold."""
     return (

src/documents/tests/search/test_json_subpath_completeness.py

@@ -0,0 +1,92 @@
+"""Every declared JSON subpath must actually be written to the index.
+
+PUBLIC_FIELDS declares each JSON field's subpaths (e.g. ``notes`` ->
+{"user", "note"}), but nothing coupled that declaration to what
+``_backend.py``'s document builder actually writes into the JSON blob at
+index time. A subpath declared but never written would be
+queryable-but-always-empty -- syntactically valid, silently matching
+nothing -- with no test failure anywhere.
+
+This indexes one real document carrying values for every JSON field
+(a Note, a CustomFieldInstance) and inspects the document's own stored
+JSON payload, rather than running field-specific queries: that way a
+future JSON field's subpaths are covered automatically, without a new
+per-subpath query having to be added by hand each time.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+import tantivy
+from django.contrib.auth.models import User
+from whoosh_compat import FieldKind
+
+from documents.models import CustomField
+from documents.models import CustomFieldInstance
+from documents.models import Document
+from documents.models import Note
+from documents.search._fields import PUBLIC_FIELDS
+
+if TYPE_CHECKING:
+    from documents.search._backend import TantivyBackend
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+
+class TestJsonSubpathsAreWrittenAtIndexTime:
+    def test_every_declared_json_subpath_appears_in_the_stored_document(
+        self,
+        backend: TantivyBackend,
+    ) -> None:
+        """
+        GIVEN:
+            - A document with a Note and a CustomFieldInstance attached
+        WHEN:
+            - The document is indexed via TantivyBackend.add_or_update
+        THEN:
+            - Every subpath PUBLIC_FIELDS declares for notes/custom_fields
+              is present as a key in the document's stored JSON payload
+        """
+        user = User.objects.create_user(username="completeness-user")
+        field = CustomField.objects.create(
+            name="Completeness Field",
+            data_type=CustomField.FieldDataType.STRING,
+        )
+        doc = Document.objects.create(
+            title="Completeness doc",
+            content="x",
+            checksum="json-subpath-completeness",
+        )
+        Note.objects.create(document=doc, user=user, note="a note")
+        CustomFieldInstance.objects.create(
+            document=doc,
+            field=field,
+            value_text="a value",
+        )
+        backend.add_or_update(doc)
+
+        index = backend._index
+        searcher = index.searcher()
+        hits = searcher.search(
+            tantivy.Query.term_query(index.schema, "id", doc.pk),
+            limit=1,
+        ).hits
+        assert hits, "the document was not indexed"
+        stored = searcher.doc(hits[0][1]).to_dict()
+
+        json_fields = [f for f in PUBLIC_FIELDS if f.kind is FieldKind.JSON]
+        assert json_fields, "no JSON fields declared - fixture is stale"
+        for field_spec in json_fields:
+            stored_values = stored.get(field_spec.name)
+            assert stored_values, (
+                f"{field_spec.name} was not written to the index at all"
+            )
+            written_keys = stored_values[0].keys()
+            for subpath in field_spec.subpaths:
+                assert subpath in written_keys, (
+                    f"{field_spec.name}.{subpath} is declared in PUBLIC_FIELDS "
+                    "but _backend.py's document builder never writes it - it "
+                    "would be queryable but always empty"
+                )

src/documents/tests/search/test_keyword_pattern_literal.py

@@ -0,0 +1,62 @@
+"""Wildcard patterns on KEYWORD fields must stay literal.
+
+``checksum`` is the only KEYWORD field: it is indexed with the raw tokenizer,
+so its terms are never lowercased, folded or stemmed. Running its wildcard
+patterns through the stemming normalizer rewrote hex prefixes ("ceded" ->
+"cede") and returned documents whose checksum did not start with what the user
+typed, which for an identity field is a wrong answer.
+
+This covers only the registry-level normalizer, which is all that exists to
+prove at this point in the stack: user queries are not yet routed through
+whoosh-compat (that lands with the query-layer PR), so the same fact proven
+end to end against real indexed documents lives in
+``test_checksum_prefix_queries.py``.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from documents.search._registry import get_field_registry
+
+if TYPE_CHECKING:
+    from whoosh_compat import FieldRegistry
+    from whoosh_compat import PatternNormalizer
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+
+def _normalizer(registry: FieldRegistry, name: str) -> PatternNormalizer:
+    ref = registry.make_ref(name)
+    assert ref is not None
+    resolved = registry.resolve(ref)
+    assert resolved is not None
+    assert resolved.spec.pattern_normalizer is not None
+    return resolved.spec.pattern_normalizer
+
+
+class TestKeywordPatternNormalizer:
+    @pytest.mark.parametrize(
+        "run",
+        [
+            pytest.param("ceded", id="stems_to_cede"),
+            pytest.param("added", id="stems_to_ad"),
+            pytest.param("cafed", id="stems_to_cafe"),
+        ],
+    )
+    def test_keyword_runs_are_folded_not_stemmed(self, run: str) -> None:
+        """
+        GIVEN:
+            - The "checksum" field's registered pattern normalizer
+              (KEYWORD kind, "en" registry)
+        WHEN:
+            - A wildcard pattern run is normalized
+        THEN:
+            - The run is returned unchanged, never widened to a stem (which
+              would return checksums that do not start with what the user
+              typed)
+        """
+        normalize = _normalizer(get_field_registry("en"), "checksum")
+        assert normalize(run) == run

src/documents/tests/search/test_pattern_normalizer.py

@@ -0,0 +1,156 @@
+"""The pattern normalizer's stem-alternates contract, and its consistency
+with the index-side analyzer.
+
+Query patterns are normalized but were not stemmed, while index terms are
+stemmed, so the natural spelling of a prefix search matched nothing:
+``invoice*`` found no document although ``invoic*`` did. v2's index was
+UNSTEMMED (whoosh ``TEXT()`` defaults to ``StandardAnalyzer``), so this
+regressed against both baselines.
+
+These are pure unit tests against ``_make_pattern_normalizer`` and
+``stem_pattern_text`` directly, no query routing involved. The end-to-end
+proof that a real wildcard query actually reaches a stemmed index term
+lives in ``test_pattern_stemming.py``.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from documents.search._registry import _make_pattern_normalizer
+from documents.search._tokenizer import ascii_fold
+from documents.search._tokenizer import paperless_text_analyzer
+from documents.search._tokenizer import stem_pattern_text
+
+if TYPE_CHECKING:
+    from whoosh_compat import PatternNormalizer
+
+
+class TestStemsMatchTheIndexAnalyzer:
+    """stem_pattern_text rebuilds paperless_text_analyzer's stemming tail rather
+    than sharing it, so a filter added to the index analyzer alone would silently
+    stop patterns from reaching the terms it produces.
+    """
+
+    @pytest.mark.parametrize(
+        "language",
+        ["en", "de", "fr", "es", "sv", None, "klingon"],
+    )
+    @pytest.mark.parametrize(
+        "word",
+        ["Copies", "copyright", "Companies", "Invoices", "laufen", "casas", "Straße"],
+    )
+    def test_stem_equals_the_index_term(self, word: str, language: str | None) -> None:
+        """
+        GIVEN:
+            - A word, across several representative index languages
+              ("en", "de", "fr", "es", "sv"), no language, and an
+              unsupported language ("klingon")
+        WHEN:
+            - `stem_pattern_text` (the pattern-side stemmer) processes the
+              folded word, and `paperless_text_analyzer` (the index-side
+              analyzer) independently processes the same word
+        THEN:
+            - The two produce the identical term. `stem_pattern_text`
+              rebuilds `paperless_text_analyzer`'s stemming tail rather
+              than sharing it, so a filter added to the index analyzer
+              alone would silently stop patterns from reaching the terms
+              it produces; this pins the two staying in sync
+        """
+        indexed = paperless_text_analyzer(language).analyze(word)[0]
+        assert stem_pattern_text(ascii_fold(word.lower()), language) == indexed
+
+
+def _forms(normalize: PatternNormalizer, text: str) -> tuple[str, ...]:
+    """The distinct forms a term may match, in order, the way the emitter reads
+    the normalizer's answer (see whoosh_compat.PatternNormalizer)."""
+    result = normalize(text)
+    if isinstance(result, str):
+        return (result,)
+    return tuple(dict.fromkeys(result))
+
+
+class TestPatternNormalizer:
+    @pytest.mark.parametrize(
+        ("text", "expected"),
+        [
+            ("Invoice", ("invoice", "invoic")),
+            ("companies", ("companies", "compani")),
+            # y -> i is a substitution, so both forms are needed: the index
+            # holds "librari" for "library" and "library" for "librarian".
+            ("library", ("library", "librari")),
+            # A run the stemmer leaves alone collapses back to one form, so it
+            # costs exactly the one regex branch it did before.
+            ("invoic", ("invoic",)),
+            ("Universit", ("universit",)),
+            ("Café", ("cafe",)),
+        ],
+    )
+    def test_offers_the_typed_run_and_its_stem(
+        self,
+        text: str,
+        expected: tuple[str, ...],
+    ) -> None:
+        """
+        GIVEN:
+            - The "en" pattern normalizer
+        WHEN:
+            - It processes a literal run (e.g. "Invoice", "library",
+              "Café")
+        THEN:
+            - It returns the folded run and, where it differs, the
+              stemmed form, as distinct alternatives; a run the stemmer
+              leaves alone (e.g. "invoic") collapses back to the single
+              folded form. "library" needs both forms since y -> i is a
+              substitution: the index holds "librari" for "library" and
+              "library" for "librarian"
+        """
+        assert _forms(_make_pattern_normalizer("en"), text) == expected
+
+    def test_run_that_yields_no_token_falls_back_to_the_typed_run(self) -> None:
+        """
+        GIVEN:
+            - The "en" pattern normalizer
+        WHEN:
+            - It processes a run past the analyzer's remove_long limit
+        THEN:
+            - The run analyzes to zero tokens, so there is no stem to
+              offer, and only the folded run remains
+        """
+        over_long = "invoices" * 20
+        assert _forms(_make_pattern_normalizer("en"), over_long) == (over_long,)
+
+    @pytest.mark.parametrize("language", [None, "klingon"])
+    def test_unstemmed_language_folds_only(self, language: str | None) -> None:
+        """
+        GIVEN:
+            - A pattern normalizer with no language configured, or one
+              this build has no stemmer for ("klingon")
+        WHEN:
+            - It processes "Invoices"
+        THEN:
+            - Only the folded form ("invoices") is offered, since with no
+              stemmer configured the index holds surface forms and the
+              pattern must keep them too
+        """
+        assert _forms(_make_pattern_normalizer(language), "Invoices") == ("invoices",)
+
+    @pytest.mark.parametrize("char", ["a", "Z", "é"])
+    def test_a_single_character_collapses_to_one_folded_form(self, char: str) -> None:
+        """
+        GIVEN:
+            - The "en" pattern normalizer
+        WHEN:
+            - It processes a single character
+        THEN:
+            - Exactly one, one-character form is returned. A bracket
+              class body is normalized one character at a time and the
+              answer is used only when it is a single one-character
+              form, so a stemmer that changed a lone character would
+              silently disable folding inside classes
+        """
+        forms = _forms(_make_pattern_normalizer("en"), char)
+        assert len(forms) == 1
+        assert len(forms[0]) == 1

src/documents/tests/search/test_registry.py

@@ -0,0 +1,224 @@
+from collections.abc import Sequence
+
+import pytest
+from whoosh_compat import FieldKind
+from whoosh_compat import FieldRegistry
+from whoosh_compat.fields import ResolvedField
+
+from documents.search._fields import PUBLIC_FIELDS
+from documents.search._registry import get_field_registry
+
+
+@pytest.fixture
+def registry() -> FieldRegistry:
+    return get_field_registry(None)
+
+
+def _resolve(registry: FieldRegistry, name: str) -> ResolvedField:
+    ref = registry.make_ref(name)
+    assert ref is not None, f"{name} is not a valid field ref"
+    resolved = registry.resolve(ref)
+    assert resolved is not None, f"{name} did not resolve"
+    return resolved
+
+
+def _distinct_forms(result: str | Sequence[str]) -> tuple[str, ...]:
+    """The forms a term may match, in order, the way whoosh-compat's emitter
+    reads a pattern_normalizer's answer: a bare str is one form, a sequence is
+    several, deduplicated."""
+    if isinstance(result, str):
+        return (result,)
+    return tuple(dict.fromkeys(result))
+
+
+class TestFieldRegistry:
+    def test_no_queryable_field_name_ends_in_id(self) -> None:
+        """
+        GIVEN:
+            - PUBLIC_FIELDS, the canonical query-syntax field table
+        WHEN:
+            - Every declared field name is inspected
+        THEN:
+            - None of them end in "_id" (internal id columns, written for
+              permission filtering and joins, must never reach the query
+              surface; checked against PUBLIC_FIELDS rather than the
+              registry so a leak is caught where it is declared)
+        """
+        leaked = [f.name for f in PUBLIC_FIELDS if f.name.endswith("_id")]
+        assert not leaked, f"internal id fields reached the query surface: {leaked}"
+
+    def test_type_alias_resolves_to_document_type(
+        self,
+        registry: FieldRegistry,
+    ) -> None:
+        """
+        GIVEN:
+            - The field registry
+        WHEN:
+            - The alias "type" is resolved
+        THEN:
+            - It resolves to the canonical "document_type" field
+        """
+        assert _resolve(registry, "type").spec.name == "document_type"
+
+    def test_path_alias_resolves_to_storage_path(self, registry: FieldRegistry) -> None:
+        """
+        GIVEN:
+            - The field registry
+        WHEN:
+            - The alias "path" is resolved
+        THEN:
+            - It resolves to the canonical "storage_path" field
+        """
+        assert _resolve(registry, "path").spec.name == "storage_path"
+
+    def test_notes_json_subpaths_resolve(self, registry: FieldRegistry) -> None:
+        """
+        GIVEN:
+            - The field registry
+        WHEN:
+            - "notes.user" is resolved
+        THEN:
+            - It resolves to the "notes" field with json_path "user"
+        """
+        resolved = _resolve(registry, "notes.user")
+        assert resolved.spec.name == "notes"
+        assert resolved.json_path == "user"
+        assert resolved.is_subpath is True
+
+    def test_custom_fields_json_subpaths_resolve(self, registry: FieldRegistry) -> None:
+        """
+        GIVEN:
+            - The field registry
+        WHEN:
+            - "custom_fields.name" and "custom_fields.value" are resolved
+        THEN:
+            - Both resolve without error
+        """
+        for raw in ("custom_fields.name", "custom_fields.value"):
+            _resolve(registry, raw)
+
+    def test_tag_is_comma_values(self, registry: FieldRegistry) -> None:
+        """
+        GIVEN:
+            - The field registry
+        WHEN:
+            - The "tag" field is resolved
+        THEN:
+            - It is marked comma_values=True
+        """
+        assert _resolve(registry, "tag").spec.comma_values is True
+
+    def test_correspondent_is_not_comma_values(self, registry: FieldRegistry) -> None:
+        """
+        GIVEN:
+            - The field registry
+        WHEN:
+            - The "correspondent" field is resolved
+        THEN:
+            - It is not marked comma_values ("tag" is the only field that
+              opts in; end to end the two readings of
+              "correspondent:foo,bar" agree anyway, since the analyzer
+              splits the literal value on the comma regardless, so this is
+              only observable at the registry level)
+        """
+        assert _resolve(registry, "correspondent").spec.comma_values is False
+
+    def test_created_is_date_kind(self, registry: FieldRegistry) -> None:
+        """
+        GIVEN:
+            - The field registry
+        WHEN:
+            - The "created" field is resolved
+        THEN:
+            - Its kind is DATE and date_only is True
+        """
+        resolved = _resolve(registry, "created")
+        assert resolved.spec.kind is FieldKind.DATE
+        assert resolved.spec.date_only is True
+
+    def test_analyzer_lowercases_and_ascii_folds(self, registry: FieldRegistry) -> None:
+        """
+        GIVEN:
+            - The field registry with no language configured (no stemmer
+              in the analyzer chain)
+        WHEN:
+            - The "title" field's analyzer processes "Café"
+        THEN:
+            - It is lowercased and ASCII-folded to the single token "cafe"
+        """
+        resolved = _resolve(registry, "title")
+        assert resolved.spec.analyzer is not None
+        assert resolved.spec.analyzer("Café") == ["cafe"]
+
+    def test_checksum_analyzer_is_identity_single_token(
+        self,
+        registry: FieldRegistry,
+    ) -> None:
+        """
+        GIVEN:
+            - The field registry
+        WHEN:
+            - The "checksum" field's analyzer (raw tokenizer, no
+              splitting) processes "ABC-123"
+        THEN:
+            - It is returned unchanged as a single token
+        """
+        resolved = _resolve(registry, "checksum")
+        assert resolved.spec.analyzer is not None
+        assert resolved.spec.analyzer("ABC-123") == ["ABC-123"]
+
+    def test_pattern_normalizer_follows_the_registry_language(
+        self,
+        registry: FieldRegistry,
+    ) -> None:
+        """
+        GIVEN:
+            - A registry with no language, and a registry built for "en"
+        WHEN:
+            - The "title" field's pattern normalizer processes "Running"
+        THEN:
+            - With no language, only the folded run is offered
+              ("running"), since the index holds surface forms
+            - With "en", the stem is offered too ("run"), since indexed
+              terms are stemmed and the pattern has to reach them
+        """
+        resolved = _resolve(registry, "title")
+        assert resolved.spec.pattern_normalizer is not None
+        assert _distinct_forms(resolved.spec.pattern_normalizer("Running")) == (
+            "running",
+        )
+
+        resolved_en = _resolve(get_field_registry("en"), "title")
+        assert resolved_en.spec.pattern_normalizer is not None
+        assert _distinct_forms(resolved_en.spec.pattern_normalizer("Running")) == (
+            "running",
+            "run",
+        )
+
+    def test_registry_is_cached_per_language(self) -> None:
+        """
+        GIVEN:
+            - Two calls to get_field_registry("en")
+        WHEN:
+            - Both calls are made
+        THEN:
+            - They return the same registry instance
+        """
+        a = get_field_registry("en")
+        b = get_field_registry("en")
+        assert a is b
+
+    def test_registry_rebuilds_on_language_change(self) -> None:
+        """
+        GIVEN:
+            - A call to get_field_registry("en") and a call to
+              get_field_registry("de")
+        WHEN:
+            - Both calls are made
+        THEN:
+            - They return different registry instances
+        """
+        a = get_field_registry("en")
+        b = get_field_registry("de")
+        assert a is not b

src/documents/tests/search/test_schema.py

@@ -5,12 +5,17 @@ from typing import TYPE_CHECKING
 
 import pytest
 
+from documents.search._fields import PUBLIC_FIELDS
 from documents.search._schema import SCHEMA_VERSION
+from documents.search._schema import build_schema
+from documents.search._schema import field_descriptors
 from documents.search._schema import needs_rebuild
+from documents.search._schema import schema_fingerprint
 
 if TYPE_CHECKING:
     from pathlib import Path
 
+    import tantivy
     from pytest_django.fixtures import Settings
 
 
@@ -30,7 +35,13 @@ class TestNeedsRebuild:
     ) -> None:
         settings.SEARCH_LANGUAGE = "en"
         (index_dir / ".index_settings.json").write_text(
-            json.dumps({"schema_version": SCHEMA_VERSION, "language": "en"}),
+            json.dumps(
+                {
+                    "schema_version": SCHEMA_VERSION,
+                    "language": "en",
+                    "schema_fingerprint": schema_fingerprint(),
+                },
+            ),
         )
         assert needs_rebuild(index_dir) is False
 
@@ -77,3 +88,61 @@ class TestNeedsRebuild:
             json.dumps({"schema_version": SCHEMA_VERSION, "language": "en"}),
         )
         assert needs_rebuild(index_dir) is True
+
+
+def _schema_fields(schema: tantivy.Schema) -> dict[str, dict]:
+    """{name: field-state} for every field declared on a tantivy Schema.
+
+    tantivy-py 0.26 exposes no public introspection API on Schema (no
+    __iter__, get_field, to_json, etc.) -- __reduce__() (used internally for
+    pickling) is the only way to recover the field list, so we lean on it
+    here for test assertions only.
+    """
+    state = schema.__reduce__()[1][0]
+    return {field["name"]: field for field in state["inner"]}
+
+
+class TestSchemaMatchesPublicFields:
+    def test_every_public_field_is_in_the_schema(self) -> None:
+        """
+        GIVEN:
+            - PUBLIC_FIELDS and the tantivy schema built by build_schema()
+        WHEN:
+            - Every field declared in PUBLIC_FIELDS is checked against the
+              schema
+        THEN:
+            - Each one is present as a field in the built schema
+        """
+        schema = build_schema()
+        schema_field_names = set(_schema_fields(schema))
+        for field in PUBLIC_FIELDS:
+            assert field.name in schema_field_names, (
+                f"{field.name} is in PUBLIC_FIELDS but missing from build_schema()"
+            )
+
+
+class TestFastFlagAgreement:
+    def test_every_public_field_fast_flag_matches_the_built_schema(self) -> None:
+        """
+        GIVEN:
+            - PUBLIC_FIELDS and field_descriptors() (the latter is exactly
+              the input build_schema()'s SchemaBuilder consumes for the
+              `fast` kwarg on every field kind, so it pins the agreement
+              without depending on a private tantivy-py pickled
+              representation)
+        WHEN:
+            - Every PUBLIC_FIELDS entry's fast flag is compared against
+              field_descriptors()' fast flag for the same field
+        THEN:
+            - They agree for every field, catching a fast=True
+              PUBLIC_FIELDS entry the builder silently ignores here
+              instead of at a user's field:* existence query, which
+              whoosh-compat's registry trusts PUBLIC_FIELDS' fast flag to
+              resolve
+        """
+        descriptor_fast = {d.name: d.fast for d in field_descriptors()}
+        for public_field in PUBLIC_FIELDS:
+            assert descriptor_fast[public_field.name] == public_field.fast, (
+                f"{public_field.name}: PUBLIC_FIELDS says fast={public_field.fast} but"
+                f" field_descriptors() says fast={descriptor_fast[public_field.name]}"
+            )

src/documents/tests/search/test_schema_fingerprint.py

@@ -0,0 +1,587 @@
+"""The schema fingerprint stamped into .index_settings.json.
+
+tantivy compares schemas by *ordered* field list, and `tantivy.Index(schema,
+path=...)` (what every write path does) raises on any difference. SCHEMA_VERSION
+is the manual guard against that, but build_schema() is edited for *parser*
+reasons - adding an alias, flipping fast=True, adding a subpath - by people not
+thinking about the on-disk index, and forgetting the bump is exactly how this
+branch's bug happened.
+
+The fingerprint is the automatic guard: it hashes the field descriptor list that
+build_schema() itself iterates, so any change to a field's name, kind, options
+or *position* forces a rebuild on its own.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import TYPE_CHECKING
+
+import pytest
+import tantivy
+
+from documents.search import _schema
+from documents.search._schema import SCHEMA_VERSION
+from documents.search._schema import FieldDescriptor
+from documents.search._schema import _write_sentinels
+from documents.search._schema import build_schema
+from documents.search._schema import field_descriptors
+from documents.search._schema import needs_rebuild
+from documents.search._schema import schema_fingerprint
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from pytest_django.fixtures import SettingsWrapper
+
+pytestmark = pytest.mark.search
+
+# The on-disk field layout of a v2 index, pinned as data. Any edit here is an
+# index-format change: it must come with a rebuild, which the fingerprint now
+# forces automatically. Reproduced from build_schema()'s output as it stood
+# before the descriptor refactor, so it also pins that the refactor changed
+# nothing.
+PINNED_DESCRIPTORS: tuple[FieldDescriptor, ...] = (
+    FieldDescriptor("id", "u64", stored=True, indexed=True, fast=True, tokenizer=None),
+    FieldDescriptor(
+        "title",
+        "text",
+        stored=True,
+        indexed=True,
+        fast=False,
+        tokenizer="paperless_text",
+    ),
+    FieldDescriptor(
+        "content",
+        "text",
+        stored=True,
+        indexed=True,
+        fast=False,
+        tokenizer="paperless_text",
+    ),
+    FieldDescriptor(
+        "correspondent",
+        "text",
+        stored=True,
+        indexed=True,
+        fast=False,
+        tokenizer="paperless_text",
+    ),
+    FieldDescriptor(
+        "document_type",
+        "text",
+        stored=True,
+        indexed=True,
+        fast=False,
+        tokenizer="paperless_text",
+    ),
+    FieldDescriptor(
+        "storage_path",
+        "text",
+        stored=True,
+        indexed=True,
+        fast=False,
+        tokenizer="paperless_text",
+    ),
+    FieldDescriptor(
+        "original_filename",
+        "text",
+        stored=True,
+        indexed=True,
+        fast=False,
+        tokenizer="paperless_text",
+    ),
+    FieldDescriptor(
+        "tag",
+        "text",
+        stored=True,
+        indexed=True,
+        fast=False,
+        tokenizer="paperless_text",
+    ),
+    FieldDescriptor(
+        "checksum",
+        "text",
+        stored=True,
+        indexed=True,
+        fast=False,
+        tokenizer="raw",
+    ),
+    FieldDescriptor("asn", "u64", stored=True, indexed=True, fast=True, tokenizer=None),
+    FieldDescriptor(
+        "page_count",
+        "u64",
+        stored=True,
+        indexed=True,
+        fast=True,
+        tokenizer=None,
+    ),
+    FieldDescriptor(
+        "num_notes",
+        "u64",
+        stored=True,
+        indexed=True,
+        fast=True,
+        tokenizer=None,
+    ),
+    FieldDescriptor(
+        "created",
+        "date",
+        stored=True,
+        indexed=True,
+        fast=True,
+        tokenizer=None,
+    ),
+    FieldDescriptor(
+        "modified",
+        "date",
+        stored=True,
+        indexed=True,
+        fast=True,
+        tokenizer=None,
+    ),
+    FieldDescriptor(
+        "added",
+        "date",
+        stored=True,
+        indexed=True,
+        fast=True,
+        tokenizer=None,
+    ),
+    FieldDescriptor(
+        "notes",
+        "json",
+        stored=True,
+        indexed=True,
+        fast=False,
+        tokenizer="paperless_text",
+    ),
+    FieldDescriptor(
+        "notes_text",
+        "text",
+        stored=True,
+        indexed=True,
+        fast=False,
+        tokenizer="paperless_text",
+    ),
+    FieldDescriptor(
+        "custom_fields",
+        "json",
+        stored=True,
+        indexed=True,
+        fast=False,
+        tokenizer="paperless_text",
+    ),
+    FieldDescriptor(
+        "title_sort",
+        "text",
+        stored=False,
+        indexed=True,
+        fast=True,
+        tokenizer="simple_analyzer",
+    ),
+    FieldDescriptor(
+        "correspondent_sort",
+        "text",
+        stored=False,
+        indexed=True,
+        fast=True,
+        tokenizer="simple_analyzer",
+    ),
+    FieldDescriptor(
+        "type_sort",
+        "text",
+        stored=False,
+        indexed=True,
+        fast=True,
+        tokenizer="simple_analyzer",
+    ),
+    FieldDescriptor(
+        "bigram_content",
+        "text",
+        stored=False,
+        indexed=True,
+        fast=False,
+        tokenizer="bigram_analyzer",
+    ),
+    FieldDescriptor(
+        "bigram_title",
+        "text",
+        stored=False,
+        indexed=True,
+        fast=False,
+        tokenizer="bigram_analyzer",
+    ),
+    FieldDescriptor(
+        "bigram_correspondent",
+        "text",
+        stored=False,
+        indexed=True,
+        fast=False,
+        tokenizer="bigram_analyzer",
+    ),
+    FieldDescriptor(
+        "bigram_document_type",
+        "text",
+        stored=False,
+        indexed=True,
+        fast=False,
+        tokenizer="bigram_analyzer",
+    ),
+    FieldDescriptor(
+        "bigram_tag",
+        "text",
+        stored=False,
+        indexed=True,
+        fast=False,
+        tokenizer="bigram_analyzer",
+    ),
+    FieldDescriptor(
+        "simple_title",
+        "text",
+        stored=False,
+        indexed=True,
+        fast=False,
+        tokenizer="simple_search_analyzer",
+    ),
+    FieldDescriptor(
+        "simple_content",
+        "text",
+        stored=False,
+        indexed=True,
+        fast=False,
+        tokenizer="simple_search_analyzer",
+    ),
+    FieldDescriptor(
+        "autocomplete_word",
+        "text",
+        stored=False,
+        indexed=True,
+        fast=False,
+        tokenizer="raw",
+    ),
+    FieldDescriptor(
+        "owner_id",
+        "u64",
+        stored=False,
+        indexed=True,
+        fast=True,
+        tokenizer=None,
+    ),
+    FieldDescriptor(
+        "viewer_id",
+        "u64",
+        stored=False,
+        indexed=True,
+        fast=True,
+        tokenizer=None,
+    ),
+    FieldDescriptor(
+        "viewer_group_id",
+        "u64",
+        stored=False,
+        indexed=True,
+        fast=True,
+        tokenizer=None,
+    ),
+)
+
+
+def _schema_fields(schema: tantivy.Schema) -> list[dict]:
+    """The tantivy-level field list, in declaration order.
+
+    tantivy-py 0.26 exposes no public introspection API on Schema, so
+    __reduce__() (its pickling hook) is the only way to recover the field list.
+    It is used here, in a test, precisely because it is the representation the
+    persisted fingerprint must NOT depend on.
+    """
+    return schema.__reduce__()[1][0]["inner"]
+
+
+def _sentinels(index_dir: Path, **overrides: object) -> None:
+    data = {
+        "schema_version": SCHEMA_VERSION,
+        "language": None,
+        "schema_fingerprint": schema_fingerprint(),
+    }
+    data.update(overrides)
+    (index_dir / ".index_settings.json").write_text(json.dumps(data))
+
+
+class TestDescriptorsDescribeTheBuiltSchema:
+    def test_descriptors_match_the_pinned_field_layout(self) -> None:
+        """
+        GIVEN:
+            - PINNED_DESCRIPTORS, a frozen snapshot of the v2 on-disk field
+              layout, reproduced from build_schema()'s output as it stood
+              before the descriptor refactor
+        WHEN:
+            - field_descriptors() is called
+        THEN:
+            - It matches the pinned layout exactly, in the same order,
+              pinning that the refactor changed nothing
+        """
+        assert tuple(field_descriptors()) == PINNED_DESCRIPTORS
+
+    def test_built_schema_matches_the_descriptors(self) -> None:
+        """
+        GIVEN:
+            - The schema built by build_schema()
+        WHEN:
+            - Its fields are read back via __reduce__() (schema.__reduce__(),
+              tantivy-py's pickling hook)
+        THEN:
+            - Every field's name, kind, stored/fast flags and tokenizer
+              match what field_descriptors() declared as input; the
+              descriptors are not a parallel description, they are the
+              input, so a descriptor edit cannot claim a shape the
+              SchemaBuilder did not actually build
+        """
+        kinds = {"text": "text", "json": "json_object", "u64": "u64", "date": "date"}
+        built = [
+            (
+                field["name"],
+                field["type"],
+                field["options"]["stored"],
+                bool(field["options"].get("fast")),
+                (field["options"].get("indexing") or {}).get("tokenizer"),
+            )
+            for field in _schema_fields(build_schema())
+        ]
+        expected = [
+            (
+                descriptor.name,
+                kinds[descriptor.kind],
+                descriptor.stored,
+                descriptor.fast,
+                descriptor.tokenizer,
+            )
+            for descriptor in field_descriptors()
+        ]
+        assert built == expected
+
+
+class TestFingerprintSensitivity:
+    def test_a_field_option_change_moves_the_fingerprint(
+        self,
+        monkeypatch: pytest.MonkeyPatch,
+    ) -> None:
+        """
+        GIVEN:
+            - The current schema fingerprint
+        WHEN:
+            - A single field descriptor's "fast" option is changed, with
+              no other change
+        THEN:
+            - The fingerprint changes
+        """
+        before = schema_fingerprint()
+        changed = field_descriptors()
+        changed[1] = changed[1]._replace(fast=True)
+        monkeypatch.setattr(_schema, "field_descriptors", lambda: changed)
+
+        assert schema_fingerprint() != before
+
+    def test_reordering_alone_moves_the_fingerprint(
+        self,
+        monkeypatch: pytest.MonkeyPatch,
+    ) -> None:
+        """
+        GIVEN:
+            - The current schema fingerprint
+        WHEN:
+            - Two field descriptors are swapped, with no other change (the
+              original bug: same fields, different declaration order)
+        THEN:
+            - The fingerprint changes; a set- or dict-based fingerprint
+              would be blind to this, and tantivy would reject every write
+              against the existing index
+        """
+        before = schema_fingerprint()
+        swapped = field_descriptors()
+        swapped[1], swapped[2] = swapped[2], swapped[1]
+        monkeypatch.setattr(_schema, "field_descriptors", lambda: swapped)
+
+        assert schema_fingerprint() != before
+
+
+class TestFingerprintIsIndependentOfTantivy:
+    def test_a_tantivy_option_key_addition_would_not_move_it(self) -> None:
+        """
+        GIVEN:
+            - The built schema's raw field list, and the same list with a
+              new tantivy-internal option key added (simulating a
+              tantivy-py upgrade)
+        WHEN:
+            - Both raw lists are hashed directly, and schema_fingerprint()
+              is compared against a hash of field_descriptors()
+        THEN:
+            - The raw hashes differ (hashing schema.__reduce__() would
+              force a global reindex on every tantivy-py upgrade), but
+              schema_fingerprint() is unaffected, since it hashes
+              field_descriptors(), never tantivy's own representation
+        """
+        fields = _schema_fields(build_schema())
+        upgraded = [
+            {**field, "options": {**field["options"], "coerce": True}}
+            for field in fields
+        ]
+        assert _hash(upgraded) != _hash(fields)
+        assert schema_fingerprint() == _fingerprint_of(field_descriptors())
+
+    def test_fingerprint_never_touches_the_schema_builder(
+        self,
+        monkeypatch: pytest.MonkeyPatch,
+    ) -> None:
+        """
+        GIVEN:
+            - tantivy.SchemaBuilder replaced with a stand-in that raises if
+              constructed
+        WHEN:
+            - build_schema() is called (and raises), then
+              schema_fingerprint() is called again
+        THEN:
+            - schema_fingerprint() still matches its earlier value,
+              proving it never consults SchemaBuilder
+        """
+        before = schema_fingerprint()
+
+        class _RemovedSchemaBuilder:
+            def __init__(self) -> None:
+                raise AssertionError("tantivy.SchemaBuilder was consulted")
+
+        monkeypatch.setattr(tantivy, "SchemaBuilder", _RemovedSchemaBuilder)
+        with pytest.raises(AssertionError):
+            build_schema()
+
+        assert schema_fingerprint() == before
+
+
+def _hash(payload: object) -> str:
+    return hashlib.blake2b(json.dumps(payload).encode()).hexdigest()
+
+
+def _fingerprint_of(descriptors: list[FieldDescriptor]) -> str:
+    return _hash([list(descriptor) for descriptor in descriptors])
+
+
+class TestNeedsRebuildOnFingerprint:
+    def test_matching_fingerprint_does_not_rebuild(
+        self,
+        index_dir: Path,
+        settings: SettingsWrapper,
+    ) -> None:
+        """
+        GIVEN:
+            - An index directory whose sentinel file records the current
+              schema_fingerprint()
+        WHEN:
+            - needs_rebuild() is called
+        THEN:
+            - It returns False
+        """
+        settings.SEARCH_LANGUAGE = None
+        _sentinels(index_dir)
+
+        assert needs_rebuild(index_dir) is False
+
+    def test_stale_fingerprint_rebuilds_despite_a_matching_version(
+        self,
+        index_dir: Path,
+        settings: SettingsWrapper,
+        monkeypatch: pytest.MonkeyPatch,
+    ) -> None:
+        """
+        GIVEN:
+            - An index directory whose sentinel matches SCHEMA_VERSION,
+              but field_descriptors() is patched to add a field the
+              fingerprint never saw (schema edited, version not bumped)
+        WHEN:
+            - needs_rebuild() is called
+        THEN:
+            - It returns True; without the fingerprint check,
+              `reindex --if-needed` would report the index up to date and
+              every subsequent write would raise
+        """
+        settings.SEARCH_LANGUAGE = None
+        _sentinels(index_dir)
+        extended = [
+            *field_descriptors(),
+            FieldDescriptor(
+                "new_field",
+                "u64",
+                stored=False,
+                indexed=True,
+                fast=True,
+                tokenizer=None,
+            ),
+        ]
+        monkeypatch.setattr(_schema, "field_descriptors", lambda: extended)
+
+        assert needs_rebuild(index_dir) is True
+
+    def test_reordered_schema_rebuilds(
+        self,
+        index_dir: Path,
+        settings: SettingsWrapper,
+        monkeypatch: pytest.MonkeyPatch,
+    ) -> None:
+        """
+        GIVEN:
+            - An index directory whose sentinel matches the current
+              fingerprint, but field_descriptors() is patched to swap two
+              fields' order
+        WHEN:
+            - needs_rebuild() is called
+        THEN:
+            - It returns True
+        """
+        settings.SEARCH_LANGUAGE = None
+        _sentinels(index_dir)
+        reordered = field_descriptors()
+        reordered[1], reordered[2] = reordered[2], reordered[1]
+        monkeypatch.setattr(_schema, "field_descriptors", lambda: reordered)
+
+        assert needs_rebuild(index_dir) is True
+
+    def test_missing_fingerprint_rebuilds(
+        self,
+        index_dir: Path,
+        settings: SettingsWrapper,
+    ) -> None:
+        """
+        GIVEN:
+            - An index directory whose sentinel has no "schema_fingerprint"
+              key at all
+        WHEN:
+            - needs_rebuild() is called
+        THEN:
+            - It returns True; an index whose schema shape nobody recorded
+              is rebuilt rather than trusted
+        """
+        settings.SEARCH_LANGUAGE = None
+        (index_dir / ".index_settings.json").write_text(
+            json.dumps({"schema_version": SCHEMA_VERSION, "language": None}),
+        )
+
+        assert needs_rebuild(index_dir) is True
+
+    def test_written_sentinels_satisfy_the_check(
+        self,
+        index_dir: Path,
+        settings: SettingsWrapper,
+    ) -> None:
+        """
+        GIVEN:
+            - An index directory whose sentinels are written by
+              _write_sentinels() itself
+        WHEN:
+            - needs_rebuild() is called
+        THEN:
+            - It returns False
+        """
+        settings.SEARCH_LANGUAGE = "en"
+        _write_sentinels(index_dir)
+
+        assert needs_rebuild(index_dir) is False

src/documents/tests/search/test_schema_version.py

@@ -0,0 +1,178 @@
+"""SCHEMA_VERSION must change whenever build_schema()'s field list or order does.
+
+tantivy compares schemas by *ordered* field list. ``Index.open()`` loads the
+schema from the index's own ``meta.json``, so reads against an index built by an
+older release keep working after a field reorder. Writes do not:
+``WriteBatch.__enter__`` calls ``tantivy.Index(build_schema(), path=...)``, an
+open-or-create that raises ``ValueError`` on any schema difference. Nothing
+catches that ValueError, so consumption, index_document and bulk edit all
+hard-fail while ``/api/status/`` still reports the index healthy.
+
+The only thing that saves such an install is ``needs_rebuild()`` noticing the
+version stamped in ``.index_settings.json`` is stale.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING
+
+import pytest
+import tantivy
+from django.conf import settings as django_settings
+
+from documents.search._schema import build_schema
+from documents.search._schema import needs_rebuild
+from documents.search._schema import open_or_rebuild_index
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+pytestmark = [pytest.mark.search]
+
+RELEASED_V1_SCHEMA_VERSION = 1
+
+
+def _build_released_v1_schema() -> tantivy.Schema:
+    """Frozen copy of build_schema() as shipped in v3.0.x (schema version 1).
+
+    Deliberately duplicated rather than imported: it must keep describing the
+    on-disk layout of already-deployed indexes even as build_schema() evolves.
+    """
+    sb = tantivy.SchemaBuilder()
+
+    sb.add_unsigned_field("id", stored=True, indexed=True, fast=True)
+    sb.add_text_field("checksum", stored=True, tokenizer_name="raw")
+
+    for field in (
+        "title",
+        "correspondent",
+        "document_type",
+        "storage_path",
+        "original_filename",
+        "content",
+    ):
+        sb.add_text_field(field, stored=True, tokenizer_name="paperless_text")
+
+    for field in ("title_sort", "correspondent_sort", "type_sort"):
+        sb.add_text_field(
+            field,
+            stored=False,
+            tokenizer_name="simple_analyzer",
+            fast=True,
+        )
+
+    for field in (
+        "bigram_content",
+        "bigram_title",
+        "bigram_correspondent",
+        "bigram_document_type",
+        "bigram_tag",
+    ):
+        sb.add_text_field(field, stored=False, tokenizer_name="bigram_analyzer")
+
+    for field in ("simple_title", "simple_content"):
+        sb.add_text_field(field, stored=False, tokenizer_name="simple_search_analyzer")
+
+    sb.add_text_field("autocomplete_word", stored=False, tokenizer_name="raw")
+    sb.add_text_field("tag", stored=True, tokenizer_name="paperless_text")
+
+    sb.add_json_field("notes", stored=True, tokenizer_name="paperless_text")
+    sb.add_text_field("notes_text", stored=True, tokenizer_name="paperless_text")
+    sb.add_json_field("custom_fields", stored=True, tokenizer_name="paperless_text")
+
+    for field in (
+        "correspondent_id",
+        "document_type_id",
+        "storage_path_id",
+        "tag_id",
+        "owner_id",
+        "viewer_id",
+        "viewer_group_id",
+    ):
+        sb.add_unsigned_field(field, stored=False, indexed=True, fast=True)
+
+    for field in ("created", "modified", "added"):
+        sb.add_date_field(field, stored=True, indexed=True, fast=True)
+
+    for field in ("asn", "page_count", "num_notes"):
+        sb.add_unsigned_field(field, stored=True, indexed=True, fast=True)
+
+    return sb.build()
+
+
+@pytest.fixture
+def released_v1_index(tmp_path: Path) -> Path:
+    """An index directory as a v3.0.x install would leave it on disk."""
+    index_dir = tmp_path / "index"
+    index_dir.mkdir()
+    tantivy.Index(_build_released_v1_schema(), path=str(index_dir))
+    (index_dir / ".index_settings.json").write_text(
+        json.dumps(
+            {
+                "schema_version": RELEASED_V1_SCHEMA_VERSION,
+                "language": django_settings.SEARCH_LANGUAGE,
+            },
+        ),
+    )
+    return index_dir
+
+
+class TestUpgradeFromReleasedV1Index:
+    def test_released_v1_index_is_flagged_for_rebuild(
+        self,
+        released_v1_index: Path,
+    ) -> None:
+        """
+        GIVEN:
+            - An index directory laid out exactly as a v3.0.x (schema
+              version 1) install would leave it
+        WHEN:
+            - needs_rebuild() is called
+        THEN:
+            - It returns True; if this fails,
+              `document_index reindex --if-needed` prints "Search index is
+              up to date" and skips, leaving the mismatched index in place
+        """
+        assert needs_rebuild(released_v1_index) is True
+
+    def test_opening_a_v1_index_leaves_it_writable(
+        self,
+        released_v1_index: Path,
+    ) -> None:
+        """
+        GIVEN:
+            - A v1 index directory
+        WHEN:
+            - open_or_rebuild_index() is called against it
+        THEN:
+            - The directory can be reopened with the current schema
+              without raising; end to end, open_or_rebuild_index must
+              hand back an index the write path can reopen. Before the
+              version bump, needs_rebuild() returned False here, and the
+              stale directory survived untouched, so every subsequent
+              write against it raised tantivy's own schema-mismatch
+              ValueError
+        """
+        open_or_rebuild_index(released_v1_index)
+
+        tantivy.Index(build_schema(), path=str(released_v1_index))
+
+    def test_rebuilt_index_is_not_rebuilt_again(
+        self,
+        released_v1_index: Path,
+    ) -> None:
+        """
+        GIVEN:
+            - A v1 index directory that has just been rebuilt by
+              open_or_rebuild_index()
+        WHEN:
+            - needs_rebuild() is called again
+        THEN:
+            - It returns False; the rebuild must stamp the version it
+              actually wrote, otherwise every startup wipes and reindexes
+              the whole corpus
+        """
+        open_or_rebuild_index(released_v1_index)
+
+        assert needs_rebuild(released_v1_index) is False

src/documents/tests/search/test_stem_pattern_text.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+import pytest
+
+from documents.search._tokenizer import stem_pattern_text
+
+pytestmark = pytest.mark.search
+
+
+class TestStemPatternText:
+    def test_unsupported_language_returns_text_unchanged(self) -> None:
+        """
+        GIVEN:
+            - A language code with no Snowball stemmer mapping
+        WHEN:
+            - A pattern run is stemmed for that language
+        THEN:
+            - The run is returned unchanged, since the stemming gate that
+              disables stemming for an unsupported language also disables
+              the pattern-side stemmer
+        """
+        assert stem_pattern_text("running", "klingon") == "running"
+
+    def test_run_past_remove_long_limit_returns_text_unchanged(self) -> None:
+        """
+        GIVEN:
+            - A supported language and a run longer than the remove_long
+              filter's limit (129 characters, matching Document.title's
+              max_length)
+        WHEN:
+            - The over-long run is stemmed
+        THEN:
+            - The remove_long filter drops the token entirely, leaving no
+              stem to substitute, so the run is returned unchanged
+        """
+        long_run = "a" * 130
+        assert stem_pattern_text(long_run, "en") == long_run

src/documents/tests/search/test_tokenizer.py

@@ -7,8 +7,8 @@ import pytest
 import tantivy
 
 from documents.search._tokenizer import _bigram_analyzer
-from documents.search._tokenizer import _paperless_text
 from documents.search._tokenizer import _simple_search_analyzer
+from documents.search._tokenizer import paperless_text_analyzer
 from documents.search._tokenizer import register_tokenizers
 
 if TYPE_CHECKING:
@@ -25,7 +25,7 @@ class TestTokenizers:
         sb.add_text_field("content", stored=True, tokenizer_name="paperless_text")
         schema = sb.build()
         idx = tantivy.Index(schema, path=None)
-        idx.register_tokenizer("paperless_text", _paperless_text(""))
+        idx.register_tokenizer("paperless_text", paperless_text_analyzer(""))
         return idx
 
     @pytest.fixture

src/documents/tests/test_regex.py

@@ -1,6 +1,5 @@
 import pytest
 import regex
-from django.conf import settings
 from pytest_mock import MockerFixture
 
 from documents.regex import safe_regex_finditer
@@ -10,12 +9,6 @@ from documents.regex import safe_regex_sub
 from documents.regex import validate_regex_pattern
 
 
-def test_regex_timeout_uses_configured_setting() -> None:
-    from documents.regex import REGEX_TIMEOUT_SECONDS
-
-    assert REGEX_TIMEOUT_SECONDS == settings.MATCH_REGEX_TIMEOUT_SECONDS
-
-
 class TestValidateRegexPattern:
     def test_valid_pattern(self) -> None:
         validate_regex_pattern(r"\d+")

src/paperless/settings/__init__.py

@@ -102,10 +102,6 @@ CLASSIFIER_MATCH_THRESHOLD: Final[float] = get_float_from_env(
     "PAPERLESS_CLASSIFIER_MATCH_THRESHOLD",
     0.6,
 )
-MATCH_REGEX_TIMEOUT_SECONDS: Final[float] = get_float_from_env(
-    "PAPERLESS_MATCH_REGEX_TIMEOUT_SECONDS",
-    0.1,
-)
 LLM_INDEX_DIR = DATA_DIR / "llm_index"
 LLM_INDEX_LOCK = LLM_INDEX_DIR / "index.lock"
 # Cross-process read/write lock guarding the LLM index compaction/migration

uv.lock

@@ -2932,6 +2932,7 @@ dependencies = [
     { name = "torch", version = "2.13.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'linux'" },
     { name = "watchfiles" },
     { name = "whitenoise" },
+    { name = "whoosh-compat", extra = ["tantivy"] },
     { name = "zxing-cpp" },
 ]
 
@@ -3090,6 +3091,7 @@ requires-dist = [
     { name = "torch", specifier = "~=2.13.0", index = "https://download.pytorch.org/whl/cpu" },
     { name = "watchfiles", specifier = ">=1.2" },
     { name = "whitenoise", specifier = "~=6.11" },
+    { name = "whoosh-compat", extras = ["tantivy"], specifier = "==0.1.0" },
     { name = "zxing-cpp", specifier = "~=3.1.0" },
 ]
 provides-extras = ["mariadb", "postgres", "webserver"]
@@ -5638,6 +5640,23 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/db/eb/d5583a11486211f3ebd4b385545ae787f32363d453c19fffd81106c9c138/whitenoise-6.12.0-py3-none-any.whl", hash = "sha256:fc5e8c572e33ebf24795b47b6a7da8da3c00cff2349f5b04c02f28d0cc5a3cc2", size = 20302, upload-time = "2026-02-27T00:05:40.086Z" },
 ]
 
+[[package]]
+name = "whoosh-compat"
+version = "0.1.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "python-dateutil" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/5d/f7/3e45f4a484afa174cd42e424ce2b8c514ae54f564830122656ad17b765e2/whoosh_compat-0.1.0.tar.gz", hash = "sha256:86935bdc159ed9b0a06a4661d17f1251d8280340b84e218cc73d915a2edaddf7", size = 577543, upload-time = "2026-08-25T15:22:42.316Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/3f/51/9a8399d0f472814e136a2884bf910c6c59843a5a2c66afc4b5133eb531ea/whoosh_compat-0.1.0-py3-none-any.whl", hash = "sha256:3e7c5f519b4d397dbf4f8d7bbe4ca1eb6004da24c8892bc701844ed393bc97e7", size = 153875, upload-time = "2026-08-25T15:22:40.787Z" },
+]
+
+[package.optional-dependencies]
+tantivy = [
+    { name = "tantivy" },
+]
+
 [[package]]
 name = "wrapt"
 version = "2.0.1"