Merge branch 'dev' into feature/ocr-templates

src/conftest.py

@@ -0,0 +1,151 @@
+"""Fixtures available to every Paperless-ngx app.
+
+Loaded automatically for every test path. Keep module-scope imports minimal:
+this file is imported for every session,  so anything heavy belongs inside
+the fixture body that needs it.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+if TYPE_CHECKING:
+    from collections.abc import Generator
+    from pathlib import Path
+
+    from django.contrib.auth.models import User
+    from pytest_django.fixtures import Settings
+    from rest_framework.test import APIClient
+
+    from paperless_testing.dirs import PaperlessDirs
+    from paperless_testing.fakes.progress import FakeProgressManager
+
+
+@pytest.fixture(scope="session", autouse=True)
+def faker_session_locale() -> str:
+    """Pin Faker's locale so generated data does not follow the host locale.
+
+    The seed itself is left to pytest-randomly, which derives one per run.
+    """
+    return "en_US"
+
+
+@pytest.fixture(autouse=True)
+def _fast_password_hasher(settings: Settings) -> None:
+    """Hash test passwords with MD5 instead of Django's default PBKDF2.
+
+    PBKDF2 is deliberately slow, and every ``admin_user`` or
+    ``create_superuser`` call pays for it: about 600 ms each. No test depends
+    on the hash format, only on ``check_password`` and on the stored value
+    changing when the password does.
+    """
+    settings.PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"]
+
+
+@pytest.fixture(autouse=True)
+def _clear_content_type_caches() -> None:
+    """Clear Django's ContentType cache and guardian's lru_cache before each test.
+
+    Tests that delete and reinsert ContentType/Permission rows (e.g. the
+    importer) corrupt both caches. Without this fixture a subsequent test on
+    the same xdist worker sees stale ContentType objects and guardian raises
+    MixedContentTypeError.
+    """
+    from django.contrib.contenttypes.models import ContentType
+    from guardian.shortcuts import clear_ct_cache
+
+    ContentType.objects.clear_cache()
+    clear_ct_cache()
+
+
+@pytest.fixture(autouse=True)
+def _clear_django_caches() -> None:
+    """Clear every configured cache before each test.
+
+    Cached values outlive the test that wrote them: the classifier keys its
+    vectorized content on a hash of the content itself, so a second test
+    generating the same fixture data takes the cache-hit path and never calls
+    the code it is asserting against.
+    """
+    from django.core.cache import caches
+
+    for cache in caches.all(initialized_only=False):
+        cache.clear()
+
+
+@pytest.fixture
+def paperless_dirs(
+    tmp_path: Path,
+    settings: Settings,
+) -> Generator[PaperlessDirs, None, None]:
+    """The standard temp directory layout, applied to Django settings."""
+    from documents.search import reset_backend
+    from paperless_testing.dirs import build_paperless_dirs
+    from paperless_testing.dirs import dirs_settings
+
+    dirs = build_paperless_dirs(tmp_path)
+    for name, value in dirs_settings(dirs).items():
+        setattr(settings, name, value)
+
+    # Not directory settings, but they are needed alongside the layout by the
+    # sanity checker tests.
+    settings.IGNORABLE_FILES = {".DS_Store", "Thumbs.db", "desktop.ini"}
+    settings.APP_LOGO = ""
+
+    reset_backend()
+    yield dirs
+    reset_backend()
+
+
+@pytest.fixture
+def rest_api_client() -> APIClient:
+    """The basic DRF APIClient, unauthenticated."""
+    from rest_framework.test import APIClient
+
+    return APIClient()
+
+
+@pytest.fixture
+def regular_user(db: None) -> User:
+    """Unprivileged user for permission boundary tests."""
+    from paperless_testing.factories import UserFactory
+
+    return UserFactory(username="regular")
+
+
+@pytest.fixture
+def admin_client(rest_api_client: APIClient, admin_user: User) -> APIClient:
+    """Admin client pre-authenticated and sending the v10 Accept header."""
+    rest_api_client.force_authenticate(user=admin_user)
+    rest_api_client.credentials(HTTP_ACCEPT="application/json; version=10")
+    return rest_api_client
+
+
+@pytest.fixture
+def v9_client(rest_api_client: APIClient, admin_user: User) -> APIClient:
+    """Admin client pre-authenticated and sending the v9 Accept header."""
+    rest_api_client.force_authenticate(user=admin_user)
+    rest_api_client.credentials(HTTP_ACCEPT="application/json; version=9")
+    return rest_api_client
+
+
+@pytest.fixture
+def user_client(rest_api_client: APIClient, regular_user: User) -> APIClient:
+    """Regular-user client pre-authenticated and sending the v10 Accept header."""
+    rest_api_client.force_authenticate(user=regular_user)
+    rest_api_client.credentials(HTTP_ACCEPT="application/json; version=10")
+    return rest_api_client
+
+
+@pytest.fixture
+def fake_progress_manager(
+    monkeypatch: pytest.MonkeyPatch,
+) -> type[FakeProgressManager]:
+    """Replace documents.tasks.ProgressManager with the fake, so consuming a file
+    in a test never tries to reach a broker."""
+    from paperless_testing.fakes.progress import FakeProgressManager
+
+    monkeypatch.setattr("documents.tasks.ProgressManager", FakeProgressManager)
+    return FakeProgressManager

src/documents/_snowball_stopwords.py

@@ -0,0 +1,213 @@
+"""
+English stop words from the Snowball project, kept as published at
+https://snowballstem.org/algorithms/english/stop.txt, under this license:
+
+Copyright (c) 2001, Dr Martin Porter
+Copyright (c) 2004,2005, Richard Boulton
+Copyright (c) 2013, Yoshiki Shibukawa
+Copyright (c) 2006-2025, Olly Betts
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+  1. Redistributions of source code must retain the above copyright notice,
+     this list of conditions and the following disclaimer.
+  2. Redistributions in binary form must reproduce the above copyright notice,
+     this list of conditions and the following disclaimer in the documentation
+     and/or other materials provided with the distribution.
+  3. Neither the name of the Snowball project nor the names of its contributors
+     may be used to endorse or promote products derived from this software
+     without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""
+
+from typing import Final
+
+ENGLISH: Final[tuple[str, ...]] = (
+    "i",
+    "me",
+    "my",
+    "myself",
+    "we",
+    "our",
+    "ours",
+    "ourselves",
+    "you",
+    "your",
+    "yours",
+    "yourself",
+    "yourselves",
+    "he",
+    "him",
+    "his",
+    "himself",
+    "she",
+    "her",
+    "hers",
+    "herself",
+    "it",
+    "its",
+    "itself",
+    "they",
+    "them",
+    "their",
+    "theirs",
+    "themselves",
+    "what",
+    "which",
+    "who",
+    "whom",
+    "this",
+    "that",
+    "these",
+    "those",
+    "am",
+    "is",
+    "are",
+    "was",
+    "were",
+    "be",
+    "been",
+    "being",
+    "have",
+    "has",
+    "had",
+    "having",
+    "do",
+    "does",
+    "did",
+    "doing",
+    "would",
+    "should",
+    "could",
+    "ought",
+    "i'm",
+    "you're",
+    "he's",
+    "she's",
+    "it's",
+    "we're",
+    "they're",
+    "i've",
+    "you've",
+    "we've",
+    "they've",
+    "i'd",
+    "you'd",
+    "he'd",
+    "she'd",
+    "we'd",
+    "they'd",
+    "i'll",
+    "you'll",
+    "he'll",
+    "she'll",
+    "we'll",
+    "they'll",
+    "isn't",
+    "aren't",
+    "wasn't",
+    "weren't",
+    "hasn't",
+    "haven't",
+    "hadn't",
+    "doesn't",
+    "don't",
+    "didn't",
+    "won't",
+    "wouldn't",
+    "shan't",
+    "shouldn't",
+    "can't",
+    "cannot",
+    "couldn't",
+    "mustn't",
+    "let's",
+    "that's",
+    "who's",
+    "what's",
+    "here's",
+    "there's",
+    "when's",
+    "where's",
+    "why's",
+    "how's",
+    "a",
+    "an",
+    "the",
+    "and",
+    "but",
+    "if",
+    "or",
+    "because",
+    "as",
+    "until",
+    "while",
+    "of",
+    "at",
+    "by",
+    "for",
+    "with",
+    "about",
+    "against",
+    "between",
+    "into",
+    "through",
+    "during",
+    "before",
+    "after",
+    "above",
+    "below",
+    "to",
+    "from",
+    "up",
+    "down",
+    "in",
+    "out",
+    "on",
+    "off",
+    "over",
+    "under",
+    "again",
+    "further",
+    "then",
+    "once",
+    "here",
+    "there",
+    "when",
+    "where",
+    "why",
+    "how",
+    "all",
+    "any",
+    "both",
+    "each",
+    "few",
+    "more",
+    "most",
+    "other",
+    "some",
+    "such",
+    "no",
+    "nor",
+    "not",
+    "only",
+    "own",
+    "same",
+    "so",
+    "than",
+    "too",
+    "very",
+)

src/documents/apps.py

@@ -29,8 +29,8 @@ class DocumentsConfig(AppConfig):
         document_consumption_finished.connect(set_document_type)
         document_consumption_finished.connect(set_tags)
         document_consumption_finished.connect(set_storage_path)
-        document_consumption_finished.connect(add_to_index)
         document_consumption_finished.connect(run_workflows_added)
+        document_consumption_finished.connect(add_to_index)
         document_consumption_finished.connect(add_or_update_document_in_llm_index)
         document_consumption_finished.connect(run_zone_ocr_extraction)
 

src/documents/barcodes.py

@@ -18,6 +18,7 @@ from documents.converters import convert_from_tiff_to_pdf
 from documents.data_models import ConsumableDocument
 from documents.data_models import DocumentMetadataOverrides
 from documents.data_models import DocumentSource
+from documents.data_models import StoredBarcode
 from documents.models import Document
 from documents.models import PaperlessTask
 from documents.models import Tag
@@ -47,6 +48,7 @@ class Barcode:
     page: int
     value: str
     settings: BarcodeConfig
+    format: str = ""
 
     @property
     def is_separator(self) -> bool:
@@ -78,6 +80,12 @@ class Barcode:
                 return True
         return False
 
+    def stored(self) -> StoredBarcode:
+        """
+        The barcode as it is stored with a document, page 1-indexed
+        """
+        return {"page": self.page + 1, "value": self.value, "format": self.format}
+
 
 class BarcodePlugin(ConsumeTaskPlugin):
     NAME: str = "BarcodePlugin"
@@ -89,16 +97,12 @@ class BarcodePlugin(ConsumeTaskPlugin):
           - ASN from barcode detection is enabled or
           - Barcode support is enabled and the mime type is supported
         """
-        if self.settings.barcode_enable_tiff_support:
-            supported_mimes: set[str] = {"application/pdf", "image/tiff"}
-        else:
-            supported_mimes = {"application/pdf"}
-
         return (
             self.settings.barcode_enable_asn
             or self.settings.barcodes_enabled
             or self.settings.barcode_enable_tag
-        ) and self.input_doc.mime_type in supported_mimes
+            or self.settings.barcode_store_values
+        ) and self.input_doc.mime_type in scannable_mime_types(self.settings)
 
     def get_settings(self) -> BarcodeConfig:
         """
@@ -161,17 +165,19 @@ class BarcodePlugin(ConsumeTaskPlugin):
         # Locate any barcodes in the files
         self.detect()
 
-        # try reading tags from barcodes
-        # If tag splitting is enabled, skip this on the original document - let each split document extract its own tags
-        # However, if we're processing a split document (original_path is set), extract tags
+        separator_pages = (
+            self.get_separation_pages() if self.settings.barcodes_enabled else {}
+        )
+
+        # Defer tags to split documents only when a split will actually happen.
         if (
             self.settings.barcode_enable_tag
             and (
                 not self.settings.barcode_tag_split
                 or self.input_doc.original_path is not None
+                or not separator_pages
             )
-            and (tags := self.tags) is not None
-            and len(tags) > 0
+            and (tags := self.tags)
         ):
             if self.metadata.tag_ids:
                 self.metadata.tag_ids += tags
@@ -180,9 +186,7 @@ class BarcodePlugin(ConsumeTaskPlugin):
             logger.info(f"Found tags in barcode: {tags}")
 
         # Lastly attempt to split documents
-        if self.settings.barcodes_enabled and (
-            separator_pages := self.get_separation_pages()
-        ):
+        if separator_pages:
             # We have pages to split against
 
             # Note this does NOT use the base_temp_dir, as that will be removed
@@ -244,6 +248,10 @@ class BarcodePlugin(ConsumeTaskPlugin):
         if self.settings.barcode_enable_asn and (located_asn := self.asn) is not None:
             self._apply_detected_asn(located_asn)
 
+        # After splitting too, so each split document keeps its own barcodes
+        if self.settings.barcode_store_values:
+            self.metadata.barcodes = [x.stored() for x in self.barcodes] or None
+
     def cleanup(self) -> None:
         self.temp_dir.cleanup()
 
@@ -262,22 +270,6 @@ class BarcodePlugin(ConsumeTaskPlugin):
         )
         self._tiff_conversion_done = True
 
-    @staticmethod
-    def read_barcodes_zxing(image: Image.Image) -> list[str]:
-        barcodes = []
-
-        import zxingcpp
-
-        detected_barcodes = zxingcpp.read_barcodes(image)
-        for barcode in detected_barcodes:
-            if barcode.text:
-                barcodes.append(barcode.text)
-                logger.debug(
-                    f"Barcode of type {barcode.format} found: {barcode.text}",
-                )
-
-        return barcodes
-
     def detect(self) -> None:
         """
         Scan all pages of the PDF as images, updating barcodes and the pages
@@ -291,60 +283,12 @@ class BarcodePlugin(ConsumeTaskPlugin):
         self.convert_from_tiff_to_pdf()
 
         try:
-            # Read number of pages from pdf
-            with Pdf.open(self.pdf_file) as pdf:
-                num_of_pages = len(pdf.pages)
-            logger.debug(f"PDF has {num_of_pages} pages")
-
-            # Get limit from configuration
-            barcode_max_pages: int = (
-                num_of_pages
-                if self.settings.barcode_max_pages == 0
-                else self.settings.barcode_max_pages
+            self.barcodes = scan_pdf(
+                self.pdf_file,
+                self.settings,
+                Path(self.temp_dir.name),
             )
 
-            if barcode_max_pages < num_of_pages:  # pragma: no cover
-                logger.debug(
-                    f"Barcodes detection will be limited to the first {barcode_max_pages} pages",
-                )
-
-            # Loop al page
-            for current_page_number in range(min(num_of_pages, barcode_max_pages)):
-                logger.debug(f"Processing page {current_page_number}")
-
-                # Convert page to image
-                page = convert_from_path(
-                    self.pdf_file,
-                    dpi=self.settings.barcode_dpi,
-                    output_folder=self.temp_dir.name,
-                    first_page=current_page_number + 1,
-                    last_page=current_page_number + 1,
-                )[0]
-
-                # Remember filename, since it is lost by upscaling
-                page_filepath = Path(page.filename)
-                logger.debug(f"Image is at {page_filepath}")
-
-                # Upscale image if configured
-                factor = self.settings.barcode_upscale
-                if factor > 1.0:
-                    logger.debug(
-                        f"Upscaling image by {factor} for better barcode detection",
-                    )
-                    x, y = page.size
-                    page = page.resize(
-                        (round(x * factor), (round(y * factor))),
-                    )
-
-                # Detect barcodes
-                for barcode_value in self.read_barcodes_zxing(page):
-                    self.barcodes.append(
-                        Barcode(current_page_number, barcode_value, self.settings),
-                    )
-
-                # Delete temporary image file
-                page_filepath.unlink()
-
         # Password protected files can't be checked
         # This is the exception raised for those
         except PasswordError as e:
@@ -534,3 +478,111 @@ class BarcodePlugin(ConsumeTaskPlugin):
                 document_paths.append(savepath)
 
             return document_paths
+
+
+def scannable_mime_types(settings: BarcodeConfig) -> set[str]:
+    """
+    The file types the barcode scan supports with the current settings
+    """
+    if settings.barcode_enable_tiff_support:
+        return {"application/pdf", "image/tiff"}
+    return {"application/pdf"}
+
+
+def read_barcodes_zxing(image: Image.Image) -> list[tuple[str, str]]:
+    """
+    Returns the text and format (zxing enum name) of each barcode found in
+    the image
+    """
+    barcodes = []
+
+    import zxingcpp
+
+    detected_barcodes = zxingcpp.read_barcodes(image)
+    for barcode in detected_barcodes:
+        if barcode.text:
+            barcodes.append((barcode.text, barcode.format.name))
+            logger.debug(
+                f"Barcode of type {barcode.format} found: {barcode.text}",
+            )
+
+    return barcodes
+
+
+def scan_pdf(pdf_path: Path, settings: BarcodeConfig, work_dir: Path) -> list[Barcode]:
+    """
+    Scans the pages of a PDF as images for barcodes. Errors are not caught,
+    so callers can tell a failed scan from one that found nothing.
+    """
+    barcodes: list[Barcode] = []
+
+    with Pdf.open(pdf_path) as pdf:
+        num_of_pages = len(pdf.pages)
+    logger.debug(f"PDF has {num_of_pages} pages")
+
+    # Get limit from configuration
+    barcode_max_pages: int = (
+        num_of_pages if settings.barcode_max_pages == 0 else settings.barcode_max_pages
+    )
+
+    if barcode_max_pages < num_of_pages:  # pragma: no cover
+        logger.debug(
+            f"Barcodes detection will be limited to the first {barcode_max_pages} pages",
+        )
+
+    for current_page_number in range(min(num_of_pages, barcode_max_pages)):
+        logger.debug(f"Processing page {current_page_number}")
+
+        # Convert page to image
+        page = convert_from_path(
+            pdf_path,
+            dpi=settings.barcode_dpi,
+            output_folder=work_dir,
+            first_page=current_page_number + 1,
+            last_page=current_page_number + 1,
+        )[0]
+
+        # Remember filename, since it is lost by upscaling
+        page_filepath = Path(page.filename)
+        logger.debug(f"Image is at {page_filepath}")
+
+        # Upscale image if configured
+        factor = settings.barcode_upscale
+        if factor > 1.0:
+            logger.debug(
+                f"Upscaling image by {factor} for better barcode detection",
+            )
+            x, y = page.size
+            page = page.resize(
+                (round(x * factor), (round(y * factor))),
+            )
+
+        for barcode_value, barcode_format in read_barcodes_zxing(page):
+            barcodes.append(
+                Barcode(current_page_number, barcode_value, settings, barcode_format),
+            )
+
+        # Delete temporary image file
+        page_filepath.unlink()
+
+    return barcodes
+
+
+def read_barcode_values(
+    path: Path,
+    mime_type: str,
+    settings: BarcodeConfig,
+    work_dir: Path,
+) -> list[StoredBarcode] | None:
+    """
+    Reads the barcodes of a file outside of the consumption plugins: for new
+    versions, which skip the barcode plugin, and when reprocessing.
+
+    Returns None if the file can't be scanned with the current settings.
+    Errors while scanning are raised.
+    """
+    if mime_type not in scannable_mime_types(settings):
+        return None
+    if mime_type == "image/tiff":
+        path = convert_from_tiff_to_pdf(path, work_dir)
+    return [x.stored() for x in scan_pdf(path, settings, work_dir)]

src/documents/bulk_edit.py

@@ -13,8 +13,14 @@ from celery import group
 from celery import shared_task
 from django.conf import settings
 from django.db import transaction
+from django.db.models import Case
+from django.db.models import F
 from django.db.models import Max
+from django.db.models import OuterRef
 from django.db.models import Q
+from django.db.models import Subquery
+from django.db.models import When
+from django.db.models.functions import Coalesce
 from django.utils import timezone
 
 from documents.data_models import ConsumableDocument
@@ -28,7 +34,7 @@ from documents.models import DocumentType
 from documents.models import PaperlessTask
 from documents.models import StoragePath
 from documents.models import Tag
-from documents.permissions import set_permissions_for_object
+from documents.permissions import set_permissions_for_objects
 from documents.plugins.helpers import DocumentsStatusManager
 from documents.tasks import bulk_update_documents
 from documents.tasks import consume_file
@@ -36,6 +42,7 @@ from documents.tasks import remove_document_from_index
 from documents.tasks import update_document_content_maybe_archive_file
 from documents.versioning import get_latest_version_for_root
 from documents.versioning import get_root_document
+from documents.versioning import versions_newest_first
 
 if TYPE_CHECKING:
     from collections.abc import Mapping
@@ -299,53 +306,55 @@ def modify_custom_fields(
 ) -> Literal["OK"]:
     qs = Document.objects.filter(id__in=doc_ids).only("pk")
     affected_docs = list(qs.values_list("pk", flat=True))
-    # Ensure add_custom_fields is a list of tuples, supports old API
+    # Ensure add_custom_fields is a list of (int, value) tuples, supports old API
     add_custom_fields = (
-        add_custom_fields.items()
+        [(int(field), value) for field, value in add_custom_fields.items()]
         if isinstance(add_custom_fields, dict)
-        else [(field, None) for field in add_custom_fields]
+        else [(int(field), None) for field in add_custom_fields]
     )
 
-    custom_fields = CustomField.objects.filter(
-        id__in=[int(field) for field, _ in add_custom_fields],
-    ).distinct()
+    # Resolved once, instead of re-querying the same field for every document
+    custom_fields_by_id: dict[int, CustomField] = CustomField.objects.in_bulk(
+        [field_id for field_id, _ in add_custom_fields],
+    )
+    # Passed to update_or_create() below rather than a bare id, so the FK is
+    # cached on the created instance and auditlog's post_save receiver does
+    # not reload it per row. Only needed for additions. content is deferred:
+    # the one field here that is both large and unused.
+    docs_by_id: dict[int, Document] = (
+        Document.objects.defer("content").in_bulk(affected_docs)
+        if add_custom_fields
+        else {}
+    )
     for field_id, value in add_custom_fields:
+        custom_field = custom_fields_by_id[field_id]
+        value_field = CustomFieldInstance.TYPE_TO_DATA_STORE_NAME_MAP[
+            custom_field.data_type
+        ]
+        is_doclink = custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK
         for doc_id in affected_docs:
-            defaults = {}
-            custom_field = custom_fields.get(id=field_id)
-            if custom_field:
-                value_field = CustomFieldInstance.TYPE_TO_DATA_STORE_NAME_MAP[
-                    custom_field.data_type
-                ]
-                defaults[value_field] = value
-                if (
-                    custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK
-                    and value
-                    and doc_id in value
-                ):
-                    # Prevent self-linking
-                    continue
+            if is_doclink and value and doc_id in value:
+                # Prevent self-linking
+                continue
             CustomFieldInstance.objects.update_or_create(
-                document_id=doc_id,
-                field_id=field_id,
-                defaults=defaults,
+                document=docs_by_id[doc_id],
+                field=custom_field,
+                defaults={value_field: value},
             )
-            if custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK:
-                doc = Document.objects.get(id=doc_id)
-                reflect_doclinks(doc, custom_field, value)
+            if is_doclink:
+                reflect_doclinks(docs_by_id[doc_id], custom_field, value)
 
-    # For doc link fields that are being removed, remove symmetrical links
+    # For doc link fields that are being removed, remove symmetrical links.
+    # select_related avoids a per-instance reload of the document and field.
     for doclink_being_removed_instance in CustomFieldInstance.objects.filter(
         document_id__in=affected_docs,
         field__id__in=remove_custom_fields,
         field__data_type=CustomField.FieldDataType.DOCUMENTLINK,
         value_document_ids__isnull=False,
-    ):
+    ).select_related("field", "document"):
         for target_doc_id in doclink_being_removed_instance.value:
             remove_doclink(
-                document=Document.objects.get(
-                    id=doclink_being_removed_instance.document.id,
-                ),
+                document=doclink_being_removed_instance.document,
                 field=doclink_being_removed_instance.field,
                 target_doc_id=target_doc_id,
             )
@@ -406,10 +415,28 @@ def reprocess(doc_ids: list[int], *, remote_ocr: bool = False) -> Literal["OK"]:
 
     Consumption workflows do not run here, so ``remote_ocr`` is how the user
     asks for the remote engine when it is not configured to handle everything.
+
+    A root document with versions reprocesses its latest version, which is the
+    file whose content, archive and thumbnail are shown for it.
     """
-    for document_id in doc_ids:
+    latest_version = versions_newest_first(
+        Document.objects.filter(root_document=OuterRef("pk")),
+    ).values("id")[:1]
+    source_ids = (
+        Document.objects.filter(id__in=doc_ids)
+        .annotate(
+            source_id=Case(
+                When(root_document__isnull=False, then=F("id")),
+                default=Coalesce(Subquery(latest_version), F("id")),
+            ),
+        )
+        .order_by()
+        .values_list("source_id", flat=True)
+        .distinct()
+    )
+    for source_id in source_ids:
         update_document_content_maybe_archive_file.apply_async(
-            kwargs={"document_id": document_id, "remote_ocr": remote_ocr},
+            kwargs={"document_id": source_id, "remote_ocr": remote_ocr},
             headers={"trigger_source": PaperlessTask.TriggerSource.MANUAL},
         )
 
@@ -431,10 +458,13 @@ def set_permissions(
     else:
         qs.update(owner=owner)
 
-    for doc in qs:
-        set_permissions_for_object(permissions=set_permissions, object=doc, merge=merge)
-
     affected_docs = list(qs.values_list("pk", flat=True))
+    set_permissions_for_objects(
+        permissions=set_permissions,
+        model=Document,
+        pks=affected_docs,
+        merge=merge,
+    )
 
     bulk_update_documents.apply_async(
         kwargs={"document_ids": affected_docs},
@@ -894,17 +924,26 @@ def edit_pdf(
     pdf_docs: list[pikepdf.Pdf] = []
 
     try:
+        if not operations:
+            raise ValueError("Output document index is out of bounds")
+
+        max_idx = max(op.get("doc", 0) for op in operations)
+        if update_document and max_idx > 0:
+            logger.error(
+                "Update requested but multiple output documents specified",
+            )
+            raise ValueError("Multiple output documents specified")
+
+        if any(
+            op.get("doc", 0) < 0 or op.get("doc", 0) >= len(operations)
+            for op in operations
+        ):
+            raise ValueError("Output document index is out of bounds")
+
         with pikepdf.open(pair.source_doc.source_path) as src:
             # prepare output documents
-            max_idx = max(op.get("doc", 0) for op in operations)
             pdf_docs = [pikepdf.new() for _ in range(max_idx + 1)]
 
-            if update_document and len(pdf_docs) > 1:
-                logger.error(
-                    "Update requested but multiple output documents specified",
-                )
-                raise ValueError("Multiple output documents specified")
-
             for op in operations:
                 dst = pdf_docs[op.get("doc", 0)]
                 page = src.pages[op["page"] - 1]
@@ -1178,10 +1217,13 @@ def remove_doclink(
     """
     Removes a 'symmetrical' link to `document` from the target document's existing custom field instance
     """
-    target_doc_field_instance = CustomFieldInstance.objects.filter(
-        document_id=target_doc_id,
-        field=field,
-    ).first()
+    # select_related: a signal receiver (auditlog) touches .document/.field on
+    # the save() below, without this that is a per-call reload query
+    target_doc_field_instance = (
+        CustomFieldInstance.objects.filter(document_id=target_doc_id, field=field)
+        .select_related("document", "field")
+        .first()
+    )
     if (
         target_doc_field_instance is not None
         and document.id in target_doc_field_instance.value

src/documents/caching.py

@@ -2,27 +2,17 @@ from __future__ import annotations
 
 import hashlib
 import logging
-import pickle
 import uuid
 from binascii import hexlify
-from collections import OrderedDict
 from dataclasses import dataclass
 from typing import TYPE_CHECKING
-from typing import Any
 from typing import Final
 
-from django.conf import settings
 from django.core.cache import cache
-from django.core.cache import caches
 
 from documents.models import Document
-from paperless.signed_pickle import SignedPickleError
-from paperless.signed_pickle import signed_pickle_dumps
-from paperless.signed_pickle import signed_pickle_loads
 
 if TYPE_CHECKING:
-    from django.core.cache.backends.base import BaseCache
-
     from documents.classifier import DocumentClassifier
 
 logger = logging.getLogger("paperless.caching")
@@ -65,82 +55,6 @@ CACHE_50_MINUTES: Final[int] = 50 * CACHE_1_MINUTE
 # Deliberately longer than any entry it names
 LLM_CACHE_GENERATION_TIMEOUT: Final[int] = 2 * CACHE_50_MINUTES
 
-read_cache = caches["read-cache"]
-
-
-class LRUCache:
-    def __init__(self, capacity: int = 128):
-        self._data = OrderedDict()
-        self.capacity = capacity
-
-    def get(self, key, default=None) -> Any | None:
-        if key in self._data:
-            self._data.move_to_end(key)
-            return self._data[key]
-        return default
-
-    def set(self, key, value) -> None:
-        self._data[key] = value
-        self._data.move_to_end(key)
-        while len(self._data) > self.capacity:
-            self._data.popitem(last=False)
-
-
-class StoredLRUCache(LRUCache):
-    """
-    LRU cache that can persist its entire contents as a single entry in a backend cache.
-
-    Useful for sharing a cache across multiple workers or processes.
-
-    Workflow:
-        1. Load the cache state from the backend using `load()`.
-        2. Use `get()` and `set()` locally as usual.
-        3. Persist changes back to the backend using `save()`.
-    """
-
-    def __init__(
-        self,
-        backend_key: str,
-        capacity: int = 128,
-        backend: BaseCache = read_cache,
-        backend_ttl=settings.CACHALOT_TIMEOUT,
-    ):
-        if backend_key is None:
-            raise ValueError("backend_key is mandatory")
-        super().__init__(capacity)
-        self._backend_key = backend_key
-        self._backend = backend
-        self.backend_ttl = backend_ttl
-
-    def load(self) -> None:
-        """
-        Load the whole cache content from backend storage.
-
-        If no valid cached data exists in the backend, the local cache is cleared.
-        """
-        serialized_data = self._backend.get(self._backend_key)
-        try:
-            self._data = (
-                signed_pickle_loads(serialized_data)
-                if serialized_data
-                else OrderedDict()
-            )
-        except (SignedPickleError, pickle.PickleError):
-            logger.warning(
-                "Cache exists in backend but could not be read (possibly invalid format)",
-            )
-
-    def save(self) -> None:
-        """Save the entire local cache to the backend as a serialized object.
-
-        The backend entry will expire after the configured TTL.
-        """
-        self._backend.set(
-            self._backend_key,
-            signed_pickle_dumps(self._data),
-            self.backend_ttl,
-        )
-
 
 def get_suggestion_cache_key(document_id: int) -> str:
     """

src/documents/classifier.py

@@ -1,9 +1,11 @@
 from __future__ import annotations
 
+import functools
 import hmac
 import logging
 import pickle
 import re
+import unicodedata
 import warnings
 from hashlib import sha256
 from pathlib import Path
@@ -13,37 +15,137 @@ if TYPE_CHECKING:
     from collections.abc import Callable
     from collections.abc import Iterator
     from datetime import datetime
+    from types import TracebackType
+    from typing import BinaryIO
+    from typing import Self
 
+    import tantivy
     from numpy import ndarray
+    from sklearn.neural_network import MLPClassifier
 
 from django.conf import settings
 from django.core.cache import cache
 from django.core.cache import caches
+from django.db.models import Prefetch
 
+from documents._snowball_stopwords import ENGLISH as ENGLISH_STOP_WORDS
 from documents.caching import CACHE_5_MINUTES
 from documents.caching import CACHE_50_MINUTES
 from documents.caching import CLASSIFIER_HASH_KEY
 from documents.caching import CLASSIFIER_MODIFIED_KEY
 from documents.caching import CLASSIFIER_VERSION_KEY
-from documents.caching import StoredLRUCache
 from documents.models import Document
 from documents.models import MatchingModel
+from documents.models import Tag
 from paperless.signed_pickle import SignedPickleError
 from paperless.signed_pickle import signed_pickle_dumps
 from paperless.signed_pickle import signed_pickle_loads
 
 logger = logging.getLogger("paperless.classifier")
 
-ADVANCED_TEXT_PROCESSING_ENABLED = (
-    settings.NLTK_LANGUAGE is not None and settings.NLTK_ENABLED
-)
+
+def _predict_with_threshold(classifier, X, threshold: float) -> int | None:
+    """
+    Return the predicted class id, or None if:
+    - the prediction is -1 (no match), or
+    - the winning class probability is below the configured threshold.
+
+    Using predict_proba() instead of predict() lets us apply a minimum-confidence
+    cutoff so that uncertain predictions are discarded rather than assigned.
+    """
+    probas = classifier.predict_proba(X)[0]
+    if len(classifier.classes_) == 1:
+        # scikit-learn treats a single-class fit as binary, so predict_proba
+        # returns [1 - p, p] for one class. Column 0 is the real class.
+        probas = probas[:1]
+    best_idx = int(probas.argmax())
+    best_class = int(classifier.classes_[best_idx])
+
+    if best_class == -1:
+        return None
+    if threshold > 0.0 and probas[best_idx] < threshold:
+        return None
+    return best_class
+
 
 read_cache = caches["read-cache"]
 
 
-RE_DIGIT = re.compile(r"\d")
 RE_WORD = re.compile(r"\b[\w]+\b")  # words that may contain digits
 
+# Documents whose content is fetched per query while training
+_CONTENT_CHUNK_SIZE = 1000
+
+
+class _SignedFileWriter:
+    """
+    Atomically writes a file made of an HMAC signature followed by the data,
+    signing the data as it streams to disk rather than holding it in memory.
+
+    The signature is only known once everything is written, so its space is
+    reserved at the start of the file and filled in on exit. The target is only
+    replaced once the file is complete; on error the partial file is removed.
+    """
+
+    def __init__(self, target: Path, mac: hmac.HMAC) -> None:
+        self._target = target
+        self._temp = target.with_name(f"{target.name}.part")
+        self._mac = mac
+        self._file: BinaryIO
+
+    def __enter__(self) -> Self:
+        self._file = self._temp.open("wb")
+        self._file.write(bytes(self._mac.digest_size))
+        return self
+
+    def write(self, data: bytes | memoryview) -> int:
+        self._mac.update(data)
+        return self._file.write(data)
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        traceback: TracebackType | None,
+    ) -> None:
+        try:
+            with self._file:
+                if exc_type is None:
+                    self._file.seek(0)
+                    self._file.write(self._mac.digest())
+            if exc_type is None:
+                self._temp.rename(self._target)
+        finally:
+            # A no-op after a successful rename, otherwise removes the partial file
+            self._temp.unlink(missing_ok=True)
+
+
+@functools.cache
+def _text_analyzer(language: str) -> tantivy.TextAnalyzer:
+    """
+    Builds the cached analyzer for a language: word tokens, lowercase, stop words, stemmer.
+
+    Long tokens are kept and accents are not folded to ASCII, since stemmers
+    for languages such as French and German rely on accents.
+    """
+    import tantivy
+
+    if language == "english":
+        # Tantivy's builtin English list is much shorter than Snowball's.
+        # Split contractions ("don't") on word characters, as content is, so they match
+        stop_words = tantivy.Filter.custom_stopword(
+            sorted({t for word in ENGLISH_STOP_WORDS for t in RE_WORD.findall(word)}),
+        )
+    else:
+        stop_words = tantivy.Filter.stopword(language)
+    return (
+        tantivy.TextAnalyzerBuilder(tantivy.Tokenizer.regex(r"\w+"))
+        .filter(tantivy.Filter.lowercase())
+        .filter(stop_words)
+        .filter(tantivy.Filter.stemmer(language))
+        .build()
+    )
+
 
 class IncompatibleClassifierVersionError(Exception):
     def __init__(self, message: str, *args: object) -> None:
@@ -102,7 +204,10 @@ class DocumentClassifier:
     # v8 - Added storage path classifier
     # v9 - Changed from hashing to time/ids for re-train check
     # v10 - HMAC-signed model file
-    FORMAT_VERSION = 10
+    # v11 - Use sample_weight for balanced training; predict_proba with threshold;
+    #       drop training-only MLP state before saving
+    #       Tantivy text preprocessing
+    FORMAT_VERSION = 11
 
     HMAC_SIZE = 32  # SHA-256 digest length
 
@@ -119,16 +224,6 @@ class DocumentClassifier:
         self.correspondent_classifier = None
         self.document_type_classifier = None
         self.storage_path_classifier = None
-        self._stemmer = None
-        # 10,000 elements roughly use 200 to 500 KB per worker,
-        # and also in the shared Redis cache,
-        # Keep this cache small to minimize lookup and I/O latency.
-        if ADVANCED_TEXT_PROCESSING_ENABLED:
-            self._stem_cache = StoredLRUCache(
-                f"stem_cache_v{self.FORMAT_VERSION}",
-                capacity=10000,
-            )
-        self._stop_words = None
 
     def _update_data_vectorizer_hash(self) -> None:
         self.data_vectorizer_hash = sha256(
@@ -136,12 +231,28 @@ class DocumentClassifier:
         ).hexdigest()
 
     @staticmethod
-    def _compute_hmac(data: bytes) -> bytes:
-        return hmac.new(
-            settings.SECRET_KEY.encode(),
-            data,
-            sha256,
-        ).digest()
+    def _new_hmac() -> hmac.HMAC:
+        return hmac.new(settings.SECRET_KEY.encode(), digestmod=sha256)
+
+    @staticmethod
+    def _strip_training_state(classifier: MLPClassifier) -> None:
+        """
+        Drop MLPClassifier state which is only used during fit(), never by predict().
+
+        The Adam optimizer keeps two moment arrays the size of the weights, and
+        without early_stopping _best_coefs/_best_intercepts are just a copy of the
+        initial random weights. Together that is 3x the size of the weights, which
+        would otherwise be pickled and loaded along with the model.
+        """
+        del classifier._optimizer
+        del classifier._best_coefs
+        del classifier._best_intercepts
+
+    @staticmethod
+    def _compute_hmac(data: bytes | memoryview) -> bytes:
+        mac = DocumentClassifier._new_hmac()
+        mac.update(data)
+        return mac.digest()
 
     def load(self) -> None:
         from sklearn.exceptions import InconsistentVersionWarning
@@ -151,8 +262,13 @@ class DocumentClassifier:
         if len(raw) <= self.HMAC_SIZE:
             raise ClassifierModelCorruptError
 
-        signature = raw[: self.HMAC_SIZE]
-        data = raw[self.HMAC_SIZE :]
+        # Slice through a memoryview so the (potentially multi-GB) payload is
+        # not copied; hmac and pickle both accept buffers directly.
+        # The whole file is still verified from memory before unpickling, rather
+        # than streamed from disk, so it cannot change between check and load.
+        view = memoryview(raw)
+        signature = view[: self.HMAC_SIZE]
+        data = view[self.HMAC_SIZE :]
 
         if not hmac.compare_digest(signature, self._compute_hmac(data)):
             raise ClassifierModelCorruptError
@@ -197,29 +313,24 @@ class DocumentClassifier:
                 raise IncompatibleClassifierVersionError("sklearn version update")
 
     def save(self) -> None:
-        target_file: Path = settings.MODEL_FILE
-        target_file_temp: Path = target_file.with_suffix(".pickle.part")
-
-        data = pickle.dumps(
-            (
-                self.FORMAT_VERSION,
-                self.last_doc_change_time,
-                self.last_auto_type_hash,
-                self.data_vectorizer,
-                self.tags_binarizer,
-                self.tags_classifier,
-                self.correspondent_classifier,
-                self.document_type_classifier,
-                self.storage_path_classifier,
-            ),
-        )
-
-        signature = self._compute_hmac(data)
-
-        with target_file_temp.open("wb") as f:
-            f.write(signature + data)
-
-        target_file_temp.rename(target_file)
+        # Stream to disk instead of building the payload in memory. Protocol 5+
+        # pickles numpy arrays without copying them (the default is 4 before 3.14).
+        with _SignedFileWriter(settings.MODEL_FILE, self._new_hmac()) as f:
+            pickle.dump(
+                (
+                    self.FORMAT_VERSION,
+                    self.last_doc_change_time,
+                    self.last_auto_type_hash,
+                    self.data_vectorizer,
+                    self.tags_binarizer,
+                    self.tags_classifier,
+                    self.correspondent_classifier,
+                    self.document_type_classifier,
+                    self.storage_path_classifier,
+                ),
+                f,
+                protocol=pickle.HIGHEST_PROTOCOL,
+            )
 
     def train(
         self,
@@ -228,29 +339,48 @@ class DocumentClassifier:
         notify = status_callback if status_callback is not None else lambda _: None
 
         # Get non-inbox documents
-        docs_queryset = (
-            Document.objects.exclude(
-                tags__is_inbox_tag=True,
-            )
-            .select_related("document_type", "correspondent", "storage_path")
-            .prefetch_related("tags")
-            .order_by("pk")
-        )
+        docs_queryset = Document.objects.exclude(
+            tags__is_inbox_tag=True,
+        ).order_by("pk")
 
         # No documents exit to train against
-        if docs_queryset.count() == 0:
+        doc_count = docs_queryset.count()
+        if doc_count == 0:
             raise ValueError("No training data available.")
 
         labels_tags = []
         labels_correspondent = []
         labels_document_type = []
         labels_storage_path = []
+        # Content is fetched separately later, for exactly these documents in this
+        # order, so it never all has to be in memory at once
+        doc_pks: list[int] = []
+        latest_doc_change: datetime | None = None
 
         # Step 1: Extract and preprocess training data from the database.
         logger.debug("Gathering data from database...")
-        notify(f"Gathering data from {docs_queryset.count()} document(s)...")
+        notify(f"Gathering data from {doc_count} document(s)...")
         hasher = sha256()
-        for doc in docs_queryset:
+        for doc in (
+            docs_queryset.defer("content")
+            .select_related("document_type", "correspondent", "storage_path")
+            .prefetch_related(
+                Prefetch(
+                    "tags",
+                    queryset=Tag.objects.filter(
+                        matching_algorithm=MatchingModel.MATCH_AUTO,
+                    )
+                    .order_by("pk")
+                    .only("pk"),
+                    to_attr="auto_tags",
+                ),
+            )
+            .iterator(chunk_size=2000)
+        ):
+            doc_pks.append(doc.pk)
+            if latest_doc_change is None or doc.modified > latest_doc_change:
+                latest_doc_change = doc.modified
+
             y = -1
             dt = doc.document_type
             if dt and dt.matching_algorithm == MatchingModel.MATCH_AUTO:
@@ -265,11 +395,7 @@ class DocumentClassifier:
             hasher.update(y.to_bytes(4, "little", signed=True))
             labels_correspondent.append(y)
 
-            tags: list[int] = list(
-                doc.tags.filter(matching_algorithm=MatchingModel.MATCH_AUTO)
-                .order_by("pk")
-                .values_list("pk", flat=True),
-            )
+            tags: list[int] = [tag.pk for tag in doc.auto_tags]
             for tag in tags:
                 hasher.update(tag.to_bytes(4, "little", signed=True))
             labels_tags.append(tags)
@@ -288,7 +414,6 @@ class DocumentClassifier:
         # Check if retraining is actually required.
         # A document has been updated since the classifier was trained
         # New auto tags, types, correspondent, storage paths exist
-        latest_doc_change = docs_queryset.latest("modified").modified
         if (
             self.last_doc_change_time is not None
             and self.last_doc_change_time >= latest_doc_change
@@ -315,7 +440,7 @@ class DocumentClassifier:
         num_storage_paths: int = len(set(labels_storage_path) | {-1}) - 1
 
         logger.debug(
-            f"{docs_queryset.count()} documents, {num_tags} tag(s), {num_correspondents} correspondent(s), "
+            f"{len(doc_pks)} documents, {num_tags} tag(s), {num_correspondents} correspondent(s), "
             f"{num_document_types} document type(s). {num_storage_paths} storage path(s)",
         )
 
@@ -324,16 +449,32 @@ class DocumentClassifier:
         from sklearn.preprocessing import LabelBinarizer
         from sklearn.preprocessing import MultiLabelBinarizer
 
+        # MLPClassifier does not support class_weight directly
+        # (https://github.com/scikit-learn/scikit-learn/issues/9113), so we use
+        # compute_sample_weight to balance classes during training and prevent
+        # over-represented correspondents from dominating predictions.
+        # https://scikit-learn.org/stable/modules/generated/sklearn.utils.class_weight.compute_sample_weight.html
+        from sklearn.utils.class_weight import compute_sample_weight
+
         # Step 2: vectorize data
         logger.debug("Vectorizing data...")
         notify("Vectorizing document content...")
 
         def content_generator() -> Iterator[str]:
             """
-            Generates the content for documents, but once at a time
+            Generates the content for documents, in the same order as the labels,
+            fetching it a chunk at a time
             """
-            for doc in docs_queryset:
-                yield self.preprocess_content(doc.content, shared_cache=False)
+            for start in range(0, len(doc_pks), _CONTENT_CHUNK_SIZE):
+                chunk = doc_pks[start : start + _CONTENT_CHUNK_SIZE]
+                docs = Document.objects.only("content").order_by().in_bulk(chunk)
+                for pk in chunk:
+                    # A document deleted since its labels were gathered still
+                    # needs a row, so labels and content stay aligned
+                    doc = docs.get(pk)
+                    yield self.preprocess_content(
+                        doc.content if doc is not None else "",
+                    )
 
         self.data_vectorizer = CountVectorizer(
             analyzer="word",
@@ -369,8 +510,9 @@ class DocumentClassifier:
                 self.tags_binarizer = MultiLabelBinarizer()
                 labels_tags_vectorized = self.tags_binarizer.fit_transform(labels_tags)
 
-            self.tags_classifier = MLPClassifier(tol=0.01)
+            self.tags_classifier = MLPClassifier(tol=0.01, random_state=0)
             self.tags_classifier.fit(data_vectorized, labels_tags_vectorized)
+            self._strip_training_state(self.tags_classifier)
         else:
             self.tags_classifier = None
             logger.debug("There are no tags. Not training tags classifier.")
@@ -380,8 +522,13 @@ class DocumentClassifier:
             notify(
                 f"Training correspondent classifier ({num_correspondents} correspondent(s))...",
             )
-            self.correspondent_classifier = MLPClassifier(tol=0.01)
-            self.correspondent_classifier.fit(data_vectorized, labels_correspondent)
+            self.correspondent_classifier = MLPClassifier(tol=0.01, random_state=0)
+            self.correspondent_classifier.fit(
+                data_vectorized,
+                labels_correspondent,
+                sample_weight=compute_sample_weight("balanced", labels_correspondent),
+            )
+            self._strip_training_state(self.correspondent_classifier)
         else:
             self.correspondent_classifier = None
             logger.debug(
@@ -393,8 +540,13 @@ class DocumentClassifier:
             notify(
                 f"Training document type classifier ({num_document_types} type(s))...",
             )
-            self.document_type_classifier = MLPClassifier(tol=0.01)
-            self.document_type_classifier.fit(data_vectorized, labels_document_type)
+            self.document_type_classifier = MLPClassifier(tol=0.01, random_state=0)
+            self.document_type_classifier.fit(
+                data_vectorized,
+                labels_document_type,
+                sample_weight=compute_sample_weight("balanced", labels_document_type),
+            )
+            self._strip_training_state(self.document_type_classifier)
         else:
             self.document_type_classifier = None
             logger.debug(
@@ -406,11 +558,13 @@ class DocumentClassifier:
                 "Training storage paths classifier...",
             )
             notify(f"Training storage path classifier ({num_storage_paths} path(s))...")
-            self.storage_path_classifier = MLPClassifier(tol=0.01)
+            self.storage_path_classifier = MLPClassifier(tol=0.01, random_state=0)
             self.storage_path_classifier.fit(
                 data_vectorized,
                 labels_storage_path,
+                sample_weight=compute_sample_weight("balanced", labels_storage_path),
             )
+            self._strip_training_state(self.storage_path_classifier)
         else:
             self.storage_path_classifier = None
             logger.debug(
@@ -429,99 +583,23 @@ class DocumentClassifier:
 
         return True
 
-    def _init_advanced_text_processing(self):
-        if self._stop_words is None or self._stemmer is None:
-            import nltk
-            from nltk.corpus import stopwords
-            from nltk.stem import SnowballStemmer
-
-            # Not really hacky, since it isn't private and is documented, but
-            # set the search path for NLTK data to the single location it should be in
-            nltk.data.path = [settings.NLTK_DIR]
-            try:
-                # Preload the corpus early, to force the lazy loader to transform
-                stopwords.ensure_loaded()
-
-                # Do some one time setup
-                # Sometimes, somehow, there's multiple threads loading the corpus
-                # and it's not thread safe, raising an AttributeError
-                self._stemmer = SnowballStemmer(settings.NLTK_LANGUAGE)
-                self._stop_words = frozenset(stopwords.words(settings.NLTK_LANGUAGE))
-            except AttributeError:
-                logger.debug("Could not initialize NLTK for advanced text processing.")
-                return False
-        return True
-
-    def stem_and_skip_stop_words(self, words: list[str], *, shared_cache=True):
-        """
-        Reduce a list of words to their stem. Stop words are converted to empty strings.
-        :param words: the list of words to stem
-        """
-
-        def _stem_and_skip_stop_word(word: str):
-            """
-            Reduce a given word to its stem. If it's a stop word, return an empty string.
-            E.g. "amazement", "amaze" and "amazed" all return "amaz".
-            """
-            cached = self._stem_cache.get(word)
-            if cached is not None:
-                return cached
-            elif word in self._stop_words:
-                return ""
-            # Assumption: words that contain numbers are never stemmed
-            elif RE_DIGIT.search(word):
-                return word
-            else:
-                result = self._stemmer.stem(word)
-                self._stem_cache.set(word, result)
-                return result
-
-        if shared_cache:
-            self._stem_cache.load()
-
-        # Stem the words and skip stop words
-        result = " ".join(
-            filter(None, (_stem_and_skip_stop_word(w) for w in words)),
-        )
-        if shared_cache:
-            self._stem_cache.save()
-        return result
-
-    def preprocess_content(
-        self,
-        content: str,
-        *,
-        shared_cache=True,
-    ) -> str:
+    def preprocess_content(self, content: str) -> str:
         """
         Process the contents of a document, distilling it down into
         words which are meaningful to the content.
-
-        A stemmer cache is shared across workers with the parameter "shared_cache".
-        This is unnecessary when training the classifier.
         """
-
-        # Lower case the document, reduce space,
-        # and keep only letters and digits.
-        content = " ".join(match.group().lower() for match in RE_WORD.finditer(content))
-
-        if ADVANCED_TEXT_PROCESSING_ENABLED:
-            from nltk.tokenize import word_tokenize
-
-            if not self._init_advanced_text_processing():
-                return content
-            # Tokenize
-            # This splits the content into tokens, roughly words
-            words = word_tokenize(content, language=settings.NLTK_LANGUAGE)
-            # Stem the words and skip stop words
-            content = self.stem_and_skip_stop_words(words, shared_cache=shared_cache)
-
-        return content
+        language = settings.CLASSIFIER_LANGUAGE
+        content = unicodedata.normalize("NFC", content)
+        if language is None:
+            return " ".join(
+                match.group().lower() for match in RE_WORD.finditer(content)
+            )
+        return " ".join(_text_analyzer(language).analyze(content))
 
     def _get_vectorizer_cache_key(self, content: str):
         hash = sha256(content.encode())
         hash.update(
-            f"|{self.FORMAT_VERSION}|{settings.NLTK_LANGUAGE}|{settings.NLTK_ENABLED}|{self.data_vectorizer_hash}".encode(),
+            f"|{self.FORMAT_VERSION}|{settings.CLASSIFIER_LANGUAGE}|{self.data_vectorizer_hash}".encode(),
         )
         return f"vectorized_content_{hash.hexdigest()}"
 
@@ -546,24 +624,24 @@ class DocumentClassifier:
     def predict_correspondent(self, content: str) -> int | None:
         if self.correspondent_classifier:
             X = self._vectorize(content)
-            correspondent_id = self.correspondent_classifier.predict(X)
-            if correspondent_id != -1:
-                return correspondent_id
-            else:
-                return None
-        else:
-            return None
+            predicted_id = _predict_with_threshold(
+                self.correspondent_classifier,
+                X,
+                settings.CLASSIFIER_MATCH_THRESHOLD,
+            )
+            return predicted_id
+        return None
 
     def predict_document_type(self, content: str) -> int | None:
         if self.document_type_classifier:
             X = self._vectorize(content)
-            document_type_id = self.document_type_classifier.predict(X)
-            if document_type_id != -1:
-                return document_type_id
-            else:
-                return None
-        else:
-            return None
+            predicted_id = _predict_with_threshold(
+                self.document_type_classifier,
+                X,
+                settings.CLASSIFIER_MATCH_THRESHOLD,
+            )
+            return predicted_id
+        return None
 
     def predict_tags(self, content: str) -> list[int]:
         from sklearn.utils.multiclass import type_of_target
@@ -589,10 +667,10 @@ class DocumentClassifier:
     def predict_storage_path(self, content: str) -> int | None:
         if self.storage_path_classifier:
             X = self._vectorize(content)
-            storage_path_id = self.storage_path_classifier.predict(X)
-            if storage_path_id != -1:
-                return storage_path_id
-            else:
-                return None
-        else:
-            return None
+            predicted_id = _predict_with_threshold(
+                self.storage_path_classifier,
+                X,
+                settings.CLASSIFIER_MATCH_THRESHOLD,
+            )
+            return predicted_id
+        return None

src/documents/consumer.py

@@ -18,6 +18,7 @@ from django.utils import timezone
 from filelock import FileLock
 from rest_framework.reverse import reverse
 
+from documents.barcodes import read_barcode_values
 from documents.classifier import load_classifier
 from documents.data_models import ConsumableDocument
 from documents.data_models import ConsumeFileSuccessResult
@@ -31,6 +32,7 @@ from documents.models import Correspondent
 from documents.models import CustomField
 from documents.models import CustomFieldInstance
 from documents.models import Document
+from documents.models import DocumentBarcode
 from documents.models import DocumentType
 from documents.models import StoragePath
 from documents.models import Tag
@@ -53,6 +55,7 @@ from documents.utils import compute_checksum
 from documents.utils import copy_basic_file_stats
 from documents.utils import copy_file_with_basic_stats
 from documents.utils import run_subprocess
+from paperless.config import BarcodeConfig
 from paperless.config import OcrConfig
 from paperless.config import RemoteOCRConfig
 from paperless.models import ArchiveFileGenerationChoices
@@ -504,6 +507,13 @@ class ConsumerPlugin(
                     f"Parser: {document_parser.name} v{document_parser.version}",
                 )
 
+                # New versions skip the barcode plugin, so read their barcodes here
+                if (
+                    self.input_doc.root_document_id is not None
+                    and self.metadata.barcodes is None
+                ):
+                    self._read_version_barcodes(mime_type, Path(tmpdir))
+
                 # Parse the document. This may take some time.
 
                 text = None
@@ -631,6 +641,8 @@ class ConsumerPlugin(
                             else:
                                 original_document.save()
 
+                            self._store_barcodes(original_document)
+
                             # Adding a version changes the effective document, so update root modified
                             Document.objects.filter(pk=root_doc.pk).update(
                                 modified=timezone.now(),
@@ -857,8 +869,9 @@ class ConsumerPlugin(
             self.log.debug(f"Creation date from parse_date: {create_date}")
         else:
             stats = Path(self.input_doc.original_file).stat()
-            create_date = timezone.make_aware(
-                datetime.datetime.fromtimestamp(stats.st_mtime),
+            create_date = datetime.datetime.fromtimestamp(
+                stats.st_mtime,
+                tz=timezone.get_current_timezone(),
             )
             self.log.debug(f"Creation date from st_mtime: {create_date}")
 
@@ -961,6 +974,32 @@ class ConsumerPlugin(
                 }
                 CustomFieldInstance.objects.create(**args)  # adds to document
 
+        self._store_barcodes(document)
+
+    def _read_version_barcodes(self, mime_type: str, work_dir: Path) -> None:
+        barcode_settings = BarcodeConfig()
+        if not barcode_settings.barcode_store_values:
+            return
+        try:
+            self.metadata.barcodes = (
+                read_barcode_values(
+                    self.working_copy,
+                    mime_type,
+                    barcode_settings,
+                    work_dir,
+                )
+                or None
+            )
+        except Exception as e:
+            self.log.warning(f"Could not read barcodes of {self.filename}: {e}")
+
+    def _store_barcodes(self, document: Document) -> None:
+        if self.metadata.barcodes:
+            DocumentBarcode.objects.bulk_create(
+                DocumentBarcode(document=document, **barcode)
+                for barcode in self.metadata.barcodes
+            )
+
     def _write(self, source, target) -> None:
         with (
             Path(source).open("rb") as read_file,

src/documents/data_models.py

@@ -9,6 +9,16 @@ from guardian.shortcuts import get_groups_with_perms
 from guardian.shortcuts import get_users_with_perms
 
 
+class StoredBarcode(TypedDict):
+    """
+    A detected barcode as it is stored with a document
+    """
+
+    page: int  # 1-indexed
+    value: str
+    format: str  # a DocumentBarcode.Format value
+
+
 @dataclasses.dataclass
 class DocumentMetadataOverrides:
     """
@@ -35,6 +45,7 @@ class DocumentMetadataOverrides:
     version_label: str | None = None
     actor_id: int | None = None
     remote_ocr: bool = False
+    barcodes: list[StoredBarcode] | None = None
 
     def update(self, other: "DocumentMetadataOverrides") -> "DocumentMetadataOverrides":
         """
@@ -119,12 +130,14 @@ class DocumentMetadataOverrides:
             get_users_with_perms(
                 doc,
                 only_with_perms_in=["view_document"],
+                with_group_users=False,
             ).values_list("id", flat=True),
         )
         overrides.change_users = list(
             get_users_with_perms(
                 doc,
                 only_with_perms_in=["change_document"],
+                with_group_users=False,
             ).values_list("id", flat=True),
         )
         overrides.custom_fields = {

src/documents/filters.py

@@ -50,6 +50,7 @@ from documents.models import ShareLink
 from documents.models import ShareLinkBundle
 from documents.models import StoragePath
 from documents.models import Tag
+from documents.permissions import annotate_authorizing_fields
 from documents.permissions import permitted_document_ids
 from documents.permissions import permitted_object_ids
 from documents.versioning import annotate_effective_content
@@ -1075,6 +1076,9 @@ class PermittedObjectsFilter(BaseFilterBackend):
 
     include_granted: bool = True
     perm_codename: str | None = None
+    # A self-referencing foreign key whose target authorizes a row, so a row is
+    # judged by its parent's owner and grants (``Document.root_document``).
+    parent_field: str | None = None
 
     def filter_queryset(self, request, queryset, view):
         # Before the superuser and owner-only paths, neither of which consults
@@ -1086,11 +1090,23 @@ class PermittedObjectsFilter(BaseFilterBackend):
         if request.user.is_superuser:
             return queryset
         if not self.include_granted:
-            return queryset.filter(Q(owner=request.user) | Q(owner__isnull=True))
+            owner_field = "owner"
+            if self.parent_field is not None:
+                queryset = annotate_authorizing_fields(queryset, self.parent_field)
+                owner_field = "authorizing_owner"
+            return queryset.filter(
+                Q(**{owner_field: request.user.pk})
+                | Q(**{f"{owner_field}__isnull": True}),
+            )
         model = queryset.model
         perm = self.perm_codename or f"view_{model._meta.model_name}"
         return queryset.filter(
-            id__in=permitted_object_ids(request.user, model, perm),
+            id__in=permitted_object_ids(
+                request.user,
+                model,
+                perm,
+                parent_field=self.parent_field,
+            ),
         )
 
 

src/documents/management/commands/base.py

@@ -21,6 +21,7 @@ from typing import ClassVar
 from typing import Generic
 from typing import TypeVar
 
+import django
 from django import db
 from django.core.management import CommandError
 from django.db.models import QuerySet
@@ -534,7 +535,10 @@ class PaperlessCommand(RichCommand):
         with self._create_progress(description) as progress:
             task_id = progress.add_task(description, total=total)
 
-            with ProcessPoolExecutor(max_workers=self.process_count) as executor:
+            with ProcessPoolExecutor(
+                max_workers=self.process_count,
+                initializer=django.setup,
+            ) as executor:
                 # Submit all tasks and map futures back to items
                 future_to_item = {executor.submit(fn, item): item for item in items}
 

src/documents/management/commands/document_exporter.py

@@ -3,6 +3,7 @@ from itertools import islice
 from pathlib import Path
 from typing import TYPE_CHECKING
 from typing import Any
+from typing import Final
 
 from allauth.mfa.models import Authenticator
 from allauth.socialaccount.models import SocialAccount
@@ -45,6 +46,7 @@ from documents.models import Correspondent
 from documents.models import CustomField
 from documents.models import CustomFieldInstance
 from documents.models import Document
+from documents.models import DocumentBarcode
 from documents.models import DocumentType
 from documents.models import Note
 from documents.models import SavedView
@@ -69,6 +71,15 @@ from paperless.models import ApplicationConfiguration
 from paperless_mail.models import MailAccount
 from paperless_mail.models import MailRule
 
+# Derived names replace the original's extension with these suffixes
+THUMBNAIL_SUFFIX: Final[str] = "-thumbnail.webp"
+ARCHIVE_SUFFIX: Final[str] = "-archive.pdf"
+# Leave room for the longest derived suffix, which replaces the original's extension
+MAX_BASE_NAME_BYTES: Final[int] = Document.MAX_PUBLIC_FILENAME_BYTES - max(
+    len(THUMBNAIL_SUFFIX),
+    len(ARCHIVE_SUFFIX),
+)
+
 
 def serialize_queryset_batched(
     queryset: "QuerySet[Any]",
@@ -353,6 +364,7 @@ class Command(CryptMixin, PaperlessCommand):
             "workflows": Workflow.objects.all(),
             "custom_fields": CustomField.objects.all(),
             "custom_field_instances": CustomFieldInstance.global_objects.all(),
+            "document_barcodes": DocumentBarcode.objects.all(),
             "app_configs": ApplicationConfiguration.objects.all(),
             "notes": Note.global_objects.all(),
             "documents": Document.global_objects.order_by("id").all(),
@@ -411,6 +423,7 @@ class Command(CryptMixin, PaperlessCommand):
                     elif self.split_manifest and key in (
                         "notes",
                         "custom_field_instances",
+                        "document_barcodes",
                     ):
                         # Written per-document in _write_split_manifest
                         pass
@@ -509,10 +522,16 @@ class Command(CryptMixin, PaperlessCommand):
                     counter=filename_counter,
                 )
             else:
-                base_name = document.get_public_filename(counter=filename_counter)
+                base_name = document.get_public_filename(
+                    counter=filename_counter,
+                    max_bytes=MAX_BASE_NAME_BYTES,
+                )
 
-            if base_name not in self.exported_files:
-                self.exported_files.add(base_name)
+            # The thumbnail and archive names are derived from the stem, so
+            # uniqueness must hold without the original's extension
+            unique_key = str(Path(base_name).with_suffix(""))
+            if unique_key not in self.exported_files:
+                self.exported_files.add(unique_key)
                 break
             else:
                 filename_counter += 1
@@ -535,7 +554,7 @@ class Command(CryptMixin, PaperlessCommand):
         document_dict[EXPORTER_FILE_NAME] = original_arc
 
         if not self.no_thumbnail:
-            thumbnail_name = base_name.parent / (base_name.stem + "-thumbnail.webp")
+            thumbnail_name = base_name.parent / (base_name.stem + THUMBNAIL_SUFFIX)
             if self.use_folder_prefix:
                 thumbnail_name = Path("thumbnails") / thumbnail_name
             thumbnail_arc = thumbnail_name.as_posix()
@@ -544,7 +563,7 @@ class Command(CryptMixin, PaperlessCommand):
             thumbnail_arc = None
 
         if not self.no_archive and document.has_archive_version:
-            archive_name = base_name.parent / (base_name.stem + "-archive.pdf")
+            archive_name = base_name.parent / (base_name.stem + ARCHIVE_SUFFIX)
             if self.use_folder_prefix:
                 archive_name = Path("archive") / archive_name
             archive_arc = archive_name.as_posix()
@@ -651,6 +670,12 @@ class Command(CryptMixin, PaperlessCommand):
                 CustomFieldInstance.global_objects.filter(document=document),
             ),
         )
+        content.extend(
+            serializers.serialize(
+                "python",
+                DocumentBarcode.objects.filter(document=document),
+            ),
+        )
         manifest_name = base_name.with_name(f"{base_name.stem}-manifest.json")
         if self.use_folder_prefix:
             manifest_name = Path("json") / manifest_name

src/documents/management/commands/document_importer.py

@@ -66,7 +66,7 @@ def iter_manifest_records(path: Path) -> Generator[dict, None, None]:
     """Yield records one at a time from a manifest JSON array via ijson."""
     try:
         with path.open("rb") as f:
-            yield from ijson.items(f, "item")
+            yield from ijson.items(f, "item", use_float=True)
     except ijson.JSONError as e:
         raise CommandError(f"Failed to parse manifest file {path}: {e}") from e
 

src/documents/management/commands/document_index.py

@@ -67,16 +67,22 @@ class Command(PaperlessCommand):
                 if options.get("recreate"):
                     wipe_index(settings.INDEX_DIR)
 
-                documents = Document.objects.select_related(
-                    "correspondent",
-                    "document_type",
-                    "storage_path",
-                    "owner",
-                ).prefetch_related(
-                    "tags",
-                    "notes__user",
-                    "custom_fields__field",
-                    "versions",
+                documents = (
+                    Document.objects.filter(root_document__isnull=True)
+                    .select_related(
+                        "correspondent",
+                        "document_type",
+                        "storage_path",
+                        "owner",
+                    )
+                    .prefetch_related(
+                        "tags",
+                        "notes__user",
+                        "custom_fields__field",
+                        "versions",
+                        "barcodes",
+                        "versions__barcodes",
+                    )
                 )
                 total = documents.count()
                 rebuild_kwargs = {}

src/documents/migrations/0027_documentbarcode.py

@@ -0,0 +1,102 @@
+# Generated by Django 5.2.16 on 2026-09-30 23:01
+
+import django.db.models.deletion
+from django.db import migrations
+from django.db import models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("documents", "0026_alter_document_archive_checksum_and_more"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="DocumentBarcode",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "page",
+                    models.PositiveIntegerField(
+                        help_text="Page of the original file, starting at 1",
+                        verbose_name="page",
+                    ),
+                ),
+                ("value", models.TextField(verbose_name="value")),
+                (
+                    "format",
+                    models.CharField(
+                        choices=[
+                            ("Codabar", "Codabar"),
+                            ("Code39", "Code 39"),
+                            ("Code39Std", "Code 39 Standard"),
+                            ("Code39Ext", "Code 39 Extended"),
+                            ("Code32", "Code 32"),
+                            ("PZN", "Pharmazentralnummer"),
+                            ("Code93", "Code 93"),
+                            ("Code128", "Code 128"),
+                            ("ITF", "ITF"),
+                            ("ITF14", "ITF-14"),
+                            ("DataBar", "DataBar"),
+                            ("DataBarOmni", "DataBar Omni"),
+                            ("DataBarStk", "DataBar Stacked"),
+                            ("DataBarStkOmni", "DataBar Stacked Omni"),
+                            ("DataBarLtd", "DataBar Limited"),
+                            ("DataBarExp", "DataBar Expanded"),
+                            ("DataBarExpStk", "DataBar Expanded Stacked"),
+                            ("EANUPC", "EAN/UPC"),
+                            ("EAN13", "EAN-13"),
+                            ("EAN8", "EAN-8"),
+                            ("EAN5", "EAN-5"),
+                            ("EAN2", "EAN-2"),
+                            ("ISBN", "ISBN"),
+                            ("UPCA", "UPC-A"),
+                            ("UPCE", "UPC-E"),
+                            ("Telepen", "Telepen"),
+                            ("TelepenAlpha", "Telepen Alpha"),
+                            ("TelepenNumeric", "Telepen Numeric"),
+                            ("OtherBarcode", "Other barcode"),
+                            ("DXFilmEdge", "DX Film Edge"),
+                            ("PDF417", "PDF417"),
+                            ("CompactPDF417", "Compact PDF417"),
+                            ("MicroPDF417", "MicroPDF417"),
+                            ("Aztec", "Aztec"),
+                            ("AztecCode", "Aztec Code"),
+                            ("AztecRune", "Aztec Rune"),
+                            ("QRCode", "QR Code"),
+                            ("QRCodeModel1", "QR Code Model 1"),
+                            ("QRCodeModel2", "QR Code Model 2"),
+                            ("MicroQRCode", "Micro QR Code"),
+                            ("RMQRCode", "rMQR Code"),
+                            ("DataMatrix", "Data Matrix"),
+                            ("MaxiCode", "MaxiCode"),
+                        ],
+                        max_length=32,
+                        verbose_name="format",
+                    ),
+                ),
+                (
+                    "document",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="barcodes",
+                        to="documents.document",
+                        verbose_name="document",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "document barcode",
+                "verbose_name_plural": "document barcodes",
+                "ordering": ("page", "id"),
+            },
+        ),
+    ]

src/documents/migrations/0028_merge_ocr_templates.py

@@ -0,0 +1,10 @@
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("documents", "0022_ocrtemplate_ocrtemplatezone"),
+        ("documents", "0027_documentbarcode"),
+    ]
+
+    operations = []

src/documents/models.py

@@ -157,6 +157,8 @@ class StoragePath(MatchingModel):
 
 class Document(SoftDeleteModel, ModelWithOwner):  # type: ignore[django-manager-missing]
     MAX_STORED_FILENAME_LENGTH: Final[int] = 1024
+    # Per-component limit of common filesystems (ext4, XFS, btrfs, ZFS, APFS)
+    MAX_PUBLIC_FILENAME_BYTES: Final[int] = 255
 
     correspondent = models.ForeignKey(
         Correspondent,
@@ -366,6 +368,16 @@ class Document(SoftDeleteModel, ModelWithOwner):  # type: ignore[django-manager-
             res += f" {self.title}"
         return res
 
+    def get_effective_barcodes(self) -> list["DocumentBarcode"]:
+        """
+        Returns the stored barcodes for the document, like
+        get_effective_content(): for root documents those of the latest
+        version when there is one, as that is the file users see.
+        """
+        from documents.versioning import latest_version
+
+        return list(latest_version(self).barcodes.all())
+
     def get_effective_content(self) -> str | None:
         """
         Returns the effective content for the document.
@@ -375,6 +387,7 @@ class Document(SoftDeleteModel, ModelWithOwner):  # type: ignore[django-manager-
         If the queryset already annotated ``effective_content``, that value is used.
         """
         # Here to avoid circular import
+        from documents.versioning import LATEST_VERSION_CONTENT_PREFETCH_ATTR
         from documents.versioning import sort_versions_newest_first
         from documents.versioning import versions_newest_first
 
@@ -384,6 +397,19 @@ class Document(SoftDeleteModel, ModelWithOwner):  # type: ignore[django-manager-
         if self.root_document_id is not None or self.pk is None:
             return self.content
 
+        latest_version_prefetch = getattr(
+            self,
+            LATEST_VERSION_CONTENT_PREFETCH_ATTR,
+            None,
+        )
+        if latest_version_prefetch is not None:
+            # Empty list means prefetch ran and found no versions — use own content.
+            return (
+                latest_version_prefetch[0].content
+                if latest_version_prefetch
+                else self.content
+            )
+
         prefetched_cache = getattr(self, "_prefetched_objects_cache", None)
         prefetched_versions = (
             prefetched_cache.get("versions")
@@ -460,28 +486,37 @@ class Document(SoftDeleteModel, ModelWithOwner):  # type: ignore[django-manager-
     def archive_file(self):
         return Path(self.archive_path).open("rb")
 
-    def get_public_filename(self, *, archive=False, counter=0, suffix=None) -> str:
+    def get_public_filename(
+        self,
+        *,
+        archive: bool = False,
+        counter: int = 0,
+        max_bytes: int = MAX_PUBLIC_FILENAME_BYTES,
+    ) -> str:
         """
         Returns a sanitized filename for the document, not including any paths.
+
+        The name fits in max_bytes (UTF-8). A long title is shortened before
+        the counter and extension are appended, because sanitize_filename
+        truncates from the end. The tail is cut only when max_bytes is
+        smaller than the tail itself.
         """
         # Root owns metadata for all versions
         context_document = (
             self.root_document if self.root_document_id is not None else self
         )
-        result = str(context_document)
 
-        if counter:
-            result += f"_{counter:02}"
+        extension = ".pdf" if archive else self.file_type
+        tail = f"_{counter:02}{extension}" if counter else extension
 
-        if suffix:
-            result += suffix
+        head_bytes = max(0, max_bytes - len(tail.encode()))
+        head = str(context_document).encode()[:head_bytes].decode(errors="ignore")
 
-        if archive:
-            result += ".pdf"
-        else:
-            result += self.file_type
-
-        return pathvalidate.sanitize_filename(result, replacement_text="-")
+        return pathvalidate.sanitize_filename(
+            head + tail,
+            replacement_text="-",
+            max_len=max_bytes,
+        )
 
     @property
     def file_type(self):
@@ -955,6 +990,87 @@ class Note(SoftDeleteModel):
         return self.note
 
 
+class DocumentBarcode(models.Model):
+    """
+    A barcode found in a document during consumption, kept so its content
+    can be shown and copied
+    """
+
+    document = models.ForeignKey(
+        Document,
+        related_name="barcodes",
+        on_delete=models.CASCADE,
+        verbose_name=_("document"),
+    )
+
+    page = models.PositiveIntegerField(
+        _("page"),
+        help_text=_("Page of the original file, starting at 1"),
+    )
+
+    value = models.TextField(_("value"))
+
+    class Format(models.TextChoices):
+        """
+        The concrete barcode formats of zxing-cpp, keyed on the enum name.
+        The labels are symbology names and aren't translated.
+        """
+
+        CODABAR = "Codabar", "Codabar"
+        CODE39 = "Code39", "Code 39"
+        CODE39_STD = "Code39Std", "Code 39 Standard"
+        CODE39_EXT = "Code39Ext", "Code 39 Extended"
+        CODE32 = "Code32", "Code 32"
+        PZN = "PZN", "Pharmazentralnummer"
+        CODE93 = "Code93", "Code 93"
+        CODE128 = "Code128", "Code 128"
+        ITF = "ITF", "ITF"
+        ITF14 = "ITF14", "ITF-14"
+        DATA_BAR = "DataBar", "DataBar"
+        DATA_BAR_OMNI = "DataBarOmni", "DataBar Omni"
+        DATA_BAR_STK = "DataBarStk", "DataBar Stacked"
+        DATA_BAR_STK_OMNI = "DataBarStkOmni", "DataBar Stacked Omni"
+        DATA_BAR_LTD = "DataBarLtd", "DataBar Limited"
+        DATA_BAR_EXP = "DataBarExp", "DataBar Expanded"
+        DATA_BAR_EXP_STK = "DataBarExpStk", "DataBar Expanded Stacked"
+        EANUPC = "EANUPC", "EAN/UPC"
+        EAN13 = "EAN13", "EAN-13"
+        EAN8 = "EAN8", "EAN-8"
+        EAN5 = "EAN5", "EAN-5"
+        EAN2 = "EAN2", "EAN-2"
+        ISBN = "ISBN", "ISBN"
+        UPCA = "UPCA", "UPC-A"
+        UPCE = "UPCE", "UPC-E"
+        TELEPEN = "Telepen", "Telepen"
+        TELEPEN_ALPHA = "TelepenAlpha", "Telepen Alpha"
+        TELEPEN_NUMERIC = "TelepenNumeric", "Telepen Numeric"
+        OTHER_BARCODE = "OtherBarcode", "Other barcode"
+        DX_FILM_EDGE = "DXFilmEdge", "DX Film Edge"
+        PDF417 = "PDF417", "PDF417"
+        COMPACT_PDF417 = "CompactPDF417", "Compact PDF417"
+        MICRO_PDF417 = "MicroPDF417", "MicroPDF417"
+        AZTEC = "Aztec", "Aztec"
+        AZTEC_CODE = "AztecCode", "Aztec Code"
+        AZTEC_RUNE = "AztecRune", "Aztec Rune"
+        QR_CODE = "QRCode", "QR Code"
+        QR_CODE_MODEL1 = "QRCodeModel1", "QR Code Model 1"
+        QR_CODE_MODEL2 = "QRCodeModel2", "QR Code Model 2"
+        MICRO_QR_CODE = "MicroQRCode", "Micro QR Code"
+        RMQR_CODE = "RMQRCode", "rMQR Code"
+        DATA_MATRIX = "DataMatrix", "Data Matrix"
+        MAXI_CODE = "MaxiCode", "MaxiCode"
+
+    format = models.CharField(_("format"), max_length=32, choices=Format.choices)
+
+    class Meta:
+        ordering = ("page", "id")
+        verbose_name = _("document barcode")
+        verbose_name_plural = _("document barcodes")
+
+    def __str__(self) -> str:  # pragma: no cover
+        return self.value
+
+
 class ShareLink(SoftDeleteModel):
     class FileVersion(models.TextChoices):
         ARCHIVE = ("archive", _("Archive"))

src/documents/permissions.py

@@ -6,7 +6,9 @@ from django.contrib.auth.models import Permission
 from django.contrib.auth.models import User
 from django.contrib.contenttypes.models import ContentType
 from django.db.models import Case
+from django.db.models import CharField
 from django.db.models import Count
+from django.db.models import F
 from django.db.models import IntegerField
 from django.db.models import Model
 from django.db.models import Q
@@ -14,6 +16,7 @@ from django.db.models import QuerySet
 from django.db.models import Value
 from django.db.models import When
 from django.db.models.functions import Cast
+from django.db.models.functions import Coalesce
 from guardian.core import ObjectPermissionChecker
 from guardian.models import GroupObjectPermission
 from guardian.models import UserObjectPermission
@@ -25,6 +28,7 @@ from rest_framework.permissions import BasePermission
 from rest_framework.permissions import DjangoObjectPermissions
 
 from documents.models import Document
+from documents.versioning import get_root_document
 
 
 class PaperlessObjectPermissions(DjangoObjectPermissions):
@@ -173,12 +177,182 @@ def set_permissions_for_object(
                         )
 
 
+def _resolve_permissions(codenames: set[str], ctype: ContentType) -> list[Permission]:
+    """
+    Resolves `codenames` to Permission rows, raising like the single-object
+    assign_perm() this bulk path replaces does (via a `.get()` internally)
+    if any codename doesn't exist. SetPermissionsSerializer rejects unknown
+    action names at the API, but a caller passing one directly would
+    otherwise get a plain `.filter()` that silently builds zero rows and
+    no-ops instead of reporting the bad input.
+    """
+    permission_objs = list(
+        Permission.objects.filter(content_type=ctype, codename__in=codenames),
+    )
+    missing = codenames - {p.codename for p in permission_objs}
+    if missing:
+        raise Permission.DoesNotExist(
+            f"Permission matching query does not exist for codename(s): "
+            f"{', '.join(sorted(missing))}",
+        )
+    return permission_objs
+
+
+def _apply_bulk_permission_entry(
+    *,
+    perm_model: type[UserObjectPermission] | type[GroupObjectPermission],
+    identity_model: type[User] | type[Group],
+    identity_field: str,
+    ids: list[int],
+    codename: str,
+    permission_objs: list[Permission],
+    ctype: ContentType,
+    object_pks: list[str],
+    merge: bool,
+) -> None:
+    # Only the ids are needed to build permission rows (via `<field>_id=`),
+    # so avoid fetching full User/Group rows for identities that may not
+    # even end up being granted anything new.
+    add_ids = set(
+        identity_model.objects.filter(id__in=ids).values_list("id", flat=True),
+    )
+
+    if not merge:
+        existing_ids = set(
+            perm_model.objects.filter(
+                content_type=ctype,
+                object_pk__in=object_pks,
+                permission__codename=codename,
+            )
+            .values_list(f"{identity_field}_id", flat=True)
+            .distinct(),
+        )
+        remove_ids = existing_ids - add_ids
+        if remove_ids:
+            perm_model.objects.filter(
+                content_type=ctype,
+                object_pk__in=object_pks,
+                permission__codename=codename,
+                **{f"{identity_field}_id__in": remove_ids},
+            ).delete()
+
+    if not add_ids:
+        return
+
+    rows = [
+        perm_model(
+            content_type=ctype,
+            object_pk=pk,
+            permission=permission_obj,
+            **{f"{identity_field}_id": identity_id},
+        )
+        for permission_obj in permission_objs
+        for pk in object_pks
+        for identity_id in add_ids
+    ]
+    # ignore_conflicts skips only rows that already exist as an exact
+    # (identity, permission, object) match -- the same de-dup the
+    # underlying (user|group, permission, object_pk) unique constraint
+    # already enforces for the single-object assign_perm() this replaces,
+    # so it doesn't change what counts as "already granted". batch_size
+    # caps how many rows go into a single INSERT statement.
+    perm_model.objects.bulk_create(rows, ignore_conflicts=True, batch_size=1000)
+
+
+def set_permissions_for_objects(
+    permissions: dict,
+    model: type[Model],
+    pks: QuerySet | list,
+    *,
+    merge: bool = False,
+) -> None:
+    """
+    Bulk equivalent of set_permissions_for_object: applies the same
+    permission changes to every object identified by `pks` at once.
+
+    Takes a model + pks (rather than model instances) deliberately -- the
+    permission rows built below only ever need `pk`, `content_type`, and
+    identity ids, so callers shouldn't have to fetch full rows (with every
+    other field) just to hand them to this function.
+
+    Deliberately does not use guardian's queryset/list-aware assign_perm:
+    passing a list as the object routes to bulk_assign_perm, which skips
+    creating a direct permission row for anyone who already has the
+    permission via ANY group membership (it checks
+    ObjectPermissionChecker.has_perm, which is group-inheritance-aware) --
+    unlike the single-object assign_perm this replaces, which always
+    ensures a direct row via get_or_create regardless of group-derived
+    access. Losing that guarantee would mean a later revocation of the
+    group's grant silently strips access an admin explicitly asked to be
+    direct. Bulk-creating rows straight against the permission models
+    instead (see _apply_bulk_permission_entry) preserves the original
+    always-create-a-direct-row semantics while still batching every object
+    and every identity into one query per action, rather than one query per
+    (object, user) pair.
+    """
+    object_pks = [str(pk) for pk in pks]
+    if not object_pks:  # pragma: no cover
+        return
+
+    model_name = model.__name__.lower()
+    ctype = ContentType.objects.get_for_model(model)
+
+    # Every action is resolved up front, before anything is written, so an
+    # unrecognized action name (see _resolve_permissions) aborts the whole
+    # call instead of leaving the actions ahead of it already applied.
+    # SetPermissionsSerializer rejects unknown actions at the API, so this
+    # guards any other caller.
+    permissions_by_action: dict[str, list[Permission]] = {}
+    for action, entry in permissions.items():
+        if "users" not in entry and "groups" not in entry:
+            continue
+        implied_codenames = {f"{action}_{model_name}"}
+        if action == "change":
+            # change gives view too
+            implied_codenames.add(f"view_{model_name}")
+        permissions_by_action[action] = _resolve_permissions(
+            implied_codenames,
+            ctype,
+        )
+
+    for action, entry in permissions.items():
+        codename = f"{action}_{model_name}"
+        permission_objs = permissions_by_action.get(action, [])
+
+        if "users" in entry:
+            _apply_bulk_permission_entry(
+                perm_model=UserObjectPermission,
+                identity_model=User,
+                identity_field="user",
+                ids=entry["users"],
+                codename=codename,
+                permission_objs=permission_objs,
+                ctype=ctype,
+                object_pks=object_pks,
+                merge=merge,
+            )
+
+        if "groups" in entry:
+            _apply_bulk_permission_entry(
+                perm_model=GroupObjectPermission,
+                identity_model=Group,
+                identity_field="group",
+                ids=entry["groups"],
+                codename=codename,
+                permission_objs=permission_objs,
+                ctype=ctype,
+                object_pks=object_pks,
+                merge=merge,
+            )
+
+
 def permitted_object_ids(
     user: User | None,
     model: type[Model],
     perm: str,
     *,
     include_deleted: bool = False,
+    parent_field: str | None = None,
 ) -> QuerySet[int]:
     """
     Generic version of ``permitted_document_ids`` for any model with an
@@ -187,6 +361,24 @@ def permitted_object_ids(
     soft-delete pattern (currently only ``Document``); for every other model
     it is accepted but has no effect, since those models have no soft-delete
     concept.
+
+    ``parent_field`` names a self-referencing foreign key whose target
+    authorizes the row (``Document.root_document``). A row with a parent is
+    visible exactly when its parent is, judged by the parent's owner and
+    grants, so the row's own owner and grants are ignored.
+
+    Guardian stores ``object_pk`` as a string, so the row key is cast to a
+    string and tested against the user's and groups' grants with a single
+    uncorrelated ``IN``. Postgres and SQLite build that set once. MariaDB
+    evaluates it as an index probe per row, which is cheap because the
+    lookups use guardian's unique indexes. Casting every ``object_pk`` to an
+    integer instead cannot use an index, and MariaDB cannot materialize it
+    inside the owner ``OR``, so it re-scans the user's grants for every row.
+    A correlated ``EXISTS`` per grant fixes MariaDB too, but Postgres and
+    SQLite re-run it for every row and end up slower than the original. The
+    user's groups are matched with an ``IN`` subquery rather than a join
+    through the membership table, which SQLite plans badly once the grant
+    tables grow.
     """
     has_soft_delete = hasattr(model, "global_objects")
     manager = (
@@ -194,8 +386,14 @@ def permitted_object_ids(
     )
     base_qs = manager.all().only("id", "owner")
 
+    owner_field, key_field = "owner", "pk"
+    if parent_field is not None:
+        owner_field, key_field = "authorizing_owner", "authorizing_id"
+        base_qs = annotate_authorizing_fields(base_qs, parent_field)
+    unowned = Q(**{f"{owner_field}__isnull": True})
+
     if user is None or not getattr(user, "is_authenticated", False):
-        return base_qs.filter(owner__isnull=True).values_list("id", flat=True)
+        return base_qs.filter(unowned).values_list("id", flat=True)
 
     # Deactivated users get nothing, deactivated superusers included, so this
     # has to come before the superuser shortcut. guardian's
@@ -219,26 +417,51 @@ def permitted_object_ids(
         "permission__content_type": content_type,
     }
 
-    user_perm_ids = (
-        UserObjectPermission.objects.filter(user=user, **perm_filter)
-        .annotate(object_pk_int=Cast("object_pk", IntegerField()))
-        .values_list("object_pk_int", flat=True)
-    )
-    group_perm_ids = (
-        GroupObjectPermission.objects.filter(group__user=user, **perm_filter)
-        .annotate(object_pk_int=Cast("object_pk", IntegerField()))
-        .values_list("object_pk_int", flat=True)
-    )
-    permitted_ids = user_perm_ids.union(group_perm_ids)
+    # Both grant sets are compared to the row key as strings, exactly as
+    # guardian stores them, and are uncorrelated, so each engine can build the
+    # set once instead of probing per row.
+    user_keys = UserObjectPermission.objects.filter(
+        user=user,
+        **perm_filter,
+    ).values_list("object_pk", flat=True)
+    group_keys = GroupObjectPermission.objects.filter(
+        group_id__in=user.groups.values("id"),
+        **perm_filter,
+    ).values_list("object_pk", flat=True)
+    permitted_keys = user_keys.union(group_keys, all=True)
 
-    return base_qs.filter(
-        Q(owner=user) | Q(owner__isnull=True) | Q(id__in=permitted_ids),
-    ).values_list("id", flat=True)
+    return (
+        base_qs.annotate(permitted_key=Cast(key_field, CharField(max_length=64)))
+        .filter(
+            Q(**{owner_field: user.pk}) | unowned | Q(permitted_key__in=permitted_keys),
+        )
+        .values_list("id", flat=True)
+    )
 
 
 ModelT = TypeVar("ModelT", bound=Model)
 
 
+def annotate_authorizing_fields(
+    queryset: QuerySet[ModelT],
+    parent_field: str,
+) -> QuerySet[ModelT]:
+    """
+    Annotate each row with ``authorizing_id`` and ``authorizing_owner``: the id
+    and owner of the row that authorizes it. A row with a parent (the
+    self-referencing foreign key ``parent_field``) is authorized by its parent,
+    any other row by itself.
+    """
+    return queryset.annotate(
+        authorizing_id=Coalesce(f"{parent_field}_id", "id"),
+        authorizing_owner=Case(
+            When(**{f"{parent_field}_id__isnull": True}, then=F("owner_id")),
+            default=F(f"{parent_field}__owner_id"),
+            output_field=IntegerField(),
+        ),
+    )
+
+
 def user_is_unrestricted(user: User | None) -> bool:
     """
     True when ``user`` means "no restriction at all" (an absent user, or an
@@ -288,6 +511,28 @@ def restrict_queryset_to_visible(
     return queryset.filter(pk__in=permitted_object_ids(user, queryset.model, perm))
 
 
+def user_is_active_superuser(user: User | None) -> bool:
+    """
+    True only for an active, authenticated superuser.
+    """
+    return user is not None and user_is_unrestricted(user)
+
+
+def restrict_queryset_to_owner_visible(
+    queryset: QuerySet[ModelT],
+    user: User | None,
+    perm: str,
+) -> QuerySet[ModelT]:
+    """
+    Like ``restrict_queryset_to_visible``, but ``user=None`` sees only unowned
+    rows (``permitted_object_ids(None, ...)`` semantics) instead of every row.
+    See ``user_is_active_superuser``.
+    """
+    if user_is_active_superuser(user):
+        return queryset
+    return queryset.filter(pk__in=permitted_object_ids(user, queryset.model, perm))
+
+
 def permitted_document_ids(
     user: User | None,
     *,
@@ -301,8 +546,17 @@ def permitted_document_ids(
     ``include_deleted=True`` for callers that need to check permission on
     soft-deleted documents (e.g. trash restore). This intentionally avoids
     ``get_objects_for_user`` to keep the subquery small and index-friendly.
+
+    A version is authorized by its root document, so a version's own owner and
+    grants never matter.
     """
-    return permitted_object_ids(user, Document, perm, include_deleted=include_deleted)
+    return permitted_object_ids(
+        user,
+        Document,
+        perm,
+        include_deleted=include_deleted,
+        parent_field="root_document",
+    )
 
 
 def get_document_count_filter_for_user(user, related_name: str = "documents"):
@@ -461,7 +715,14 @@ def has_perms_owner_aware(user, perms, obj):
     single-object check still has many production callers. Several callers
     remain across ``documents/``, ``paperless_mail/``, and ``paperless_ai/``
     -- grep for this function name before removing it.
+
+    A document version is authorized by its root document, like in
+    ``permitted_document_ids``, so a version's own owner and grants never
+    matter. Fetch the root with ``select_related("root_document__owner")`` to
+    avoid extra queries.
     """
+    if isinstance(obj, Document):
+        obj = get_root_document(obj)
     checker = ObjectPermissionChecker(user)
     return obj.owner is None or obj.owner == user or checker.has_perm(perms, obj)
 
@@ -484,16 +745,41 @@ class ViewDocumentsPermissions(BasePermission):
         return request.user.has_perms(self.perms_map.get(request.method, []))
 
 
+class TrashPermissions(BasePermission):
+    """Check the global document permission for each trash operation."""
+
+    perms_map = {
+        "OPTIONS": ["documents.view_document"],
+        "HEAD": ["documents.view_document"],
+        "GET": ["documents.view_document"],
+        "POST": ["documents.delete_document"],
+    }
+
+    def has_permission(self, request, view):
+        if not request.user or not request.user.is_authenticated:  # pragma: no cover
+            return False
+
+        return request.user.has_perms(self.perms_map.get(request.method, []))
+
+
 class PaperlessNotePermissions(BasePermission):
     """
     Permissions class that checks for model permissions for Notes.
     """
 
     perms_map = {
-        "OPTIONS": ["documents.view_note"],
-        "GET": ["documents.view_note"],
-        "POST": ["documents.add_note"],
-        "DELETE": ["documents.delete_note"],
+        "OPTIONS": ["documents.view_note", "documents.view_document"],
+        "GET": ["documents.view_note", "documents.view_document"],
+        "POST": [
+            "documents.add_note",
+            "documents.view_document",
+            "documents.change_document",
+        ],
+        "DELETE": [
+            "documents.delete_note",
+            "documents.view_document",
+            "documents.change_document",
+        ],
     }
 
     def has_permission(self, request, view):

src/documents/search/__init__.py

@@ -6,13 +6,20 @@ from documents.search._backend import TantivyRelevanceList
 from documents.search._backend import WriteBatch
 from documents.search._backend import get_backend
 from documents.search._backend import reset_backend
+from documents.search._errors import InvalidDateQuery
+from documents.search._errors import InvalidNumberQuery
+from documents.search._errors import MultipleSearchQueryErrors
+from documents.search._errors import QueryTooLongError
+from documents.search._errors import SearchQueryError
+from documents.search._errors import search_query_error_messages
 from documents.search._schema import needs_rebuild
 from documents.search._schema import wipe_index
-from documents.search._translate import InvalidDateQuery
-from documents.search._translate import SearchQueryError
 
 __all__ = [
     "InvalidDateQuery",
+    "InvalidNumberQuery",
+    "MultipleSearchQueryErrors",
+    "QueryTooLongError",
     "SearchHit",
     "SearchIndexLockError",
     "SearchMode",
@@ -23,5 +30,6 @@ __all__ = [
     "get_backend",
     "needs_rebuild",
     "reset_backend",
+    "search_query_error_messages",
     "wipe_index",
 ]

src/documents/search/_backend.py

@@ -7,6 +7,7 @@ import threading
 import time
 from datetime import UTC
 from datetime import datetime
+from datetime import timedelta
 from enum import StrEnum
 from itertools import islice
 from typing import TYPE_CHECKING
@@ -22,8 +23,8 @@ import tantivy
 from django.conf import settings
 from django.utils.timezone import get_current_timezone
 
-from documents.search._query import build_permission_filter
 from documents.search._query import extract_cjk_text
+from documents.search._query import normalize_search_text
 from documents.search._query import parse_simple_text_highlight_query
 from documents.search._query import parse_simple_text_query
 from documents.search._query import parse_simple_title_query
@@ -31,6 +32,7 @@ from documents.search._query import parse_user_query
 from documents.search._schema import _write_sentinels
 from documents.search._schema import build_schema
 from documents.search._schema import open_or_rebuild_index
+from documents.search._schema import rebuild_in_progress
 from documents.search._schema import wipe_index
 from documents.search._tokenizer import ascii_fold
 from documents.search._tokenizer import autocomplete_tokens
@@ -40,6 +42,7 @@ from documents.utils import QuerySetStream
 from documents.utils import identity
 
 if TYPE_CHECKING:
+    from collections.abc import Iterable
     from collections.abc import Iterator
     from collections.abc import Sequence
     from pathlib import Path
@@ -53,6 +56,19 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger("paperless.search")
 
+# tantivy stores dates as signed 64-bit nanoseconds since the Unix epoch, which
+# covers 1677-09-21T00:12:43 to 2262-04-11T23:47:16 UTC
+_INDEX_DATE_NANOS_MIN: Final[int] = -(2**63)
+_INDEX_DATE_NANOS_MAX: Final[int] = 2**63 - 1
+_UNIX_EPOCH: Final[datetime] = datetime(1970, 1, 1, tzinfo=UTC)
+
+
+def _is_indexable_date(value: datetime) -> bool:
+    """Whether value, at whole-second precision, fits tantivy's date range."""
+    nanos = ((value - _UNIX_EPOCH) // timedelta(seconds=1)) * 1_000_000_000
+    return _INDEX_DATE_NANOS_MIN <= nanos <= _INDEX_DATE_NANOS_MAX
+
+
 _LOCK_TIMEOUT_SECONDS: Final[float] = 10.0  # per-attempt acquire timeout
 _LOCK_RETRY_ATTEMPTS: Final[int] = 4  # total attempts (1 initial + 3 retries)
 _LOCK_BACKOFF_BASE: Final[float] = 1.0  # seconds
@@ -195,52 +211,49 @@ class WriteBatch:
         return self._raw_writer
 
     def __enter__(self) -> Self:
-        if self._backend._path is not None:
-            lock_path = self._backend._path / ".tantivy.lock"
-            self._lock = filelock.FileLock(str(lock_path))
-            for attempt in range(_LOCK_RETRY_ATTEMPTS):
-                try:
-                    self._lock.acquire(timeout=self._lock_timeout)
-                    break
-                except filelock.Timeout:
-                    if attempt == _LOCK_RETRY_ATTEMPTS - 1:
-                        raise SearchIndexLockError(
-                            f"Could not acquire index lock after {_LOCK_RETRY_ATTEMPTS} "
-                            f"attempts (timeout={self._lock_timeout}s each)",
-                        )
-                    sleep_s = random.uniform(
-                        0,
-                        min(_LOCK_BACKOFF_CAP, _LOCK_BACKOFF_BASE * (2**attempt)),
+        lock_path = self._backend._path / ".tantivy.lock"
+        self._lock = filelock.FileLock(str(lock_path))
+        for attempt in range(_LOCK_RETRY_ATTEMPTS):
+            try:
+                self._lock.acquire(timeout=self._lock_timeout)
+                break
+            except filelock.Timeout:
+                if attempt == _LOCK_RETRY_ATTEMPTS - 1:
+                    raise SearchIndexLockError(
+                        f"Could not acquire index lock after {_LOCK_RETRY_ATTEMPTS} "
+                        f"attempts (timeout={self._lock_timeout}s each)",
                     )
-                    logger.debug(
-                        "Index lock contention; retrying in %.2fs (attempt %d/%d)",
-                        sleep_s,
-                        attempt + 1,
-                        _LOCK_RETRY_ATTEMPTS,
-                    )
-                    time.sleep(sleep_s)
+                sleep_s = random.uniform(
+                    0,
+                    min(_LOCK_BACKOFF_CAP, _LOCK_BACKOFF_BASE * (2**attempt)),
+                )
+                logger.debug(
+                    "Index lock contention; retrying in %.2fs (attempt %d/%d)",
+                    sleep_s,
+                    attempt + 1,
+                    _LOCK_RETRY_ATTEMPTS,
+                )
+                time.sleep(sleep_s)
 
-            # Open a fresh Index (and thus a fresh Tantivy ManagedDirectory)
-            # for the write, rather than reusing the process-local cached
-            # index. ManagedDirectory loads its GC bookkeeping (.managed.json)
-            # once, at construction, and never re-reads it; paperless runs
-            # several long-lived processes (Granian workers, Celery workers)
-            # that take turns writing under the file lock above. A cached,
-            # long-lived writer index would carry a stale managed-files view
-            # and, on commit, overwrite .managed.json with that stale view -
-            # permanently losing track of segment files other processes
-            # registered in the meantime, so they can never be garbage
-            # collected. Reopening fresh here always picks up the current
-            # on-disk state. The long-lived self._backend._index is used for
-            # reads only and is reloaded (not reopened) after commit below.
-            write_index = tantivy.Index(
-                build_schema(),
-                path=str(self._backend._path),
-            )
-            register_tokenizers(write_index, settings.SEARCH_LANGUAGE)
-            self._raw_writer = write_index.writer()
-        else:
-            self._raw_writer = self._backend._index.writer()
+        # Open a fresh Index (and thus a fresh Tantivy ManagedDirectory)
+        # for the write, rather than reusing the process-local cached
+        # index. ManagedDirectory loads its GC bookkeeping (.managed.json)
+        # once, at construction, and never re-reads it; paperless runs
+        # several long-lived processes (Granian workers, Celery workers)
+        # that take turns writing under the file lock above. A cached,
+        # long-lived writer index would carry a stale managed-files view
+        # and, on commit, overwrite .managed.json with that stale view -
+        # permanently losing track of segment files other processes
+        # registered in the meantime, so they can never be garbage
+        # collected. Reopening fresh here always picks up the current
+        # on-disk state. The long-lived self._backend._index is used for
+        # reads only and is reloaded (not reopened) after commit below.
+        write_index = tantivy.Index(
+            build_schema(),
+            path=str(self._backend._path),
+        )
+        register_tokenizers(write_index, settings.SEARCH_LANGUAGE)
+        self._raw_writer = write_index.writer()
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
@@ -271,9 +284,14 @@ class WriteBatch:
         and adding the new version. This ensures stale document data (e.g., after
         permission changes) doesn't persist in the index.
 
+        Only root documents are indexed, with their effective content, so a
+        version is indexed as its root document.
+
         Args:
             document: Django Document instance to index
         """
+        if document.root_document_id is not None:
+            document = document.root_document
         self.remove(document.pk)
         doc = self._backend._build_tantivy_doc(document)
         self._writer.add_document(doc)
@@ -298,22 +316,30 @@ class WriteBatch:
         An id with no matching document (e.g. deleted between the caller
         collecting ids and the batch running) is silently skipped, matching
         ``add_or_update()``'s existing single-document deferred-task behavior
-        rather than erroring or leaving a stale index entry.
+        rather than erroring or leaving a stale index entry. The id of a
+        version stands for its root document.
 
         Args:
             ids: Primary keys of Document instances to index
         """
         from documents.models import Document
         from documents.versioning import annotate_effective_content
+        from documents.versioning import root_document_ids
 
         ids = list(ids)
         if not ids:
             return
 
         queryset = annotate_effective_content(
-            Document.objects.filter(pk__in=ids)
+            Document.objects.filter(pk__in=root_document_ids(ids))
             .select_related("correspondent", "document_type", "storage_path", "owner")
-            .prefetch_related("tags", "notes__user", "custom_fields__field"),
+            .prefetch_related(
+                "tags",
+                "notes__user",
+                "custom_fields__field",
+                "barcodes",
+                "versions__barcodes",
+            ),
         )
         for document, grant in _DocumentViewerStream(queryset, chunk_size=1000):
             self.remove(document.pk)
@@ -325,14 +351,54 @@ class WriteBatch:
             self._writer.add_document(doc)
 
 
+def build_permission_filter(
+    schema: tantivy.Schema,
+    user: AbstractUser,
+    viewer_group_ids: Iterable[int] = (),
+) -> tantivy.Query:
+    """
+    Build a query filter for user document permissions.
+
+    Creates a query that matches only documents visible to the specified user
+    according to paperless-ngx permission rules:
+    - Public documents (no owner) are visible to all users
+    - Private documents are visible to their owner
+    - Documents explicitly shared with the user are visible
+    - Documents shared with one of the user's current groups are visible
+
+    Args:
+        schema: Tantivy schema for field validation
+        user: User to check permissions for
+        viewer_group_ids: Current group memberships for the user
+
+    Returns:
+        Tantivy query that filters results to visible documents
+    """
+    owner_any = tantivy.Query.exists_query("owner_id")
+    no_owner = tantivy.Query.boolean_query(
+        [
+            (tantivy.Occur.Must, tantivy.Query.all_query()),
+            (tantivy.Occur.MustNot, owner_any),
+        ],
+    )
+    owned = tantivy.Query.term_query(schema, "owner_id", user.pk)
+    shared = tantivy.Query.term_query(schema, "viewer_id", user.pk)
+    group_shared = [
+        tantivy.Query.term_query(schema, "viewer_group_id", group_id)
+        for group_id in viewer_group_ids
+    ]
+    return tantivy.Query.disjunction_max_query(
+        [no_owner, owned, shared, *group_shared],
+    )
+
+
 class TantivyBackend:
     """
     Tantivy search backend with explicit lifecycle management.
 
     Provides full-text search capabilities using the Tantivy search engine.
-    Supports in-memory indexes (for testing) and persistent on-disk indexes
-    (for production use). Handles document indexing, search queries, autocompletion,
-    and "more like this" functionality.
+    Keeps a persistent on-disk index. Handles document indexing, search queries,
+    autocompletion, and "more like this" functionality.
 
     The backend manages its own connection lifecycle and can be reset when
     the underlying index directory changes (e.g., during test isolation).
@@ -366,9 +432,7 @@ class TantivyBackend:
         },
     )
 
-    def __init__(self, path: Path | None = None):
-        # path=None → in-memory index (for tests)
-        # path=some_dir → on-disk index (for production)
+    def __init__(self, path: Path):
         self._path = path
         self._raw_index: tantivy.Index | None = None
         self._raw_schema: tantivy.Schema | None = None
@@ -387,16 +451,13 @@ class TantivyBackend:
         """
         Open or rebuild the index as needed.
 
-        For disk-based indexes, checks if rebuilding is needed due to schema
-        version or language changes. Registers custom tokenizers after opening.
+        Checks if rebuilding is needed due to schema version or language
+        changes. Registers custom tokenizers after opening.
         Safe to call multiple times - subsequent calls are no-ops.
         """
         if self._raw_index is not None:
             return  # pragma: no cover
-        if self._path is not None:
-            self._raw_index = open_or_rebuild_index(self._path)
-        else:
-            self._raw_index = tantivy.Index(build_schema())
+        self._raw_index = open_or_rebuild_index(self._path)
         register_tokenizers(self._raw_index, settings.SEARCH_LANGUAGE)
         self._raw_schema = self._raw_index.schema
 
@@ -421,6 +482,7 @@ class TantivyBackend:
     ) -> tantivy.Query:
         """Parse a user query string into a Tantivy Query object."""
         tz = get_current_timezone()
+        query = normalize_search_text(query)
         if search_mode is SearchMode.TEXT:
             return parse_simple_text_query(self._index, query)
         elif search_mode is SearchMode.TITLE:
@@ -469,79 +531,94 @@ class TantivyBackend:
         from guardian.shortcuts import get_groups_with_perms
         from guardian.shortcuts import get_users_with_perms
 
-        content = document.get_effective_content() or ""
+        # Every searchable string is normalized on the way in, and every
+        # query string on the way out (_parse_query), so the two agree on
+        # how a composed character is spelled. See normalize_search_text.
+        content = normalize_search_text(document.get_effective_content() or "")
+        title = normalize_search_text(document.title)
 
         doc = tantivy.Document()
 
         # Basic fields
         doc.add_unsigned("id", document.pk)
         doc.add_text("checksum", document.checksum)
-        doc.add_text("title", document.title)
-        doc.add_text("title_sort", document.title)
-        doc.add_text("simple_title", document.title)
+        doc.add_text("title", title)
+        doc.add_text("title_sort", title)
+        doc.add_text("simple_title", title)
         doc.add_text("content", content)
         doc.add_text("simple_content", content)
         # Bigram (character-ngram) fields exist for CJK substring search,
         # no need to bloat the bigram index with latin characters.
-        if cjk_title := extract_cjk_text(document.title):
+        if cjk_title := extract_cjk_text(title):
             doc.add_text("bigram_title", cjk_title)
         if content and (cjk_content := extract_cjk_text(content)):
             doc.add_text("bigram_content", cjk_content)
 
         # Original filename - only add if not None/empty
         if document.original_filename:
-            doc.add_text("original_filename", document.original_filename)
+            doc.add_text(
+                "original_filename",
+                normalize_search_text(document.original_filename),
+            )
 
         # Correspondent
         if document.correspondent:
-            doc.add_text("correspondent", document.correspondent.name)
-            doc.add_text("correspondent_sort", document.correspondent.name)
-            if cjk_corr := extract_cjk_text(document.correspondent.name):
+            correspondent = normalize_search_text(document.correspondent.name)
+            doc.add_text("correspondent", correspondent)
+            doc.add_text("correspondent_sort", correspondent)
+            if cjk_corr := extract_cjk_text(correspondent):
                 doc.add_text("bigram_correspondent", cjk_corr)
-            doc.add_unsigned("correspondent_id", document.correspondent_id)
 
         # Document type
         if document.document_type:
-            doc.add_text("document_type", document.document_type.name)
-            doc.add_text("type_sort", document.document_type.name)
-            if cjk_type := extract_cjk_text(document.document_type.name):
+            document_type = normalize_search_text(document.document_type.name)
+            doc.add_text("document_type", document_type)
+            doc.add_text("type_sort", document_type)
+            if cjk_type := extract_cjk_text(document_type):
                 doc.add_text("bigram_document_type", cjk_type)
-            doc.add_unsigned("document_type_id", document.document_type_id)
 
         # Storage path
         if document.storage_path:
-            doc.add_text("storage_path", document.storage_path.name)
-            doc.add_unsigned("storage_path_id", document.storage_path_id)
+            doc.add_text(
+                "storage_path",
+                normalize_search_text(document.storage_path.name),
+            )
 
         # Tags — collect names for autocomplete in the same pass
         tag_names: list[str] = []
         for tag in document.tags.all():
-            doc.add_text("tag", tag.name)
-            if cjk_tag := extract_cjk_text(tag.name):
+            tag_name = normalize_search_text(tag.name)
+            doc.add_text("tag", tag_name)
+            if cjk_tag := extract_cjk_text(tag_name):
                 doc.add_text("bigram_tag", cjk_tag)
-            doc.add_unsigned("tag_id", tag.pk)
-            tag_names.append(tag.name)
+            tag_names.append(tag_name)
 
         # Notes — JSON for structured queries (notes.user:alice, notes.note:text).
         # notes_text is a plain-text companion for snippet/highlight generation;
-        # tantivy's SnippetGenerator does not support JSON fields.
+        # tantivy's SnippetGenerator does not support JSON fields. It is not in
+        # _DEFAULT_SEARCH_FIELDS, so an unqualified query never searches it: a
+        # note matches through the JSON field or not at all.
         num_notes = 0
         note_texts: list[str] = []
         for note in document.notes.all():
             num_notes += 1
+            note_text = normalize_search_text(note.note)
             doc.add_json(
                 "notes",
                 {
-                    "note": note.note,
-                    "user": note.user.username if note.user else None,
+                    "note": note_text,
+                    "user": (
+                        normalize_search_text(note.user.username) if note.user else None
+                    ),
                 },
             )
-            note_texts.append(note.note)
+            note_texts.append(note_text)
         if note_texts:
             doc.add_text("notes_text", " ".join(note_texts))
 
-        # Custom fields — JSON for structured queries (custom_fields.name:x, custom_fields.value:y),
-        # companion text field for default full-text search.
+        # Custom fields: JSON for structured queries (custom_fields.name:x,
+        # custom_fields.value:y). There is no companion text field here, unlike
+        # notes: custom field values are reachable only through the JSON field.
         for cfi in document.custom_fields.all():
             search_value = cfi.value_for_search
             # Skip fields where there is no value yet
@@ -550,8 +627,18 @@ class TantivyBackend:
             doc.add_json(
                 "custom_fields",
                 {
-                    "name": cfi.field.name,
-                    "value": search_value,
+                    "name": normalize_search_text(cfi.field.name),
+                    "value": normalize_search_text(search_value),
+                },
+            )
+
+        # Barcodes: JSON field like custom_fields, only filled when stored
+        for barcode in document.get_effective_barcodes():
+            doc.add_json(
+                "barcodes",
+                {
+                    "value": normalize_search_text(barcode.value),
+                    "format": normalize_search_text(barcode.format),
                 },
             )
 
@@ -562,7 +649,15 @@ class TantivyBackend:
             document.created.day,
             tzinfo=UTC,
         )
-        doc.add_date("created", created_date)
+        if _is_indexable_date(created_date):
+            doc.add_date("created", created_date)
+        else:
+            logger.warning(
+                "Document %s has a created date (%s) outside the range the search "
+                "index can store; it will be indexed without a created date",
+                document.pk,
+                document.created,
+            )
         doc.add_date("modified", document.modified)
         doc.add_date("added", document.added)
 
@@ -605,11 +700,11 @@ class TantivyBackend:
             doc.add_unsigned("viewer_group_id", viewer_group_id)
 
         # Autocomplete words
-        text_sources = [document.title, content]
+        text_sources = [title, content]
         if document.correspondent:
-            text_sources.append(document.correspondent.name)
+            text_sources.append(correspondent)
         if document.document_type:
-            text_sources.append(document.document_type.name)
+            text_sources.append(document_type)
         text_sources.extend(tag_names)
 
         for word in sorted(_extract_autocomplete_words(text_sources)):
@@ -706,9 +801,22 @@ class TantivyBackend:
 
         self._ensure_open()
         user_query = self._parse_query(query, search_mode)
+        # _parse_query normalizes its own copy; the snippet queries below are
+        # built from the string directly, so normalize it here too.
+        query = normalize_search_text(query)
         highlight_query = user_query
         if search_mode is SearchMode.TEXT:
-            highlight_query = parse_simple_text_highlight_query(self._index, query)
+            try:
+                highlight_query = parse_simple_text_highlight_query(
+                    self._index,
+                    query,
+                )
+            except ValueError:
+                logger.debug(
+                    "Skipping simple text highlight query: token string is not "
+                    "valid tantivy query syntax: %r",
+                    query,
+                )
 
         # For notes_text snippet generation, we need a query that targets the
         # notes_text field directly. user_query may contain JSON-field terms
@@ -818,6 +926,7 @@ class TantivyBackend:
         sort_field: str | None = None,
         sort_reverse: bool = False,
         search_mode: SearchMode = SearchMode.QUERY,
+        filter_query: str | None = None,
         limit: int | None = None,
     ) -> list[int]:
         """
@@ -833,6 +942,7 @@ class TantivyBackend:
             sort_field: Field to sort by (None for relevance ranking)
             sort_reverse: Whether to reverse the sort order
             search_mode: Query parsing mode (QUERY, TEXT, or TITLE)
+            filter_query: Optional advanced query results must also match
             limit: Maximum number of IDs to return (None = all matching docs)
 
         Returns:
@@ -840,6 +950,13 @@ class TantivyBackend:
         """
         self._ensure_open()
         user_query = self._parse_query(query, search_mode)
+        if filter_query is not None:
+            user_query = user_query.and_must_match(
+                tantivy.Query.const_score_query(
+                    self._parse_query(filter_query, SearchMode.QUERY),
+                    0.0,  # zero so doesn't affect relevance
+                ),
+            )
         final_query = self._apply_permission_filter(user_query, user)
 
         searcher = self._index.searcher()
@@ -935,16 +1052,21 @@ class TantivyBackend:
         excluded from results.
 
         Args:
-            doc_id: Primary key of the reference document
+            doc_id: Primary key of the reference document, or of one of its
+                versions
             user: User for permission filtering (None for no filtering)
             limit: Maximum number of IDs to return (None = all matching docs)
 
         Returns:
             List of similar document IDs (excluding the original)
         """
+        from documents.versioning import root_document_ids
+
         self._ensure_open()
         searcher = self._index.searcher()
 
+        # Only root documents are indexed, so a version stands for its root
+        doc_id = next(iter(root_document_ids([doc_id])), doc_id)
         id_query = tantivy.Query.term_query(self._schema, "id", doc_id)
         results = searcher.search(id_query, limit=1)
 
@@ -1031,44 +1153,45 @@ class TantivyBackend:
                 writer's threads). Larger values buffer more docs in RAM before
                 flushing a segment, deferring merge work; they do not avoid it.
         """
-        # Create new index (on-disk or in-memory)
-        if self._path is not None:
-            wipe_index(self._path)
+        wipe_index(self._path)
+        # The marker covers the window where the empty index is already stamped
+        # as current but not yet populated, so an interrupted rebuild is retried.
+        with rebuild_in_progress(self._path):
             new_index = tantivy.Index(build_schema(), path=str(self._path))
             _write_sentinels(self._path)
-        else:
-            new_index = tantivy.Index(build_schema())
-        register_tokenizers(new_index, settings.SEARCH_LANGUAGE)
+            register_tokenizers(new_index, settings.SEARCH_LANGUAGE)
 
-        # Point instance at the new index so _build_tantivy_doc uses it
-        old_index, old_schema = self._raw_index, self._raw_schema
-        self._raw_index = new_index
-        self._raw_schema = new_index.schema
-        # Stream documents one-by-one (so the progress bar advances per
-        # document) while fetching viewer permissions one SQL query per chunk.
-        # The stream is Sized, so iter_wrapper can still discover the total.
-        documents_stream = _DocumentViewerStream(documents, chunk_size=1000)
-        try:
-            writer = new_index.writer(heap_size=writer_heap_bytes)
-            for document, (viewer_ids, viewer_group_ids) in iter_wrapper(
-                documents_stream,
-            ):
-                doc = self._build_tantivy_doc(
-                    document,
-                    viewer_ids=viewer_ids,
-                    viewer_group_ids=viewer_group_ids,
-                )
-                writer.add_document(doc)
-            writer.commit()
-            # Wait for background merge threads to finish so all segments are
-            # fully merged and persisted before the index is considered rebuilt.
-            writer.wait_merging_threads()
-            new_index.reload()
-        except BaseException:  # pragma: no cover
-            # Restore old index on failure so the backend remains usable
-            self._raw_index = old_index
-            self._raw_schema = old_schema
-            raise
+            # Point instance at the new index so _build_tantivy_doc uses it
+            old_index, old_schema = self._raw_index, self._raw_schema
+            self._raw_index = new_index
+            self._raw_schema = new_index.schema
+            # Stream documents one-by-one (so the progress bar advances per
+            # document) while fetching viewer permissions one SQL query per
+            # chunk. The stream is Sized, so iter_wrapper can still discover
+            # the total.
+            documents_stream = _DocumentViewerStream(documents, chunk_size=1000)
+            try:
+                writer = new_index.writer(heap_size=writer_heap_bytes)
+                for document, (viewer_ids, viewer_group_ids) in iter_wrapper(
+                    documents_stream,
+                ):
+                    doc = self._build_tantivy_doc(
+                        document,
+                        viewer_ids=viewer_ids,
+                        viewer_group_ids=viewer_group_ids,
+                    )
+                    writer.add_document(doc)
+                writer.commit()
+                # Wait for background merge threads to finish so all segments
+                # are fully merged and persisted before the index is considered
+                # rebuilt.
+                writer.wait_merging_threads()
+                new_index.reload()
+            except BaseException:  # pragma: no cover
+                # Restore old index on failure so the backend remains usable
+                self._raw_index = old_index
+                self._raw_schema = old_schema
+                raise
 
 
 def chunked(iterable, size):

src/documents/search/_dates.py

@@ -1,171 +0,0 @@
-from __future__ import annotations
-
-from datetime import UTC
-from datetime import date
-from datetime import datetime
-from datetime import timedelta
-from typing import TYPE_CHECKING
-from typing import Final
-
-from dateutil.relativedelta import relativedelta
-
-if TYPE_CHECKING:
-    from datetime import tzinfo
-
-_DATE_ONLY_FIELDS = frozenset({"created"})
-
-_TODAY: Final[str] = "today"
-_YESTERDAY: Final[str] = "yesterday"
-_PREVIOUS_WEEK: Final[str] = "previous week"
-_THIS_MONTH: Final[str] = "this month"
-_PREVIOUS_MONTH: Final[str] = "previous month"
-_THIS_YEAR: Final[str] = "this year"
-_PREVIOUS_YEAR: Final[str] = "previous year"
-_PREVIOUS_QUARTER: Final[str] = "previous quarter"
-
-_DATE_KEYWORDS = frozenset(
-    {
-        _TODAY,
-        _YESTERDAY,
-        _PREVIOUS_WEEK,
-        _THIS_MONTH,
-        _PREVIOUS_MONTH,
-        _THIS_YEAR,
-        _PREVIOUS_YEAR,
-        _PREVIOUS_QUARTER,
-    },
-)
-
-
-def _fmt(dt: datetime) -> str:
-    """Format a datetime as an ISO 8601 UTC string for use in Tantivy range queries."""
-    return dt.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-
-def _iso_range(lo: datetime, hi: datetime) -> str:
-    """
-    Format a half-open ``[lo TO hi)`` range in ISO 8601 for Tantivy query syntax.
-
-    ``hi`` is always the exclusive ceiling of a computed period (the start of
-    the *next* day/week/month/quarter/year), so the closing bracket must be
-    the Tantivy exclusive-range brace ``}`` rather than ``]`` — otherwise the
-    first instant of the following period (e.g. the 1st of next month) is
-    incorrectly included in the match.
-    """
-    return f"[{_fmt(lo)} TO {_fmt(hi)}}}"
-
-
-def _quarter_start(d: date) -> date:
-    """Return the first day of the calendar quarter containing ``d``."""
-    return date(d.year, ((d.month - 1) // 3) * 3 + 1, 1)
-
-
-def _midnight(d: date, tz: tzinfo) -> datetime:
-    """Convert a calendar date at local-timezone midnight to a UTC datetime."""
-    return datetime(d.year, d.month, d.day, tzinfo=tz).astimezone(UTC)
-
-
-def _keyword_bounds(keyword: str, tz: tzinfo) -> tuple[date, date]:
-    """
-    Map a relative date keyword to ``(start, exclusive_end)`` calendar dates.
-
-    ``tz`` only determines what "today" is; the caller decides how the returned
-    dates become UTC datetime boundaries (date-only vs. local-midnight offset).
-    """
-    today = datetime.now(tz).date()
-    if keyword == _TODAY:
-        return today, today + timedelta(days=1)
-    if keyword == _YESTERDAY:
-        return today - timedelta(days=1), today
-    if keyword == _PREVIOUS_WEEK:
-        this_monday = today - timedelta(days=today.weekday())
-        return this_monday - timedelta(weeks=1), this_monday
-    if keyword == _THIS_MONTH:
-        first = today.replace(day=1)
-        return first, first + relativedelta(months=1)
-    if keyword == _PREVIOUS_MONTH:
-        this_first = today.replace(day=1)
-        return this_first - relativedelta(months=1), this_first
-    if keyword == _THIS_YEAR:
-        return date(today.year, 1, 1), date(today.year + 1, 1, 1)
-    if keyword == _PREVIOUS_YEAR:
-        return date(today.year - 1, 1, 1), date(today.year, 1, 1)
-    if keyword == _PREVIOUS_QUARTER:
-        this_quarter = _quarter_start(today)
-        return this_quarter - relativedelta(months=3), this_quarter
-    raise ValueError(f"Unknown keyword: {keyword}")
-
-
-def _date_only_range(keyword: str, tz: tzinfo) -> str:
-    """
-    For `created` (DateField): use the local calendar date, converted to
-    midnight UTC boundaries. No offset arithmetic — date only.
-    """
-    start, end = _keyword_bounds(keyword, tz)
-    lo = datetime(start.year, start.month, start.day, tzinfo=UTC)
-    hi = datetime(end.year, end.month, end.day, tzinfo=UTC)
-    return _iso_range(lo, hi)
-
-
-def _datetime_range(keyword: str, tz: tzinfo) -> str:
-    """
-    For `added` / `modified` (DateTimeField, stored as UTC): convert local day
-    boundaries to UTC — full offset arithmetic required.
-    """
-    start, end = _keyword_bounds(keyword, tz)
-    return _iso_range(_midnight(start, tz), _midnight(end, tz))
-
-
-def _precision_bounds(digits: str) -> tuple[date, date] | None:
-    """
-    Map a 4/6/8-digit date token to (start, exclusive_end) calendar dates.
-
-    YYYY -> whole year, YYYYMM -> whole month, YYYYMMDD -> single day.
-    Returns None for any unparsable or out-of-range value (e.g. month 23),
-    so callers can emit a no-match clause instead of erroring (Whoosh parity).
-    """
-    try:
-        if len(digits) == 4:
-            year = int(digits)
-            return date(year, 1, 1), date(year + 1, 1, 1)
-        if len(digits) == 6:
-            year, month = int(digits[:4]), int(digits[4:6])
-            start = date(year, month, 1)
-            end = date(year + 1, 1, 1) if month == 12 else date(year, month + 1, 1)
-            return start, end
-        if len(digits) == 8:
-            start = date(int(digits[:4]), int(digits[4:6]), int(digits[6:8]))
-            return start, start + timedelta(days=1)
-    except ValueError:
-        return None
-    return None
-
-
-def _utc_bounds_for_field(
-    field: str,
-    start: date,
-    end: date,
-    tz: tzinfo,
-) -> tuple[datetime, datetime]:
-    """
-    Convert calendar-date bounds to UTC datetimes per the field's storage type.
-
-    For DateField (``created``) the bounds are UTC midnight (no offset). For
-    DateTimeField (``added``/``modified``) the bounds are local-tz midnight
-    converted to UTC, matching how each field is indexed.
-    """
-    if field in _DATE_ONLY_FIELDS:
-        return (
-            datetime(start.year, start.month, start.day, tzinfo=UTC),
-            datetime(end.year, end.month, end.day, tzinfo=UTC),
-        )
-    return (
-        datetime(start.year, start.month, start.day, tzinfo=tz).astimezone(UTC),
-        datetime(end.year, end.month, end.day, tzinfo=tz).astimezone(UTC),
-    )
-
-
-def _field_range_from_dates(field: str, start: date, end: date, tz: tzinfo) -> str:
-    """Build a Tantivy ``field:[lo TO hi]`` ISO range from calendar-date bounds."""
-    lo, hi = _utc_bounds_for_field(field, start, end, tz)
-    return f"{field}:{_iso_range(lo, hi)}"

src/documents/search/_errors.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+
+class SearchQueryError(ValueError):
+    """
+    Base for user-fixable search query errors.
+
+    Carries a message safe to surface to the user (no internal details). The
+    view layer catches this and returns an HTTP 400, so any future subclass
+    gets the same treatment.
+    """
+
+
+class InvalidDateQuery(SearchQueryError):
+    """Raised when a date field value or range bound cannot be parsed."""
+
+    def __init__(self, field: str | None, value: str | None) -> None:
+        self.field = field
+        self.value = value
+        super().__init__(f"Invalid date value {value!r} for field {field!r}.")
+
+
+class InvalidNumberQuery(SearchQueryError):
+    """Raised when a numeric field value or range bound cannot be parsed."""
+
+    def __init__(self, field: str | None, value: str | None) -> None:
+        self.field = field
+        self.value = value
+        super().__init__(f"Invalid numeric value {value!r} for field {field!r}.")
+
+
+class QueryTooLongError(SearchQueryError):
+    """Raised when a query string exceeds the maximum allowed length.
+
+    whoosh-compat's fieldname tagger is O(n^2) in plain word characters, so an
+    unbounded query is a CPU-exhaustion vector against a single request
+    handler. This is a hard boundary, not a validation nicety.
+    """
+
+    def __init__(self, length: int, limit: int) -> None:
+        self.length = length
+        self.limit = limit
+        super().__init__(
+            f"The search query is too long ({length} characters). "
+            f"The maximum allowed length is {limit} characters.",
+        )
+
+
+class MultipleSearchQueryErrors(SearchQueryError):
+    """Aggregates every user-fixable error from one parse, not just the first."""
+
+    def __init__(self, errors: Sequence[SearchQueryError]) -> None:
+        self.errors = tuple(errors)
+        super().__init__("; ".join(str(e) for e in self.errors))
+
+
+def search_query_error_messages(e: SearchQueryError) -> list[str]:
+    """The user-facing message list for a SearchQueryError.
+
+    Every offending value's message, not just the first, so the user can
+    fix them all in one round-trip. Shared by every view that maps
+    SearchQueryError to an HTTP 400.
+    """
+    if isinstance(e, MultipleSearchQueryErrors):
+        return [str(sub) for sub in e.errors]
+    return [str(e)]

src/documents/search/_fields.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from whoosh_compat import FieldKind
+from whoosh_compat import FieldSpec
+from whoosh_compat import SubpathSpec
+
+# Internal-only schema fields with no query-syntax meaning of their own
+# (sort shadow fields, bigram CJK fields, simple_title/simple_content,
+# autocomplete_word, notes_text) are NOT represented here, they are
+# declared in _schema.py's field_descriptors().
+#
+# analyzer/pattern_normalizer are deliberately left at FieldSpec's default
+# (None): they're language-specific and only meaningful to whoosh-compat's
+# parser, so _registry.py attaches them per-language via dataclasses.replace()
+# rather than PUBLIC_FIELDS declaring them itself. _schema.py only reads
+# name/kind/fast and never sees the analyzer at all.
+PUBLIC_FIELDS: tuple[FieldSpec, ...] = (
+    FieldSpec("title", FieldKind.TEXT),
+    FieldSpec("content", FieldKind.TEXT),
+    FieldSpec("correspondent", FieldKind.TEXT),
+    FieldSpec("document_type", FieldKind.TEXT, aliases=("type",)),
+    FieldSpec("storage_path", FieldKind.TEXT, aliases=("path",)),
+    FieldSpec("original_filename", FieldKind.TEXT),
+    FieldSpec("tag", FieldKind.TEXT, comma_values=True),
+    FieldSpec("checksum", FieldKind.KEYWORD),
+    FieldSpec("asn", FieldKind.U64, fast=True),
+    FieldSpec("page_count", FieldKind.U64, fast=True),
+    FieldSpec("num_notes", FieldKind.U64, fast=True),
+    FieldSpec("created", FieldKind.DATE, date_only=True, fast=True),
+    FieldSpec("modified", FieldKind.DATETIME, fast=True),
+    FieldSpec("added", FieldKind.DATETIME, fast=True),
+    FieldSpec(
+        "notes",
+        FieldKind.JSON,
+        subpaths={"user": SubpathSpec(), "note": SubpathSpec(default=True)},
+    ),
+    FieldSpec(
+        "custom_fields",
+        FieldKind.JSON,
+        subpaths={"name": SubpathSpec(), "value": SubpathSpec(default=True)},
+    ),
+    FieldSpec(
+        "barcodes",
+        FieldKind.JSON,
+        subpaths={"value": SubpathSpec(default=True), "format": SubpathSpec()},
+    ),
+)

src/documents/search/_query.py

@@ -1,32 +1,142 @@
 from __future__ import annotations
 
+import functools
 import logging
+import unicodedata
+from functools import cache
 from typing import TYPE_CHECKING
 from typing import Final
 
 import regex
 import tantivy
+import whoosh_compat as wc
 from django.conf import settings
+from whoosh_compat.emitters.tantivy_ import emit as tantivy_emit
+from whoosh_compat.errors import Cause
+from whoosh_compat.errors import Diagnostic
+from whoosh_compat.errors import DiagnosticKind
+from whoosh_compat.errors import QueryError
 
+from documents.search._errors import InvalidDateQuery
+from documents.search._errors import InvalidNumberQuery
+from documents.search._errors import MultipleSearchQueryErrors
+from documents.search._errors import SearchQueryError
+from documents.search._registry import get_field_registry
+from documents.search._tokenizer import _bigram_analyzer
+from documents.search._tokenizer import paperless_text_analyzer
 from documents.search._tokenizer import simple_search_tokens
-from documents.search._translate import SearchQueryError
-from documents.search._translate import translate_query
 
 if TYPE_CHECKING:
-    from collections.abc import Iterable
     from datetime import tzinfo
 
-    from django.contrib.auth.base_user import AbstractBaseUser
-
 logger = logging.getLogger("paperless.search")
 
-# Maximum seconds any single regex substitution may run.
-# Prevents ReDoS on adversarial user-supplied query strings.
+# Maximum seconds any single regex substitution over user-supplied query text
+# may run. The one remaining use is a character class, which cannot backtrack,
+# so the bound is an upper limit on that substitution's cost, not the ReDoS
+# guard it was originally written as.
 _REGEX_TIMEOUT: Final[float] = 1.0
 
 # Matches CJK/Hangul characters so queries can be routed to bigram fields.
 # Uses Unicode properties to cover all blocks including Extension B+ planes.
-_CJK_RE: Final = regex.compile(r"[\p{Han}\p{Hiragana}\p{Katakana}\p{Hangul}]+")
+# The marks that sit inside a Japanese word are listed explicitly, because
+# their Unicode script is Common and the script classes therefore miss
+# them: the katakana prolonged sound mark ー (U+30FC) and its halfwidth form
+# ｰ (U+FF70), the closing mark 〆 (U+3006), and the halfwidth voiced and
+# semi-voiced sound marks ﾞ (U+FF9E) and ﾟ (U+FF9F). Without them a word
+# splits into one-character runs, which have no bigrams: コーヒー becomes
+# コ + ヒ, and halfwidth ﾊﾟﾝ becomes ﾊ + ﾝ, leaving nothing to index or
+# search at all.
+#
+# The combining marks U+3099/U+309A are deliberately absent: everything
+# entering the index and every query string is put through
+# normalize_search_text first, so decomposed kana is composed away before
+# this pattern ever sees it. The halfwidth marks are not, and cannot be:
+# unlike パ (U+30D1), halfwidth katakana has no precomposed voiced form, so
+# NFC leaves ﾊ + ﾟ as two codepoints where it folds か + U+3099 into が.
+_CJK_RE: Final = regex.compile(
+    r"[\p{Han}\p{Hiragana}\p{Katakana}\p{Hangul}ーｰ〆ﾞﾟ]+",
+)
+
+
+def normalize_search_text(text: str) -> str:
+    """Put text into the one Unicode normal form the index is built in.
+
+    Both the indexed text and the query string go through this, because a
+    bigram is a pair of codepoints: NFD がっこう is four where NFC is three,
+    so an unnormalized document never matches a normalized query.
+
+    NFC, not NFKC: folding ﾊﾟﾝ to パン would be a search-behavior decision
+    rather than an encoding one.
+    """
+    return unicodedata.normalize("NFC", text)
+
+
+def _user_facing_emit_message(d: Diagnostic) -> str:
+    """A user-safe message for an emit-time QueryError's Diagnostic.
+
+    Built from the Diagnostic's structured fields (kind, field), never from
+    d.message: whoosh-compat documents that as developer/log output with no
+    stability guarantee, and PATTERN_TOO_COMPLEX embeds the raw backend
+    error text in it. SCHEMA_FIELD_MISSING never reaches here: _map_emit_error
+    re-raises it before calling this function, the same as INTERNAL.
+    """
+    field = str(d.field) if d.field is not None else None
+    if d.kind is DiagnosticKind.EXISTS_REQUIRES_FAST:
+        return f"Existence searches (field:*) are not supported for field {field!r}."
+    if d.kind is DiagnosticKind.TEXT_RANGE:
+        return f"Range searches are not supported for field {field!r}."
+    if d.kind is DiagnosticKind.PATTERN_TOO_COMPLEX:
+        return f"The wildcard pattern for field {field!r} is too complex."
+    logger.warning(
+        "Unmapped emit diagnostic %s: %s",
+        d.kind,
+        d.message,
+    )  # pragma: no cover
+    return "The search query could not be executed."  # pragma: no cover
+
+
+def _map_emit_error(e: QueryError) -> SearchQueryError:
+    """Route an emit-time QueryError by its Diagnostic's Cause.
+
+    INVALID_INPUT/UNSUPPORTED are user-input errors, exactly like a parse
+    diagnostic, and map to a 400. INTERNAL means a defect in whoosh-compat
+    or in our own AST handling, never the user's query, so the QueryError is
+    re-raised rather than converted, reaching the generic 500 handler instead
+    of blaming the query. MISCONFIGURED other than EXISTS_REQUIRES_FAST is
+    treated the same way as INTERNAL: the registry and the index schema
+    disagree, which only an operator can fix, and the exact same query would
+    succeed on its own once the index is rebuilt. That makes it a transient
+    server-side condition, not a permanently bad request, so it is logged as
+    an error and re-raised rather than converted to a 400: telling the
+    client their query is invalid would be wrong, it would work once the
+    index catches up, and a 400 also hides the condition from monitoring
+    that only watches 5xx rates.
+
+    EXISTS_REQUIRES_FAST is the one MISCONFIGURED kind that is not a
+    disagreement. whoosh-compat derives it from the registry's own FieldSpec
+    (kind plus fast) without ever consulting the index schema, so it fires
+    whenever a non-fast field of a kind that cannot answer "exists" is asked
+    to: for us that is only the JSON fields, which field_descriptors() builds
+    non-fast on purpose. "notes:*" and the five other spellings of it are
+    ordinary user error that no operator action can clear, so they get the
+    400 without the alert.
+    """
+    d = e.diagnostic
+    if d.cause is Cause.INTERNAL:
+        raise e
+    if (
+        d.cause is Cause.MISCONFIGURED
+        and d.kind is not DiagnosticKind.EXISTS_REQUIRES_FAST
+    ):
+        logger.error(
+            "Search index misconfiguration for field %s (%s): %s",
+            d.field,
+            d.kind.name,
+            d.message,
+        )
+        raise e
+    return SearchQueryError(_user_facing_emit_message(d))
 
 
 def _has_cjk(text: str) -> bool:
@@ -37,14 +147,35 @@ def _has_cjk(text: str) -> bool:
 def extract_cjk_text(text: str) -> str:
     """Join the CJK runs in ``text`` for indexing into bigram (char-ngram) fields.
 
-    Mirrors the query side (``_build_cjk_query``): only CJK runs are ever searched
-    against the bigram fields, so only CJK runs are worth indexing there. Latin
-    text fed to a character-bigram field is never matched and only bloats the
+    Mirrors the query side, which extracts the CJK runs of whatever it is
+    about to search for (the raw string in simple modes, each CJK term's
+    own text in query mode): only CJK runs are ever searched against
+    the bigram fields, so only CJK runs are worth indexing there. Latin text
+    fed to a character-bigram field is never matched and only bloats the
     index and slows indexing/merge. Returns "" when there is no CJK text.
     """
     return " ".join(_CJK_RE.findall(text))
 
 
+def _parse_cjk_text(
+    index: tantivy.Index,
+    cjk_text: str,
+    fields: list[str],
+) -> tantivy.Query | None:
+    """Parse a plain CJK run string against ``fields``, or None if it won't parse."""
+    try:
+        return index.parse_query(cjk_text, fields)
+    except Exception:
+        # Broad on purpose: cjk_text isn't filtered to a guaranteed-safe
+        # token set, so the exact failure mode tantivy could raise here
+        # isn't pinned down.
+        logger.debug(
+            "Skipping CJK search clause: could not parse CJK text: %r",
+            cjk_text,
+        )
+        return None
+
+
 def _build_cjk_query(
     index: tantivy.Index,
     raw_query: str,
@@ -52,91 +183,314 @@ def _build_cjk_query(
 ) -> tantivy.Query | None:
     """Build a bigram-field query from the CJK runs in ``raw_query``.
 
-    Only the CJK character runs are extracted and parsed; ASCII field prefixes,
-    boolean operators and date keywords are discarded. This keeps the CJK clause
-    plain-text and consistent across query/simple modes (no leaked ``field:``
-    semantics, no parse failures from spaced ``-``/``+``), and avoids feeding
-    Latin tokens into the character-bigram matcher (which would produce spurious
-    matches against unrelated Latin text). Returns None when there is no CJK
-    text or the parse fails.
+    For the simple (TEXT/TITLE) modes, whose input is plain text and carries
+    no query grammar to respect. Only the CJK character runs are extracted, so
+    a stray ``field:`` prefix or ``-``/``+`` in the input can neither leak
+    field semantics nor fail the parse, and no Latin token reaches the
+    character-bigram matcher (where it would produce spurious matches against
+    unrelated Latin text). Returns None when there is no CJK text or the parse
+    fails.
     """
-    cjk_text = " ".join(_CJK_RE.findall(raw_query))
+    cjk_text = extract_cjk_text(raw_query)
     if not cjk_text:
         return None
-    try:
-        return index.parse_query(cjk_text, fields)
-    except Exception:
-        return None
+    return _parse_cjk_text(index, cjk_text, fields)
 
 
-def build_permission_filter(
-    schema: tantivy.Schema,
-    user: AbstractBaseUser,
-    viewer_group_ids: Iterable[int] = (),
-) -> tantivy.Query:
-    """
-    Build a query filter for user document permissions.
-
-    Creates a query that matches only documents visible to the specified user
-    according to paperless-ngx permission rules:
-    - Public documents (no owner) are visible to all users
-    - Private documents are visible to their owner
-    - Documents explicitly shared with the user are visible
-    - Documents shared with one of the user's current groups are visible
-
-    Args:
-        schema: Tantivy schema for field validation
-        user: User to check permissions for
-        viewer_group_ids: Current group memberships for the user
-
-    Returns:
-        Tantivy query that filters results to visible documents
-    """
-    owner_any = tantivy.Query.exists_query("owner_id")
-    no_owner = tantivy.Query.boolean_query(
-        [
-            (tantivy.Occur.Must, tantivy.Query.all_query()),
-            (tantivy.Occur.MustNot, owner_any),
-        ],
-    )
-    owned = tantivy.Query.term_query(schema, "owner_id", user.pk)
-    shared = tantivy.Query.term_query(schema, "viewer_id", user.pk)
-    group_shared = [
-        tantivy.Query.term_query(schema, "viewer_group_id", group_id)
-        for group_id in viewer_group_ids
-    ]
-    return tantivy.Query.disjunction_max_query(
-        [no_owner, owned, shared, *group_shared],
-    )
-
-
-DEFAULT_SEARCH_FIELDS = [
+_DEFAULT_SEARCH_FIELDS: Final[list[str]] = [
     "title",
     "content",
     "correspondent",
     "document_type",
     "tag",
 ]
-SIMPLE_SEARCH_FIELDS = ["simple_title", "simple_content"]
-TITLE_SEARCH_FIELDS = ["simple_title"]
-_CJK_ALL_FIELDS: Final[list[str]] = [
-    "bigram_content",
-    "bigram_title",
-    "bigram_correspondent",
-    "bigram_document_type",
-    "bigram_tag",
-]
+_SIMPLE_SEARCH_FIELDS: Final[list[str]] = ["simple_title", "simple_content"]
+_TITLE_SEARCH_FIELDS: Final[list[str]] = ["simple_title"]
+# The bigram (character-ngram) companion of each default search field.
+_CJK_BIGRAM_FIELDS: Final[dict[str, str]] = {
+    field: f"bigram_{field}" for field in _DEFAULT_SEARCH_FIELDS
+}
 _CJK_CONTENT_FIELDS: Final[list[str]] = ["bigram_content"]
 _CJK_TITLE_FIELDS: Final[list[str]] = ["bigram_title"]
 _FIELD_BOOSTS = {"title": 2.0}
 _SIMPLE_FIELD_BOOSTS = {"simple_title": 2.0}
 
 
-def _simple_query_tokens(raw_query: str) -> list[str]:
-    # Tokenize and fold via the same analyzer used to index simple_title /
-    # simple_content, so query terms fold identically to the indexed terms
-    # (single source of truth for ASCII folding).
-    return simple_search_tokens(raw_query)
+@cache
+def _get_emit_field_registry(language: str | None) -> wc.FieldRegistry:
+    """The parse registry plus the CJK bigram fields, for analyzing and
+    emitting a query whose leaves _widen_leaf has widened. Cached per
+    language, on the same trigger get_field_registry() rebuilds on.
+
+    Never used to parse: the bigram fields are internal (absent from
+    PUBLIC_FIELDS), and queries are still parsed against
+    get_field_registry(), so ``bigram_content:...`` never becomes query
+    syntax.
+
+    The bigram specs set ``multitoken=Multitoken.AND`` explicitly. A
+    widened leaf's bigram side always sits inside the widening Or, so under
+    Multitoken.DEFAULT a run's bigrams would inherit that Or, and 東京都
+    would match a document containing only 京都.
+    """
+    bigram_analyze = _bigram_analyzer().analyze
+    return wc.FieldRegistry(
+        [
+            *get_field_registry(language),
+            *(
+                wc.FieldSpec(
+                    bigram_field,
+                    wc.FieldKind.TEXT,
+                    analyzer=bigram_analyze,
+                    multitoken=wc.Multitoken.AND,
+                )
+                for bigram_field in _CJK_BIGRAM_FIELDS.values()
+            ),
+        ],
+    )
+
+
+# Splits text exactly where the content analyzer's simple tokenizer does,
+# with none of its filters, so each piece is the raw text of one token the
+# index could hold. No remove_long either: a long CJK run must still reach
+# the bigram side.
+_TOKEN_SPLITTER: Final = tantivy.TextAnalyzerBuilder(tantivy.Tokenizer.simple()).build()
+
+
+def _collapse(
+    node_cls: type[wc.ast.Node],
+    children: list[wc.ast.Node],
+    span: dict[str, int | None],
+) -> wc.ast.Node:
+    """Return the single child as it is, or wrap several in node_cls."""
+    if len(children) == 1:
+        return children[0]
+    return node_cls(children=tuple(children), **span)
+
+
+def _leaf_span(leaf: wc.ast.Term | wc.ast.Phrase) -> dict[str, int | None]:
+    """The startchar/endchar kwargs a leaf's alternatives are built with,
+    so a rewritten leaf still points at the same span of the original
+    query text."""
+    return {"startchar": leaf.startchar, "endchar": leaf.endchar}
+
+
+def _cjk_alternative(leaf: wc.ast.Term | wc.ast.Phrase) -> wc.ast.Node | None:
+    """Build the bigram alternative for a CJK leaf, or None if it gets none.
+
+    The content analyzer keeps an unspaced CJK run as one token, so only
+    the bigram fields can find a CJK term inside running text.
+
+    The alternative is built from the leaf's own text, split where the
+    content analyzer would split it. What each kind of piece contributes,
+    and how the pieces combine, is commented at the step that decides it.
+    None means there was nothing to build one from.
+    """
+    if leaf.field is None or leaf.field.name not in _CJK_BIGRAM_FIELDS:
+        return None
+    text = str(leaf.text)
+    if not _has_cjk(text):
+        return None
+    span = _leaf_span(leaf)
+    bigram_field = wc.FieldRef(_CJK_BIGRAM_FIELDS[leaf.field.name])
+    cjk_terms: list[wc.ast.Node] = []
+    latin_terms: list[wc.ast.Node] = []
+    for token in _TOKEN_SPLITTER.analyze(text):
+        runs = _CJK_RE.findall(token)
+        if runs:
+            # One bigram Term per run, never a joined string, which would
+            # produce bigrams spanning the join. A run's own bigrams stay
+            # jointly required through multitoken=AND on the bigram
+            # FieldSpec (see _get_emit_field_registry), which no enclosing
+            # group can loosen. A one-character run has no bigram at all
+            # and analyzes away to nothing.
+            cjk_terms.extend(
+                wc.ast.Term(field=bigram_field, text=run, **span) for run in runs
+            )
+        else:
+            # Latin the analyzer split off on its own. Latin glued to CJK
+            # inside one token (東京report) never reaches here, and must
+            # not: the index holds it only inside that whole unspaced
+            # token, so requiring it would lose documents the run finds.
+            latin_terms.append(wc.ast.Term(field=leaf.field, text=token, **span))
+    # A Term's runs are alternatives to each other, the way the separate
+    # bigram clause treated them. A Phrase's are required together: quoting
+    # asks for more than the bare words, and the parser's default group is
+    # And, so an Or here would make "東京都 大阪府" match strictly more
+    # than 東京都 大阪府 does. And is also the tightest thing available,
+    # since the bigram analyzer puts every token at position 0 and no
+    # alternative built from it can enforce adjacency.
+    cjk_group = wc.ast.And if isinstance(leaf, wc.ast.Phrase) else wc.ast.Or
+    pieces: list[wc.ast.Node] = []
+    if cjk_terms:
+        pieces.append(_collapse(cjk_group, cjk_terms, span))
+    pieces.extend(latin_terms)
+    if not pieces:
+        # A few hundred codepoints match _CJK_RE but yield no token at all
+        # from the simple tokenizer (CJK radicals, circled and squared
+        # forms), leaving nothing to widen with. The caller then leaves the
+        # leaf as it is, and it analyzes to the same nothing it does today.
+        return None
+    # Separated latin is required alongside the CJK side, which is what
+    # stops "invoice NOT 東京-report" from excluding every 東京 document.
+    return _collapse(wc.ast.And, pieces, span)
+
+
+# The index analyzer minus stemming: the same word boundaries and the same
+# drops (remove_long, characters the simple tokenizer discards). The field's
+# pattern_normalizer does the one stemming step.
+_FUZZY_WORD_SPLITTER: Final = paperless_text_analyzer(None)
+
+
+def _fuzzy_alternative(leaf: wc.ast.Term | wc.ast.Phrase) -> wc.ast.Node | None:
+    """Build the near-match alternative for a leaf, or None if it gets none.
+
+    Each of the leaf's words becomes a Fuzzy leaf on the leaf's own field.
+    A Term's words are OR'd, which is the per-word recall the old clause
+    had and what lets ``COVID-19`` match on one half. A Phrase's are
+    AND-ed: quoting asks for more than the bare words, so an Or there
+    would make a quoted phrase match strictly more than the same words
+    unquoted. Adjacency is out of reach either way, so requiring every
+    word is the floor.
+
+    Words come from the index analyzer minus its stemmer, so a leaf gets a
+    fuzzy side exactly when its exact side has tokens. A regex split would
+    keep words the index never holds (``__``, or a word past remove_long),
+    whose exact side analyzes to nothing, leaving a required fuzzy clause
+    that can never match.
+
+    Words of one character are skipped: with prefix matching, a
+    one-character fuzzy term matches every term in the field.
+
+    CJK words are skipped entirely. The content analyzer keeps an unspaced
+    CJK run as one token, so a prefix Fuzzy over it matches any run within
+    one edit of its start: ``東京`` would match a document holding only
+    ``京都の観光案内``, the very thing the bigram fields' multitoken=AND
+    exists to prevent (see _get_emit_field_registry). A two-character CJK
+    word is as broad here as the one-character word the length guard
+    already rejects, and _cjk_alternative supplies the in-run recall
+    anyway, so there is nothing to gain and precision to lose.
+
+    Fuzzy text goes through the field's pattern_normalizer rather than its
+    analyzer, and the splitter's output is already lowercased and folded,
+    so the word is stemmed exactly once.
+    """
+    words = [
+        word
+        for word in _FUZZY_WORD_SPLITTER.analyze(str(leaf.text))
+        if len(word) > 1 and not _has_cjk(word)
+    ]
+    if not words:
+        return None
+    span = _leaf_span(leaf)
+    group = wc.ast.And if isinstance(leaf, wc.ast.Phrase) else wc.ast.Or
+    leaves: list[wc.ast.Node] = [
+        wc.ast.Fuzzy(field=leaf.field, text=word, distance=1, prefix=True, **span)
+        for word in words
+    ]
+    return _collapse(group, leaves, span)
+
+
+def _widen_leaf(
+    leaf: wc.ast.Term | wc.ast.Phrase,
+    *,
+    fuzzy: bool,
+    negated: frozenset[int],
+) -> wc.ast.Node:
+    """``rewrite_leaf`` hook for emit(): widen a leaf on a default search
+    field to ``Or(leaf, alternatives...)`` where it sits, and leave every
+    other leaf as it is.
+
+    Widening in place, rather than OR-ing a separate clause in at the top,
+    keeps every AND, NOT, REQUIRE, boost, field restriction and positive
+    filter around the leaf applying to its widened match too.
+
+    A leaf can gain a CJK alternative, a fuzzy one, or both. A negated
+    leaf keeps its CJK alternative, because NOT X should exclude exactly
+    what X matches, but gets no fuzzy one: with prefix matching, NOT tax
+    would otherwise exclude "taxi" and "taxonomy". Negated leaves are
+    identified by identity through the pre-scan, since the hook cannot see
+    a leaf's context.
+
+    analyze() only offers Term and Phrase leaves, so Prefix and Wildcard
+    patterns are never widened.
+    """
+    if leaf.field is None or leaf.field.name not in _DEFAULT_SEARCH_FIELDS:
+        return leaf
+    span = _leaf_span(leaf)
+    alternatives: list[wc.ast.Node] = []
+    cjk = _cjk_alternative(leaf)
+    if cjk is not None:
+        alternatives.append(cjk)
+    if fuzzy and id(leaf) not in negated:
+        near = _fuzzy_alternative(leaf)
+        if near is not None:
+            alternatives.append(wc.ast.Boosted(child=near, boost=0.1, **span))
+    if not alternatives:
+        return leaf
+    # The leaf itself, not a copy: analyze() then keeps it combined the way
+    # its enclosing group says rather than the way this Or would, and a long
+    # run its analyzer drops to nothing leaves just the alternative.
+    return wc.ast.Or(children=(leaf, *alternatives), **span)
+
+
+def _negated_leaf_ids(node: wc.ast.Node) -> frozenset[int]:
+    """Return the id() of every Term and Phrase under a negation.
+
+    A negated leaf keeps its CJK alternative but gets no fuzzy one, and
+    the hook cannot see a leaf's context, so the tree is walked once here
+    and the hook compares by identity. analyze() guarantees it is handed
+    the input tree's own leaf objects, which is what makes identity work.
+
+    Negative positions are Not.child and AndNot.negative, and nothing
+    else in the node set. Leaves are collected at any depth and under any
+    number of negations: over-collecting costs a widening, while missing
+    a negated leaf would let NOT tax exclude "taxi".
+
+    Iterative, and total over node types: this runs outside emit()'s
+    error conversion, so an exception here would reach the generic 500
+    handler.
+    """
+    negated: set[int] = set()
+    stack: list[tuple[wc.ast.Node, bool]] = [(node, False)]
+    while stack:
+        current, under_negation = stack.pop()
+        if isinstance(current, (wc.ast.Term, wc.ast.Phrase)):
+            if under_negation:
+                negated.add(id(current))
+        elif isinstance(current, wc.ast.Not):
+            stack.append((current.child, True))
+        elif isinstance(current, wc.ast.AndNot):
+            stack.append((current.positive, under_negation))
+            stack.append((current.negative, True))
+        elif isinstance(current, wc.ast.AndMaybe):
+            stack.append((current.required, under_negation))
+            stack.append((current.optional, under_negation))
+        elif isinstance(current, wc.ast.Require):
+            stack.append((current.scored, under_negation))
+            stack.append((current.filter_only, under_negation))
+        elif isinstance(current, wc.ast.Boosted):
+            stack.append((current.child, under_negation))
+        elif isinstance(current, (wc.ast.And, wc.ast.Or)):
+            stack.extend((child, under_negation) for child in current.children)
+    return frozenset(negated)
+
+
+# Weight of the tiebreak clause that restores relevance ordering among
+# near-miss results. The widened tree is const-scored so the threshold
+# cannot cut a near-miss by the BM25 spread of the query's correctly
+# spelled words; this small share of its real score orders them again.
+# A BM25 spread above 0.1/_FUZZY_TIEBREAK can still cut one.
+_FUZZY_TIEBREAK: Final[float] = 0.01
+
+
+def _any_of(clauses: list[tuple[tantivy.Occur, tantivy.Query]]) -> tantivy.Query:
+    """Collapse a clause list: none -> empty, one -> itself (no wasted
+    single-clause boolean_query wrapping), many -> boolean_query(clauses)."""
+    if not clauses:
+        return tantivy.Query.empty_query()
+    if len(clauses) == 1:
+        return clauses[0][1]
+    return tantivy.Query.boolean_query(clauses)
 
 
 def _build_simple_token_query(
@@ -168,9 +522,7 @@ def _build_simple_token_query(
             query = tantivy.Query.boost_query(query, boost)
         field_queries.append((tantivy.Occur.Should, query))
 
-    if len(field_queries) == 1:
-        return field_queries[0][1]
-    return tantivy.Query.boolean_query(field_queries)
+    return _any_of(field_queries)
 
 
 def parse_user_query(
@@ -179,78 +531,144 @@ def parse_user_query(
     tz: tzinfo,
 ) -> tantivy.Query:
     """
-    Parse user query through the complete preprocessing pipeline.
+    Parse user query through whoosh-compat, then widen its leaves and emit,
+    once or twice depending on whether fuzzy matching is on.
 
-    Transforms the raw user query through multiple stages:
-    1. Date keyword rewriting (today → ISO 8601 ranges)
-    2. Query normalization (comma expansion, whitespace cleanup)
-    3. Tantivy parsing with field boosts
-    4. Optional fuzzy query blending (if ADVANCED_FUZZY_SEARCH_THRESHOLD set)
-
-    Args:
-        index: Tantivy index with registered tokenizers
-        raw_query: Original user query string
-        tz: Timezone for date boundary calculations
-
-    Returns:
-        Parsed Tantivy query ready for execution
-
-    Note:
-        When ADVANCED_FUZZY_SEARCH_THRESHOLD is configured, adds a low-priority
-        fuzzy query as a Should clause (0.1 boost) to catch approximate matches
-        while keeping exact matches ranked higher. The threshold value is applied
-        as a post-search score filter, not during query construction.
+    1. wc.parse() against the shared FieldRegistry (whoosh grammar -> AST).
+       Bare notes:/custom_fields: prefixes resolve to their default subpath
+       (notes.note:/custom_fields.value:) directly in the registry, via
+       each JSON field's SubpathSpec(default=True).
+    2. Any diagnostics (bad dates/numbers) map to SearchQueryError subclasses
+       and raise, the view returns HTTP 400 with every offending field
+       listed, not just the first.
+    3. With fuzzy off, the AST is emitted once. If the query has CJK text,
+       emit()'s rewrite_leaf hook (_widen_leaf) rewrites each CJK term in
+       the AST to also match its bigram field, in place, so the rest of
+       the query constrains the bigram match too. With fuzzy on, the AST
+       is emitted twice through the same hook: once with fuzzy=True for
+       the widened tree used as a filter, and once with fuzzy=False for a
+       normally scored tree, and the two are blended into a boolean query
+       (see the comment above the blend for why). Either way the tree is
+       analyzed and emitted against _get_emit_field_registry() whenever
+       CJK or fuzzy widening applies, which adds the bigram fields; the
+       query itself was parsed without them.
+       emit() turns the AST into a tantivy.Query directly (no string
+       round-trip). A QueryError is routed by its Diagnostic's Cause
+       (_map_emit_error): a construct that parses but can't execute against
+       tantivy (e.g. a text-field range) is a 400, a registry/schema
+       mismatch is logged and re-raised, and an INTERNAL defect is
+       re-raised.
     """
+    registry = get_field_registry(settings.SEARCH_LANGUAGE)
+    result = wc.parse(
+        raw_query,
+        registry=registry,
+        default_fields=_DEFAULT_SEARCH_FIELDS,
+        field_boosts=_FIELD_BOOSTS,
+        tz=tz,
+    )
+    if result.diagnostics:
+        raise _diagnostics_to_error(result.diagnostics)
+
+    fuzzy_on = settings.ADVANCED_FUZZY_SEARCH_THRESHOLD is not None
+    cjk = _has_cjk(raw_query)
+    emit_registry = (
+        _get_emit_field_registry(settings.SEARCH_LANGUAGE)
+        if cjk or fuzzy_on
+        else registry
+    )
+    negated = _negated_leaf_ids(result.ast) if fuzzy_on else frozenset()
+
+    def emit_widened(*, fuzzy: bool) -> tantivy.Query:
+        hook = (
+            functools.partial(_widen_leaf, fuzzy=fuzzy, negated=negated)
+            if cjk or fuzzy
+            else None
+        )
+        return tantivy_emit(
+            result.ast,
+            index=index,
+            registry=emit_registry,
+            rewrite_leaf=hook,
+        )
 
     try:
-        query_str = translate_query(raw_query, tz)
-    except SearchQueryError:
-        # Intentional, user-fixable error (e.g. an unparsable date). Propagate so
-        # the view can return a 400 with a helpful message rather than falling
-        # back to the raw (still-invalid) query.
-        raise
-    except Exception:  # pragma: no cover - defensive
-        logger.warning("Query translation failed; using raw query", exc_info=True)
-        query_str = raw_query
+        if not fuzzy_on:
+            return emit_widened(fuzzy=False)
+        widened = emit_widened(fuzzy=True)
+        scored = emit_widened(fuzzy=False)
+    except QueryError as e:
+        raise _map_emit_error(e) from e
 
-    exact = index.parse_query(
-        query_str,
-        DEFAULT_SEARCH_FIELDS,
-        field_boosts=_FIELD_BOOSTS,
+    # The widened tree supplies the matched set at a flat score, so a
+    # near-miss is not ranked by how well the query's correctly spelled
+    # words matched. The CJK-only tree adds real scoring back for what
+    # matched exactly, and the last clause reintroduces the widened tree's
+    # own scoring at a small weight so near-misses still rank among
+    # themselves. const_score_query discards every boost inside it, the
+    # title field's 2.0 included, so a title match earns its boost through
+    # the second and third clauses rather than the first.
+    return tantivy.Query.boolean_query(
+        [
+            (tantivy.Occur.Must, tantivy.Query.const_score_query(widened, 0.1)),
+            (tantivy.Occur.Should, scored),
+            (
+                tantivy.Occur.Should,
+                tantivy.Query.boost_query(widened, _FUZZY_TIEBREAK),
+            ),
+        ],
     )
 
-    # The standard analyzer keeps a whitespace-free CJK run as a single token,
-    # so substring queries can't match content/title (and long runs are dropped
-    # by remove_long). Route CJK queries to the bigram fields, whose ngram
-    # tokenizer indexes overlapping 2-grams for substring matching.
-    cjk_query = (
-        _build_cjk_query(index, raw_query, _CJK_ALL_FIELDS)
-        if _has_cjk(raw_query)
-        else None
-    )
 
-    clauses: list[tuple[tantivy.Occur, tantivy.Query]] = [
-        (tantivy.Occur.Should, exact),
-    ]
+# The three whoosh-compat kinds for a wildcard on a field that cannot
+# carry one. d.field_kind supplies the discriminator, so naming the field's
+# type needs no second trip through the registry.
+_PATTERN_ON_KINDS: Final = frozenset(
+    {
+        DiagnosticKind.PATTERN_ON_NUMERIC,
+        DiagnosticKind.PATTERN_ON_BOOLEAN_EXISTS,
+        DiagnosticKind.PATTERN_ON_SUBPATH,
+    },
+)
 
-    threshold = settings.ADVANCED_FUZZY_SEARCH_THRESHOLD
-    if threshold is not None:
-        fuzzy = index.parse_query(
-            query_str,
-            DEFAULT_SEARCH_FIELDS,
-            field_boosts=_FIELD_BOOSTS,
-            # (prefix=True, distance=1, transposition_cost_one=True) — edit-distance fuzziness
-            fuzzy_fields={f: (True, 1, True) for f in DEFAULT_SEARCH_FIELDS},
+
+def _diagnostics_to_error(diagnostics: tuple[Diagnostic, ...]) -> SearchQueryError:
+    errors = [_single_diagnostic_to_error(d) for d in diagnostics]
+    return errors[0] if len(errors) == 1 else MultipleSearchQueryErrors(errors)
+
+
+def _single_diagnostic_to_error(d: Diagnostic) -> SearchQueryError:
+    # d.field is a FieldRef, not a str: str(d.field) gives the canonical
+    # dotted name (an aliased query, e.g. type:, reports document_type).
+    field_name = str(d.field) if d.field is not None else None
+    if d.kind is DiagnosticKind.BAD_DATE:
+        return InvalidDateQuery(field_name, d.raw_value)
+    if d.kind is DiagnosticKind.BAD_NUMBER:
+        return InvalidNumberQuery(field_name, d.raw_value)
+    if d.kind is DiagnosticKind.TOO_DEEP:
+        return SearchQueryError("The search query is nested too deeply.")
+    if d.kind in _PATTERN_ON_KINDS:
+        kind_label = f" ({d.field_kind.name.lower()})" if d.field_kind else ""
+        return SearchQueryError(
+            f"Wildcard patterns are not supported for field "
+            f"{field_name!r}{kind_label}.",
         )
-        # 0.1 boost keeps fuzzy hits ranked below exact matches (intentional)
-        clauses.append((tantivy.Occur.Should, tantivy.Query.boost_query(fuzzy, 0.1)))
-
-    if cjk_query is not None:
-        clauses.append((tantivy.Occur.Should, cjk_query))
-
-    if len(clauses) == 1:
-        return exact
-    return tantivy.Query.boolean_query(clauses)
+    if d.kind is DiagnosticKind.SINGLE_CHAR_BRACKET_RANGE:
+        field_label = f" for field {field_name!r}" if field_name else ""
+        return SearchQueryError(
+            f"{d.raw_value!r} looks like a bracket range{field_label}, but "
+            "'[' is not a wildcard character on its own. Combine it with a "
+            "wildcard, e.g. a trailing '*', or double-quote the value to "
+            "search it as literal text.",
+        )
+    logger.warning(
+        "Unmapped parse diagnostic %s: %s",
+        d.kind,
+        d.message,
+    )  # pragma: no cover
+    return SearchQueryError(
+        "The search query could not be executed.",
+    )  # pragma: no cover
 
 
 def parse_simple_query(
@@ -268,7 +686,7 @@ def parse_simple_query(
     CJK substrings the simple analyzer can't (long whitespace-free runs are
     dropped by remove_long).
     """
-    tokens = _simple_query_tokens(raw_query)
+    tokens = simple_search_tokens(raw_query)
 
     clauses: list[tuple[tantivy.Occur, tantivy.Query]] = []
     if tokens:
@@ -291,23 +709,14 @@ def parse_simple_query(
             )
             for token in tokens
         ]
-        simple_query = (
-            token_queries[0][1]
-            if len(token_queries) == 1
-            else tantivy.Query.boolean_query(token_queries)
-        )
-        clauses.append((tantivy.Occur.Should, simple_query))
+        clauses.append((tantivy.Occur.Should, _any_of(token_queries)))
 
     if cjk_fields and _has_cjk(raw_query):
         cjk_q = _build_cjk_query(index, raw_query, cjk_fields)
         if cjk_q is not None:
             clauses.append((tantivy.Occur.Should, cjk_q))
 
-    if not clauses:
-        return tantivy.Query.empty_query()
-    if len(clauses) == 1:
-        return clauses[0][1]
-    return tantivy.Query.boolean_query(clauses)
+    return _any_of(clauses)
 
 
 def parse_simple_text_highlight_query(
@@ -322,13 +731,21 @@ def parse_simple_text_highlight_query(
 
     # Strip Tantivy operator chars before tokenizing: this is a plain-text
     # highlight query, not a structured boolean query, so +/- are separators.
-    tokens = _simple_query_tokens(
+    tokens = simple_search_tokens(
         regex.sub(r"[-+]", " ", raw_query, timeout=_REGEX_TIMEOUT),
     )
     if not tokens:
         return tantivy.Query.empty_query()
 
-    return index.parse_query(" ".join(tokens), ["content"])
+    # Quote each token as its own phrase, escaping backslashes and embedded
+    # quotes. simple search tokens can carry arbitrary Tantivy syntax
+    # characters (`"`, `:`, `(`, `[`, `/`, ...) that the query-string parser
+    # would otherwise interpret as query grammar rather than literal text.
+    quoted_tokens = [
+        '"' + token.replace("\\", "\\\\").replace('"', '\\"') + '"' for token in tokens
+    ]
+
+    return index.parse_query(" ".join(quoted_tokens), ["content"])
 
 
 def parse_simple_text_query(
@@ -342,7 +759,7 @@ def parse_simple_text_query(
     return parse_simple_query(
         index,
         raw_query,
-        SIMPLE_SEARCH_FIELDS,
+        _SIMPLE_SEARCH_FIELDS,
         cjk_fields=_CJK_CONTENT_FIELDS,
     )
 
@@ -358,6 +775,6 @@ def parse_simple_title_query(
     return parse_simple_query(
         index,
         raw_query,
-        TITLE_SEARCH_FIELDS,
+        _TITLE_SEARCH_FIELDS,
         cjk_fields=_CJK_TITLE_FIELDS,
     )

src/documents/search/_registry.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+import dataclasses
+from typing import TYPE_CHECKING
+
+from whoosh_compat import FieldKind
+from whoosh_compat import FieldRegistry
+
+from documents.search._fields import PUBLIC_FIELDS
+from documents.search._tokenizer import ascii_fold
+from documents.search._tokenizer import paperless_text_analyzer
+from documents.search._tokenizer import stem_pattern_text
+
+if TYPE_CHECKING:
+    from whoosh_compat import PatternNormalizer
+
+_registry_cache: dict[str | None, FieldRegistry] = {}
+
+
+def _identity_analyzer(text: str) -> list[str]:
+    """Analyzer for KEYWORD fields indexed with the raw tokenizer (no splitting)."""
+    return [text]
+
+
+def _fold_normalizer(text: str) -> str:
+    """Wildcard/regex literal-run normalizer for fields indexed without stemming."""
+    return ascii_fold(text.lower())
+
+
+def _make_pattern_normalizer(language: str | None) -> PatternNormalizer:
+    """Build the wildcard/regex literal-run normalizer for a search language."""
+
+    def _pattern_normalizer(text: str) -> tuple[str, ...]:
+        """Normalize a literal run into the forms a term may match.
+
+        TEXT index terms go through lowercase -> ascii_fold -> stem, so a
+        pattern that skips stemming can never match one: "invoice*" would look
+        for a term starting with "invoice" while the index holds "invoic". The
+        run is therefore offered stemmed as well. KEYWORD fields are indexed
+        raw and get _fold_normalizer instead, so their patterns stay literal.
+
+        Both forms are returned, as alternatives, because neither is a prefix
+        of the other in general: English stemming substitutes as well as
+        truncates ("copy" -> "copi"), so the stem alone loses the compounds
+        the typed run reaches ("copyright") while the typed run alone loses
+        the inflections the stem reaches ("copies"). whoosh-compat ORs the
+        alternatives per literal run and deduplicates them, so a run the
+        stemmer leaves alone costs exactly the one branch it did before.
+
+        Inside a bracket class the emitter calls this once per character and
+        uses the answer only if it is a single one-character form; two forms
+        there leave the character as typed. A stemmer does not change a lone
+        character, so the two forms deduplicate to one and the class body is
+        folded as before.
+        """
+        folded = ascii_fold(text.lower())
+        stemmed = stem_pattern_text(folded, language)
+        return (folded, stemmed)
+
+    return _pattern_normalizer
+
+
+def get_field_registry(language: str | None) -> FieldRegistry:
+    """Build (or return the cached) FieldRegistry for the given search language.
+
+    Cached keyed by language, rebuilt on the same trigger register_tokenizers()
+    uses (settings.SEARCH_LANGUAGE change). A fresh call with a new language
+    builds and caches a new registry rather than mutating the old one.
+    """
+    if language in _registry_cache:
+        return _registry_cache[language]
+
+    text_analyzer = paperless_text_analyzer(language).analyze
+    pattern_normalizer = _make_pattern_normalizer(language)
+
+    specs = [
+        dataclasses.replace(
+            field,
+            analyzer=_identity_analyzer
+            if field.kind is FieldKind.KEYWORD
+            else text_analyzer,
+            pattern_normalizer=_fold_normalizer
+            if field.kind is FieldKind.KEYWORD
+            else pattern_normalizer,
+        )
+        for field in PUBLIC_FIELDS
+    ]
+
+    registry = FieldRegistry(specs)
+    _registry_cache[language] = registry
+    return registry

src/documents/search/_schema.py

@@ -1,22 +1,215 @@
 from __future__ import annotations
 
+import hashlib
 import json
 import logging
 import shutil
+from contextlib import contextmanager
 from typing import TYPE_CHECKING
 from typing import Final
+from typing import NamedTuple
 from typing import cast
 
 import tantivy
 from django.conf import settings
+from whoosh_compat import FieldKind
+
+from documents.search._fields import PUBLIC_FIELDS
 
 if TYPE_CHECKING:
+    from collections.abc import Iterator
     from pathlib import Path
 
 logger = logging.getLogger("paperless.search")
 
 # v1 - Initial tantivy schema format
-SCHEMA_VERSION: Final[int] = 1
+# v2 - build_schema() derived from PUBLIC_FIELDS, changing the field declaration
+#      order, and the write-only correspondent/document_type/storage_path/tag id
+#      columns dropped. tantivy compares schemas by ordered field list, so an
+#      index built by v1 rejects every write against the v2 schema.
+# v3 - barcodes JSON field for stored barcode contents
+# v4 - root documents only. Earlier indexes may hold document versions under
+#      their own id and metadata, so they are rebuilt without them.
+SCHEMA_VERSION: Final[int] = 4
+
+# Present in the index directory from the moment a full rebuild starts until it
+# finishes. If a rebuild is interrupted it is left behind, so the half-built
+# index is not mistaken for a complete one.
+REBUILD_MARKER: Final[str] = ".rebuilding"
+
+
+class FieldDescriptor(NamedTuple):
+    """One tantivy field, in declaration order.
+
+    The descriptor vocabulary is paperless', not tantivy-py's: it is both the
+    input to the SchemaBuilder and the input to schema_fingerprint(), so the
+    persisted fingerprint cannot move under a tantivy-py upgrade.
+    """
+
+    name: str
+    kind: str
+    stored: bool
+    indexed: bool
+    fast: bool
+    tokenizer: str | None
+
+
+# (schema kind, tokenizer) for the FieldKind -> FieldDescriptor mapping that
+# doesn't need special-casing. JSON is handled separately below since it can
+# emit a second, synthetic descriptor.
+_KIND_TABLE: Final[dict[FieldKind, tuple[str, str | None]]] = {
+    FieldKind.TEXT: ("text", "paperless_text"),
+    FieldKind.KEYWORD: ("text", "raw"),
+    FieldKind.U64: ("u64", None),
+    FieldKind.DATE: ("date", None),
+    FieldKind.DATETIME: ("date", None),
+}
+# Kinds whose fast-field flag follows FieldSpec.fast rather than always False.
+_FAST_FROM_FIELD: Final[frozenset[FieldKind]] = frozenset(
+    {FieldKind.U64, FieldKind.DATE, FieldKind.DATETIME},
+)
+
+
+def _public_field_descriptors() -> list[FieldDescriptor]:
+    """Descriptors for the query-visible fields declared in PUBLIC_FIELDS."""
+    descriptors: list[FieldDescriptor] = []
+    for field in PUBLIC_FIELDS:
+        if field.kind is FieldKind.JSON:
+            descriptors.append(
+                FieldDescriptor(
+                    field.name,
+                    "json",
+                    stored=True,
+                    indexed=True,
+                    fast=False,
+                    tokenizer="paperless_text",
+                ),
+            )
+            if field.name == "notes":
+                # Plain-text companion for snippet generation: tantivy's
+                # SnippetGenerator does not support JSON fields. Schema-only,
+                # no query-syntax meaning, not in PUBLIC_FIELDS.
+                descriptors.append(
+                    FieldDescriptor(
+                        "notes_text",
+                        "text",
+                        stored=True,
+                        indexed=True,
+                        fast=False,
+                        tokenizer="paperless_text",
+                    ),
+                )
+            continue
+        schema_kind, tokenizer = _KIND_TABLE[field.kind]
+        descriptors.append(
+            FieldDescriptor(
+                field.name,
+                schema_kind,
+                stored=True,
+                indexed=True,
+                fast=field.fast if field.kind in _FAST_FROM_FIELD else False,
+                tokenizer=tokenizer,
+            ),
+        )
+    return descriptors
+
+
+def field_descriptors() -> list[FieldDescriptor]:
+    """Every field of the document index, in the order tantivy declares them.
+
+    tantivy compares schemas by *ordered* field list, so the order here is
+    part of the on-disk contract: schema_fingerprint() hashes it and
+    needs_rebuild() acts on the result.
+    """
+    return [
+        FieldDescriptor(
+            "id",
+            "u64",
+            stored=True,
+            indexed=True,
+            fast=True,
+            tokenizer=None,
+        ),
+        *_public_field_descriptors(),
+        # Shadow sort fields - fast, not stored
+        *(
+            FieldDescriptor(
+                name,
+                "text",
+                stored=False,
+                indexed=True,
+                fast=True,
+                tokenizer="simple_analyzer",
+            )
+            for name in ("title_sort", "correspondent_sort", "type_sort")
+        ),
+        # CJK support - not stored, indexed only
+        *(
+            FieldDescriptor(
+                name,
+                "text",
+                stored=False,
+                indexed=True,
+                fast=False,
+                tokenizer="bigram_analyzer",
+            )
+            for name in (
+                "bigram_content",
+                "bigram_title",
+                "bigram_correspondent",
+                "bigram_document_type",
+                "bigram_tag",
+            )
+        ),
+        # Simple substring search support for title/content - not stored,
+        # indexed only
+        *(
+            FieldDescriptor(
+                name,
+                "text",
+                stored=False,
+                indexed=True,
+                fast=False,
+                tokenizer="simple_search_analyzer",
+            )
+            for name in ("simple_title", "simple_content")
+        ),
+        # Autocomplete prefix scan via terms_with_prefix, which walks the
+        # field's term dictionary - so the field must be indexed (term dict),
+        # not stored. The stored value is never read back, so storing it only
+        # wastes space.
+        FieldDescriptor(
+            "autocomplete_word",
+            "text",
+            stored=False,
+            indexed=True,
+            fast=False,
+            tokenizer="raw",
+        ),
+        # Permission filter columns, read by build_permission_filter.
+        *(
+            FieldDescriptor(
+                name,
+                "u64",
+                stored=False,
+                indexed=True,
+                fast=True,
+                tokenizer=None,
+            )
+            for name in ("owner_id", "viewer_id", "viewer_group_id")
+        ),
+    ]
+
+
+def schema_fingerprint() -> str:
+    """Hash of the field descriptors, stamped into .index_settings.json.
+
+    Changes whenever a field is added, removed, retyped, re-optioned or
+    reordered, so an index built from a different schema shape is detected
+    even when SCHEMA_VERSION was not bumped.
+    """
+    payload = json.dumps([list(descriptor) for descriptor in field_descriptors()])
+    return hashlib.blake2b(payload.encode()).hexdigest()
 
 
 def build_schema() -> tantivy.Schema:
@@ -32,85 +225,37 @@ def build_schema() -> tantivy.Schema:
     """
     sb = tantivy.SchemaBuilder()
 
-    sb.add_unsigned_field("id", stored=True, indexed=True, fast=True)
-    sb.add_text_field("checksum", stored=True, tokenizer_name="raw")
-
-    for field in (
-        "title",
-        "correspondent",
-        "document_type",
-        "storage_path",
-        "original_filename",
-        "content",
-    ):
-        sb.add_text_field(field, stored=True, tokenizer_name="paperless_text")
-
-    # Shadow sort fields - fast, not stored/indexed
-    for field in ("title_sort", "correspondent_sort", "type_sort"):
-        sb.add_text_field(
-            field,
-            stored=False,
-            tokenizer_name="simple_analyzer",
-            fast=True,
-        )
-
-    # CJK support - not stored, indexed only
-    sb.add_text_field("bigram_content", stored=False, tokenizer_name="bigram_analyzer")
-    sb.add_text_field("bigram_title", stored=False, tokenizer_name="bigram_analyzer")
-    sb.add_text_field(
-        "bigram_correspondent",
-        stored=False,
-        tokenizer_name="bigram_analyzer",
-    )
-    sb.add_text_field(
-        "bigram_document_type",
-        stored=False,
-        tokenizer_name="bigram_analyzer",
-    )
-    sb.add_text_field("bigram_tag", stored=False, tokenizer_name="bigram_analyzer")
-
-    # Simple substring search support for title/content - not stored, indexed only
-    sb.add_text_field(
-        "simple_title",
-        stored=False,
-        tokenizer_name="simple_search_analyzer",
-    )
-    sb.add_text_field(
-        "simple_content",
-        stored=False,
-        tokenizer_name="simple_search_analyzer",
-    )
-
-    # Autocomplete prefix scan via terms_with_prefix, which walks the field's
-    # term dictionary - so the field must be indexed (term dict), not stored.
-    # The stored value is never read back, so storing it only wastes space.
-    sb.add_text_field("autocomplete_word", stored=False, tokenizer_name="raw")
-
-    sb.add_text_field("tag", stored=True, tokenizer_name="paperless_text")
-
-    # JSON fields — structured queries: notes.user:alice, custom_fields.name:invoice
-    sb.add_json_field("notes", stored=True, tokenizer_name="paperless_text")
-    # Plain-text companion for notes — tantivy's SnippetGenerator does not support
-    # JSON fields, so highlights require a text field with the same content.
-    sb.add_text_field("notes_text", stored=True, tokenizer_name="paperless_text")
-    sb.add_json_field("custom_fields", stored=True, tokenizer_name="paperless_text")
-
-    for field in (
-        "correspondent_id",
-        "document_type_id",
-        "storage_path_id",
-        "tag_id",
-        "owner_id",
-        "viewer_id",
-        "viewer_group_id",
-    ):
-        sb.add_unsigned_field(field, stored=False, indexed=True, fast=True)
-
-    for field in ("created", "modified", "added"):
-        sb.add_date_field(field, stored=True, indexed=True, fast=True)
-
-    for field in ("asn", "page_count", "num_notes"):
-        sb.add_unsigned_field(field, stored=True, indexed=True, fast=True)
+    for descriptor in field_descriptors():
+        if descriptor.kind == "text":
+            sb.add_text_field(
+                descriptor.name,
+                stored=descriptor.stored,
+                fast=descriptor.fast,
+                tokenizer_name=cast("str", descriptor.tokenizer),
+            )
+        elif descriptor.kind == "json":
+            sb.add_json_field(
+                descriptor.name,
+                stored=descriptor.stored,
+                fast=descriptor.fast,
+                tokenizer_name=cast("str", descriptor.tokenizer),
+            )
+        elif descriptor.kind == "u64":
+            sb.add_unsigned_field(
+                descriptor.name,
+                stored=descriptor.stored,
+                indexed=descriptor.indexed,
+                fast=descriptor.fast,
+            )
+        elif descriptor.kind == "date":
+            sb.add_date_field(
+                descriptor.name,
+                stored=descriptor.stored,
+                indexed=descriptor.indexed,
+                fast=descriptor.fast,
+            )
+        else:
+            raise ValueError(f"Unknown schema field kind: {descriptor.kind}")
 
     return sb.build()
 
@@ -119,9 +264,9 @@ def needs_rebuild(index_dir: Path) -> bool:
     """
     Check if the search index needs rebuilding.
 
-    Reads .index_settings.json to compare the stored schema version and
-    search language against the current configuration. Returns True if the
-    file is missing, unparsable, or either value mismatches.
+    True if a previous full rebuild never finished (the rebuild marker is still
+    present), or if the index's stamped settings no longer match the current
+    configuration. See _settings_mismatch().
 
     Args:
         index_dir: Path to the search index directory
@@ -129,6 +274,40 @@ def needs_rebuild(index_dir: Path) -> bool:
     Returns:
         True if the index needs rebuilding, False if it's up to date
     """
+    if (index_dir / REBUILD_MARKER).exists():
+        logger.warning("Previous search index rebuild did not finish - rebuilding.")
+        return True
+    return _settings_mismatch(index_dir)
+
+
+@contextmanager
+def rebuild_in_progress(index_dir: Path) -> Iterator[None]:
+    """
+    Flag the index as incomplete for the duration of a full rebuild.
+
+    The marker is cleared only if the block exits cleanly. There is deliberately
+    no try/finally: an exception must leave the marker behind so the next
+    needs_rebuild() check retries the rebuild.
+    """
+    marker = index_dir / REBUILD_MARKER
+    marker.touch()
+    yield
+    marker.unlink(missing_ok=True)
+
+
+def _settings_mismatch(index_dir: Path) -> bool:
+    """
+    Check the stamped settings against the current configuration.
+
+    Reads .index_settings.json to compare the stored schema version, search
+    language and schema fingerprint. Returns True if the file is missing,
+    unparsable, or any value mismatches.
+
+    This deliberately ignores the rebuild marker: open_or_rebuild_index() uses it
+    so that a process opening the index while another process is mid-rebuild
+    (or after one died) does not wipe the partial index out from under it.
+    Repopulating is the job of ``document_index reindex``.
+    """
     settings_file = index_dir / ".index_settings.json"
     if not settings_file.exists():
         return True
@@ -140,6 +319,9 @@ def needs_rebuild(index_dir: Path) -> bool:
         if "language" not in data or data["language"] != settings.SEARCH_LANGUAGE:
             logger.info("Search index language changed - rebuilding.")
             return True
+        if data.get("schema_fingerprint") != schema_fingerprint():
+            logger.info("Search index schema fingerprint mismatch - rebuilding.")
+            return True
     except ValueError:
         return True
     return False
@@ -170,6 +352,7 @@ def _write_sentinels(index_dir: Path) -> None:
             {
                 "schema_version": SCHEMA_VERSION,
                 "language": settings.SEARCH_LANGUAGE,
+                "schema_fingerprint": schema_fingerprint(),
             },
         ),
     )
@@ -193,9 +376,18 @@ def open_or_rebuild_index(index_dir: Path | None = None) -> tantivy.Index:
         index_dir = cast("Path", settings.INDEX_DIR)
     if not index_dir.exists():
         return tantivy.Index(build_schema())
-    if needs_rebuild(index_dir):
+    if _settings_mismatch(index_dir):
+        wipe_index(index_dir)
+        idx = tantivy.Index(build_schema(), path=str(index_dir))
+        _write_sentinels(index_dir)
+        return idx
+    try:
+        return tantivy.Index.open(str(index_dir))
+    except ValueError:
+        logger.exception(
+            "Search index is corrupted or incomplete - rebuilding from scratch.",
+        )
         wipe_index(index_dir)
         idx = tantivy.Index(build_schema(), path=str(index_dir))
         _write_sentinels(index_dir)
         return idx
-    return tantivy.Index.open(str(index_dir))

src/documents/search/_tokenizer.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+from functools import cache
 from typing import Final
 
 import tantivy
@@ -71,7 +72,7 @@ def register_tokenizers(index: tantivy.Index, language: str | None) -> None:
         use fast=True and Tantivy requires fast-field tokenizers to exist
         even for documents that omit those fields.
     """
-    index.register_tokenizer("paperless_text", _paperless_text(language))
+    index.register_tokenizer("paperless_text", paperless_text_analyzer(language))
     index.register_tokenizer("simple_analyzer", _simple_analyzer())
     index.register_tokenizer("bigram_analyzer", _bigram_analyzer())
     index.register_tokenizer("simple_search_analyzer", _simple_search_analyzer())
@@ -79,7 +80,7 @@ def register_tokenizers(index: tantivy.Index, language: str | None) -> None:
     index.register_fast_field_tokenizer("simple_analyzer", _simple_analyzer())
 
 
-def _paperless_text(language: str | None) -> tantivy.TextAnalyzer:
+def paperless_text_analyzer(language: str | None) -> tantivy.TextAnalyzer:
     """Main full-text tokenizer for content, title, etc: simple -> remove_long(129) -> lowercase -> ascii_fold [-> stemmer]"""
     builder = (
         tantivy.TextAnalyzerBuilder(tantivy.Tokenizer.simple())
@@ -100,6 +101,54 @@ def _paperless_text(language: str | None) -> tantivy.TextAnalyzer:
     return builder.build()
 
 
+@cache
+def _pattern_stemmer(language: str | None) -> tantivy.TextAnalyzer | None:
+    """The stemming tail of paperless_text_analyzer, over a whole literal run.
+
+    Same language gate and same Snowball stemmer paperless_text_analyzer
+    applies at index time, so query patterns follow SEARCH_LANGUAGE. Returns
+    None when that gate disables stemming; paperless_text_analyzer already
+    warns about an unsupported language, so this stays quiet.
+
+    The raw tokenizer keeps the run whole (a wildcard literal is a fragment,
+    not necessarily a word), and remove_long is kept so an over-long run is
+    treated the same way the index treats it.
+    """
+    if not language:
+        return None
+    tantivy_lang = _LANGUAGE_MAP.get(language.lower())
+    if tantivy_lang is None:
+        return None
+    return (
+        tantivy.TextAnalyzerBuilder(tantivy.Tokenizer.raw())
+        .filter(tantivy.Filter.remove_long(_TOKEN_REMOVE_LONG_LIMIT))
+        .filter(tantivy.Filter.stemmer(tantivy_lang))
+        .build()
+    )
+
+
+def stem_pattern_text(text: str, language: str | None) -> str:
+    """Stem an already lowercased/ascii-folded run the way index terms are.
+
+    Returns text unchanged when stemming is disabled for language, and also
+    when the stem step does not yield exactly one token: remove_long drops a run
+    past the length limit, leaving no stem to substitute. Falling back to the
+    text as typed is the safe direction for a pattern prefix, since it can only
+    be as narrow as it was before stemming was considered.
+
+    The raw tokenizer emits one token whatever the input and the stemmer is
+    1-to-1, so only the zero-token case can fire today; the guard covers both
+    counts so a tokenizer change cannot turn this into an IndexError.
+    """
+    analyzer = _pattern_stemmer(language)
+    if analyzer is None:
+        return text
+    tokens = analyzer.analyze(text)
+    if len(tokens) != 1:
+        return text
+    return tokens[0]
+
+
 def _simple_analyzer() -> tantivy.TextAnalyzer:
     """Tokenizer for shadow sort fields (title_sort, correspondent_sort, type_sort): simple -> lowercase -> ascii_fold."""
     return (

src/documents/search/_translate.py

@@ -1,610 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-from datetime import UTC
-from datetime import datetime
-from datetime import timedelta
-from typing import TYPE_CHECKING
-from typing import TypeAlias
-
-import regex
-from dateutil.relativedelta import relativedelta
-
-from documents.search._dates import _DATE_KEYWORDS
-from documents.search._dates import _DATE_ONLY_FIELDS
-from documents.search._dates import _date_only_range
-from documents.search._dates import _datetime_range
-from documents.search._dates import _field_range_from_dates
-from documents.search._dates import _fmt
-from documents.search._dates import _precision_bounds
-from documents.search._dates import _utc_bounds_for_field
-
-# Compiled regex that matches any known multi-word (or single-word) date keyword
-# at the start of a match position, longest alternatives first so "previous week"
-# wins over a hypothetical shorter "previous".
-_KEYWORD_VALUE_RE = regex.compile(
-    "|".join(sorted((regex.escape(k) for k in _DATE_KEYWORDS), key=len, reverse=True)),
-    regex.IGNORECASE,
-)
-
-if TYPE_CHECKING:
-    from datetime import tzinfo
-
-# TODO: this module translates date queries into Tantivy *string* syntax, which
-# forces a workaround for something Tantivy's string parser cannot express on
-# date fields: open-ended ranges use far-past/far-future string sentinels
-# (OPEN_LO/OPEN_HI). These can be replaced with a real tantivy.Query object
-# (Query.range_query(..., None) for open bounds) once tantivy-py accepts Python
-# datetimes in range_query/term_query on Date fields. That support exists on
-# tantivy-py master (PRs #655 + #666) but postdates the pinned 0.26.0 wheel, so
-# it is blocked only on a published release > 0.26.0 and a dependency bump.
-# (Unparsable dates now raise InvalidDateQuery -> HTTP 400 rather than using a
-# no-match string sentinel.)
-
-# Fields that store exact, non-analyzed comma-joined tokens in the index and so
-# need explicit comma->AND expansion (Whoosh KEYWORD(commas=True) set).
-MULTI_VALUE_FIELDS = frozenset({"tag", "tag_id", "viewer_id"})
-
-# Date fields whose values/ranges get rewritten to RFC3339 Tantivy ranges.
-DATE_FIELDS = frozenset({"created", "modified", "added"})
-
-# Field aliases: Whoosh (v2) field names that were renamed in the Tantivy schema.
-# Preserved here so v2 queries using the old names continue to work without 400
-# errors instead of silently failing. Applied by _render to non-date field tokens.
-FIELD_ALIASES: dict[str, str] = {
-    "type": "document_type",
-    "type_id": "document_type_id",
-    "path": "storage_path",
-    "path_id": "storage_path_id",
-}
-
-# Known schema fields: a comma immediately followed by ``<known>:`` is a clause
-# separator. Restricting to known fields prevents URL-like ``http:`` misfires.
-KNOWN_FIELDS = frozenset(
-    {
-        "title",
-        "content",
-        "correspondent",
-        "document_type",
-        "type",  # v2 alias -> document_type
-        "storage_path",
-        "path",  # v2 alias -> storage_path
-        "tag",
-        "tag_id",
-        "correspondent_id",
-        "document_type_id",
-        "type_id",  # v2 alias -> document_type_id
-        "storage_path_id",
-        "path_id",  # v2 alias -> storage_path_id
-        "owner_id",
-        "viewer_id",
-        "asn",
-        "page_count",
-        "num_notes",
-        "created",
-        "modified",
-        "added",
-        "original_filename",
-        "checksum",
-        "notes",
-        "custom_fields",
-    },
-)
-
-_FIELD_RE = regex.compile(r"(?P<field>\w+):")
-
-# Matches the TO separator inside a range bracket. Handles three forms:
-#   middle:   "lo TO hi"   (either lo or hi may be empty)
-#   trailing: "lo TO"      (open upper bound)
-#   leading:  "TO hi"      (open lower bound)
-# Bounds MAY contain internal spaces (e.g. "-7 days"), so we use .*? / .+?
-# and split on the whitespace-delimited " TO " / " to " separator.
-_RANGE_RE = regex.compile(
-    r"^\s*(?P<lo>.*?)\s+[Tt][Oo]\s+(?P<hi>.+?)\s*$"
-    r"|"
-    r"^\s*(?P<lo2>.+?)\s+[Tt][Oo]\s*$"
-    r"|"
-    r"^\s*[Tt][Oo]\s+(?P<hi2>.+?)\s*$",
-)
-
-
-@dataclass(frozen=True, slots=True)
-class FieldValue:
-    field: str
-    value: str
-
-
-# Produced by the comma-resolution pass (not by scan()).
-@dataclass(frozen=True, slots=True)
-class FieldValueList:
-    field: str
-    values: tuple[str, ...]
-
-
-@dataclass(frozen=True, slots=True)
-class FieldRange:
-    field: str
-    open: str
-    lo: str
-    hi: str
-    close: str
-
-
-# Produced by the comma-resolution pass (not by scan()).
-@dataclass(frozen=True, slots=True)
-class Comma:
-    pass
-
-
-@dataclass(frozen=True, slots=True)
-class Passthrough:
-    raw: str
-
-
-Token: TypeAlias = FieldValue | FieldValueList | FieldRange | Comma | Passthrough
-
-_CLOSE: dict[str, str] = {"[": "]", "{": "}"}
-
-
-def scan(query: str) -> list[Token]:
-    """
-    Tokenize a raw query into date/comma-aware tokens, leaving everything else
-    as verbatim ``Passthrough`` runs. Non-recursive: finds the first matching
-    close bracket/quote. Nested brackets are not valid Tantivy range syntax and
-    pass through verbatim on mismatch.
-    """
-    tokens: list[Token] = []
-    buf: list[str] = []  # accumulates passthrough chars
-    i, n = 0, len(query)
-    while i < n:
-        matched = _match_field_token(query, i)
-        if matched is None:
-            buf.append(query[i])
-            i += 1
-            continue
-        token, i = matched
-        if buf and buf[-1] == ",":
-            buf.pop()
-            _flush(buf, tokens)
-            tokens.append(Comma())
-        else:
-            _flush(buf, tokens)
-        tokens.append(token)
-        i = _maybe_comma(query, i, tokens)
-    _flush(buf, tokens)
-    return tokens
-
-
-def _flush(buf: list[str], tokens: list[Token]) -> None:
-    """Emit any accumulated passthrough characters as a single token."""
-    if buf:
-        tokens.append(Passthrough("".join(buf)))
-        buf.clear()
-
-
-def _at_word_boundary(query: str, i: int) -> bool:
-    """A field token may begin only at the start or after a non-word character."""
-    return i == 0 or not (query[i - 1].isalnum() or query[i - 1] == "_")
-
-
-def _match_field_token(query: str, i: int) -> tuple[Token, int] | None:
-    """
-    If a known ``field:`` token starts at ``i``, consume it and return
-    ``(token, end_index)``; otherwise return None so the caller treats the
-    character as passthrough. Handles both ``field:[range]`` and ``field:value``,
-    and returns None when the range/value cannot be consumed.
-    """
-    m = _FIELD_RE.match(query, i)
-    if m is None or m.group("field") not in KNOWN_FIELDS:
-        return None
-    if not _at_word_boundary(query, i):
-        return None
-    field = m.group("field")
-    j = m.end()
-    if j < len(query) and query[j] in "[{":
-        return _consume_range(query, j, field)
-    consumed = _consume_field_value(query, field, j)
-    if consumed is None:
-        return None
-    value, end = consumed
-    return FieldValue(field, value), end
-
-
-def _consume_field_value(query: str, field: str, start: int) -> tuple[str, int] | None:
-    """
-    Consume a field value starting at ``start``: a multi-word date keyword phrase
-    (date fields only), or a bare/quoted value, then absorb any comma-joined
-    continuation that is not a clause separator. ``resolve_commas`` later splits a
-    multi-value field's joined value into a ``FieldValueList``; for other fields
-    the comma stays literal.
-    """
-    n = len(query)
-    consumed = None
-    if field in DATE_FIELDS:
-        km = _KEYWORD_VALUE_RE.match(query, start)
-        if km is not None and (km.end() >= n or query[km.end()] in " \t),"):
-            consumed = (km.group(0), km.end())
-    if consumed is None:
-        consumed = _consume_value(query, start)
-    if consumed is None:
-        return None
-    value, k = consumed
-    while k < n and query[k] == ",":
-        if _looks_like_known_field(query, k + 1):
-            break  # clause separator: left for _maybe_comma to emit a Comma()
-        more = _consume_value(query, k + 1)
-        if more is None:
-            break
-        value = f"{value},{more[0]}"
-        k = more[1]
-    return value, k
-
-
-def _consume_range(
-    query: str,
-    start: int,
-    field: str,
-) -> tuple[FieldRange, int] | None:
-    """Consume ``[lo TO hi]`` / ``{lo TO hi}`` from ``start`` (the bracket)."""
-    open_br = query[start]
-    close_br = _CLOSE[open_br]
-    end = query.find(close_br, start + 1)
-    if end == -1:
-        return None
-    inner = query[start + 1 : end]
-    m = _RANGE_RE.match(inner)
-    if m is not None:
-        if m.group("lo") is not None or m.group("hi") is not None:
-            # Middle form: "lo TO hi" (either may be empty string)
-            lo = (m.group("lo") or "").strip()
-            hi = (m.group("hi") or "").strip()
-        elif m.group("lo2") is not None:
-            # Trailing form: "lo TO"
-            lo = m.group("lo2").strip()
-            hi = ""
-        else:
-            # Leading form: "TO hi"
-            lo = ""
-            hi = (m.group("hi2") or "").strip()
-    else:
-        lo, hi = inner.strip(), ""
-    return FieldRange(field, open_br, lo, hi, close_br), end + 1
-
-
-def _consume_value(query: str, start: int) -> tuple[str, int] | None:
-    """Consume a bare or quoted field value from ``start``, stopping at comma."""
-    n = len(query)
-    if start >= n or query[start] in " \t":
-        return None
-    if query[start] in "\"'":
-        quote = query[start]
-        end = query.find(quote, start + 1)
-        if end == -1:
-            return None
-        return query[start : end + 1], end + 1
-    j = start
-    while j < n and query[j] not in " \t),":
-        j += 1
-    return query[start:j], j
-
-
-def _looks_like_known_field(query: str, pos: int) -> bool:
-    """True if a known ``field:`` token starts at ``pos``."""
-    m = _FIELD_RE.match(query, pos)
-    return bool(m and m.group("field") in KNOWN_FIELDS)
-
-
-def _maybe_comma(query: str, i: int, tokens: list) -> int:
-    """If a clause-separator comma follows at ``i``, emit ``Comma()`` and advance."""
-    if i < len(query) and query[i] == "," and _looks_like_known_field(query, i + 1):
-        tokens.append(Comma())
-        return i + 1
-    return i
-
-
-def resolve_commas(tokens: list) -> list:
-    """
-    Collapse value-list commas into ``FieldValueList`` and keep clause-separator
-    commas as ``Comma``. (Clause-sep commas are already emitted by ``scan`` via
-    the value-stop logic; this pass folds value-lists.)
-    """
-    out: list = []
-    for tok in tokens:
-        if (
-            isinstance(tok, FieldValue)
-            and tok.field in MULTI_VALUE_FIELDS
-            and "," in tok.value
-        ):
-            values = tuple(v for v in tok.value.split(",") if v)
-            out.append(FieldValueList(tok.field, values))
-        else:
-            out.append(tok)
-    return out
-
-
-class SearchQueryError(ValueError):
-    """
-    Base for user-fixable search query errors.
-
-    Carries a message safe to surface to the user (no internal details). The view
-    layer catches this and returns an HTTP 400, so any future subclass (unknown
-    field, malformed range, wrapped parser errors) gets the same treatment.
-    """
-
-
-class InvalidDateQuery(SearchQueryError):
-    """Raised when a date field value or range bound cannot be parsed."""
-
-    def __init__(self, field: str, value: str) -> None:
-        self.field = field
-        self.value = value
-        super().__init__(f"Invalid date value {value!r} for field {field!r}.")
-
-
-_DIGITS_RE = regex.compile(r"^\d{4}(?:\d{2}){0,2}$")
-_ISO_RE = regex.compile(r"^\d{4}(?:-\d{2}(?:-\d{2})?)?$")
-
-
-def translate_scalar(field: str, value: str, tz: tzinfo) -> str:
-    """Translate a bare date-field value to a Tantivy range string."""
-    bare = value.strip("\"'").lower()
-    if bare in _DATE_KEYWORDS:
-        if field in _DATE_ONLY_FIELDS:
-            return f"{field}:{_date_only_range(bare, tz)}"
-        return f"{field}:{_datetime_range(bare, tz)}"
-    digits = value.replace("-", "")
-    if _DIGITS_RE.match(value) or _ISO_RE.match(value):
-        bounds = _precision_bounds(digits)
-        if bounds is None:
-            raise InvalidDateQuery(field, value)
-        return _field_range_from_dates(field, bounds[0], bounds[1], tz)
-    if regex.fullmatch(r"\d{14}", value):
-        try:
-            dt = datetime(
-                int(value[0:4]),
-                int(value[4:6]),
-                int(value[6:8]),
-                int(value[8:10]),
-                int(value[10:12]),
-                int(value[12:14]),
-                tzinfo=UTC,
-            )
-        except ValueError:
-            raise InvalidDateQuery(field, value) from None
-        iso = _fmt(dt)
-        return f"{field}:[{iso} TO {iso}]"
-    # Unrecognized shape -> tell the user their date is malformed rather than
-    # silently matching nothing or emitting invalid Tantivy syntax.
-    raise InvalidDateQuery(field, value)
-
-
-# Open-bound sentinels for date ranges. These far-past/far-future strings allow
-# open-ended ranges to be expressed as Tantivy string queries until tantivy-py
-# exposes Query.range_query(..., None) on Date fields (see module TODO).
-OPEN_LO = "0001-01-01T00:00:00Z"
-OPEN_HI = "9999-12-31T23:59:59Z"
-
-
-# Matches compact now-offset tokens like now-7d, now+1h, now-30m.
-_NOW_COMPACT_RE = regex.compile(
-    r"^now(?P<sign>[+-])(?P<n>\d+)(?P<unit>[dhm])$",
-    regex.IGNORECASE,
-)
-
-# Matches "±N <unit>" Whoosh-style offsets (e.g. -7 days, -1 week, +3 hours).
-# Whoosh's own date parser (qparser.dateparse.PlusMinus) additionally accepted
-# abbreviated unit spellings (e.g. "yrs", "yr", "y", "mos", "wks", "hrs", "mins",
-# "secs"); saved views/searches created under the old Whoosh backend can still
-# contain those tokens (e.g. "-999yrs"), so they are accepted here too and
-# normalized to a canonical unit via _UNIT_ALIASES below.
-_NOW_SPACED_RE = regex.compile(
-    r"^(?P<sign>[+-])(?P<n>\d+)\s*"
-    r"(?P<unit>years|year|yrs|yr|ys|y"
-    r"|months|month|mons|mon|mos|mo"
-    r"|weeks|week|wks|wk|ws|w"
-    r"|days|day|dys|dy|ds|d"
-    r"|hours|hour|hrs|hr|hs|h"
-    r"|minutes|minute|mins|min|ms|m"
-    r"|seconds|second|secs|sec|s)$",
-    regex.IGNORECASE,
-)
-
-# Maps every accepted unit spelling (including Whoosh-era abbreviations) to the
-# canonical unit name used as a key into the delta map in _resolve_relative_bound.
-_UNIT_ALIASES: dict[str, str] = {
-    alias: canonical
-    for canonical, aliases in {
-        "year": ("years", "year", "yrs", "yr", "ys", "y"),
-        "month": ("months", "month", "mons", "mon", "mos", "mo"),
-        "week": ("weeks", "week", "wks", "wk", "ws", "w"),
-        "day": ("days", "day", "dys", "dy", "ds", "d"),
-        "hour": ("hours", "hour", "hrs", "hr", "hs", "h"),
-        "minute": ("minutes", "minute", "mins", "min", "ms", "m"),
-        "second": ("seconds", "second", "secs", "sec", "s"),
-    }.items()
-    for alias in aliases
-}
-
-
-def _resolve_relative_bound(token: str) -> datetime | None:
-    """
-    Resolve a relative bound token to an exact UTC instant, or return None.
-
-    Supported forms:
-      - ``now``            -> current UTC instant
-      - ``now+/-<n>d/h/m`` -> now +/- timedelta (d=days, h=hours, m=minutes)
-      - ``±N <unit>``     -> now +/- delta; month/year use relativedelta;
-                              unit also accepts Whoosh-era abbreviations
-                              (e.g. "yrs", "mos", "wks", "hrs", "mins", "secs")
-    """
-    stripped = token.strip()
-    low = stripped.lower()
-    now = datetime.now(UTC)
-
-    if low == "now":
-        return now
-
-    m = _NOW_COMPACT_RE.match(stripped)
-    if m:
-        sign = 1 if m.group("sign") == "+" else -1
-        n = int(m.group("n"))
-        unit = m.group("unit").lower()
-        delta = (
-            sign
-            * {
-                "d": timedelta(days=n),
-                "h": timedelta(hours=n),
-                "m": timedelta(minutes=n),
-            }[unit]
-        )
-        return now + delta
-
-    m = _NOW_SPACED_RE.match(stripped)
-    if m:
-        sign = 1 if m.group("sign") == "+" else -1
-        n = int(m.group("n"))
-        unit = _UNIT_ALIASES[m.group("unit").lower()]
-        delta_map: dict[str, timedelta | relativedelta] = {
-            "second": timedelta(seconds=n),
-            "minute": timedelta(minutes=n),
-            "hour": timedelta(hours=n),
-            "day": timedelta(days=n),
-            "week": timedelta(weeks=n),
-            "month": relativedelta(months=n),
-            "year": relativedelta(years=n),
-        }
-        return now - delta_map[unit] if sign == -1 else now + delta_map[unit]
-
-    return None
-
-
-def _bound_datetimes(
-    field: str,
-    token: str,
-    tz: tzinfo,
-) -> tuple[datetime, datetime] | None:
-    """
-    Return (floor_dt, ceil_dt) UTC datetimes for a single range bound token, or
-    None if the token is unparsable. ``now`` and relative offsets resolve to the
-    current instant (floor == ceil == that instant; no day-flooring).
-    """
-    token = token.strip()
-
-    # Try relative/now forms first (before stripping hyphens which would mangle them).
-    rel = _resolve_relative_bound(token)
-    if rel is not None:
-        return rel, rel
-
-    # Full ISO datetime token (contains "T"): parse directly and return an exact
-    # instant (floor == ceil). Python 3.11+ datetime.fromisoformat accepts trailing Z.
-    if "T" in token:
-        try:
-            dt = datetime.fromisoformat(token)
-            # Ensure timezone-aware UTC result.
-            dt = dt.replace(tzinfo=UTC) if dt.tzinfo is None else dt.astimezone(UTC)
-            return dt, dt
-        except ValueError:
-            return None
-
-    digits = token.replace("-", "")
-    bounds = _precision_bounds(digits)
-    if bounds is None:
-        return None
-    start, end = bounds
-    return _utc_bounds_for_field(field, start, end, tz)
-
-
-def _render(tok: Token, tz: tzinfo) -> str:
-    """Render a single token back to a Tantivy query string fragment."""
-    if isinstance(tok, Passthrough):
-        return tok.raw
-    if isinstance(tok, Comma):
-        return " AND "
-    if isinstance(tok, FieldValueList):
-        field = FIELD_ALIASES.get(tok.field, tok.field)
-        return " AND ".join(f"{field}:{v}" for v in tok.values)
-    if isinstance(tok, FieldValue):
-        field = FIELD_ALIASES.get(tok.field, tok.field)
-        if field in DATE_FIELDS:
-            return translate_scalar(field, tok.value, tz)
-        return f"{field}:{tok.value}"
-    if isinstance(tok, FieldRange):
-        field = FIELD_ALIASES.get(tok.field, tok.field)
-        if field in DATE_FIELDS:
-            return translate_range(field, tok.lo, tok.hi, tz)
-        return f"{field}:{tok.open}{tok.lo} TO {tok.hi}{tok.close}"
-    return ""  # pragma: no cover
-
-
-# Post-render operator normalization patterns: collapse repeated whitespace and
-# strip spaced/trailing Tantivy boolean operators that would otherwise be invalid.
-_MULTI_SPACE_RE = regex.compile(r" {2,}")
-_TRAILING_OP_RE = regex.compile(r"\s+[-+]+\s*$")
-_SPACED_OP_RE = regex.compile(r"\s+[-+]\s+")
-
-
-def _normalize_operators(text: str) -> str:
-    """
-    Collapse multiple spaces, strip trailing dangling operators, and replace
-    spaced operators (`` - `` / `` + ``) with a single space.
-
-    Applied only to Passthrough fragments (the rendered output is scanned for
-    operator artifacts outside bracketed ranges) via a post-render pass on the
-    full rendered string. This preserves date ranges (``[... TO ...]``) verbatim
-    while cleaning natural-language separators in the surrounding text.
-    """
-    text = _MULTI_SPACE_RE.sub(" ", text)
-    text = _TRAILING_OP_RE.sub("", text).strip()
-    text = _SPACED_OP_RE.sub(" ", text).strip()
-    return text
-
-
-def translate_query(raw: str, tz: tzinfo) -> str:
-    """Translate a raw Whoosh-style query into Tantivy-compatible syntax."""
-    tokens = resolve_commas(scan(raw))
-    rendered = "".join(_render(t, tz) for t in tokens)
-    return _normalize_operators(rendered)
-
-
-def translate_range(field: str, lo: str, hi: str, tz: tzinfo) -> str:
-    """Translate a date-field ``[lo TO hi]`` range to a Tantivy ISO range string.
-
-    Handles partial-date bounds (YYYY, YYYYMM, YYYYMMDD, ISO dash variants),
-    open bounds (empty string -> OPEN_LO/OPEN_HI), ``now``, and reversed ranges
-    (swaps tokens before computing floor/ceil so the span is always correct).
-    """
-    lo_s = lo.strip()
-    hi_s = hi.strip()
-
-    # Parse both bounds to (floor, ceil) pairs when present.
-    lo_pair: tuple[datetime, datetime] | None = None
-    hi_pair: tuple[datetime, datetime] | None = None
-
-    if lo_s:
-        lo_pair = _bound_datetimes(field, lo_s, tz)
-        if lo_pair is None:
-            raise InvalidDateQuery(field, lo_s)
-    if hi_s:
-        hi_pair = _bound_datetimes(field, hi_s, tz)
-        if hi_pair is None:
-            raise InvalidDateQuery(field, hi_s)
-
-    # Detect a reversed range: only swap when BOTH bounds are present.
-    if lo_pair is not None and hi_pair is not None and lo_pair[0] > hi_pair[0]:
-        lo_pair, hi_pair = hi_pair, lo_pair
-
-    lo_iso = _fmt(lo_pair[0]) if lo_pair is not None else OPEN_LO
-
-    # A bound resolves to (floor, ceil) where floor == ceil for an exact instant
-    # (a full ISO datetime, "now", or a "+/-N unit" offset) and floor != ceil for
-    # a coarser period token (year/month/day precision). Only the latter needs a
-    # half-open close: its ceil is the start of the *next* period and must be
-    # excluded, or that instant (e.g. the 1st of next month) wrongly matches.
-    if hi_pair is not None:
-        hi_iso = _fmt(hi_pair[1])
-        hi_close = "]" if hi_pair[0] == hi_pair[1] else "}"
-    else:
-        hi_iso = OPEN_HI
-        hi_close = "]"
-
-    return f"{field}:[{lo_iso} TO {hi_iso}{hi_close}"

src/documents/serialisers.py

@@ -61,6 +61,7 @@ from documents.models import Correspondent
 from documents.models import CustomField
 from documents.models import CustomFieldInstance
 from documents.models import Document
+from documents.models import DocumentBarcode
 from documents.models import DocumentType
 from documents.models import MatchingModel
 from documents.models import Note
@@ -92,6 +93,7 @@ from documents.templating.utils import convert_format_str_to_template_format
 from documents.templating.workflows import validate_workflow_template
 from documents.validators import uri_validator
 from documents.validators import url_validator
+from documents.versioning import has_prefetched_effective_content
 from documents.versioning import sort_versions_newest_first
 
 if TYPE_CHECKING:
@@ -212,6 +214,9 @@ class MatchingModelSerializer(serializers.ModelSerializer[Any]):
         return match
 
 
+PERMISSION_ACTIONS = ("view", "change")
+
+
 class SetPermissionsMixin:
     def _validate_user_ids(self, user_ids):
         users = User.objects.none()
@@ -234,12 +239,9 @@ class SetPermissionsMixin:
         return groups
 
     def validate_set_permissions(self, set_permissions=None):
-        permissions_dict = {
-            "view": {},
-            "change": {},
-        }
+        permissions_dict = {action: {} for action in PERMISSION_ACTIONS}
         if set_permissions is not None:
-            for action in ["view", "change"]:
+            for action in PERMISSION_ACTIONS:
                 if action in set_permissions:
                     if "users" in set_permissions[action]:
                         users = set_permissions[action]["users"]
@@ -267,41 +269,31 @@ class SerializerWithPerms(serializers.Serializer[dict[str, Any]]):
         super().__init__(*args, **kwargs)
 
 
-@extend_schema_field(
-    field={
-        "type": "object",
-        "properties": {
-            "view": {
-                "type": "object",
-                "properties": {
-                    "users": {
-                        "type": "array",
-                        "items": {"type": "integer"},
-                    },
-                    "groups": {
-                        "type": "array",
-                        "items": {"type": "integer"},
-                    },
-                },
-            },
-            "change": {
-                "type": "object",
-                "properties": {
-                    "users": {
-                        "type": "array",
-                        "items": {"type": "integer"},
-                    },
-                    "groups": {
-                        "type": "array",
-                        "items": {"type": "integer"},
-                    },
-                },
-            },
-        },
-    },
-)
-class SetPermissionsSerializer(serializers.DictField):
-    pass
+class PermissionSetSerializer(serializers.Serializer[dict[str, Any]]):
+    users = serializers.ListField(
+        child=serializers.IntegerField(),
+        required=False,
+        allow_null=True,
+    )
+    groups = serializers.ListField(
+        child=serializers.IntegerField(),
+        required=False,
+        allow_null=True,
+    )
+
+
+class SetPermissionsSerializer(serializers.Serializer[dict[str, Any]]):
+    view = PermissionSetSerializer(required=False)
+    change = PermissionSetSerializer(required=False)
+
+    def to_internal_value(self, data):
+        if isinstance(data, dict):
+            unknown_keys = set(data) - set(PERMISSION_ACTIONS)
+            if unknown_keys:
+                raise serializers.ValidationError(
+                    {key: "Unknown permission action." for key in sorted(unknown_keys)},
+                )
+        return super().to_internal_value(data)
 
 
 class OwnedObjectSerializer(
@@ -472,7 +464,6 @@ class OwnedObjectSerializer(
 
     set_permissions = SetPermissionsSerializer(
         label="Set permissions",
-        allow_empty=True,
         required=False,
         write_only=True,
     )
@@ -1000,6 +991,12 @@ class BasicUserSerializer(serializers.ModelSerializer[User]):
         fields = ["id", "username", "first_name", "last_name"]
 
 
+class DocumentBarcodeSerializer(serializers.ModelSerializer[DocumentBarcode]):
+    class Meta:
+        model = DocumentBarcode
+        fields = ["page", "value", "format"]
+
+
 class NotesSerializer(serializers.ModelSerializer[Note]):
     user = BasicUserSerializer(read_only=True)
 
@@ -1092,6 +1089,16 @@ class DocumentSerializer(
     )
 
     def get_page_count(self, obj) -> int | None:
+        # Like content versions get their own page count from the newest version,
+        # use the prefetched versions cache to avoid an extra query
+        prefetched_cache = getattr(obj, "_prefetched_objects_cache", None)
+        prefetched_versions = (
+            prefetched_cache.get("versions")
+            if isinstance(prefetched_cache, dict)
+            else None
+        )
+        if obj.root_document_id is None and prefetched_versions:
+            return sort_versions_newest_first(prefetched_versions)[0].page_count
         return obj.page_count
 
     @extend_schema_field(DuplicateDocumentSummarySerializer(many=True))
@@ -1155,8 +1162,14 @@ class DocumentSerializer(
 
     def to_representation(self, instance):
         doc = super().to_representation(instance)
-        if "content" in self.fields and hasattr(instance, "effective_content"):
-            doc["content"] = getattr(instance, "effective_content") or ""
+        if "content" in self.fields and has_prefetched_effective_content(instance):
+            # Only resolve version-aware content when it's cheap: an SQL
+            # annotation or a versions prefetch is already on the instance.
+            # A caller that set up neither (e.g. TrashView, GlobalSearchView,
+            # which build their own querysets) gets the document's own,
+            # unresolved content instead of paying for an extra per-instance
+            # query -- same as before effective_content resolution existed.
+            doc["content"] = instance.get_effective_content() or ""
         if self.truncate_content and "content" in self.fields:
             doc["content"] = doc.get("content")[0:550]
         return doc
@@ -1250,30 +1263,31 @@ class DocumentSerializer(
 
             validated_data["tags"] = list(final_tags)
         if validated_data.get("remove_inbox_tags"):
-            tag_ids_being_added = (
-                [
-                    tag.id
-                    for tag in validated_data["tags"]
-                    if tag not in instance.tags.all()
-                ]
+            current_tag_ids = {t.pk for t in instance.tags.all()}
+            tags = (
+                validated_data["tags"]
                 if "tags" in validated_data
-                else []
+                else list(instance.tags.all())
             )
-            inbox_tags_not_being_added = Tag.objects.filter(is_inbox_tag=True).exclude(
-                id__in=tag_ids_being_added,
-            )
-            if "tags" in validated_data:
-                validated_data["tags"] = [
-                    tag
-                    for tag in validated_data["tags"]
-                    if tag not in inbox_tags_not_being_added
-                ]
-            else:
-                validated_data["tags"] = [
-                    tag
-                    for tag in instance.tags.all()
-                    if tag not in inbox_tags_not_being_added
-                ]
+
+            # Tags newly added in this update, plus their ancestors, are kept
+            keep_ids: set[int] = set()
+            for tag in tags:
+                if tag.pk not in current_tag_ids:
+                    keep_ids.add(tag.pk)
+                    keep_ids.update(int(pk) for pk in tag.get_ancestors_pks())
+
+            # Remove inbox tags and their descendants, except those being kept
+            remove_ids: set[int] = set()
+            for inbox_tag in (
+                Tag.objects.filter(is_inbox_tag=True)
+                .exclude(pk__in=keep_ids)
+                .only("pk", "tn_descendants_pks")
+            ):
+                remove_ids.add(inbox_tag.pk)
+                remove_ids.update(int(pk) for pk in inbox_tag.get_descendants_pks())
+
+            validated_data["tags"] = [t for t in tags if t.pk not in remove_ids]
 
         if settings.AUDIT_LOG_ENABLED:
             with set_actor(self.user):
@@ -1330,6 +1344,7 @@ class DocumentSerializer(
             "root_document",
             "versions",
         )
+        read_only_fields = ("deleted_at",)
         list_serializer_class = OwnedObjectListSerializer
 
 
@@ -1652,11 +1667,23 @@ class DocumentSelectionSerializer(DocumentListSerializer):
         write_only=True,
     )
 
+    excluded_documents = serializers.ListField(
+        required=False,
+        default=list,
+        write_only=True,
+        child=serializers.IntegerField(),
+    )
+
     def validate(self, attrs):
         if attrs.get("all", False):
             attrs.setdefault("documents", [])
             return attrs
 
+        if attrs["excluded_documents"]:
+            raise serializers.ValidationError(
+                "excluded_documents is only supported when all is true.",
+            )
+
         if "documents" not in attrs:
             raise serializers.ValidationError(
                 "documents is required unless all is true.",
@@ -1674,6 +1701,13 @@ class SourceModeValidationMixin:
         return source_mode
 
 
+def _validate_rotation_degrees(degrees: int, field: str = "degrees") -> int:
+    # QPDF refuses any other angle, which would otherwise fail inside the task
+    if degrees % 90 != 0:
+        raise serializers.ValidationError(f"{field} must be a multiple of 90")
+    return degrees
+
+
 class RotateDocumentsSerializer(DocumentSelectionSerializer, SourceModeValidationMixin):
     degrees = serializers.IntegerField(required=True)
     source_mode = serializers.CharField(
@@ -1682,6 +1716,9 @@ class RotateDocumentsSerializer(DocumentSelectionSerializer, SourceModeValidatio
     )
     from_webui = serializers.BooleanField(required=False, default=False)
 
+    def validate_degrees(self, value: int) -> int:
+        return _validate_rotation_degrees(value)
+
 
 class MergeDocumentsSerializer(DocumentListSerializer, SourceModeValidationMixin):
     metadata_document_id = serializers.IntegerField(
@@ -1743,8 +1780,21 @@ class MergeDocumentsAsVersionsSerializer(DocumentListSerializer):
         return attrs
 
 
+class PdfEditOperationSerializer(serializers.Serializer[dict[str, int]]):
+    page = serializers.IntegerField(min_value=1)
+    rotate = serializers.IntegerField(required=False)
+    doc = serializers.IntegerField(required=False, min_value=0)
+
+    def validate_rotate(self, value: int) -> int:
+        return _validate_rotation_degrees(value, field="rotate")
+
+
 class EditPdfDocumentsSerializer(DocumentListSerializer, SourceModeValidationMixin):
-    operations = serializers.ListField(required=True)
+    operations = serializers.ListField(
+        child=PdfEditOperationSerializer(),
+        required=True,
+        allow_empty=False,
+    )
     delete_original = serializers.BooleanField(required=False, default=False)
     update_document = serializers.BooleanField(required=False, default=False)
     include_metadata = serializers.BooleanField(required=False, default=True)
@@ -1762,18 +1812,9 @@ class EditPdfDocumentsSerializer(DocumentListSerializer, SourceModeValidationMix
             )
 
         operations = attrs["operations"]
-        if not isinstance(operations, list):
-            raise serializers.ValidationError("operations must be a list")
 
-        for op in operations:
-            if not isinstance(op, dict):
-                raise serializers.ValidationError("invalid operation entry")
-            if "page" not in op or not isinstance(op["page"], int):
-                raise serializers.ValidationError("page must be an integer")
-            if "rotate" in op and not isinstance(op["rotate"], int):
-                raise serializers.ValidationError("rotate must be an integer")
-            if "doc" in op and not isinstance(op["doc"], int):
-                raise serializers.ValidationError("doc must be an integer")
+        if any(op.get("doc", 0) >= len(operations) for op in operations):
+            raise serializers.ValidationError("doc index is out of bounds")
 
         if attrs["update_document"]:
             max_idx = max(op.get("doc", 0) for op in operations)
@@ -1785,7 +1826,7 @@ class EditPdfDocumentsSerializer(DocumentListSerializer, SourceModeValidationMix
         doc = Document.objects.get(id=documents[0])
         if doc.page_count:
             for op in operations:
-                if op["page"] < 1 or op["page"] > doc.page_count:
+                if op["page"] > doc.page_count:
                     raise serializers.ValidationError(
                         f"Page {op['page']} is out of bounds for document with {doc.page_count} pages.",
                     )
@@ -2030,32 +2071,39 @@ class BulkEditSerializer(
         else:
             raise serializers.ValidationError("remove_custom_fields not specified")
 
-    def _validate_owner(self, owner):
-        ownerUser = User.objects.get(pk=owner)
-        if ownerUser is None:
-            raise serializers.ValidationError("Specified owner cannot be found")
-        return ownerUser
+    def _validate_owner(self, owner) -> User:
+        owner_field = serializers.PrimaryKeyRelatedField(queryset=User.objects.all())
+        try:
+            return owner_field.run_validation(owner)
+        except serializers.ValidationError as e:
+            raise serializers.ValidationError(
+                "Specified owner cannot be found",
+            ) from e
 
     def _validate_parameters_set_permissions(self, parameters) -> None:
         if "set_permissions" not in parameters:
             raise serializers.ValidationError("set_permissions not specified")
+        set_permissions = parameters["set_permissions"]
+        if set_permissions is not None:
+            set_permissions = SetPermissionsSerializer().run_validation(
+                set_permissions,
+            )
         parameters["set_permissions"] = self.validate_set_permissions(
-            parameters["set_permissions"],
+            set_permissions,
         )
         if "owner" in parameters and parameters["owner"] is not None:
-            self._validate_owner(parameters["owner"])
+            parameters["owner"] = self._validate_owner(parameters["owner"]).pk
         if "merge" not in parameters:
             parameters["merge"] = False
 
     def _validate_parameters_rotate(self, parameters) -> None:
-        try:
-            if (
-                "degrees" not in parameters
-                or not float(parameters["degrees"]).is_integer()
-            ):
-                raise serializers.ValidationError("invalid rotation degrees")
-        except ValueError:
+        if "degrees" not in parameters:
             raise serializers.ValidationError("invalid rotation degrees")
+        try:
+            degrees = serializers.IntegerField().run_validation(parameters["degrees"])
+        except serializers.ValidationError as e:
+            raise serializers.ValidationError("invalid rotation degrees") from e
+        parameters["degrees"] = _validate_rotation_degrees(degrees)
 
     def _validate_source_mode(self, parameters) -> None:
         source_mode = parameters.get(
@@ -2064,28 +2112,27 @@ class BulkEditSerializer(
         )
         parameters["source_mode"] = self.validate_source_mode(source_mode)
 
-    def _validate_parameters_split(self, parameters) -> None:
+    def _validate_parameters_split(self, parameters, document_id) -> None:
         if "pages" not in parameters:
             raise serializers.ValidationError("pages not specified")
-        try:
-            pages = []
-            docs = parameters["pages"].split(",")
-            for doc in docs:
-                if "-" in doc:
-                    pages.append(
-                        [
-                            x
-                            for x in range(
-                                int(doc.split("-")[0]),
-                                int(doc.split("-")[1]) + 1,
-                            )
-                        ],
-                    )
-                else:
-                    pages.append([int(doc)])
-            parameters["pages"] = pages
-        except ValueError:
+        if not isinstance(parameters["pages"], str):
             raise serializers.ValidationError("invalid pages specified")
+        page_count = Document.objects.get(id=document_id).page_count
+        if not page_count:
+            raise serializers.ValidationError("document page count is unknown")
+        pages = []
+        for group in parameters["pages"].split(","):
+            start, is_range, end = group.partition("-")
+            try:
+                first = int(start)
+                last = int(end) if is_range else first
+            except ValueError as e:
+                raise serializers.ValidationError("invalid pages specified") from e
+            # Bound the range before building it, a huge one would exhaust memory
+            if not 1 <= first <= last <= page_count:
+                raise serializers.ValidationError("invalid pages specified")
+            pages.append(list(range(first, last + 1)))
+        parameters["pages"] = pages
 
         if "delete_originals" in parameters:
             if not isinstance(parameters["delete_originals"], bool):
@@ -2116,17 +2163,18 @@ class BulkEditSerializer(
     def _validate_parameters_edit_pdf(self, parameters, document_id) -> None:
         if "operations" not in parameters:
             raise serializers.ValidationError("operations not specified")
-        if not isinstance(parameters["operations"], list):
-            raise serializers.ValidationError("operations must be a list")
-        for op in parameters["operations"]:
-            if not isinstance(op, dict):
-                raise serializers.ValidationError("invalid operation entry")
-            if "page" not in op or not isinstance(op["page"], int):
-                raise serializers.ValidationError("page must be an integer")
-            if "rotate" in op and not isinstance(op["rotate"], int):
-                raise serializers.ValidationError("rotate must be an integer")
-            if "doc" in op and not isinstance(op["doc"], int):
-                raise serializers.ValidationError("doc must be an integer")
+        operations_field = serializers.ListField(
+            child=PdfEditOperationSerializer(),
+            allow_empty=False,
+        )
+        try:
+            operations = operations_field.run_validation(parameters["operations"])
+        except serializers.ValidationError as e:
+            # Key the errors under "operations" so they match what the
+            # dedicated edit_pdf endpoint returns
+            raise serializers.ValidationError({"operations": e.detail}) from e
+        parameters["operations"] = operations
+
         if "update_document" in parameters:
             if not isinstance(parameters["update_document"], bool):
                 raise serializers.ValidationError("update_document must be a boolean")
@@ -2138,8 +2186,11 @@ class BulkEditSerializer(
         else:
             parameters["include_metadata"] = True
 
+        if any(op.get("doc", 0) >= len(operations) for op in operations):
+            raise serializers.ValidationError("doc index is out of bounds")
+
         if parameters["update_document"]:
-            max_idx = max(op.get("doc", 0) for op in parameters["operations"])
+            max_idx = max(op.get("doc", 0) for op in operations)
             if max_idx > 0:
                 raise serializers.ValidationError(
                     "update_document only allowed with a single output document",
@@ -2148,8 +2199,8 @@ class BulkEditSerializer(
         doc = Document.objects.get(id=document_id)
         # doc existence is already validated
         if doc.page_count:
-            for op in parameters["operations"]:
-                if op["page"] < 1 or op["page"] > doc.page_count:
+            for op in operations:
+                if op["page"] > doc.page_count:
                     raise serializers.ValidationError(
                         f"Page {op['page']} is out of bounds for document with {doc.page_count} pages.",
                     )
@@ -2208,7 +2259,7 @@ class BulkEditSerializer(
                 raise serializers.ValidationError(
                     "Split method only supports one document",
                 )
-            self._validate_parameters_split(parameters)
+            self._validate_parameters_split(parameters, attrs["documents"][0])
         elif method == bulk_edit.delete_pages:
             if len(attrs["documents"]) > 1:
                 raise serializers.ValidationError(
@@ -2818,6 +2869,11 @@ class AcknowledgeTasksViewSerializer(serializers.Serializer[dict[str, Any]]):
 
 
 class ShareLinkSerializer(OwnedObjectSerializer):
+    document_title = serializers.CharField(
+        source="document.title",
+        read_only=True,
+    )
+
     class Meta:
         model = ShareLink
         fields = (
@@ -2826,6 +2882,7 @@ class ShareLinkSerializer(OwnedObjectSerializer):
             "expiration",
             "slug",
             "document",
+            "document_title",
             "file_version",
         )
 
@@ -2834,10 +2891,14 @@ class ShareLinkSerializer(OwnedObjectSerializer):
         return super().create(validated_data)
 
     def validate_document(self, document):
-        if self.user is not None and has_perms_owner_aware(
-            self.user,
-            "view_document",
-            document,
+        if (
+            self.user is not None
+            and self.user.has_perm("documents.view_document")
+            and has_perms_owner_aware(
+                self.user,
+                "view_document",
+                document,
+            )
         ):
             return document
         raise PermissionDenied(
@@ -2991,9 +3052,8 @@ class BulkEditObjectsSerializer(SerializerWithPerms, SetPermissionsMixin):
         allow_null=True,
     )
 
-    permissions = serializers.DictField(
+    permissions = SetPermissionsSerializer(
         label="Set permissions",
-        allow_empty=False,
         required=False,
         write_only=True,
     )
@@ -3029,8 +3089,8 @@ class BulkEditObjectsSerializer(SerializerWithPerms, SetPermissionsMixin):
             )
         return objects
 
-    def _validate_permissions(self, permissions) -> None:
-        self.validate_set_permissions(
+    def _validate_permissions(self, permissions) -> dict:
+        return self.validate_set_permissions(
             permissions,
         )
 
@@ -3054,7 +3114,11 @@ class BulkEditObjectsSerializer(SerializerWithPerms, SetPermissionsMixin):
         if operation == "set_permissions":
             permissions = attrs.get("permissions")
             if permissions is not None:
-                self._validate_permissions(permissions)
+                if not permissions:
+                    raise serializers.ValidationError(
+                        "permissions must not be empty",
+                    )
+                attrs["permissions"] = self._validate_permissions(permissions)
 
         return attrs
 
@@ -3598,6 +3662,8 @@ class WorkflowSerializer(serializers.ModelSerializer[Workflow]):
 
         if "actions" in validated_data:
             actions = validated_data.pop("actions")
+            for action in actions:
+                action.pop("id", None)
 
         instance = super().create(validated_data)
 

src/documents/signals/handlers.py

@@ -56,6 +56,7 @@ from documents.permissions import get_objects_for_user_owner_aware
 from documents.plugins.helpers import DocumentsStatusManager
 from documents.templating.utils import convert_format_str_to_template_format
 from documents.utils import compute_checksum
+from documents.utils import copy_file_with_basic_stats
 from documents.workflows.actions import build_workflow_action_context
 from documents.workflows.actions import execute_email_action
 from documents.workflows.actions import execute_move_to_trash_action
@@ -363,7 +364,11 @@ def cleanup_document_deletion(sender, instance, **kwargs) -> None:
 
             logger.debug(f"Moving {instance.source_path} to trash at {new_file_path}")
             try:
-                shutil.move(instance.source_path, new_file_path)
+                shutil.move(
+                    instance.source_path,
+                    new_file_path,
+                    copy_function=copy_file_with_basic_stats,
+                )
             except OSError as e:
                 logger.error(
                     f"Failed to move {instance.source_path} to trash at "
@@ -1189,13 +1194,18 @@ def before_task_publish_handler(
         trigger_source = _determine_trigger_source(headers)
         owner_id = _extract_owner_id(task_type, task_kwargs)
 
-        PaperlessTask.objects.create(
+        # A retried task is republished with the same task_id, so this fires
+        # again for it; get_or_create keeps the original PENDING record
+        # instead of raising a duplicate-key IntegrityError on the retry.
+        PaperlessTask.objects.get_or_create(
             task_id=task_id,
-            task_type=task_type,
-            trigger_source=trigger_source,
-            status=PaperlessTask.Status.PENDING,
-            input_data=input_data,
-            owner_id=owner_id,
+            defaults={
+                "task_type": task_type,
+                "trigger_source": trigger_source,
+                "status": PaperlessTask.Status.PENDING,
+                "input_data": input_data,
+                "owner_id": owner_id,
+            },
         )
     except Exception:  # pragma: no cover
         logger.exception("Creating PaperlessTask failed")

src/documents/tasks.py

@@ -20,6 +20,7 @@ from filelock import FileLock
 
 from documents import sanity_checker
 from documents.barcodes import BarcodePlugin
+from documents.barcodes import read_barcode_values
 from documents.bulk_download import ArchiveOnlyStrategy
 from documents.bulk_download import OriginalsOnlyStrategy
 from documents.caching import clear_document_caches
@@ -36,6 +37,7 @@ from documents.data_models import ConsumeFileDuplicateResult
 from documents.data_models import ConsumeFileStoppedResult
 from documents.data_models import ConsumeFileSuccessResult
 from documents.data_models import DocumentMetadataOverrides
+from documents.data_models import StoredBarcode
 from documents.double_sided import CollatePlugin
 from documents.file_handling import create_source_path_directory
 from documents.file_handling import generate_unique_filename
@@ -43,6 +45,7 @@ from documents.matching import prefilter_documents_by_workflowtrigger
 from documents.models import Correspondent
 from documents.models import CustomFieldInstance
 from documents.models import Document
+from documents.models import DocumentBarcode
 from documents.models import DocumentType
 from documents.models import PaperlessTask
 from documents.models import ShareLink
@@ -67,6 +70,7 @@ from documents.utils import identity
 from documents.versioning import annotate_effective_content
 from documents.workflows.utils import get_workflows_for_trigger
 from paperless.config import AIConfig
+from paperless.config import BarcodeConfig
 from paperless.config import RemoteOCRConfig
 from paperless.logging import consume_task_id
 from paperless.parsers import ParserContext
@@ -341,6 +345,29 @@ def bulk_update_documents(document_ids) -> None:
         )
 
 
+def _read_barcodes_for_reprocess(document: Document) -> list[StoredBarcode] | None:
+    """
+    Reads the barcodes of the original again, e.g. for documents consumed
+    before storing them was enabled. Returns None if they should be left as
+    they are: storing is off, the file can't be scanned with the current
+    settings, or the scan failed.
+    """
+    barcode_settings = BarcodeConfig()
+    if not barcode_settings.barcode_store_values:
+        return None
+    try:
+        with TemporaryDirectory(dir=settings.SCRATCH_DIR) as tmpdir:
+            return read_barcode_values(
+                document.source_path,
+                document.mime_type,
+                barcode_settings,
+                Path(tmpdir),
+            )
+    except Exception as e:
+        logger.warning(f"Could not read barcodes of document {document}: {e}")
+        return None
+
+
 @shared_task
 def update_document_content_maybe_archive_file(
     document_id,
@@ -387,6 +414,8 @@ def update_document_content_maybe_archive_file(
                 produce_archive=produce_archive,
             )
 
+            barcodes = _read_barcodes_for_reprocess(document)
+
             thumbnail = parser.get_thumbnail(document.source_path, mime_type)
 
             with transaction.atomic():
@@ -443,6 +472,17 @@ def update_document_content_maybe_archive_file(
                             action=LogEntry.Action.UPDATE,
                         )
 
+                if barcodes is not None:
+                    document.barcodes.all().delete()
+                    DocumentBarcode.objects.bulk_create(
+                        DocumentBarcode(document=document, **barcode)
+                        for barcode in barcodes
+                    )
+                    # metadata_etag includes modified
+                    Document.objects.filter(pk=document.pk).update(
+                        modified=timezone.now(),
+                    )
+
                 with FileLock(settings.MEDIA_LOCK):
                     if parser.get_archive_path():
                         create_source_path_directory(document.archive_path)
@@ -462,6 +502,8 @@ def update_document_content_maybe_archive_file(
                 llm_index_add_or_update_document(document)
 
             clear_document_caches(document.pk)
+            if document.root_document_id is not None:
+                clear_document_caches(document.root_document_id)
 
         except Exception:
             logger.exception(

src/documents/templates/account/login.html

@@ -15,7 +15,7 @@
 {% endblock form_top_content %}
 
 {% block form_content %}
-		{% if FIRST_INSTALL %}
+		{% if FIRST_INSTALL and not DISABLE_REGULAR_LOGIN %}
 			<script type="text/javascript">
 				// forward to the signup page if no users exist
 				window.location.href = "{{ signup_url }}";

src/documents/templates/index.html

@@ -12,7 +12,7 @@
 	<meta name="robots" content="noindex,nofollow">
 	<meta name="author" content="The Paperless-ngx Team">
 	<link rel="icon" type="image/x-icon" href="favicon.ico">
-	<link rel="manifest" href="{% static webmanifest %}">
+	<link rel="manifest" href="{% static webmanifest %}" crossorigin="use-credentials">
 	<link rel="stylesheet" href="{% static styles_css %}">
 	<link rel="apple-touch-icon" href="{% static apple_touch_icon %}">
 </head>

src/documents/tests/conftest.py

@@ -1,88 +1,41 @@
 import shutil
-import zoneinfo
-from collections.abc import Generator
-from dataclasses import dataclass
 from pathlib import Path
 from typing import TYPE_CHECKING
 
 import filelock
 import pytest
-from django.contrib.auth import get_user_model
-from django.contrib.contenttypes.models import ContentType
-from guardian.shortcuts import clear_ct_cache
-from pytest_django.fixtures import Settings
-from rest_framework.test import APIClient
 
-from documents.tests.factories import DocumentFactory
-
-UserModelT = get_user_model()
+from paperless_testing.factories import DocumentFactory
 
 if TYPE_CHECKING:
     from documents.models import Document
-
-
-@dataclass(frozen=True, slots=True)
-class PaperlessDirs:
-    """Standard Paperless-ngx directory layout for tests."""
-
-    media: Path
-    originals: Path
-    archive: Path
-    thumbnails: Path
+    from paperless_testing.dirs import PaperlessDirs
 
 
 @pytest.fixture(scope="session")
-def samples_dir() -> Path:
+def document_samples_dir() -> Path:
     """Path to the shared test sample documents."""
     return Path(__file__).parent / "samples" / "documents"
 
 
-@pytest.fixture()
-def paperless_dirs(tmp_path: Path) -> PaperlessDirs:
-    """Create and return the directory structure for testing."""
-    media = tmp_path / "media"
-    dirs = PaperlessDirs(
-        media=media,
-        originals=media / "documents" / "originals",
-        archive=media / "documents" / "archive",
-        thumbnails=media / "documents" / "thumbnails",
-    )
-    for d in (dirs.originals, dirs.archive, dirs.thumbnails):
-        d.mkdir(parents=True)
-    return dirs
-
-
-@pytest.fixture()
-def _media_settings(paperless_dirs: PaperlessDirs, settings) -> None:
-    """Configure Django settings to point at temp directories."""
-    settings.MEDIA_ROOT = paperless_dirs.media
-    settings.ORIGINALS_DIR = paperless_dirs.originals
-    settings.ARCHIVE_DIR = paperless_dirs.archive
-    settings.THUMBNAIL_DIR = paperless_dirs.thumbnails
-    settings.MEDIA_LOCK = paperless_dirs.media / "media.lock"
-    settings.IGNORABLE_FILES = {".DS_Store", "Thumbs.db", "desktop.ini"}
-    settings.APP_LOGO = ""
-
-
 @pytest.fixture()
 def sample_doc(
-    paperless_dirs: PaperlessDirs,
-    _media_settings: None,
-    samples_dir: Path,
+    paperless_dirs: "PaperlessDirs",
+    document_samples_dir: Path,
 ) -> "Document":
     """Create a document with valid files and matching checksums."""
-    with filelock.FileLock(paperless_dirs.media / "media.lock"):
+    with filelock.FileLock(paperless_dirs.media_lock):
         shutil.copy(
-            samples_dir / "originals" / "0000001.pdf",
-            paperless_dirs.originals / "0000001.pdf",
+            document_samples_dir / "originals" / "0000001.pdf",
+            paperless_dirs.originals_dir / "0000001.pdf",
         )
         shutil.copy(
-            samples_dir / "archive" / "0000001.pdf",
-            paperless_dirs.archive / "0000001.pdf",
+            document_samples_dir / "archive" / "0000001.pdf",
+            paperless_dirs.archive_dir / "0000001.pdf",
         )
         shutil.copy(
-            samples_dir / "thumbnails" / "0000001.webp",
-            paperless_dirs.thumbnails / "0000001.webp",
+            document_samples_dir / "thumbnails" / "0000001.webp",
+            paperless_dirs.thumbnail_dir / "0000001.webp",
         )
 
     return DocumentFactory(
@@ -97,88 +50,22 @@ def sample_doc(
     )
 
 
-@pytest.fixture()
-def _search_index(
-    tmp_path: Path,
-    settings: Settings,
-) -> Generator[None, None, None]:
-    """Create a temp index directory and point INDEX_DIR at it.
+@pytest.fixture
+def _search_index(paperless_dirs: "PaperlessDirs") -> None:
+    """Point the search backend at a fresh, empty index directory.
 
-    Resets the backend singleton before and after so each test gets a clean
-    index rather than reusing a stale singleton from another test.
+    paperless_dirs owns INDEX_DIR and resets the backend singleton on both
+    sides of the test, so requesting it is all that is needed.
     """
-    from documents.search import reset_backend
-
-    index_dir = tmp_path / "index"
-    index_dir.mkdir()
-    settings.INDEX_DIR = index_dir
-    reset_backend()
-    yield
-    reset_backend()
-
-
-@pytest.fixture()
-def settings_timezone(settings: Settings) -> zoneinfo.ZoneInfo:
-    return zoneinfo.ZoneInfo(settings.TIME_ZONE)
 
 
 @pytest.fixture
-def rest_api_client():
+def searchable_document(_search_index: None) -> "Document":
+    """One searchable document, for tests about what the search endpoint
+    returns rather than about what it finds.
     """
-    The basic DRF ApiClient
-    """
-    yield APIClient()
+    from documents.search import get_backend
 
-
-@pytest.fixture()
-def regular_user(django_user_model: type[UserModelT]) -> UserModelT:
-    """Unprivileged authenticated user for permission boundary tests."""
-    return django_user_model.objects.create_user(username="regular", password="regular")
-
-
-@pytest.fixture()
-def admin_client(rest_api_client: APIClient, admin_user: UserModelT) -> APIClient:
-    """Admin client pre-authenticated and sending the v10 Accept header."""
-    rest_api_client.force_authenticate(user=admin_user)
-    rest_api_client.credentials(HTTP_ACCEPT="application/json; version=10")
-    return rest_api_client
-
-
-@pytest.fixture()
-def v9_client(rest_api_client: APIClient, admin_user: UserModelT) -> APIClient:
-    """Admin client pre-authenticated and sending the v9 Accept header."""
-    rest_api_client.force_authenticate(user=admin_user)
-    rest_api_client.credentials(HTTP_ACCEPT="application/json; version=9")
-    return rest_api_client
-
-
-@pytest.fixture()
-def user_client(rest_api_client: APIClient, regular_user: UserModelT) -> APIClient:
-    """Regular-user client pre-authenticated and sending the v10 Accept header."""
-    rest_api_client.force_authenticate(user=regular_user)
-    rest_api_client.credentials(HTTP_ACCEPT="application/json; version=10")
-    return rest_api_client
-
-
-@pytest.fixture(autouse=True)
-def _clear_content_type_caches() -> None:
-    """Clear Django's ContentType cache and guardian's lru_cache before each test.
-
-    Tests that delete and reinsert ContentType/Permission rows (e.g. the
-    importer) corrupt both caches. Without this fixture a subsequent test on
-    the same xdist worker sees stale ContentType objects and guardian raises
-    MixedContentTypeError.
-    """
-    ContentType.objects.clear_cache()
-    clear_ct_cache()
-
-
-@pytest.fixture(scope="session", autouse=True)
-def faker_session_locale():
-    """Set Faker locale for reproducibility."""
-    return "en_US"
-
-
-@pytest.fixture(scope="session", autouse=True)
-def faker_seed():
-    return 12345
+    doc = DocumentFactory.create(title="quarterly invoice", content="acme corp")
+    get_backend().add_or_update(doc)
+    return doc

src/documents/tests/helpers.py

@@ -0,0 +1,10 @@
+import re
+
+
+def dummy_preprocess(content: str) -> str:
+    """
+    Simpler, faster pre-processing for testing purposes
+    """
+    content = content.lower().strip()
+    content = re.sub(r"\s+", " ", content)
+    return content

src/documents/tests/management/test_management_sanity_checker.py

@@ -15,11 +15,11 @@ from rich.console import Console
 
 from documents.management.commands.document_sanity_checker import Command
 from documents.sanity_checker import SanityCheckMessages
-from documents.tests.factories import DocumentFactory
+from paperless_testing.factories import DocumentFactory
 
 if TYPE_CHECKING:
     from documents.models import Document
-    from documents.tests.conftest import PaperlessDirs
+    from paperless_testing.dirs import PaperlessDirs
 
 
 def _render_to_string(messages: SanityCheckMessages) -> str:
@@ -71,7 +71,7 @@ class TestRenderResultsWithIssues:
         assert "INFO" in output
         assert "No OCR data" in output
 
-    @pytest.mark.usefixtures("_media_settings")
+    @pytest.mark.usefixtures("paperless_dirs")
     def test_global_message(self) -> None:
         msgs = SanityCheckMessages()
         msgs.warning(None, "Orphaned file: /tmp/stray.pdf")
@@ -87,7 +87,7 @@ class TestRenderResultsWithIssues:
         assert "Thumbnail missing" in output
         assert "Checksum mismatch" in output
 
-    @pytest.mark.usefixtures("_media_settings")
+    @pytest.mark.usefixtures("paperless_dirs")
     def test_unknown_doc_pk(self) -> None:
         msgs = SanityCheckMessages()
         msgs.error(99999, "Ghost document")
@@ -184,7 +184,6 @@ class TestDocumentSanityCheckerCommand:
         assert "ERROR" in output
         assert "Original of document does not exist" in output
 
-    @pytest.mark.usefixtures("_media_settings")
     def test_checksum_mismatch(self, paperless_dirs: PaperlessDirs) -> None:
         """Lightweight document with zero-byte files triggers checksum mismatch."""
         doc = DocumentFactory(

src/documents/tests/samples/preprocessed_content_advanced.txt

@@ -1 +1 @@
-sampl textual document content includ mani charact possibl check classifi vector hey 00 test0707 content exampl document creat 2025 06 25 digit 0123456789 punctuat english text quick brown fox jump lazi dog english stop word accent latin diacrit àâäæçéèêëîïôœùûüÿñ arab لقد قام المترجم بعمل جيد greek αλφα βήτα γάμμα δέλτα ωμέγα cyril привет как дела добро пожаловать chines simplifi 你好 世界 今天的天气很好 chines tradit 歡迎來到世界 今天天氣很好 japanes kanji hiragana katakana 東京へ行きます カタカナ ひらがな 漢字 korean hangul 안녕하세요 오늘 날씨 어때요 arab مرحب ا كيف حالك hebrew שלום מה שלומך emoji symbol µ math ₀ x² dx π 3 14159 e ρ ε₀ currenc 1 date format 25 06 2025 june 25 2025 2025年6月25日 quot french bonjour ça va quot german guten tag wie geht newlin test r n r tab ttest tspace 192 33601010101 end document
+sampl textual document content includ mani charact possibl check classifi vector hey 00 test0707 content exampl document creat 2025 06 25 digit 0123456789 punctuat english text quick brown fox jump lazi dog english stop word accent latin diacrit àâäæçéèêëîïôœùûüÿñ arab لقد قام المترجم بعمل جيد greek αλφα βήτα γάμμα δέλτα ωμέγα cyril привет как дела добро пожаловать chines simplifi 你好 世界 今天的天气很好 chines tradit 歡迎來到世界 今天天氣很好 japanes kanji hiragana katakana 東京へ行きます カタカナ ひらがな 漢字 korean hangul 안녕하세요 오늘 날씨 어때요 arab مرحبًا كيف حالك hebrew שלום מה שלומך emoji ️ symbol µ math x dx π 3 14159 e ρ ε currenc 1 date format 25 06 2025 june 25 2025 2025年6月25日 quot french bonjour ça va quot german guten tag wie geht newlin test r n r tab ttest tspace 192 33601010101 end document

src/documents/tests/search/_ast_helpers.py

@@ -0,0 +1,30 @@
+"""Field references and leaf constructors shared by the unit-level tests.
+
+These are pure whoosh-compat AST values with no state and no database
+behind them, so they stay plain module-level functions rather than
+fixtures. ``NOTES`` is the reason this module exists at all: notes is a
+JSON field, so addressing its text means naming a subpath, and spelling
+``wc.FieldRef("notes", "note")`` out once per test file invites two of
+them to disagree.
+"""
+
+from __future__ import annotations
+
+import whoosh_compat as wc
+import whoosh_compat.ast as wc_ast
+
+CONTENT = wc.FieldRef("content")
+TITLE = wc.FieldRef("title")
+NOTES = wc.FieldRef("notes", "note")
+BIGRAM_CONTENT = wc.FieldRef("bigram_content")
+BIGRAM_TITLE = wc.FieldRef("bigram_title")
+
+
+def content(text: str) -> wc_ast.Term:
+    """A Term on the content field, the default leaf these tests widen."""
+    return wc_ast.Term(field=CONTENT, text=text)
+
+
+def bigram(text: str) -> wc_ast.Term:
+    """A Term on the bigram side of content."""
+    return wc_ast.Term(field=BIGRAM_CONTENT, text=text)

src/documents/tests/search/conftest.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import tempfile
 from typing import TYPE_CHECKING
 
 import pytest
@@ -10,25 +9,21 @@ from documents.search._backend import TantivyBackend
 from documents.search._backend import reset_backend
 from documents.search._schema import build_schema
 from documents.search._tokenizer import register_tokenizers
+from paperless_testing.factories import DocumentFactory
 
 if TYPE_CHECKING:
+    from collections.abc import Callable
     from collections.abc import Generator
-    from pathlib import Path
 
     from pytest_django.fixtures import Settings
 
-
-@pytest.fixture
-def index_dir(tmp_path: Path, settings: Settings) -> Path:
-    path = tmp_path / "index"
-    path.mkdir()
-    settings.INDEX_DIR = path
-    return path
+    from documents.models import Document
+    from paperless_testing.dirs import PaperlessDirs
 
 
 @pytest.fixture
-def backend() -> Generator[TantivyBackend, None, None]:
-    b = TantivyBackend()  # path=None → in-memory index
+def backend(paperless_dirs: PaperlessDirs) -> Generator[TantivyBackend, None, None]:
+    b = TantivyBackend(path=paperless_dirs.index_dir)
     b.open()
     try:
         yield b
@@ -37,9 +32,58 @@ def backend() -> Generator[TantivyBackend, None, None]:
         reset_backend()
 
 
+@pytest.fixture
+def long_cjk_run() -> str:
+    """A CJK run too long for the content analyzer to keep.
+
+    48 characters, 144 UTF-8 bytes: one token longer than remove_long's
+    129-byte limit, so the content side analyzes to nothing and only the
+    bigram field can match it.
+    """
+    return "東京都の公共文書について" * 4
+
+
+@pytest.fixture
+def index_document(backend: TantivyBackend) -> Callable[..., Document]:
+    """Build a Document with DocumentFactory and add it to the index.
+
+    The factory supplies a unique checksum, so a test only passes the
+    fields its assertion actually depends on.
+    """
+
+    def _index(**kwargs: object) -> Document:
+        doc = DocumentFactory(**kwargs)
+        backend.add_or_update(doc)
+        return doc
+
+    return _index
+
+
+@pytest.fixture
+def matched_ids(backend: TantivyBackend) -> Callable[[str], set[int]]:
+    """Run a query as no particular user and return the matching pks."""
+
+    def _matched_ids(query: str) -> set[int]:
+        return set(backend.search_ids(query, user=None))
+
+    return _matched_ids
+
+
 @pytest.fixture(scope="module")
-def index() -> tantivy.Index:
-    """A real Tantivy index for parse-acceptance tests (module scope for speed)."""
-    idx = tantivy.Index(build_schema(), path=tempfile.mkdtemp())
-    register_tokenizers(idx, "english")
+def query_index() -> tantivy.Index:
+    """An in-memory, unstemmed index for the parse-only tests.
+
+    These never index a document, so one index per module is shared
+    read-only across that module's tests.
+    """
+    idx = tantivy.Index(build_schema(), path=None)
+    register_tokenizers(idx, "")
     return idx
+
+
+@pytest.fixture
+def fuzzy_enabled(settings: Settings) -> None:
+    """Enable the fuzzy blend clause. The threshold doubles as a minimum
+    score filter, so it is set to 0.0: every hit passes and the test sees
+    the clause's matching behaviour, not the filter's."""
+    settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.0

src/documents/tests/search/test_acceptance.py

@@ -0,0 +1,508 @@
+"""Result-level acceptance corpus: real documents indexed via build_schema(),
+real queries run through parse_user_query(), matched-document-ID sets
+asserted, not intermediate ASTs or query strings. This is paperless-ngx's
+analogue of whoosh-compat's own tests/emitter/test_acceptance_e2e.py.
+
+Supersedes test_query.py's TestParseUserQuery result-level cases.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC
+from datetime import datetime
+from typing import TYPE_CHECKING
+
+import pytest
+import time_machine
+
+from documents.models import CustomField
+from documents.models import CustomFieldInstance
+from documents.models import DocumentType
+from documents.models import Note
+from documents.models import StoragePath
+from documents.search._query import parse_user_query
+from paperless_testing.factories import DocumentFactory
+from paperless_testing.factories import UserFactory
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from documents.models import Document
+    from documents.search._backend import TantivyBackend
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+FROZEN_NOW = datetime(2026, 6, 15, 12, 0, tzinfo=UTC)
+
+
+@pytest.fixture
+def indexed_documents(index_document: Callable[..., Document]) -> dict[str, int]:
+    """Index a small fixture set, return {label: doc_id} for corpus queries."""
+    docs = {
+        "invoice_2020": index_document(
+            title="Invoice 2020",
+            content="invoice total due",
+            archive_serial_number=100,
+        ),
+        "invoice_2021": index_document(
+            title="Invoice 2021",
+            content="invoice total due",
+            archive_serial_number=101,
+        ),
+        "invoice_2023": index_document(
+            title="Invoice 2023",
+            content="invoice total due",
+            archive_serial_number=102,
+        ),
+        "receipt_2022": index_document(
+            title="Receipt 2022",
+            content="receipt total due",
+            archive_serial_number=103,
+        ),
+    }
+    return {label: doc.pk for label, doc in docs.items()}
+
+
+class TestIssue13568BracketWildcard:
+    """paperless-ngx#13568: title:202[0-3]* must keep its character class,
+    not fold to a prefix query that silently drops it."""
+
+    def test_bracket_class_wildcard_matches_only_in_range_years(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        indexed_documents: dict[str, int],
+    ) -> None:
+        """
+        GIVEN:
+            - Four indexed documents titled Invoice 2020/2021/2023 and
+              Receipt 2022
+        WHEN:
+            - "title:202[0-1]*" is searched ([0-1], not [0-3], is
+              deliberate: the fixture's trailing digits are 0/1/2/3, so a
+              [0-3] class would match all four and pass even if the
+              character class were silently dropped and folded to an
+              unconstrained "202*" prefix; [0-1] partitions the fixture
+              into a genuine in-range/out-of-range split)
+        THEN:
+            - Only the 2020 and 2021 documents match, proving the bracket
+              character class survived (issue #13568's original bug)
+        """
+        matched = matched_ids("title:202[0-1]*")
+        expected = {
+            indexed_documents["invoice_2020"],
+            indexed_documents["invoice_2021"],
+        }
+        assert matched == expected, (
+            "title:202[0-1]* must match 2020/2021 titles and exclude 2022/2023 "
+            "- if this matches everything, the wildcard's character class was "
+            "silently dropped (issue #13568's original bug)"
+        )
+
+
+class TestFieldBoosts:
+    def test_title_boost_ranks_title_match_above_content_only_match(
+        self,
+        backend: TantivyBackend,
+        index_document: Callable[..., Document],
+    ) -> None:
+        """
+        GIVEN:
+            - One document whose title contains the query word and another
+              whose content (not title) contains it
+        WHEN:
+            - The query word is searched unfielded
+        THEN:
+            - The title match ranks first, proving our title field boost
+              actually affects ranking
+        """
+        title_match = index_document(
+            title="urgent",
+            content="nothing else relevant",
+        )
+        index_document(
+            title="nothing",
+            content="urgent matter here",
+        )
+        query = parse_user_query(backend._index, "urgent", UTC)
+        searcher = backend._index.searcher()
+        results = searcher.search(query, limit=10)
+        ranked_ids = [
+            searcher.doc(addr).to_dict()["id"][0] for _score, addr in results.hits
+        ]
+        assert ranked_ids[0] == title_match.pk
+
+
+class TestJsonSubpaths:
+    def test_notes_user_matches_document_with_that_note_author(
+        self,
+        backend: TantivyBackend,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document with a Note authored by "alice" and a second,
+              unrelated document with no note
+        WHEN:
+            - "notes.user:alice" is searched
+        THEN:
+            - Only the document with alice's note matches
+        """
+        alice = UserFactory(username="alice")
+        doc_with_note = DocumentFactory(
+            title="Has note",
+            content="x",
+        )
+        Note.objects.create(document=doc_with_note, user=alice, note="reminder")
+        backend.add_or_update(doc_with_note)
+        index_document(title="No note", content="x")
+        matched = matched_ids("notes.user:alice")
+        assert matched == {doc_with_note.pk}
+
+    def test_custom_fields_name_and_value_combine(
+        self,
+        backend: TantivyBackend,
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document with a "Contract Number" custom field valued
+              "policy", and a second document with a differently-named
+              custom field also valued "policy"
+        WHEN:
+            - 'custom_fields.name:"Contract Number" custom_fields.value:policy'
+              is searched
+        THEN:
+            - Only the document whose field name AND value both match is
+              returned
+        """
+        field = CustomField.objects.create(
+            name="Contract Number",
+            data_type=CustomField.FieldDataType.STRING,
+        )
+        other_field = CustomField.objects.create(
+            name="Other Field",
+            data_type=CustomField.FieldDataType.STRING,
+        )
+        matching = DocumentFactory(
+            title="Matching",
+            content="x",
+        )
+        CustomFieldInstance.objects.create(
+            document=matching,
+            field=field,
+            value_text="policy",
+        )
+        backend.add_or_update(matching)
+        non_matching = DocumentFactory(
+            title="Non-matching",
+            content="x",
+        )
+        CustomFieldInstance.objects.create(
+            document=non_matching,
+            field=other_field,
+            value_text="policy",
+        )
+        backend.add_or_update(non_matching)
+        matched = matched_ids(
+            'custom_fields.name:"Contract Number" custom_fields.value:policy',
+        )
+        assert matched == {matching.pk}
+
+
+class TestUnregisteredIdFieldFoldsToLiteralText:
+    """tag_id, owner_id, etc. are intentionally excluded from the
+    FieldRegistry - always internal index columns, never meant to be
+    query-addressable. Prove an unregistered field folds to a literal
+    text search that matches nothing, rather than erroring."""
+
+    def test_tag_id_query_matches_nothing(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        indexed_documents: dict[str, int],
+    ) -> None:
+        """
+        GIVEN:
+            - A real indexed corpus and "tag_id", a field intentionally
+              excluded from the FieldRegistry (an internal index column,
+              never meant to be query-addressable)
+        WHEN:
+            - "tag_id:5" is searched
+        THEN:
+            - It folds to a literal text search and matches nothing,
+              rather than erroring
+        """
+        matched = matched_ids("tag_id:5")
+        assert matched == set()
+
+
+class TestFuzzyBlendSurvivesWhooshGrammar:
+    """A query mixing whoosh-only grammar (a date keyword) with a typo'd
+    free-text word must still fuzzy-match the intended document when
+    ADVANCED_FUZZY_SEARCH_THRESHOLD is enabled. Fuzzy widening happens
+    inside the already-parsed AST (_widen_leaf, via emit()), never by
+    re-parsing the raw query string, so whoosh grammar that tantivy's own
+    parser rejects cannot knock the fuzzy side out."""
+
+    def test_typo_fuzzy_matches_alongside_date_keyword(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+        settings,
+    ) -> None:
+        """
+        GIVEN:
+            - ADVANCED_FUZZY_SEARCH_THRESHOLD enabled, and a document
+              indexed with content "receipt total due"
+        WHEN:
+            - The query blends whoosh-only grammar tantivy's own parser
+              rejects ("added:today") with a one-transposition misspelling
+              of a word in the indexed content
+        THEN:
+            - The document still matches, because fuzzy widening happens
+              inside the already-parsed AST, never by re-parsing the raw
+              query string, so grammar tantivy's parser cannot handle
+              cannot knock the fuzzy side out
+        """
+        settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5
+        with time_machine.travel(FROZEN_NOW, tick=False):
+            doc = index_document(
+                title="Receipt March",
+                content="receipt total due",
+                archive_serial_number=900,
+            )
+            # Sanity: the exact spelling matches through the exact clause.
+            assert doc.pk in matched_ids("added:today receipt")
+            # The regression: the misspelling (one transposition) only
+            # matches via the fuzzy clause, and "added:today" is
+            # whoosh-only grammar tantivy's parser rejects, so raw-string
+            # fuzzy parsing skips the clause entirely and this returns
+            # nothing. The typo is deliberate; keep codespell away from it.
+            typo_query = "added:today reciept"  # codespell:ignore reciept
+            assert doc.pk in matched_ids(typo_query)
+
+    def test_negated_words_do_not_fuzzy_match(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+        settings,
+    ) -> None:
+        """
+        GIVEN:
+            - ADVANCED_FUZZY_SEARCH_THRESHOLD enabled, and a document
+              containing the NOT'd word ("receipt") but not the positive
+              word ("total"), so nothing matches the exact clause -- the
+              shape a naive fuzzy string built from ALL words (including
+              the NOT'd one) would make this document the sole hit,
+              normalize its score to 1.0, and survive any threshold (a
+              shape with an exact-matching sibling document would NOT
+              discriminate: normalization would rank the resurfaced
+              document far below the exact match and the threshold would
+              cut it even for a naive implementation)
+        WHEN:
+            - "added:today total NOT receipt" is searched
+        THEN:
+            - The document does not match; a term the user excluded must
+              not resurface through the fuzzy clause
+        """
+        settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5
+        with time_machine.travel(FROZEN_NOW, tick=False):
+            index_document(
+                title="Receipt Archive",
+                content="receipt archived stack",
+                archive_serial_number=901,
+            )
+            assert matched_ids("added:today total NOT receipt") == set()
+
+
+class TestUnquotedDateKeywordPhrases:
+    """The unquoted spelling (added:previous month) is honored natively by
+    whoosh-compat's own grammar for this closed phrase vocabulary, no
+    app-level rewrite is involved. Pins that the historically supported
+    spelling keeps working now that paperless no longer pre-quotes it."""
+
+    @pytest.fixture
+    def period_documents(
+        self,
+        index_document: Callable[..., Document],
+    ) -> dict[str, int]:
+        with time_machine.travel(FROZEN_NOW, tick=False):
+            in_may = index_document(
+                title="May Doc",
+                content="statement",
+                archive_serial_number=910,
+                added=datetime(2026, 5, 20, 12, 0, tzinfo=UTC),
+            )
+            in_june = index_document(
+                title="June Doc",
+                content="statement",
+                archive_serial_number=911,
+                added=datetime(2026, 6, 10, 12, 0, tzinfo=UTC),
+            )
+        return {"in_may": in_may.pk, "in_june": in_june.pk}
+
+    @pytest.mark.parametrize(
+        "query",
+        [
+            pytest.param("added:previous month", id="unquoted"),
+            pytest.param('added:"previous month"', id="quoted"),
+            pytest.param("added:Previous Month", id="unquoted-mixed-case"),
+        ],
+    )
+    def test_unquoted_matches_the_same_documents_as_quoted(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        period_documents: dict[str, int],
+        query: str,
+    ) -> None:
+        """
+        GIVEN:
+            - Two documents added in different months, time frozen so
+              only one falls in "previous month"
+        WHEN:
+            - The same date-keyword phrase is spelled unquoted, quoted,
+              and unquoted with mixed case
+        THEN:
+            - All three spellings match the same document; paperless no
+              longer pre-quotes this phrase before parsing, relying on
+              whoosh-compat's own grammar to accept it unquoted natively
+        """
+        with time_machine.travel(FROZEN_NOW, tick=False):
+            assert matched_ids(query) == {period_documents["in_may"]}
+
+    @pytest.mark.parametrize(
+        "query",
+        [
+            pytest.param("added:this month", id="this-month"),
+            pytest.param("added:this year", id="this-year"),
+            pytest.param("added:previous week", id="previous-week"),
+            pytest.param("added:previous quarter", id="previous-quarter"),
+            pytest.param("added:previous year", id="previous-year"),
+            pytest.param("created:previous month", id="created-field"),
+            pytest.param("modified:previous month", id="modified-field"),
+        ],
+    )
+    def test_every_phrase_and_date_field_parses_without_error(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        period_documents: dict[str, int],
+        query: str,
+    ) -> None:
+        """
+        GIVEN:
+            - Our real schema and every date-keyword phrase in the
+              vocabulary, against every date field we expose (added,
+              created, modified)
+        WHEN:
+            - Each combination is searched
+        THEN:
+            - It parses and searches cleanly against our schema (no
+              SearchQueryError, so no HTTP 400); exact window semantics
+              are whoosh-compat's own and are pinned in its own suite
+        """
+        with time_machine.travel(FROZEN_NOW, tick=False):
+            matched_ids(query)
+
+    def test_text_field_keyword_words_are_ordinary_text(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+        period_documents: dict[str, int],
+    ) -> None:
+        """
+        GIVEN:
+            - period_documents (indexed by added-date) and a third
+              document whose title literally contains the words
+              "previous month"
+        WHEN:
+            - "title:previous month" is searched
+        THEN:
+            - Only the document whose title contains those words matches;
+              "previous month" after a TEXT field (or unfielded) is
+              ordinary text, not a date phrase, so the date-window
+              documents do not match
+        """
+        with time_machine.travel(FROZEN_NOW, tick=False):
+            wordy = index_document(
+                title="Notes from the previous month",
+                content="meeting notes",
+                archive_serial_number=912,
+            )
+            assert matched_ids("title:previous month") == {wordy.pk}
+
+
+class TestFieldAliases:
+    """type:/path: are registry aliases for document_type:/storage_path:.
+    The only other alias coverage is parse-shape; these prove resolution
+    end-to-end against a real index."""
+
+    def test_type_alias_and_canonical_name_match_the_same_document(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document with document_type "invoice", and a decoy
+              document with no type whose content merely mentions
+              "invoice" (document_type is itself a default search field,
+              so if alias resolution ever broke and "type:invoice"
+              demoted to unfielded text, the token would STILL match the
+              typed document through the field value; the decoy carrying
+              the query word in content is what makes a demoted search
+              distinguishable, since it would then match both documents
+              and fail the exact-set assertion -- the title avoids
+              stemming to "type": English stems Typed -> type)
+        WHEN:
+            - "type:invoice" and "document_type:invoice" are each
+              searched
+        THEN:
+            - Both resolve to the same document, proving the "type" alias
+              and its canonical field name agree end-to-end against a
+              real index
+        """
+        invoice_type = DocumentType.objects.create(name="invoice")
+        typed = index_document(
+            title="First",
+            content="quarterly statement",
+            document_type=invoice_type,
+        )
+        index_document(
+            title="Second",
+            content="invoice mentioned in body",
+        )
+        assert matched_ids("type:invoice") == {typed.pk}
+        assert matched_ids("document_type:invoice") == {typed.pk}
+
+    def test_path_alias_and_canonical_name_match_the_same_document(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document stored under storage_path "archive", and a decoy
+              document with no storage_path whose content merely mentions
+              "archive" (storage_path is NOT a default search field
+              today, so a demoted "path:archive" already matches nothing;
+              the content decoy keeps this test discriminating even if
+              storage_path ever joins the defaults)
+        WHEN:
+            - "path:archive" and "storage_path:archive" are each searched
+        THEN:
+            - Both resolve to the same document, proving the "path" alias
+              and its canonical field name agree end-to-end against a
+              real index
+        """
+        archive = StoragePath.objects.create(name="archive", path="archive/{title}")
+        stored = index_document(
+            title="Stored",
+            content="quarterly statement",
+            storage_path=archive,
+        )
+        index_document(
+            title="Loose",
+            content="archive mentioned in body",
+        )
+        assert matched_ids("path:archive") == {stored.pk}
+        assert matched_ids("storage_path:archive") == {stored.pk}

src/documents/tests/search/test_any_of.py

@@ -0,0 +1,43 @@
+"""_any_of, the clause-list helper.
+
+One test, for the empty-list case its callers never produce but which the
+helper still has to answer for.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from documents.search._query import _any_of
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from documents.models import Document
+    from documents.search._backend import TantivyBackend
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+
+class TestAnyOf:
+    def test_an_empty_clause_list_matches_nothing(
+        self,
+        backend: TantivyBackend,
+        index_document: Callable[..., Document],
+    ) -> None:
+        """
+        GIVEN:
+            - An indexed document, and no clauses at all
+        WHEN:
+            - _any_of is called with an empty list and the result is run
+        THEN:
+            - It matches no documents, rather than raising or matching
+              everything
+        """
+        index_document(title="x", content="x")
+
+        results = backend._index.searcher().search(_any_of([]), limit=10)
+
+        assert len(results.hits) == 0

src/documents/tests/search/test_backend.py

@@ -1,12 +1,12 @@
 import json
+import logging
+from datetime import date
 from pathlib import Path
 
 import pytest
 from django.contrib.auth.models import Group
-from django.contrib.auth.models import User
 from django.db import connection
 from django.test.utils import CaptureQueriesContext
-from guardian.shortcuts import assign_perm
 from pytest_mock import MockerFixture
 
 from documents.models import CustomField
@@ -18,12 +18,16 @@ from documents.search._backend import TantivyBackend
 from documents.search._backend import WriteBatch
 from documents.search._backend import get_backend
 from documents.search._backend import reset_backend
+from documents.search._schema import REBUILD_MARKER
+from documents.search._schema import needs_rebuild
 from documents.signals.handlers import add_to_index
-from documents.tests.factories import CorrespondentFactory
-from documents.tests.factories import DocumentFactory
-from documents.tests.factories import DocumentTypeFactory
-from documents.tests.factories import TagFactory
-from documents.tests.factories import UserFactory
+from paperless_testing.dirs import PaperlessDirs
+from paperless_testing.factories import CorrespondentFactory
+from paperless_testing.factories import DocumentFactory
+from paperless_testing.factories import DocumentTypeFactory
+from paperless_testing.factories import TagFactory
+from paperless_testing.factories import UserFactory
+from paperless_testing.permissions import grant_object
 
 pytestmark = [pytest.mark.search, pytest.mark.django_db]
 
@@ -189,7 +193,7 @@ class TestAddOrUpdateIds:
             pk=1,
             owner=owner,
         )
-        assign_perm("view_document", user, doc)
+        grant_object(user, doc, "view_document")
 
         with backend.batch_update() as batch:
             batch.add_or_update_ids([doc.pk])
@@ -209,7 +213,7 @@ class TestAddOrUpdateIds:
             pk=1,
             owner=owner,
         )
-        assign_perm("view_document", group, doc)
+        grant_object(group, doc, "view_document")
 
         with backend.batch_update() as batch:
             batch.add_or_update_ids([doc.pk])
@@ -289,6 +293,80 @@ class TestAddOrUpdateIds:
         assert backend.search_ids("updated", user=None) == [doc.pk]
 
 
+class TestVersionsAreIndexedAsTheirRoot:
+    """Only root documents are indexed, with their effective content, so
+    every write path that is handed a version indexes its root instead."""
+
+    @staticmethod
+    def _root_with_version() -> tuple[Document, Document]:
+        root = DocumentFactory(title="Statement", content="stale text")
+        version = DocumentFactory(
+            title="Statement",
+            content="latest text",
+            root_document=root,
+            version_index=1,
+        )
+        return root, version
+
+    def test_add_or_update_indexes_the_root_of_a_version(
+        self,
+        backend: TantivyBackend,
+    ) -> None:
+        """
+        GIVEN:
+            - A root document with a version
+        WHEN:
+            - The version is passed to add_or_update
+        THEN:
+            - The root is indexed with the version's text, and the version is not
+        """
+        root, version = self._root_with_version()
+
+        backend.add_or_update(version)
+
+        assert backend.search_ids("latest", user=None) == [root.pk]
+        assert backend.search_ids("stale", user=None) == []
+
+    def test_add_or_update_ids_indexes_each_root_once(
+        self,
+        backend: TantivyBackend,
+    ) -> None:
+        """
+        GIVEN:
+            - A root document with a version
+        WHEN:
+            - Both ids are passed to add_or_update_ids
+        THEN:
+            - The root is indexed once and the version is not indexed
+        """
+        root, version = self._root_with_version()
+
+        with backend.batch_update() as batch:
+            batch.add_or_update_ids([version.pk, root.pk])
+
+        assert backend.search_ids("Statement", user=None) == [root.pk]
+        assert backend.search_ids("latest", user=None) == [root.pk]
+
+    def test_add_or_update_ids_resolves_a_lone_version(
+        self,
+        backend: TantivyBackend,
+    ) -> None:
+        """
+        GIVEN:
+            - A root document with a version
+        WHEN:
+            - Only the version's id is passed to add_or_update_ids
+        THEN:
+            - The root is indexed
+        """
+        root, version = self._root_with_version()
+
+        with backend.batch_update() as batch:
+            batch.add_or_update_ids([version.pk])
+
+        assert backend.search_ids("latest", user=None) == [root.pk]
+
+
 class TestSearch:
     """Test search query parsing and matching via search_ids."""
 
@@ -763,8 +841,8 @@ class TestSearchIds:
 
     def test_respects_permission_filter(self, backend: TantivyBackend) -> None:
         """search_ids must respect user permission filtering."""
-        owner = User.objects.create_user("ids_owner")
-        other = User.objects.create_user("ids_other")
+        owner = UserFactory(username="ids_owner")
+        other = UserFactory(username="ids_other")
         doc = Document.objects.create(
             title="private doc",
             content="secret keyword",
@@ -824,6 +902,53 @@ class TestRebuild:
         backend.rebuild(Document.objects.all(), iter_wrapper=wrapper)
         assert 30 in seen
 
+    def test_successful_rebuild_leaves_index_up_to_date(
+        self,
+        backend: TantivyBackend,
+        paperless_dirs: PaperlessDirs,
+    ) -> None:
+        """
+        GIVEN:
+            - A backend and one document
+        WHEN:
+            - rebuild() completes
+        THEN:
+            - needs_rebuild() is False and no rebuild marker remains
+        """
+        DocumentFactory.create()
+
+        backend.rebuild(Document.objects.all())
+
+        assert needs_rebuild(paperless_dirs.index_dir) is False
+        assert not (paperless_dirs.index_dir / REBUILD_MARKER).exists()
+
+    def test_interrupted_rebuild_is_retried(
+        self,
+        backend: TantivyBackend,
+        paperless_dirs: PaperlessDirs,
+    ) -> None:
+        """
+        GIVEN:
+            - A rebuild that dies while indexing documents (e.g. the database
+              connection is lost)
+        WHEN:
+            - needs_rebuild() is checked afterwards
+        THEN:
+            - It is True, even though the empty index was already stamped with
+              current settings, so the next start rebuilds instead of reporting
+              the index as up to date
+        """
+        DocumentFactory.create()
+
+        def die(pairs):
+            raise RuntimeError("terminating connection due to administrator command")
+            yield  # pragma: no cover
+
+        with pytest.raises(RuntimeError):
+            backend.rebuild(Document.objects.all(), iter_wrapper=die)
+
+        assert needs_rebuild(paperless_dirs.index_dir) is True
+
     def test_includes_group_granted_viewers(self, backend: TantivyBackend) -> None:
         """Rebuild must index viewer ids for group-only grants, not just direct ones.
 
@@ -843,7 +968,7 @@ class TestRebuild:
             content="group secret keyword",
             owner=owner,
         )
-        assign_perm("view_document", group, doc)
+        grant_object(group, doc, "view_document")
 
         backend.rebuild(Document.objects.all())
 
@@ -855,6 +980,79 @@ class TestRebuild:
         assert ids == [doc.pk]
 
 
+class TestCreatedDateOutOfRange:
+    """The index stores dates as nanosecond i64 values (1677-09-22 to 2262-04-11).
+
+    A document whose created date falls outside that window must not abort
+    indexing: it is indexed without a created value and a warning names it.
+    """
+
+    @pytest.mark.parametrize(
+        ("created", "expected_warnings"),
+        [
+            pytest.param(date(1677, 9, 22), 0, id="first-representable-day"),
+            pytest.param(date(2262, 4, 11), 0, id="last-representable-day"),
+            pytest.param(date(1677, 9, 21), 1, id="day-before-first"),
+            pytest.param(date(2262, 4, 12), 1, id="day-after-last"),
+            pytest.param(date(16, 8, 30), 1, id="two-digit-year-read-as-year-16"),
+            pytest.param(date(9999, 12, 31), 1, id="max-python-date"),
+        ],
+    )
+    def test_add_or_update_indexes_document_and_warns_when_out_of_range(
+        self,
+        backend: TantivyBackend,
+        caplog: pytest.LogCaptureFixture,
+        created: date,
+        expected_warnings: int,
+    ) -> None:
+        """
+        GIVEN:
+            - A document with a created date at or beyond the index date limits
+        WHEN:
+            - The document is added to the index
+        THEN:
+            - The document is indexed and searchable either way
+            - A warning naming the document is logged only for out-of-range dates
+        """
+        doc = DocumentFactory(created=created, content="boundarycontent")
+
+        with caplog.at_level(logging.WARNING, logger="paperless.search"):
+            backend.add_or_update(doc)
+
+        assert backend.search_ids("boundarycontent", user=None) == [doc.pk]
+        warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
+        assert len(warnings) == expected_warnings
+        if expected_warnings:
+            assert f"Document {doc.pk}" in warnings[0].getMessage()
+
+    def test_rebuild_continues_past_out_of_range_document(
+        self,
+        backend: TantivyBackend,
+        caplog: pytest.LogCaptureFixture,
+    ) -> None:
+        """
+        GIVEN:
+            - A document with an unrepresentable created date among valid ones
+        WHEN:
+            - The index is rebuilt
+        THEN:
+            - Rebuild completes and every document is searchable
+            - A warning names the offending document
+        """
+        good = DocumentFactory(created=date(2016, 8, 30), content="rebuildcontent")
+        bad = DocumentFactory(created=date(16, 8, 30), content="rebuildcontent")
+
+        with caplog.at_level(logging.WARNING, logger="paperless.search"):
+            backend.rebuild(Document.objects.all())
+
+        assert sorted(backend.search_ids("rebuildcontent", user=None)) == sorted(
+            [good.pk, bad.pk],
+        )
+        warnings = [r for r in caplog.records if r.levelno == logging.WARNING]
+        assert len(warnings) == 1
+        assert f"Document {bad.pk}" in warnings[0].getMessage()
+
+
 class TestAutocomplete:
     """Test autocomplete functionality."""
 
@@ -938,6 +1136,34 @@ class TestMoreLikeThis:
         assert 150 not in ids
         assert 151 in ids
 
+    def test_more_like_this_ids_seeded_by_version_uses_root(
+        self,
+        backend: TantivyBackend,
+    ) -> None:
+        """A version is not indexed, so it must be looked up as its root."""
+        root = DocumentFactory.create(
+            title="Important document",
+            content="financial information report",
+        )
+        version = DocumentFactory.create(
+            title="Important document",
+            content="financial information report",
+            root_document=root,
+            version_index=1,
+        )
+        other = DocumentFactory.create(
+            title="Another document",
+            content="financial information report",
+        )
+        backend.add_or_update(root)
+        backend.add_or_update(other)
+
+        ids = backend.more_like_this_ids(doc_id=version.pk, user=None)
+
+        assert other.pk in ids
+        assert root.pk not in ids
+        assert version.pk not in ids
+
 
 class TestSingleton:
     """Test get_backend() and reset_backend() singleton lifecycle."""
@@ -948,7 +1174,8 @@ class TestSingleton:
         yield
         reset_backend()
 
-    def test_returns_same_instance_on_repeated_calls(self, index_dir) -> None:
+    @pytest.mark.usefixtures("paperless_dirs")
+    def test_returns_same_instance_on_repeated_calls(self) -> None:
         """Singleton pattern: repeated calls to get_backend() must return the same instance."""
         assert get_backend() is get_backend()
 
@@ -965,7 +1192,8 @@ class TestSingleton:
         assert b1 is not b2
         assert b2._path == tmp_path / "b"
 
-    def test_reset_forces_new_instance(self, index_dir) -> None:
+    @pytest.mark.usefixtures("paperless_dirs")
+    def test_reset_forces_new_instance(self) -> None:
         """reset_backend() must force creation of a new backend instance on next get_backend() call."""
         b1 = get_backend()
         reset_backend()
@@ -1071,7 +1299,7 @@ class TestFieldHandling:
 
     def test_notes_include_user_information(self, backend: TantivyBackend) -> None:
         """Notes must be indexed with user information when available for structured queries."""
-        user = User.objects.create_user("notewriter")
+        user = UserFactory(username="notewriter")
         doc = Document.objects.create(
             title="Doc with notes",
             content="test",
@@ -1173,7 +1401,7 @@ class TestHighlightHits:
         notes.note: prefix so the query targets notes content directly, but
         the snippet is generated from notes_text which stores the same text.
         """
-        user = User.objects.create_user("hl_noteuser")
+        user = UserFactory(username="hl_noteuser")
         doc = Document.objects.create(
             title="Doc with matching note",
             content="unrelated content",

src/documents/tests/search/test_barcode_search.py

@@ -0,0 +1,95 @@
+"""Stored barcode contents in the search index.
+
+Barcodes are a JSON field like notes and custom fields: barcodes: resolves to
+barcodes.value:, and a plain query without the prefix does not look at them.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from documents.models import DocumentBarcode
+from paperless_testing.factories import DocumentBarcodeFactory
+from paperless_testing.factories import DocumentFactory
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from documents.models import Document
+    from documents.search._backend import TantivyBackend
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+
+class TestBarcodeSearch:
+    @pytest.fixture
+    def with_barcodes(self, backend: TantivyBackend) -> Document:
+        document = DocumentFactory(title="Letter", content="x")
+        DocumentBarcodeFactory(
+            document=document,
+            value="WIFI:T:WPA;S:Guest-WLAN;P:crocodile123;;",
+        )
+        DocumentBarcodeFactory(
+            document=document,
+            page=2,
+            value="DE89370400440532013000",
+            format=DocumentBarcode.Format.CODE128,
+        )
+        backend.add_or_update(document)
+        return document
+
+    def test_bare_barcodes_prefix_searches_values(
+        self,
+        with_barcodes: Document,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document with stored barcodes, and a decoy document whose
+              content (not a barcode) contains the same word
+        WHEN:
+            - A bare "barcodes:" prefix query is run
+        THEN:
+            - Only the document whose barcode matches is returned, also for a
+              part of a barcode between separators
+        """
+        index_document(title="Decoy", content="crocodile123 in the text")
+
+        assert matched_ids("barcodes:crocodile123") == {with_barcodes.pk}
+        assert matched_ids("barcodes.value:DE89370400440532013000") == {
+            with_barcodes.pk,
+        }
+
+    def test_barcodes_format_subpath(
+        self,
+        with_barcodes: Document,
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document with a QR code and a Code 128 barcode
+        WHEN:
+            - The format subpath is queried
+        THEN:
+            - The document is found by its barcode formats
+        """
+        assert matched_ids("barcodes.format:qrcode") == {with_barcodes.pk}
+        assert matched_ids("barcodes.format:aztec") == set()
+
+    def test_plain_query_ignores_barcodes(
+        self,
+        with_barcodes: Document,
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document with a barcode value not found in its text
+        WHEN:
+            - The value is searched without a field prefix
+        THEN:
+            - Nothing is found, as with notes and custom fields
+        """
+        assert matched_ids("crocodile123") == set()

src/documents/tests/search/test_checksum_prefix_queries.py

@@ -0,0 +1,82 @@
+"""``checksum`` wildcard patterns stay literal end to end, once user queries
+route through whoosh-compat.
+
+The registry-level fact (the pattern normalizer folds a KEYWORD pattern
+rather than stemming it) is pinned on its own in
+``test_keyword_pattern_literal.py``. This proves it actually reaches a real
+query: ``checksum:ceded*`` must match only the document whose checksum
+starts with "ceded", not the one whose checksum stems to the same run.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from documents.models import Document
+
+if TYPE_CHECKING:
+    from documents.search._backend import TantivyBackend
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+CEDEF00D = "cedef00ddeadbeef0123456789abcdef01234567"
+CEDEDEAD = "cededeadbeef567801234567" + "89abcdef01234567"
+
+
+class TestChecksumPrefixQueries:
+    @pytest.fixture
+    def indexed(self, backend: TantivyBackend) -> None:
+        for i, checksum in enumerate((CEDEF00D, CEDEDEAD)):
+            doc = Document.objects.create(
+                title=f"Checksum doc {i}",
+                content="invoices for the quarter",
+                checksum=checksum,
+                archive_serial_number=940 + i,
+            )
+            backend.add_or_update(doc)
+
+    def _ids(self, backend: TantivyBackend, query: str) -> set[int]:
+        return set(backend.search_ids(query, user=None))
+
+    def test_prefix_matches_only_the_document_that_starts_with_it(
+        self,
+        backend: TantivyBackend,
+        indexed: None,
+    ) -> None:
+        """
+        GIVEN:
+            - Two documents indexed with checksums that share a stem when
+              run through the English stemmer ("cedef00d..." and
+              "cededead...") but only one literally starts with "ceded"
+        WHEN:
+            - "checksum:ceded*" is searched
+        THEN:
+            - Only the document whose checksum literally starts with
+              "ceded" matches; the pattern normalizer folds a KEYWORD
+              pattern rather than stemming it, so this reaches a real
+              query end to end
+        """
+        matched = self._ids(backend, "checksum:ceded*")
+        expected = Document.objects.get(checksum=CEDEDEAD).pk
+        assert matched == {expected}
+
+    def test_text_prefix_still_reaches_the_stemmed_index(
+        self,
+        backend: TantivyBackend,
+        indexed: None,
+    ) -> None:
+        """
+        GIVEN:
+            - Two documents indexed with content "invoices for the
+              quarter"
+        WHEN:
+            - "invoice*" is searched against the TEXT content field
+        THEN:
+            - Both documents match, confirming the checksum field's
+              literal-pattern behavior is specific to KEYWORD fields and
+              does not affect TEXT field wildcard matching against
+              stemmed terms
+        """
+        assert len(self._ids(backend, "invoice*")) == 2

src/documents/tests/search/test_cjk_clause.py

@@ -0,0 +1,193 @@
+"""CJK bigram matching in QUERY-mode searches.
+
+The bigram fields exist so CJK runs are matchable at all (the default
+analyzers keep a whitespace-free CJK run as one indivisible token), but
+matching them must not widen the query beyond what the user asked for: a
+CJK term the query excludes, or restricts to one field, must not come back
+through them.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from pytest_django.fixtures import SettingsWrapper
+
+    from documents.models import Document
+
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+
+class TestCjkParseFailureDegradesGracefully:
+    def test_a_cjk_run_tantivy_cannot_parse_drops_the_clause_only(self) -> None:
+        """
+        GIVEN:
+            - A CJK run and an index-like object whose parse_query is
+              forced to raise
+        WHEN:
+            - _parse_cjk_text is called
+        THEN:
+            - It returns None instead of propagating, so a CJK run tantivy
+              cannot parse only drops the bigram clause rather than
+              failing the whole query. Broad on purpose (bare except
+              Exception), unlike the fuzzy blend's narrower ValueError
+              guard: a CJK run is not filtered to a guaranteed-safe token
+              set the way the fuzzy blend's word string is, so the exact
+              failure mode tantivy could raise here is not pinned down
+        """
+        from documents.search._query import _parse_cjk_text
+
+        class _RaisingIndex:
+            def parse_query(self, *args: object, **kwargs: object) -> object:
+                raise RuntimeError("synthetic parse failure")
+
+        assert _parse_cjk_text(_RaisingIndex(), "東京", ["bigram_content"]) is None
+
+    def test_no_cjk_text_at_all_returns_none_without_parsing(self) -> None:
+        """
+        GIVEN:
+            - A raw query string with no CJK characters at all
+        WHEN:
+            - _build_cjk_query (the simple TEXT/TITLE-mode builder) is
+              called directly
+        THEN:
+            - It returns None without ever attempting to parse anything.
+              The only real caller already guards this with _has_cjk(),
+              so this is defensive: it keeps the function safe to call on
+              its own, not a path a real search currently reaches
+        """
+        from documents.search._query import _build_cjk_query
+
+        assert _build_cjk_query(None, "invoice total due", ["bigram_content"]) is None
+
+
+class TestCjkClauseFollowsTheParsedQuery:
+    def test_negated_cjk_term_is_excluded(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - Two documents both matching "invoice", one whose content
+              also contains 漢字
+        WHEN:
+            - "invoice NOT 漢字" is searched
+        THEN:
+            - Only the document without 漢字 matches; 'invoice NOT 漢字'
+              must not return the document containing 漢字
+        """
+        with_cjk = index_document(
+            title="Invoice A",
+            content="invoice total 漢字",
+        )
+        without_cjk = index_document(
+            title="Invoice B",
+            content="invoice total only",
+        )
+
+        assert matched_ids("invoice") == {with_cjk.pk, without_cjk.pk}
+        assert matched_ids("invoice NOT 漢字") == {without_cjk.pk}
+
+    @pytest.mark.parametrize(
+        ("threshold", "expected"),
+        [
+            pytest.param(None, {"titled"}, id="fuzzy_off"),
+            pytest.param(0.0, {"titled"}, id="fuzzy_on"),
+        ],
+    )
+    def test_fielded_cjk_term_searches_only_that_field(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+        settings: SettingsWrapper,
+        threshold: float | None,
+        expected: set[str],
+    ) -> None:
+        """
+        GIVEN:
+            - One document with 東京 in its title, another with 東京 only
+              in its content, and ADVANCED_FUZZY_SEARCH_THRESHOLD either
+              off or on
+        WHEN:
+            - "title:東京" is searched
+        THEN:
+            - Only the titled document matches, whether fuzzy is on or
+              off: both the CJK alternative and the fuzzy alternative are
+              widened in place on the fielded leaf, so 'title:東京' still
+              must not match a document whose 東京 is only in the content
+        """
+        settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = threshold
+        content_only = index_document(
+            title="Tokyo report",
+            content="東京都の人口は約1400万人です",
+        )
+        titled = index_document(
+            title="東京都の報告書",
+            content="an english summary",
+        )
+        pks = {"titled": titled.pk, "content_only": content_only.pk}
+
+        assert matched_ids("東京") == set(pks.values())
+        assert matched_ids("title:東京") == {pks[label] for label in expected}
+
+    def test_cjk_on_a_non_default_field_is_not_widened(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document with 東京 in its content
+        WHEN:
+            - "notes:東京" is searched (a field outside the default
+              search fields)
+        THEN:
+            - Nothing matches; a CJK term restricted to a field outside
+              the default search fields has no bigram companion to widen
+              to, so it must not fall back to matching 東京 in the
+              content
+        """
+        index_document(
+            title="Tokyo report",
+            content="東京都の人口は約1400万人です",
+        )
+
+        assert matched_ids("notes:東京") == set()
+
+    def test_bare_cjk_term_still_matches_every_default_field(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - One document with 重要 in its content, another with 重要 in
+              its title
+        WHEN:
+            - "重要" and "重要 OR report" are each searched unfielded
+        THEN:
+            - Both documents match either way; bigram matching exists
+              so that an unfielded CJK run matches wherever it is
+              indexed, and does so alongside a latin term
+        """
+        in_content = index_document(
+            title="report",
+            content="本文に重要な情報",
+        )
+        in_title = index_document(
+            title="重要な報告書",
+            content="english only",
+        )
+
+        assert matched_ids("重要") == {in_content.pk, in_title.pk}
+        assert matched_ids("重要 OR report") == {
+            in_content.pk,
+            in_title.pk,
+        }

src/documents/tests/search/test_cjk_emit_registry.py

@@ -0,0 +1,120 @@
+"""The emit-only FieldRegistry a CJK-widened query tree is emitted against.
+
+It is the parse registry plus the internal bigram fields. These tests pin
+what the bigram half has to get right on its own: combining a run's bigrams
+with AND wherever the leaf sits, producing exactly the terms the index
+holds, and naming only fields the index schema actually has.
+"""
+
+from __future__ import annotations
+
+import pytest
+import tantivy
+import whoosh_compat.ast as wc_ast
+
+from documents.search._query import _CJK_BIGRAM_FIELDS
+from documents.search._query import _get_emit_field_registry
+from documents.search._schema import field_descriptors
+from documents.search._tokenizer import register_tokenizers
+from documents.tests.search._ast_helpers import BIGRAM_CONTENT
+from documents.tests.search._ast_helpers import CONTENT
+
+pytestmark = pytest.mark.search
+
+
+class TestEmitFieldRegistry:
+    def test_a_multi_bigram_run_requires_every_bigram_even_under_an_or(
+        self,
+    ) -> None:
+        """
+        GIVEN:
+            - A three-character CJK run on bigram_content, sitting inside
+              an Or (where the CJK widener always places it)
+        WHEN:
+            - The tree is analyzed against the emit registry
+        THEN:
+            - The run's two bigrams are combined with And, not Or. Left
+              to Multitoken.DEFAULT they would inherit the enclosing Or,
+              so "東京都" would match a document containing only 京都
+        """
+        tree = wc_ast.Or(
+            children=(
+                wc_ast.Term(field=BIGRAM_CONTENT, text="東京都"),
+                wc_ast.Term(field=CONTENT, text="report"),
+            ),
+        )
+
+        analyzed = wc_ast.analyze(tree, _get_emit_field_registry(None))
+
+        assert analyzed == wc_ast.Or(
+            children=(
+                wc_ast.And(
+                    children=(
+                        wc_ast.Term(field=BIGRAM_CONTENT, text="東京"),
+                        wc_ast.Term(field=BIGRAM_CONTENT, text="京都"),
+                    ),
+                ),
+                wc_ast.Term(field=CONTENT, text="report"),
+            ),
+        )
+
+    @pytest.mark.parametrize(
+        "text",
+        [
+            pytest.param("東京都の公共文書", id="japanese"),
+            pytest.param("北京市人民政府", id="chinese"),
+            pytest.param("서울특별시", id="korean"),
+        ],
+    )
+    def test_every_query_bigram_is_a_term_the_index_holds(self, text: str) -> None:
+        """
+        GIVEN:
+            - A document whose bigram_content was indexed by the tokenizer
+              register_tokenizers() installs
+        WHEN:
+            - The emit registry's bigram analyzer tokenizes the same text
+        THEN:
+            - Every token it produces is found as an indexed term. The
+              query side and the index side each build their own bigram
+              analyzer, so this pins the two staying equivalent
+        """
+        sb = tantivy.SchemaBuilder()
+        sb.add_text_field(
+            "bigram_content",
+            stored=False,
+            tokenizer_name="bigram_analyzer",
+        )
+        index = tantivy.Index(sb.build(), path=None)
+        register_tokenizers(index, None)
+        writer = index.writer()
+        doc = tantivy.Document()
+        doc.add_text("bigram_content", text)
+        writer.add_document(doc)
+        writer.commit()
+        index.reload()
+
+        resolved = _get_emit_field_registry(None).resolve(BIGRAM_CONTENT)
+        assert resolved is not None
+        tokens = resolved.spec.analyzer(text)
+
+        assert tokens
+        searcher = index.searcher()
+        for token in tokens:
+            query = tantivy.Query.term_query(index.schema, "bigram_content", token)
+            assert searcher.search(query, limit=1).count == 1, token
+
+    def test_every_bigram_field_is_in_the_index_schema(self) -> None:
+        """
+        GIVEN:
+            - The bigram fields the emit registry declares, and the index
+              schema's field descriptors
+        WHEN:
+            - Each bigram field name is looked up in the schema
+        THEN:
+            - Every one is present. Every index is built from these
+              descriptors, or rebuilt when its stored fingerprint of them
+              differs, so this is what guarantees each widened bigram leaf
+              names a field the index has
+        """
+        schema_names = {descriptor.name for descriptor in field_descriptors()}
+        assert set(_CJK_BIGRAM_FIELDS.values()) <= schema_names

src/documents/tests/search/test_cjk_leaf_rewrite.py

@@ -0,0 +1,498 @@
+"""_widen_cjk_leaf, the rewrite_leaf hook that makes CJK runs matchable in
+QUERY mode.
+
+Unit tests against hand-built trees, no index needed: the hook on its own,
+then the tree wc.analyze() builds around it. Result-level proof against a
+real corpus lives in test_cjk_widening.py.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+import whoosh_compat as wc
+import whoosh_compat.ast as wc_ast
+
+from documents.search._query import _DEFAULT_SEARCH_FIELDS
+from documents.search._query import _FIELD_BOOSTS
+from documents.search._query import _get_emit_field_registry
+from documents.search._query import _widen_leaf
+from documents.search._registry import get_field_registry
+from documents.tests.search._ast_helpers import BIGRAM_CONTENT
+from documents.tests.search._ast_helpers import BIGRAM_TITLE
+from documents.tests.search._ast_helpers import CONTENT
+from documents.tests.search._ast_helpers import NOTES
+from documents.tests.search._ast_helpers import TITLE
+from documents.tests.search._ast_helpers import bigram
+from documents.tests.search._ast_helpers import content
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+pytestmark = pytest.mark.search
+
+
+def _widened(original: wc_ast.Node, bigram_side: wc_ast.Node) -> wc_ast.Or:
+    return wc_ast.Or(children=(original, bigram_side))
+
+
+def _widen_cjk_leaf(leaf: wc_ast.Term | wc_ast.Phrase) -> wc_ast.Node:
+    """The hook as the CJK work used it: no fuzzy side, nothing negated."""
+    return _widen_leaf(leaf, fuzzy=False, negated=frozenset())
+
+
+def _analyze(tree: wc_ast.Node) -> wc_ast.Node:
+    return wc.analyze(
+        tree,
+        _get_emit_field_registry(None),
+        rewrite_leaf=_widen_cjk_leaf,
+    )
+
+
+class TestTheHook:
+    @pytest.mark.parametrize(
+        "leaf",
+        [
+            pytest.param(content("invoice"), id="latin_term"),
+            pytest.param(
+                wc_ast.Term(field=NOTES, text="東京"),
+                id="non_default_field",
+            ),
+            pytest.param(wc_ast.Term(field=None, text="東京"), id="unfielded"),
+        ],
+    )
+    def test_leaf_is_returned_unchanged(self, leaf: wc_ast.Term) -> None:
+        """
+        GIVEN:
+            - A leaf with no CJK text, a CJK term on a field outside the
+              default search fields, or an unfielded CJK term
+        WHEN:
+            - The hook is called with it
+        THEN:
+            - It returns the very same object, which tells analyze() to
+              keep the leaf's ordinary analysis
+        """
+        assert _widen_cjk_leaf(leaf) is leaf
+
+    def test_a_leaf_the_tokenizer_drops_entirely_is_returned_unchanged(self) -> None:
+        """
+        GIVEN:
+            - A term whose text matches _CJK_RE (U+2E80, a CJK Radicals
+              Supplement codepoint, so _has_cjk is True) but which the
+              content analyzer's simple tokenizer yields no token for at
+              all: a few hundred codepoints across the CJK blocks share
+              this gap between the regex and the tokenizer
+        WHEN:
+            - The hook is called with it
+        THEN:
+            - Neither cjk_terms nor latin_terms gains a piece, and the
+              leaf is returned unchanged rather than becoming an empty Or
+        """
+        leaf = content("⺀")
+
+        assert _widen_cjk_leaf(leaf) is leaf
+
+    def test_the_original_leaf_object_is_the_first_alternative(self) -> None:
+        """
+        GIVEN:
+            - A CJK term on content
+        WHEN:
+            - The hook widens it
+        THEN:
+            - The Or's first child is the leaf object itself, not an equal
+              copy: analyze() only keeps the leaf's own enclosing-group
+              analysis for that exact object
+        """
+        leaf = content("東京")
+
+        widened = _widen_cjk_leaf(leaf)
+
+        assert isinstance(widened, wc_ast.Or)
+        assert widened.children[0] is leaf
+
+    def test_a_cjk_term_gains_a_bigram_alternative_on_its_own_field(self) -> None:
+        """
+        GIVEN:
+            - A CJK term on title
+        WHEN:
+            - The hook widens it
+        THEN:
+            - The bigram side targets bigram_title, the leaf's own field's
+              companion, so a fielded term stays fielded
+        """
+        leaf = wc_ast.Term(field=TITLE, text="東京")
+
+        assert _widen_cjk_leaf(leaf) == _widened(
+            leaf,
+            wc_ast.Term(field=BIGRAM_TITLE, text="東京"),
+        )
+
+    def test_each_cjk_run_gets_its_own_bigram_leaf(self) -> None:
+        """
+        GIVEN:
+            - One term whose text holds two CJK runs split by an
+              interpunct (東京・大阪)
+        WHEN:
+            - The hook widens it
+        THEN:
+            - The bigram side is Or(bigram 東京, bigram 大阪), one leaf
+              per run. Joining the runs with a space into one leaf would
+              produce bigrams spanning the space, which only exist when
+              the two runs sit side by side in a document. The runs are
+              alternatives, not requirements: a multi-run term matches on
+              any one of its runs, which is what the separate bigram
+              clause did before this change
+        """
+        leaf = content("東京・大阪")
+
+        assert _widen_cjk_leaf(leaf) == _widened(
+            leaf,
+            wc_ast.Or(children=(bigram("東京"), bigram("大阪"))),
+        )
+
+    def test_each_run_keeps_its_own_bigrams_required(self) -> None:
+        """
+        GIVEN:
+            - A term whose two runs each produce more than one bigram
+              (東京都・大阪府)
+        WHEN:
+            - The hook widens it and the tree is analyzed
+        THEN:
+            - Each run's bigrams stay And-combined inside the cross-run
+              Or. The Or is between runs only; multitoken=AND on the
+              bigram FieldSpec is what requires a run's own bigrams, and
+              it does not inherit from the enclosing group. Without this,
+              東京都 would match a document holding only 京都
+        """
+        assert _analyze(content("東京都・大阪府")) == wc_ast.Or(
+            children=(
+                wc_ast.And(children=(content("東京都"), content("大阪府"))),
+                wc_ast.And(children=(bigram("東京"), bigram("京都"))),
+                wc_ast.And(children=(bigram("大阪"), bigram("阪府"))),
+            ),
+        )
+
+    def test_a_phrase_widens_to_a_bigram_term_not_a_bigram_phrase(self) -> None:
+        """
+        GIVEN:
+            - A quoted CJK phrase on content
+        WHEN:
+            - The hook widens it
+        THEN:
+            - The bigram side is a plain Term: the bigram analyzer puts
+              every token at position 0, so a positional Phrase against it
+              could never match
+        """
+        leaf = wc_ast.Phrase(field=CONTENT, text="東京都")
+
+        assert _widen_cjk_leaf(leaf) == _widened(leaf, bigram("東京都"))
+
+    def test_a_multi_word_phrase_requires_every_run(self) -> None:
+        """
+        GIVEN:
+            - A quoted CJK phrase holding two words
+        WHEN:
+            - The hook widens it
+        THEN:
+            - The runs are And-combined, where a Term's runs are
+              Or-combined (test_each_cjk_run_gets_its_own_bigram_leaf).
+              The parser's default group is And, so an Or here would let
+              the quoted phrase match strictly more than the same two
+              words unquoted, and quoting must not widen a search
+        """
+        leaf = wc_ast.Phrase(field=CONTENT, text="東京都 大阪府")
+
+        assert _widen_cjk_leaf(leaf) == _widened(
+            leaf,
+            wc_ast.And(children=(bigram("東京都"), bigram("大阪府"))),
+        )
+
+    @pytest.mark.parametrize(
+        ("leaf", "alternative"),
+        [
+            pytest.param(
+                content("東京-report"),
+                wc_ast.And(children=(bigram("東京"), content("report"))),
+                id="separated_term",
+            ),
+            pytest.param(
+                wc_ast.Phrase(field=CONTENT, text="東京 report"),
+                wc_ast.And(children=(bigram("東京"), content("report"))),
+                id="phrase",
+            ),
+            pytest.param(content("東京report"), bigram("東京"), id="glued"),
+        ],
+    )
+    def test_latin_is_required_only_where_the_analyzer_splits_it_off(
+        self,
+        leaf: wc_ast.Term | wc_ast.Phrase,
+        alternative: wc_ast.Node,
+    ) -> None:
+        """
+        GIVEN:
+            - A leaf mixing CJK and latin text, either split into separate
+              tokens by the content analyzer (東京-report, "東京 report")
+              or glued into one token (東京report)
+        WHEN:
+            - The hook widens it
+        THEN:
+            - A separate latin token is required on the leaf's own field
+              beside the CJK run's bigrams. Glued latin is not required:
+              the index only holds it inside the whole unspaced token, so
+              requiring it would lose documents the CJK run finds
+        """
+        assert _widen_cjk_leaf(leaf) == _widened(leaf, alternative)
+
+    def test_only_latin_pieces_after_a_tokenizer_drop_still_widens(self) -> None:
+        """
+        GIVEN:
+            - A term combining a tokenizer-dropped CJK codepoint (U+2E80)
+              with a latin word (⺀report): _has_cjk matches on the
+              leading codepoint, but the tokenizer discards it and keeps
+              only the "report" token, so no run ever reaches cjk_terms
+        WHEN:
+            - The hook widens it
+        THEN:
+            - The alternative is the lone latin term itself, not wrapped
+              in And (a single piece collapses to itself), even though
+              the CJK side contributed nothing
+        """
+        leaf = content("⺀report")
+
+        assert _widen_cjk_leaf(leaf) == _widened(leaf, content("report"))
+
+    def test_new_nodes_carry_the_leaf_span(self) -> None:
+        """
+        GIVEN:
+            - A CJK term with a source span, split into two pieces
+        WHEN:
+            - The hook widens it
+        THEN:
+            - The Or, the alternative and each piece carry the leaf's
+              span, so an emit-time diagnostic still points into the
+              query text
+        """
+        leaf = wc_ast.Term(field=CONTENT, text="東京・大阪", startchar=3, endchar=8)
+
+        widened = _widen_cjk_leaf(leaf)
+
+        assert isinstance(widened, wc_ast.Or)
+        alternative = widened.children[1]
+        assert isinstance(alternative, wc_ast.Or)
+        spans = {
+            (node.startchar, node.endchar)
+            for node in (widened, alternative, *alternative.children)
+        }
+        assert spans == {(3, 8)}
+
+
+class TestAnalyzedTree:
+    @pytest.mark.parametrize(
+        "node",
+        [
+            pytest.param(wc_ast.Prefix(field=CONTENT, text="東京"), id="cjk_prefix"),
+            pytest.param(
+                wc_ast.Wildcard(field=CONTENT, pattern="東*"),
+                id="cjk_wildcard",
+            ),
+        ],
+    )
+    def test_patterns_are_never_widened(self, node: wc_ast.Node) -> None:
+        """
+        GIVEN:
+            - A CJK prefix or wildcard pattern on content
+        WHEN:
+            - The tree is analyzed with the hook
+        THEN:
+            - It comes back unchanged: analyze() only offers Term and
+              Phrase leaves to the hook
+        """
+        assert _analyze(node) == node
+
+    @pytest.mark.parametrize(
+        "build",
+        [
+            pytest.param(lambda node: node, id="bare"),
+            pytest.param(lambda node: wc_ast.Not(child=node), id="negated"),
+        ],
+    )
+    def test_a_run_past_the_length_limit_keeps_its_bigram_side(
+        self,
+        build: Callable[[wc_ast.Node], wc_ast.Node],
+        long_cjk_run: str,
+    ) -> None:
+        """
+        GIVEN:
+            - A CJK run longer than the content analyzer's remove_long
+              limit, bare or under NOT
+        WHEN:
+            - The tree is analyzed with the hook
+        THEN:
+            - The content side drops out and the bigram side stands alone,
+              still negated under NOT. A NOT left holding nothing would
+              turn into "match everything" instead
+        """
+        resolved = _get_emit_field_registry(None).resolve(BIGRAM_CONTENT)
+        assert resolved is not None
+        bigrams = resolved.spec.analyzer(long_cjk_run)
+
+        assert _analyze(build(content(long_cjk_run))) == build(
+            wc_ast.And(
+                children=tuple(bigram(token) for token in dict.fromkeys(bigrams)),
+            ),
+        )
+
+    def test_a_one_character_run_drops_out_of_the_alternative(self) -> None:
+        """
+        GIVEN:
+            - A term whose runs are 東 (one character) and 大阪
+        WHEN:
+            - The tree is analyzed with the hook
+        THEN:
+            - Only 大阪 is required on the bigram side: a one-character run
+              has no bigram. An accepted gap, the same on the positive and
+              the negated side
+        """
+        analyzed = _analyze(content("東・大阪"))
+
+        assert analyzed == _widened(
+            wc_ast.And(children=(content("東"), content("大阪"))),
+            bigram("大阪"),
+        )
+
+    def test_the_title_boost_wraps_the_widened_title_leaf(self) -> None:
+        """
+        GIVEN:
+            - An unfielded CJK query parsed the way parse_user_query parses
+              it: copied onto every default field, title boosted
+        WHEN:
+            - The tree is analyzed with the hook
+        THEN:
+            - Each field copy is widened on its own bigram field, and the
+              title boost wraps the title copy's whole widened Or, so a
+              title bigram match scores like a title match
+        """
+        parsed = wc.parse(
+            "東京",
+            registry=get_field_registry(None),
+            default_fields=_DEFAULT_SEARCH_FIELDS,
+            field_boosts=_FIELD_BOOSTS,
+        ).ast
+
+        analyzed = _analyze(parsed)
+
+        assert isinstance(analyzed, wc_ast.Or)
+        assert (
+            wc_ast.Boosted(
+                child=_widened(
+                    wc_ast.Term(field=TITLE, text="東京"),
+                    wc_ast.Term(field=BIGRAM_TITLE, text="東京"),
+                ),
+                boost=_FIELD_BOOSTS["title"],
+            )
+            in analyzed.children
+        )
+        assert {BIGRAM_CONTENT, wc.FieldRef("bigram_tag")} <= {
+            child.field for child in analyzed.children if isinstance(child, wc_ast.Term)
+        }
+
+    def test_under_and_the_original_side_keeps_and(self) -> None:
+        """
+        GIVEN:
+            - A term the content analyzer splits into two tokens
+              (東京・大阪), inside an And group
+        WHEN:
+            - The tree is analyzed with the hook
+        THEN:
+            - The original side still ANDs its two tokens. Wrapping the
+              leaf in the hook's Or must not turn that into an Or of them.
+              The cross-run Or flattens into the widening Or, so the
+              alternative is not a single child here and _widened() does
+              not fit; the tree is written out literally
+        """
+        tree = wc_ast.And(children=(content("東京・大阪"), content("report")))
+
+        assert _analyze(tree) == wc_ast.And(
+            children=(
+                wc_ast.Or(
+                    children=(
+                        wc_ast.And(children=(content("東京"), content("大阪"))),
+                        bigram("東京"),
+                        bigram("大阪"),
+                    ),
+                ),
+                content("report"),
+            ),
+        )
+
+    def test_under_or_the_original_side_keeps_or(self) -> None:
+        """
+        GIVEN:
+            - The same two-token term inside an Or group
+        WHEN:
+            - The tree is analyzed with the hook
+        THEN:
+            - The original side ORs its tokens, as the group says, and the
+              hook's Or flattens into the group
+        """
+        tree = wc_ast.Or(children=(content("東京・大阪"), content("report")))
+
+        assert _analyze(tree) == wc_ast.Or(
+            children=(
+                content("東京"),
+                content("大阪"),
+                bigram("東京"),
+                bigram("大阪"),
+                content("report"),
+            ),
+        )
+
+    @pytest.mark.parametrize(
+        "build",
+        [
+            pytest.param(lambda leaf: wc_ast.Not(child=leaf), id="not"),
+            pytest.param(
+                lambda leaf: wc_ast.AndNot(positive=content("invoice"), negative=leaf),
+                id="andnot",
+            ),
+            pytest.param(
+                lambda leaf: wc_ast.AndMaybe(
+                    required=content("invoice"),
+                    optional=leaf,
+                ),
+                id="andmaybe",
+            ),
+            pytest.param(
+                lambda leaf: wc_ast.Require(
+                    scored=leaf,
+                    filter_only=content("invoice"),
+                ),
+                id="require",
+            ),
+            pytest.param(
+                lambda leaf: wc_ast.Boosted(child=leaf, boost=2.0),
+                id="boosted",
+            ),
+        ],
+    )
+    def test_the_widened_leaf_stays_where_it_was(
+        self,
+        build: Callable[[wc_ast.Node], wc_ast.Node],
+        long_cjk_run: str,
+    ) -> None:
+        """
+        GIVEN:
+            - A CJK term under Not, AndNot, AndMaybe, Require or Boosted,
+              beside a latin term
+        WHEN:
+            - The tree is analyzed with the hook
+        THEN:
+            - The node keeps its type, its latin operand is untouched, and
+              only the CJK leaf is replaced by its widened Or, in the same
+              position. A negated CJK leaf is widened too, so a negation
+              excludes exactly what the positive search would match
+        """
+        assert _analyze(build(content("東京"))) == build(
+            _widened(content("東京"), bigram("東京")),
+        )

src/documents/tests/search/test_cjk_run_extraction.py

@@ -0,0 +1,200 @@
+"""Which characters count as part of a CJK run.
+
+_CJK_RE decides both what is indexed into the bigram fields
+(extract_cjk_text) and how a query's CJK text is cut into runs
+(_widen_leaf). A character it misses splits a word in two on both
+sides. For the katakana prolonged sound mark that leaves one-character
+runs, which have no bigrams, so the word could not be found through the
+bigram fields at all.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from documents.search._query import extract_cjk_text
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from documents.models import Document
+
+
+pytestmark = pytest.mark.search
+
+
+class TestRunExtraction:
+    @pytest.mark.parametrize(
+        ("text", "expected"),
+        [
+            pytest.param("コーヒーを飲む", "コーヒーを飲む", id="prolonged_sound_mark"),
+            pytest.param("ｺｰﾋｰ", "ｺｰﾋｰ", id="halfwidth_prolonged_sound_mark"),
+            pytest.param("ｺﾞﾙﾌ", "ｺﾞﾙﾌ", id="halfwidth_voiced_mark"),
+            pytest.param("ﾊﾟﾝ", "ﾊﾟﾝ", id="halfwidth_semi_voiced_mark"),
+            pytest.param("締め切り〆日", "締め切り〆日", id="closing_mark"),
+            pytest.param("人々の生活", "人々の生活", id="iteration_mark"),
+            pytest.param("東京・大阪", "東京 大阪", id="interpunct_still_splits"),
+            pytest.param("東京、大阪", "東京 大阪", id="comma_still_splits"),
+        ],
+    )
+    def test_japanese_word_marks_stay_inside_the_run(
+        self,
+        text: str,
+        expected: str,
+    ) -> None:
+        """
+        GIVEN:
+            - Japanese text containing ー (U+30FC), its halfwidth form
+              (U+FF70), the halfwidth voiced and semi-voiced marks
+              (U+FF9E, U+FF9F), 〆 or 々, or a separator between two runs
+        WHEN:
+            - Its CJK runs are extracted for the bigram fields
+        THEN:
+            - The marks stay inside their word's run. Every one of them
+              has Unicode script Common, so the script classes alone miss
+              them. Punctuation such as ・ and 、 still separates runs
+        """
+        assert extract_cjk_text(text) == expected
+
+
+@pytest.mark.django_db
+class TestProlongedSoundMarkSearch:
+    @pytest.mark.parametrize(
+        ("query", "content"),
+        [
+            pytest.param("コーヒー", "コーヒーを飲む", id="fullwidth"),
+            pytest.param("サーバー", "サーバーの設定を変更", id="fullwidth_two_marks"),
+            pytest.param("ｺｰﾋｰ", "ｺｰﾋｰを飲む", id="halfwidth"),
+        ],
+    )
+    def test_a_word_with_the_mark_is_found_inside_running_text(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+        query: str,
+        content: str,
+    ) -> None:
+        """
+        GIVEN:
+            - A document containing a katakana word with ー inside a
+              longer unspaced run, and an unrelated latin document
+        WHEN:
+            - The word is searched
+        THEN:
+            - The document matches through the bigram fields. Without the
+              mark in _CJK_RE, the word splits into one-character runs
+              with no bigrams, and only a standalone content token could
+              match
+        """
+        match = index_document(title="A", content=content)
+        index_document(title="B", content="invoice only")
+
+        assert matched_ids(query) == {match.pk}
+
+    def test_not_excludes_a_word_with_the_mark(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - An "invoice" document containing コーヒー inside a longer
+              run, and a latin-only "invoice" document
+        WHEN:
+            - "invoice NOT コーヒー" is searched
+        THEN:
+            - Only the latin document matches: the negation excludes what
+              the positive search finds
+        """
+        index_document(
+            title="A",
+            content="コーヒーを飲む invoice",
+        )
+        latin = index_document(title="B", content="invoice only")
+
+        assert matched_ids("invoice NOT コーヒー") == {latin.pk}
+
+    def test_a_different_word_sharing_only_the_edges_does_not_match(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document with コード and ヒント, two separate words
+        WHEN:
+            - "コーヒー" is searched
+        THEN:
+            - Nothing matches. With ー outside _CJK_RE, both sides reduce
+              to runs like コ and ヒ joined by spaces, and the space
+              bigrams that produces could match unrelated words
+        """
+        index_document(title="A", content="コード ヒント")
+
+        assert matched_ids("コーヒー") == set()
+
+
+@pytest.mark.django_db
+class TestHalfwidthVoicedMarkSearch:
+    """Halfwidth katakana, as legacy systems and bank statements write it.
+
+    Its voiced marks are separate codepoints with no precomposed form, so
+    NFC leaves them where they are and the character class has to cover
+    them. A word like ﾊﾟﾝ is three codepoints, and splitting it at the mark
+    leaves two one-character runs with no bigrams, so it is not merely
+    imprecise but unfindable.
+    """
+
+    @pytest.mark.parametrize(
+        ("query", "content"),
+        [
+            pytest.param("ｺﾞﾙﾌ", "ｺﾞﾙﾌ場の利用料金", id="voiced_mark"),
+            pytest.param("ﾊﾟﾝ", "ﾊﾟﾝと牛乳の購入", id="semi_voiced_mark"),
+        ],
+    )
+    def test_a_word_with_the_mark_is_found_inside_running_text(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+        query: str,
+        content: str,
+    ) -> None:
+        """
+        GIVEN:
+            - A document holding a halfwidth katakana word inside a longer
+              unspaced run, and an unrelated latin document
+        WHEN:
+            - The word is searched
+        THEN:
+            - The document matches. Without the mark in _CJK_RE the word
+              splits at it, and ﾊﾟﾝ in particular loses both halves to the
+              one-character rule and can never be found
+        """
+        match = index_document(title="A", content=content)
+        index_document(title="B", content="invoice only")
+
+        assert matched_ids(query) == {match.pk}
+
+    def test_the_whole_word_is_searched_not_just_the_tail(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document holding ｺﾞﾙﾌ, and one holding ﾙﾌﾅﾞｰ , which
+              shares the ﾙﾌ pair that splitting ｺﾞﾙﾌ at its mark leaves
+              behind
+        WHEN:
+            - "ｺﾞﾙﾌ" is searched
+        THEN:
+            - Only the first matches. Splitting at the mark would search
+              the ﾙﾌ fragment alone, which silently degrades the query
+              into a two-character substring search
+        """
+        golf = index_document(title="A", content="ｺﾞﾙﾌ場の利用料金")
+        index_document(title="B", content="ﾙﾌﾅﾞｰの修理")
+
+        assert matched_ids("ｺﾞﾙﾌ") == {golf.pk}

src/documents/tests/search/test_cjk_widening.py

@@ -0,0 +1,622 @@
+"""CJK terms in QUERY-mode searches, matched through their bigram fields
+in place inside the parsed query.
+
+The content analyzer keeps an unspaced CJK run as one token, so a CJK
+term is only findable through the bigram fields. Each CJK leaf is widened
+where it sits, so everything around it (AND, NOT, REQUIRE, boosts,
+fielding) constrains the bigram match exactly as it constrains the
+original term.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from documents.models import Correspondent
+from documents.models import Document
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from pytest_django.fixtures import SettingsWrapper
+
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+
+class TestNegationSymmetry:
+    def test_not_excludes_what_the_positive_term_matches(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document whose 東京 sits inside a longer unspaced run
+              (indexed as one content token), and a latin-only document,
+              both matching "invoice"
+        WHEN:
+            - "東京" and "invoice NOT 東京" are searched
+        THEN:
+            - 東京 finds the CJK document, and NOT 東京 excludes that same
+              document. A content-only negation never fires here, since
+              the content field holds the whole run, not 東京
+        """
+        cjk = index_document(
+            title="A",
+            content="東京都の公共文書について invoice",
+        )
+        latin = index_document(title="B", content="invoice only")
+
+        assert matched_ids("東京") == {cjk.pk}
+        assert matched_ids("invoice NOT 東京") == {latin.pk}
+
+    def test_bigram_approximation_is_the_same_on_both_sides(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document mentioning 東京 and 京都 separately, never 東京都
+        WHEN:
+            - "東京都" and "invoice NOT 東京都" are searched
+        THEN:
+            - 東京都 matches it (both of its bigrams are present, and the
+              bigram match ignores position), and NOT 東京都 excludes it.
+              An accepted approximation: a query and its negation agree
+        """
+        separate = index_document(
+            title="A",
+            content="東京 と 京都 invoice",
+        )
+        latin = index_document(title="B", content="invoice only")
+
+        assert matched_ids("東京都") == {separate.pk}
+        assert matched_ids("invoice NOT 東京都") == {latin.pk}
+
+    @pytest.mark.parametrize(
+        "query",
+        [
+            pytest.param("invoice NOT 東京都", id="term"),
+            pytest.param('invoice NOT "東京都"', id="phrase"),
+        ],
+    )
+    def test_a_negated_multi_bigram_run_needs_every_bigram_to_exclude(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+        query: str,
+    ) -> None:
+        """
+        GIVEN:
+            - A document containing 東京都, and one containing only 京都
+              (one of 東京都's two bigrams)
+        WHEN:
+            - 東京都 is excluded, as a bare term or a quoted phrase
+        THEN:
+            - Only the 東京都 document is excluded. Sharing one bigram
+              with the excluded run is not enough to be excluded
+        """
+        full = index_document(
+            title="A",
+            content="東京都の公共文書について invoice",
+        )
+        partial = index_document(
+            title="B",
+            content="京都の invoice",
+        )
+
+        assert matched_ids("invoice") == {full.pk, partial.pk}
+        assert matched_ids(query) == {partial.pk}
+
+    def test_a_run_past_the_length_limit_is_matched_and_excluded(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+        long_cjk_run: str,
+    ) -> None:
+        """
+        GIVEN:
+            - A document holding a CJK run longer than the content
+              analyzer's remove_long limit, and a latin-only document
+        WHEN:
+            - The long run is searched bare, and excluded with NOT
+        THEN:
+            - Bare, it matches the document; negated, it excludes that
+              document and nothing else. The content side of the run
+              analyzes to nothing, which must not erase the NOT
+        """
+        long = index_document(
+            title="A",
+            content=f"{long_cjk_run} invoice",
+        )
+        latin = index_document(title="B", content="invoice only")
+
+        assert matched_ids(long_cjk_run) == {long.pk}
+        assert matched_ids(f"invoice NOT {long_cjk_run}") == {latin.pk}
+
+
+class TestTokenizerDroppedCodepoints:
+    @pytest.fixture
+    def invoice(self, index_document: Callable[..., Document]) -> Document:
+        """One latin "invoice" document, with no U+2E80 anywhere in it."""
+        return index_document(title="A", content="invoice only")
+
+    def test_a_regex_only_match_finds_nothing(
+        self,
+        invoice: Document,
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document whose content has nothing resembling U+2E80 (a
+              CJK Radicals Supplement codepoint that matches _CJK_RE but
+              that the content analyzer's simple tokenizer yields no
+              token for at all)
+        WHEN:
+            - "⺀" is searched
+        THEN:
+            - Nothing matches: _widen_leaf finds no CJK pieces to widen
+              with and returns the leaf as it is, which analyzes to the
+              same nothing it always did
+        """
+        assert matched_ids("⺀") == set()
+
+    def test_not_a_regex_only_match_excludes_nothing(
+        self,
+        invoice: Document,
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - An "invoice" document with no U+2E80 in it
+        WHEN:
+            - "invoice NOT ⺀" is searched
+        THEN:
+            - The document still matches: the negated leaf widens to
+              nothing extra, exactly as an unwidened NOT of a term with
+              no matches would behave
+        """
+        assert matched_ids("invoice NOT ⺀") == {invoice.pk}
+
+
+class TestStructureConstrainsTheCjkMatch:
+    @pytest.fixture
+    def pks(self, index_document: Callable[..., Document]) -> dict[str, int]:
+        """One document with both 東京 and "invoice", one with only 東京."""
+        return {
+            "both": index_document(title="A", content="東京都の請求書 invoice").pk,
+            "cjk_only": index_document(title="B", content="東京 report").pk,
+        }
+
+    @pytest.mark.parametrize(
+        ("query", "expected"),
+        [
+            pytest.param("東京 AND invoice", {"both"}, id="and"),
+            pytest.param("東京 REQUIRE invoice", {"both"}, id="require"),
+            pytest.param("東京^2 AND invoice", {"both"}, id="boosted"),
+            pytest.param("東京 ANDMAYBE invoice", {"both", "cjk_only"}, id="andmaybe"),
+        ],
+    )
+    def test_a_latin_operand_still_constrains_the_cjk_term(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        pks: dict[str, int],
+        query: str,
+        expected: set[str],
+    ) -> None:
+        """
+        GIVEN:
+            - One document with both 東京 and "invoice", one with only 東京
+        WHEN:
+            - 東京 is combined with "invoice" by AND, REQUIRE, a boosted
+              AND, or ANDMAYBE
+        THEN:
+            - AND and REQUIRE (boosted or not) need "invoice" too, so only
+              the document with both matches; ANDMAYBE leaves "invoice"
+              optional, so both match
+        """
+        assert matched_ids(query) == {pks[label] for label in expected}
+
+
+class TestFieldedTerms:
+    def test_a_fielded_multi_run_term_matches_through_either_run(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document with 東京 and 大阪, and one with only 東京, both
+              containing "report"
+        WHEN:
+            - "content:東京・大阪 AND report" is searched (the content
+              analyzer splits 東京・大阪 into two tokens)
+        THEN:
+            - Both match. The original side still ANDs its two content
+              tokens, pinned in test_cjk_leaf_rewrite.py, but the bigram
+              side accepts either run, so the 東京-only document comes
+              back through it. The AND still binds: "report" is required
+              of both. This is what the separate bigram clause returned
+              before this change
+        """
+        both = index_document(title="A", content="東京 大阪 report")
+        one = index_document(title="B", content="東京 report")
+
+        assert matched_ids("content:東京・大阪 AND report") == {
+            both.pk,
+            one.pk,
+        }
+
+    def test_a_run_is_matched_wherever_it_appears_in_the_text(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document holding 大阪 and 東京 apart, inside one unspaced
+              run (大阪府と東京都), a document with only 東京, and one
+              with neither
+        WHEN:
+            - "content:東京・大阪" is searched
+        THEN:
+            - The first two match and the third does not. Each run is
+              searched on its own, so no bigram spanning the gap between
+              the runs is required, and either run on its own is enough
+        """
+        apart = index_document(title="A", content="大阪府と東京都")
+        one = index_document(title="B", content="東京 report")
+        index_document(title="C", content="report only")
+
+        assert matched_ids("content:東京・大阪") == {apart.pk, one.pk}
+
+    def test_an_unfielded_multi_run_term_matches_on_either_run(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document with both CJK runs, one with only the first run,
+              and one with neither, all containing "invoice"
+        WHEN:
+            - The two runs are searched as one unfielded term (東京・大阪)
+        THEN:
+            - Both CJK documents match. An unfielded term is copied across
+              the default fields inside an Or, so the leaf's own semantics
+              are "either"; the bigram side matches that, and returns what
+              the separate bigram clause returned before this change
+        """
+        both = index_document(
+            title="A",
+            content="大阪府と東京都の報告 invoice",
+        )
+        one = index_document(
+            title="B",
+            content="東京都の報告書 invoice",
+        )
+        index_document(title="C", content="invoice only")
+
+        assert matched_ids("東京・大阪") == {both.pk, one.pk}
+
+    def test_a_negated_multi_run_term_excludes_on_either_run(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - Documents containing both runs of a glued mixed term, only
+              its first run, only its second run, and neither, all
+              containing "invoice"
+        WHEN:
+            - "invoice NOT 東京report資料" is searched
+        THEN:
+            - Only the document with neither run survives. The negation
+              excludes exactly what the positive term matches, and the
+              positive term matches on either run, so the exclusion is
+              wider than an "only documents holding both" reading. Today
+              this query excludes nothing at all
+        """
+        index_document(
+            title="A",
+            content="東京都のreport資料です invoice",
+        )
+        index_document(title="B", content="東京都の報告 invoice")
+        index_document(title="C", content="参考資料の一覧 invoice")
+        neither = index_document(title="D", content="invoice only")
+
+        assert matched_ids("invoice NOT 東京report資料") == {neither.pk}
+
+    def test_a_bigram_field_name_is_not_query_syntax(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document holding 東京 in its content
+        WHEN:
+            - A query naming an internal bigram field as a field prefix is
+              searched end to end (bigram_content:東京)
+        THEN:
+            - Nothing matches. The prefix is not query syntax, so the
+              whole thing is plain text: the literal token
+              "bigram_content" is required on the leaf's own field, and no
+              document holds it. Asserted through the real search path, so
+              that parsing with the emit registry instead of the public
+              one would fail this test. A unit-level wc.parse() assertion
+              would not: it pins whoosh-compat's handling of an unknown
+              prefix rather than our choice of parse registry
+        """
+        index_document(title="A", content="東京都の報告書")
+
+        assert matched_ids("bigram_content:東京") == set()
+
+
+class TestMixedScript:
+    def test_separated_latin_is_required(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document with 東京 and "report" as separate words, and one
+              with 東京 but no "report"
+        WHEN:
+            - "東京-report" is searched (the content analyzer splits it into
+              東京 and report)
+        THEN:
+            - Only the document with both matches: the latin piece is
+              required beside the CJK run's bigrams
+        """
+        both = index_document(title="A", content="東京都の report")
+        index_document(title="B", content="東京都の invoice")
+
+        assert matched_ids("東京-report") == {both.pk}
+
+    def test_glued_latin_is_not_required(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document where the latin text is glued to other CJK text
+              (東京都のreport資料, one content token), and one with 東京
+              but no "report" at all
+        WHEN:
+            - "東京report" is searched (one token: latin glued to CJK)
+        THEN:
+            - Both match through 東京's bigrams. Glued latin is only ever
+              indexed inside a whole unspaced token, so requiring it would
+              have lost the first document
+        """
+        glued = index_document(
+            title="A",
+            content="東京都のreport資料",
+        )
+        other = index_document(
+            title="B",
+            content="東京都の invoice",
+        )
+
+        assert matched_ids("東京report") == {glued.pk, other.pk}
+
+    def test_not_a_separated_mixed_term_excludes_only_documents_with_both(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - Three "invoice" documents: with 東京 and "report", with 東京
+              only, and latin only
+        WHEN:
+            - "invoice NOT 東京-report" is searched
+        THEN:
+            - Only the document with both is excluded, not every document
+              mentioning 東京
+        """
+        index_document(
+            title="A",
+            content="東京都の report invoice",
+        )
+        cjk_only = index_document(
+            title="B",
+            content="東京都の invoice",
+        )
+        latin = index_document(title="C", content="invoice only")
+
+        assert matched_ids("invoice NOT 東京-report") == {
+            cjk_only.pk,
+            latin.pk,
+        }
+
+    def test_a_one_character_run_is_not_required(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document containing 大阪 but not 東
+        WHEN:
+            - "content:東・大阪" is searched
+        THEN:
+            - It matches: a one-character run has no bigram, so only 大阪
+              is required. An accepted gap, the same under NOT
+        """
+        osaka = index_document(title="A", content="大阪府の報告書")
+
+        assert matched_ids("content:東・大阪") == {osaka.pk}
+
+
+class TestQuotedPhrase:
+    """A quoted CJK phrase must not match more than the same words unquoted.
+
+    The bigram fields put every token at position 0, so no alternative
+    built from them can enforce adjacency. Requiring both runs is the
+    tightest thing available, and the floor: the parser's default group is
+    And, so anything looser would make quoting widen the search.
+    """
+
+    @pytest.fixture
+    def pks(self, index_document: Callable[..., Document]) -> dict[str, int]:
+        return {
+            "tokyo": index_document(
+                title="A",
+                content="東京都の報告書 invoice",
+            ).pk,
+            "osaka": index_document(
+                title="B",
+                content="大阪府の報告書 invoice",
+            ).pk,
+            "both": index_document(
+                title="C",
+                content="東京都と大阪府の報告書 invoice",
+            ).pk,
+        }
+
+    @pytest.mark.parametrize(
+        "query",
+        [
+            pytest.param("東京都 大阪府", id="unquoted"),
+            pytest.param('"東京都 大阪府"', id="quoted"),
+        ],
+    )
+    def test_both_words_are_required_either_way(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        pks: dict[str, int],
+        query: str,
+    ) -> None:
+        """
+        GIVEN:
+            - Three documents, holding only the first word, only the
+              second, and both
+        WHEN:
+            - The two words are searched unquoted and as a quoted phrase
+        THEN:
+            - Only the document holding both matches, either way. Quoting
+              asks for more than the bare words, never less
+        """
+        assert matched_ids(query) == {pks["both"]}
+
+    @pytest.mark.parametrize(
+        "query",
+        [
+            pytest.param("invoice NOT (東京都 大阪府)", id="unquoted"),
+            pytest.param('invoice NOT "東京都 大阪府"', id="quoted"),
+        ],
+    )
+    def test_the_negation_excludes_only_what_the_phrase_matches(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        pks: dict[str, int],
+        query: str,
+    ) -> None:
+        """
+        GIVEN:
+            - The same three documents, all matching "invoice"
+        WHEN:
+            - The two words are excluded, grouped and as a quoted phrase
+        THEN:
+            - Only the document holding both words is excluded. The
+              negation is the mirror of the positive side, so a phrase
+              that required either word would take the two single-word
+              documents out with it
+        """
+        assert matched_ids(query) == {pks["tokyo"], pks["osaka"]}
+
+
+class TestOtherDefaultFields:
+    @pytest.mark.parametrize("where", ["title", "correspondent"])
+    def test_not_excludes_a_cjk_match_outside_content(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+        where: str,
+    ) -> None:
+        """
+        GIVEN:
+            - An "invoice" document whose only 東京 is inside a longer run
+              in its title or its correspondent, and a latin-only one
+        WHEN:
+            - "invoice NOT 東京" is searched
+        THEN:
+            - The CJK document is excluded: every default field's bigram
+              companion is widened, not only content's
+        """
+        kwargs: dict[str, object] = {
+            "title": "A",
+            "content": "invoice",
+        }
+        if where == "title":
+            kwargs["title"] = "東京都の報告書"
+        else:
+            kwargs["correspondent"] = Correspondent.objects.create(name="東京電力")
+        index_document(**kwargs)
+        latin = index_document(
+            title="B",
+            content="invoice only",
+        )
+
+        assert matched_ids("invoice NOT 東京") == {latin.pk}
+
+
+class TestFuzzyDoesNotReadmit:
+    @pytest.mark.parametrize(
+        ("query", "threshold", "expected"),
+        [
+            pytest.param("invoice NOT 東京", None, {"latin"}, id="not_fuzzy_off"),
+            pytest.param("invoice NOT 東京", 0.0, {"latin"}, id="not_fuzzy_on"),
+            pytest.param("東京 AND invoice", None, {"cjk"}, id="and_fuzzy_off"),
+            pytest.param(
+                "東京 AND invoice",
+                0.0,
+                {"cjk"},
+                id="and_fuzzy_on",
+            ),
+        ],
+    )
+    def test_fuzzy_on_matches_fuzzy_off_for_structure(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+        settings: SettingsWrapper,
+        query: str,
+        threshold: float | None,
+        expected: set[str],
+    ) -> None:
+        """
+        GIVEN:
+            - An "invoice" document whose 東京 is inside a longer unspaced
+              run, a 東京-only document and a latin-only "invoice"
+              document, with fuzzy search off or on
+        WHEN:
+            - "invoice NOT 東京" or "東京 AND invoice" is searched
+        THEN:
+            - The result is the same whether fuzzy is on or off: fuzzy
+              widening happens inside the tree now, so a CJK NOT or AND
+              still constrains both the exact and the fuzzy side, and
+              neither re-admits what the structure excludes
+        """
+        settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = threshold
+        cjk = index_document(
+            title="A",
+            content="東京都の公共文書について invoice",
+        )
+        cjk_only = index_document(
+            title="B",
+            content="東京都の報告書",
+        )
+        latin = index_document(title="C", content="invoice only")
+        pks = {"cjk": cjk.pk, "cjk_only": cjk_only.pk, "latin": latin.pk}
+
+        assert matched_ids(query) == {pks[label] for label in expected}

src/documents/tests/search/test_compact_date_forms.py

@@ -0,0 +1,72 @@
+"""Whoosh's compact, separator-free date spelling, resolved end to end.
+
+whoosh-compat owns both widths of this spelling and asserts both forms'
+bounds directly in its own test suite: the 8-digit form as a whole calendar
+day (lower bound, upper bound and exclusivity), and the 14-digit form as a
+single instant. The 14-digit form is kept here as the single representative
+because it is the one that exercises paperless's ``added`` DATETIME fast
+field at full precision: the corpus separates a document at the named
+instant from one on the same calendar day at another hour and one on the
+next day at the same hour, so a query that degrades into a whole-day
+window, or drops the time of day, matches the wrong set rather than passing
+on a corpus that could not tell the difference.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC
+from datetime import datetime
+from typing import TYPE_CHECKING
+
+import pytest
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from documents.models import Document
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+
+@pytest.fixture
+def docs(index_document: Callable[..., Document]) -> dict[str, int]:
+    return {
+        "instant": index_document(
+            title="On the instant",
+            content="x",
+            added=datetime(2005, 3, 4, 15, 30, tzinfo=UTC),
+        ).pk,
+        "same_day": index_document(
+            title="Same day, other hour",
+            content="x",
+            added=datetime(2005, 3, 4, 9, 0, tzinfo=UTC),
+        ).pk,
+        "next_day": index_document(
+            title="Next day, same hour",
+            content="x",
+            added=datetime(2005, 3, 5, 15, 30, tzinfo=UTC),
+        ).pk,
+    }
+
+
+class TestCompactDateForms:
+    def test_fourteen_digits_is_a_single_instant(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        docs: dict[str, int],
+    ) -> None:
+        """
+        GIVEN:
+            - Three documents indexed on the ``added`` DATETIME fast field:
+              one at 2005-03-04T15:30:00, one on the same calendar day at a
+              different hour, and one on the next day at the same hour
+        WHEN:
+            - Searching with the 14-digit compact date form
+              ``added:20050304153000``
+        THEN:
+            - Only the document at that exact instant matches; the same-day
+              document is what tells this apart from the 8-digit day-window
+              form, and the next-day document from a form that ignored the
+              time of day altogether
+        """
+        assert matched_ids("added:20050304153000") == {docs["instant"]}

src/documents/tests/search/test_date_keyword_phrase_removal.py

@@ -0,0 +1,78 @@
+"""Pins the correctness gained by deleting the pre-parse
+_quote_date_keyword_phrases rewrite.
+
+That rewrite matched date-keyword phrases (e.g. "previous month" after a
+date field) anywhere in the raw query string, including inside an
+unrelated quoted string, and inserted quotes mid-phrase there too. Its
+own docstring gave ``title:"see added:previous month notes"`` as the
+example of what it corrupted. whoosh-compat's grammar accepts the same
+phrase vocabulary unquoted natively (see TestUnquotedDateKeywordPhrases
+in test_acceptance.py), so the rewrite was redundant everywhere it was
+safe and actively wrong everywhere it was not. This is the one case that
+tells the two apart: a literal title phrase that happens to contain
+"added:previous month" as running text.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from documents.models import Document
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+
+class TestQuotedStringContainingDateKeywordText:
+    """A quoted title phrase containing the literal text
+    "added:previous month" as running words must match on that literal
+    text alone, never spill into an unfielded search for "previous" and
+    "month" across the default search fields the way the deleted rewrite
+    would have decomposed it into."""
+
+    def test_matches_only_the_literal_phrase(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document whose title literally contains "see
+              added:previous month notes", and a decoy document whose
+              title/content carry the individual fragments the deleted
+              _quote_date_keyword_phrases rewrite would have decomposed
+              the phrase into (the decoy would incorrectly match under
+              the deleted rewrite: its title contains the "see added:"
+              and " notes" fragments the corrupted parse required as
+              title phrases, and its content supplies "previous" and
+              "month" as the decomposed word-match clauses the rewrite
+              turned the middle of the phrase into)
+        WHEN:
+            - 'title:"see added:previous month notes"' is searched
+        THEN:
+            - Only the document with the literal phrase matches; it must
+              never spill into an unfielded search for "previous" and
+              "month" across the default search fields
+        """
+        literal = index_document(
+            title="see added:previous month notes",
+            content="quarterly filing",
+            archive_serial_number=920,
+        )
+        # Under the deleted rewrite, this decoy would incorrectly match:
+        # its title contains the "see added:" and " notes" fragments the
+        # corrupted parse required as title phrases, and its content
+        # supplies "previous" and "month" as the decomposed word-match
+        # clauses the rewrite turned the middle of the phrase into.
+        decoy = index_document(
+            title="see added: quarterly report notes",
+            content="we reviewed the previous statement about month end",
+            archive_serial_number=921,
+        )
+        query = 'title:"see added:previous month notes"'
+        assert matched_ids(query) == {literal.pk}
+        assert decoy.pk not in matched_ids(query)

src/documents/tests/search/test_date_keyword_timezone.py

@@ -0,0 +1,87 @@
+"""Date keyword phrases (``today``, etc.) resolved in a non-UTC timezone,
+end to end.
+
+paperless's own ``tz=get_current_timezone()`` plumbing
+(``TantivyBackend._parse_query``) is exercised elsewhere only for
+relative *ranges* (``added:[-1 week to now]``, in
+documents/tests/test_api_search.py). This covers a date *keyword*
+(``today``), whose day boundary depends on the active timezone the same
+way but goes through whoosh-compat's DateParserPlugin resolution instead
+of an explicit range.
+
+Discriminating shape: frozen at 2026-06-15T02:00 UTC, which is
+2026-06-14T22:00 in America/New_York -- still "today" (06-14) there, but
+already "today" (06-15) in UTC. Two documents pin both directions of the
+mistake a hardcoded-UTC bug would make:
+
+- ``in_ny_today`` (added 2026-06-14T20:00 UTC = 2026-06-14T16:00 NY) is
+  inside New York's "today" window and outside a naive UTC-calendar-day
+  window. A ``tz``-ignoring bug would miss it.
+- ``in_utc_calendar_day_only`` (added 2026-06-15T10:00 UTC =
+  2026-06-15T06:00 NY) is inside a naive UTC-calendar-day window but
+  outside New York's actual "today" window. A ``tz``-ignoring bug would
+  wrongly match it.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC
+from datetime import datetime
+from typing import TYPE_CHECKING
+
+import pytest
+import time_machine
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from pytest_django.fixtures import SettingsWrapper
+
+    from documents.models import Document
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+FROZEN_NOW = datetime(2026, 6, 15, 2, 0, tzinfo=UTC)
+
+
+class TestDateKeywordUsesTheActiveTimezone:
+    def test_today_matches_the_new_york_calendar_day_not_the_utc_one(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+        settings: SettingsWrapper,
+    ) -> None:
+        """
+        GIVEN:
+            - TIME_ZONE set to America/New_York, time frozen at
+              2026-06-15T02:00 UTC (2026-06-14T22:00 NY -- still "today"
+              there, but already "today" in UTC), and two documents: one
+              added inside New York's "today" window but outside a naive
+              UTC-calendar-day window, the other the reverse (inside a
+              naive UTC-calendar-day window but outside New York's actual
+              "today")
+        WHEN:
+            - "added:today" is searched
+        THEN:
+            - Only the document inside New York's actual "today" window
+              matches, proving our tz=get_current_timezone() plumbing
+              resolves the date keyword in the active timezone rather
+              than a hardcoded UTC calendar day
+        """
+        settings.TIME_ZONE = "America/New_York"
+        with time_machine.travel(FROZEN_NOW, tick=False):
+            in_ny_today = index_document(
+                title="NY today",
+                content="x",
+                added=datetime(2026, 6, 14, 20, 0, tzinfo=UTC),
+            )
+            # Not captured: the exact-set assertion below already proves
+            # this document (inside a naive UTC-calendar-day window, but
+            # outside New York's actual "today") does not match.
+            index_document(
+                title="UTC calendar day only",
+                content="x",
+                added=datetime(2026, 6, 15, 10, 0, tzinfo=UTC),
+            )
+
+            assert matched_ids("added:today") == {in_ny_today.pk}

src/documents/tests/search/test_default_search_fields_guard.py

@@ -0,0 +1,32 @@
+"""``_DEFAULT_SEARCH_FIELDS`` must stay a subset of the registered public
+field names.
+
+Nothing enforced this before: a rename in PUBLIC_FIELDS not mirrored in
+``_DEFAULT_SEARCH_FIELDS`` (documents/search/_query.py) would 400 every
+unfielded search at request time, since ``index.parse_query``, the
+fuzzy clause builder and the CJK bigram widening are handed a field name
+the schema no longer has.
+"""
+
+from __future__ import annotations
+
+from documents.search._fields import PUBLIC_FIELDS
+from documents.search._query import _DEFAULT_SEARCH_FIELDS
+
+
+class TestDefaultSearchFieldsAreRegistered:
+    def test_every_default_search_field_is_a_public_field(self) -> None:
+        """
+        GIVEN:
+            - PUBLIC_FIELDS and _DEFAULT_SEARCH_FIELDS, our own field
+              tables
+        WHEN:
+            - Every name in _DEFAULT_SEARCH_FIELDS is checked against the
+              registered public field names
+        THEN:
+            - Every one is present; a rename in PUBLIC_FIELDS not
+              mirrored here would 400 every unfielded search at request
+              time
+        """
+        public_field_names = {f.name for f in PUBLIC_FIELDS}
+        assert set(_DEFAULT_SEARCH_FIELDS) <= public_field_names

src/documents/tests/search/test_documented_syntax.py

@@ -0,0 +1,453 @@
+"""Pins the search syntax that ``docs/usage.md`` promises users.
+
+Every query here is syntax the "Document searches" section of
+``docs/usage.md`` documents, either spelled as the docs spell it or as a
+concrete instance of a form the docs describe. Each case indexes real
+documents and asserts on matched document IDs rather than on the parsed
+query, because a query that parses cleanly is not necessarily a query that
+means what the documentation says it means: ``added:now`` parses without a
+single diagnostic and then matches nothing, because it resolves to an
+instant rather than to a span.
+
+The negative cases matter as much as the positive ones. They pin the
+behaviours the docs explicitly warn about, so that if any of them ever
+starts working the warning can be removed deliberately rather than being
+left standing as a lie.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC
+from datetime import datetime
+from typing import TYPE_CHECKING
+
+import pytest
+import time_machine
+
+from documents.models import Note
+from documents.models import Tag
+from documents.search._errors import InvalidDateQuery
+from paperless_testing.factories import DocumentFactory
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+    from collections.abc import Generator
+
+    from django.contrib.auth.models import User
+
+    from documents.models import Document
+    from documents.search._backend import TantivyBackend
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+# A Monday, so that "next monday"/"last monday" land a clean week either side.
+FROZEN_NOW = datetime(2026, 6, 15, 12, 0, tzinfo=UTC)
+
+# The checksum used in the docs' `checksum:` example.
+DOC_CHECKSUM = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"
+
+
+class TestLogicalExpressions:
+    @pytest.fixture
+    def docs(self, index_document: Callable[..., Document]) -> dict[str, int]:
+        return {
+            "secret": index_document(
+                title="Invoice one",
+                content="invoice secret contents",
+            ).pk,
+            "plain": index_document(
+                title="Invoice two",
+                content="invoice ordinary contents",
+            ).pk,
+        }
+
+    def test_not_excludes_a_term(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        docs: dict[str, int],
+    ) -> None:
+        """
+        GIVEN:
+            - Two indexed documents, one containing "secret" and one not
+        WHEN:
+            - "invoice NOT secret" is searched, as docs/usage.md documents
+        THEN:
+            - Only the document without "secret" matches
+        """
+        assert matched_ids("invoice NOT secret") == {docs["plain"]}
+
+    def test_leading_hyphen_requires_the_term_instead_of_excluding_it(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        docs: dict[str, int],
+    ) -> None:
+        """
+        GIVEN:
+            - Two indexed documents, one containing "secret" and one not
+        WHEN:
+            - "invoice -secret" is searched (a leading hyphen, not "NOT")
+        THEN:
+            - Only the document containing "secret" matches, because
+              separators are stripped at index time, so "-secret" is
+              indexed as the plain term "secret" and the query becomes an
+              AND rather than an exclusion, exactly as the docs warn
+        """
+        assert matched_ids("invoice -secret") == {docs["secret"]}
+
+    def test_or_inside_parentheses_matches_either_branch(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        docs: dict[str, int],
+    ) -> None:
+        """
+        GIVEN:
+            - Two indexed documents, one containing "secret" and one
+              containing "ordinary"
+        WHEN:
+            - "invoice AND (secret OR ordinary)" is searched
+        THEN:
+            - Both documents match
+        """
+        matched = matched_ids("invoice AND (secret OR ordinary)")
+        assert matched == {docs["secret"], docs["plain"]}
+
+
+class TestPhraseSearch:
+    def test_quoted_phrase_requires_the_words_in_order(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document whose content contains "the quick brown fox jumps"
+        WHEN:
+            - A quoted phrase is searched, in order and out of order
+        THEN:
+            - The in-order phrase matches, and the same words reordered do
+              not
+        """
+        doc = index_document(
+            title="Phrase",
+            content="the quick brown fox jumps",
+        )
+        assert matched_ids('"quick brown fox"') == {doc.pk}
+        assert matched_ids('"brown quick fox"') == set()
+
+
+class TestTagCommaList:
+    """``tag:bills,unpaid`` is published syntax (docs/usage.md), so this checks
+    that the documented spelling still returns what the docs promise: only the
+    document carrying every listed tag.
+
+    It is deliberately not proof of paperless's field configuration, and must
+    not be read as such. Removing ``comma_values`` from the ``tag`` FieldSpec
+    leaves this test passing, because paperless's analyzer splits the literal
+    value "bills,unpaid" into the same two tokens the value-list reading
+    produces, so the two readings select the same documents. The registry fact
+    -- that ``tag`` opts in and no other field does -- is observable only at
+    the registry, and is owned by test_registry.py's
+    ``test_tag_is_comma_values``/``test_correspondent_is_not_comma_values``.
+    """
+
+    def test_comma_list_requires_every_listed_tag(
+        self,
+        backend: TantivyBackend,
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document carrying both "bills" and "unpaid" tags, and a
+              second document carrying only "bills" (plus "archived")
+        WHEN:
+            - "tag:bills,unpaid" is searched
+        THEN:
+            - Only the document carrying every listed tag matches, and a
+              single-tag "tag:bills" search still matches both documents
+        """
+        bills = Tag.objects.create(name="bills")
+        unpaid = Tag.objects.create(name="unpaid")
+        archived = Tag.objects.create(name="archived")
+
+        both = DocumentFactory(title="Both tags", content="body")
+        both.tags.add(bills, unpaid)
+        backend.add_or_update(both)
+
+        one = DocumentFactory(title="One tag", content="body")
+        one.tags.add(bills, archived)
+        backend.add_or_update(one)
+
+        assert matched_ids("tag:bills,unpaid") == {both.pk}
+        assert matched_ids("tag:bills") == {both.pk, one.pk}
+
+
+class TestArchiveMetadataFields:
+    @pytest.fixture
+    def doc(self, backend: TantivyBackend, admin_user: User) -> Document:
+        doc = DocumentFactory(
+            title="Metadata",
+            content="body",
+            checksum=DOC_CHECKSUM,
+            archive_serial_number=100,
+            page_count=12,
+            original_filename="invoice.pdf",
+        )
+        Note.objects.create(document=doc, user=admin_user, note="a note")
+        backend.add_or_update(doc)
+        return doc
+
+    @pytest.mark.parametrize(
+        "query",
+        [
+            "asn:100",
+            "asn:[50 to 150]",
+            "page_count:12",
+            "page_count:[10 to 20]",
+            "num_notes:1",
+            "num_notes:[1 to 5]",
+            "original_filename:invoice.pdf",
+            f"checksum:{DOC_CHECKSUM}",
+            "checksum:9f86d081*",
+            # A checksum term is stored verbatim, but a checksum *pattern* is
+            # lowercased before it is matched, so an uppercase prefix pattern
+            # still matches even though the uppercase term in the negative
+            # list below does not.
+            "checksum:9F86D081*",
+        ],
+    )
+    def test_documented_metadata_query_matches(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        doc: Document,
+        query: str,
+    ) -> None:
+        """
+        GIVEN:
+            - A document with an ASN, page count, a note, an original
+              filename and a known checksum
+        WHEN:
+            - Every documented metadata-field spelling (exact value,
+              range, and, for checksum, a lowercase prefix pattern
+              regardless of the case the pattern itself is typed in) is
+              searched
+        THEN:
+            - Each one matches the document
+        """
+        assert matched_ids(query) == {doc.pk}
+
+    @pytest.mark.parametrize(
+        "query",
+        [
+            # The docs say only a complete, lowercase checksum matches.
+            "checksum:9f86d081",
+            f"checksum:{DOC_CHECKSUM.upper()}",
+        ],
+    )
+    def test_partial_or_uppercase_checksum_matches_nothing(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        doc: Document,
+        query: str,
+    ) -> None:
+        """
+        GIVEN:
+            - A document with a known, complete, lowercase checksum
+        WHEN:
+            - An exact-value search is run with a partial or uppercase
+              spelling of that checksum
+        THEN:
+            - Nothing matches, as the docs say only a complete, lowercase
+              checksum matches as an exact value
+        """
+        assert matched_ids(query) == set()
+
+
+class TestDocumentedDateForms:
+    @pytest.fixture(autouse=True)
+    def frozen_now(self) -> Generator[None, None, None]:
+        with time_machine.travel(FROZEN_NOW, tick=False):
+            yield
+
+    @pytest.fixture
+    def dated(self, backend: TantivyBackend) -> dict[str, int]:
+        stamps = {
+            "today": datetime(2026, 6, 15, 9, 0, tzinfo=UTC),
+            "yesterday": datetime(2026, 6, 14, 9, 0, tzinfo=UTC),
+            "tomorrow": datetime(2026, 6, 16, 9, 0, tzinfo=UTC),
+            "next_monday": datetime(2026, 6, 22, 10, 0, tzinfo=UTC),
+            "last_monday": datetime(2026, 6, 8, 10, 0, tzinfo=UTC),
+            "january": datetime(2026, 1, 10, 10, 0, tzinfo=UTC),
+            "old": datetime(2005, 3, 4, 15, 30, tzinfo=UTC),
+        }
+        docs = {
+            label: DocumentFactory(title=label, content="dated body", added=stamp)
+            for label, stamp in stamps.items()
+        }
+        with backend.batch_update() as batch:
+            for doc in docs.values():
+                batch.add_or_update(doc)
+        return {label: doc.pk for label, doc in docs.items()}
+
+    @pytest.mark.parametrize(
+        ("query", "label"),
+        [
+            ("added:today", "today"),
+            ("added:yesterday", "yesterday"),
+            ("added:tomorrow", "tomorrow"),
+            ('added:"next monday"', "next_monday"),
+            ('added:"last monday"', "last_monday"),
+            ("added:january", "january"),
+            ("added:2005-03-04", "old"),
+            ("added:2005-03", "old"),
+            ("added:[2005-01-01 to 2005-12-31]", "old"),
+            ("added:[2005 to 2009]", "old"),
+            # A full timestamp works, but only quoted when it stands alone,
+            # and only unquoted when it is a range bound. The bare standalone
+            # spelling is pinned as a non-match below.
+            ('added:"2005-03-04T15:30:00Z"', "old"),
+            ("added:[2005-03-04T09:00:00Z to 2005-03-04T17:00:00Z]", "old"),
+            # A quoted range bound works when the quotes are single ones; the
+            # double-quoted spelling is pinned as an error below.
+            ("added:['2005-03-04' to 2005-03-05]", "old"),
+        ],
+    )
+    def test_documented_date_form_matches_its_day_or_month(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        dated: dict[str, int],
+        query: str,
+        label: str,
+    ) -> None:
+        """
+        GIVEN:
+            - Documents dated today, yesterday, tomorrow, next/last
+              Monday, in January, and on an old fixed date, indexed
+              against a frozen "now" (a Monday)
+        WHEN:
+            - Every documented date-form spelling is searched: relative
+              keywords, quoted multi-word phrases, a bare year-month, an
+              explicit range, a quoted full timestamp standing alone, an
+              unquoted full timestamp as a range bound, and a
+              single-quoted range bound
+        THEN:
+            - Each form matches exactly the document dated on its day or
+              within its month
+        """
+        assert matched_ids(query) == {dated[label]}
+
+    @pytest.mark.parametrize(
+        "query",
+        [
+            # Zero-width: these resolve to a single instant, not a span, so
+            # nothing in a realistic corpus lands on them. The docs warn
+            # about them rather than presenting them as usable.
+            "added:now",
+            "added:noon",
+            "added:midnight",
+            # Quoting is what rescues the other multi-word date expressions,
+            # so pin that it does not rescue these: the problem is the width
+            # of the resulting range, not the way the value is delimited.
+            # One quoted spelling is enough for that; which keyword sits
+            # inside the quotes is grammar whoosh-compat owns.
+            'added:"now"',
+            # A relative offset, which the warning in the docs names by this
+            # exact spelling. Standing alone it is an instant like the rest of
+            # this list; the same offset used as a range bound is a real
+            # window, pinned by the test below.
+            'added:"-1 week"',
+        ],
+    )
+    def test_forms_the_docs_warn_about_match_nothing(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        dated: dict[str, int],
+        query: str,
+    ) -> None:
+        """
+        GIVEN:
+            - A realistic dated corpus (see the `dated` fixture)
+        WHEN:
+            - A zero-width date form ("now", "noon", "midnight", a quoted
+              "now") or a standalone relative offset ("-1 week") is
+              searched: each resolves to a single instant rather than a
+              span, and quoting does not rescue them the way it rescues
+              other multi-word date expressions, since the problem is the
+              width of the resulting range, not how the value is
+              delimited
+        THEN:
+            - Nothing matches, exactly as the docs warn, rather than
+              presenting these as usable spellings
+        """
+        assert matched_ids(query) == set()
+
+    def test_bare_timestamp_is_rejected_rather_than_matching_nothing(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        dated: dict[str, int],
+    ) -> None:
+        """
+        GIVEN:
+            - A realistic dated corpus, including a document dated at a
+              known full timestamp
+        WHEN:
+            - The bare, unquoted spelling of that full timestamp is
+              searched (the quoted and range-bound spellings pinned above
+              do work and match this fixture's document)
+        THEN:
+            - `InvalidDateQuery` is raised rather than the query silently
+              matching nothing, since this is a user-fixable error the
+              docs tell the user to quote, and the reported value is the
+              whole contiguous fragment the user typed, not just the
+              prefix the date grammar's tokenizer first split on
+        """
+        with pytest.raises(InvalidDateQuery) as exc_info:
+            matched_ids("added:2005-03-04T15:30:00Z")
+        assert exc_info.value.field == "added"
+        assert exc_info.value.value == "2005-03-04T15:30:00Z"
+
+    def test_relative_offset_as_a_range_bound_is_a_real_window(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        dated: dict[str, int],
+    ) -> None:
+        """
+        GIVEN:
+            - A realistic dated corpus, including a document dated two
+              hours before a "last Monday to now" window opens, and
+              documents dated today and yesterday, inside that window
+        WHEN:
+            - "added:['-1 week' to now]" is searched: the same offset
+              that matches nothing standing alone (see the test above),
+              used here as a range bound instead
+        THEN:
+            - The window matches today and yesterday but excludes the
+              document two hours before it opens, showing the bound is
+              the offset itself and not a whole-day rounding of it, as
+              the docs say next to the warning about the standalone form
+        """
+        assert matched_ids("added:['-1 week' to now]") == {
+            dated["today"],
+            dated["yesterday"],
+        }
+
+    def test_double_quoted_range_bound_is_rejected(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        dated: dict[str, int],
+    ) -> None:
+        """
+        GIVEN:
+            - A realistic dated corpus
+        WHEN:
+            - A range bound is double-quoted rather than single-quoted
+              ("added:[\"2005-03-04\" to 2005-03-05]")
+        THEN:
+            - `InvalidDateQuery` is raised, pinning which of the two
+              quote characters fails: quoting a range bound is allowed,
+              but only with single quotes, since the double-quoted
+              spelling reaches the date grammar with its quotes still
+              attached and is not a recognizable date
+        """
+        with pytest.raises(InvalidDateQuery) as exc_info:
+            matched_ids('added:["2005-03-04" to 2005-03-05]')
+        assert exc_info.value.value == '"2005-03-04"'

src/documents/tests/search/test_error_routing.py

@@ -0,0 +1,358 @@
+"""Diagnostics route by Cause, and user-facing messages are host-owned.
+
+whoosh-compat documents ``Diagnostic.message`` as developer output with no
+stability guarantee, so it must never reach an HTTP response body.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import UTC
+from typing import TYPE_CHECKING
+
+import pytest
+from whoosh_compat.errors import Diagnostic
+from whoosh_compat.errors import DiagnosticKind
+from whoosh_compat.errors import QueryError
+from whoosh_compat.errors import cause_for
+from whoosh_compat.fields import FieldKind
+from whoosh_compat.fields import FieldRef
+
+from documents.search._errors import SearchQueryError
+from documents.search._query import _map_emit_error
+from documents.search._query import _single_diagnostic_to_error
+from documents.search._query import parse_user_query
+
+if TYPE_CHECKING:
+    import tantivy
+
+
+pytestmark = pytest.mark.search
+
+_LIBRARY_PROSE = "INTERNAL LIBRARY WORDING WITH raw tantivy detail"
+
+
+def _diagnostic(
+    kind: DiagnosticKind,
+    *,
+    field: FieldRef | None = FieldRef("title"),
+    field_kind: FieldKind | None = FieldKind.TEXT,
+) -> Diagnostic:
+    """A Diagnostic shaped like the emitter's, with the library's own
+    kind -> cause mapping rather than a hand-picked cause."""
+    return Diagnostic(
+        kind=kind,
+        cause=cause_for(kind),
+        message=_LIBRARY_PROSE,
+        field=field,
+        field_kind=field_kind,
+    )
+
+
+class TestEmitErrorRouting:
+    """Every Cause gets a distinguishable treatment, not just "a 400"."""
+
+    @pytest.mark.parametrize(
+        "kind",
+        [
+            DiagnosticKind.BACKEND_REJECTED,
+            DiagnosticKind.AST_INVALID_SHAPE,
+            DiagnosticKind.AST_UNKNOWN_FIELD,
+        ],
+    )
+    def test_internal_cause_is_not_converted(self, kind: DiagnosticKind) -> None:
+        """
+        GIVEN:
+            - A QueryError wrapping a Diagnostic whose Cause is INTERNAL
+              (BACKEND_REJECTED/AST_INVALID_SHAPE/AST_UNKNOWN_FIELD)
+        WHEN:
+            - _map_emit_error processes it
+        THEN:
+            - The original QueryError propagates unchanged, so it surfaces
+              as a 500 monitoring can see, never a 400 blaming the user
+        """
+        error = QueryError(_diagnostic(kind))
+        with pytest.raises(QueryError) as excinfo:
+            _map_emit_error(error)
+        assert excinfo.value is error
+
+    def test_misconfigured_cause_is_logged_and_reraised(
+        self,
+        caplog: pytest.LogCaptureFixture,
+    ) -> None:
+        """
+        GIVEN:
+            - A QueryError for SCHEMA_FIELD_MISSING naming field "asn"
+        WHEN:
+            - _map_emit_error processes it
+        THEN:
+            - Exactly one ERROR log record is emitted naming the field and
+              the diagnostic kind, and the original QueryError propagates
+              unchanged: a registry/schema disagreement is transient (the
+              exact same query succeeds once the index is rebuilt), so it
+              surfaces as a 500 an operator can see rather than a 400
+              telling the client their query is permanently invalid
+        """
+        kind = DiagnosticKind.SCHEMA_FIELD_MISSING
+        error = QueryError(_diagnostic(kind, field=FieldRef("asn")))
+        with (
+            caplog.at_level(logging.ERROR, logger="paperless.search"),
+            pytest.raises(QueryError) as excinfo,
+        ):
+            _map_emit_error(error)
+        assert excinfo.value is error
+        errors = [r for r in caplog.records if r.levelno == logging.ERROR]
+        assert len(errors) == 1
+        assert "asn" in errors[0].getMessage()
+        assert kind.name in errors[0].getMessage()
+
+    @pytest.mark.parametrize(
+        "kind",
+        [
+            DiagnosticKind.TEXT_RANGE,
+            DiagnosticKind.PATTERN_TOO_COMPLEX,
+            DiagnosticKind.EXISTS_REQUIRES_FAST,
+        ],
+    )
+    def test_unsupported_cause_is_a_400_with_no_operator_log(
+        self,
+        kind: DiagnosticKind,
+        caplog: pytest.LogCaptureFixture,
+    ) -> None:
+        """
+        GIVEN:
+            - A QueryError for a query tantivy cannot run
+              (TEXT_RANGE/PATTERN_TOO_COMPLEX/EXISTS_REQUIRES_FAST)
+        WHEN:
+            - _map_emit_error processes it
+        THEN:
+            - It becomes a SearchQueryError with no log record at WARNING
+              or above; a query tantivy cannot run is the user's to fix,
+              not an operator alert. EXISTS_REQUIRES_FAST is nominally
+              MISCONFIGURED but belongs here: it is decided from the
+              registry's own FieldSpec, so it never reports a disagreement
+              anyone could resolve
+        """
+        with caplog.at_level(logging.WARNING, logger="paperless.search"):
+            error = _map_emit_error(QueryError(_diagnostic(kind)))
+        assert isinstance(error, SearchQueryError)
+        assert caplog.records == []
+
+    @pytest.mark.parametrize(
+        "kind",
+        [
+            DiagnosticKind.TEXT_RANGE,
+            DiagnosticKind.PATTERN_TOO_COMPLEX,
+            DiagnosticKind.EXISTS_REQUIRES_FAST,
+        ],
+    )
+    def test_user_facing_message_never_echoes_library_prose(
+        self,
+        kind: DiagnosticKind,
+    ) -> None:
+        """
+        GIVEN:
+            - A QueryError carrying whoosh-compat's own developer-facing
+              message text (SCHEMA_FIELD_MISSING excluded: it is now
+              re-raised rather than converted, so it never produces a
+              user-facing message at all, see
+              test_misconfigured_cause_is_logged_and_reraised)
+        WHEN:
+            - _map_emit_error processes it
+        THEN:
+            - The resulting error's string never contains that library
+              prose
+        """
+        error = _map_emit_error(QueryError(_diagnostic(kind)))
+        assert _LIBRARY_PROSE not in str(error)
+
+    @pytest.mark.parametrize(
+        "kind",
+        [
+            DiagnosticKind.TEXT_RANGE,
+            DiagnosticKind.PATTERN_TOO_COMPLEX,
+            DiagnosticKind.EXISTS_REQUIRES_FAST,
+        ],
+    )
+    def test_user_facing_message_names_the_field(
+        self,
+        kind: DiagnosticKind,
+    ) -> None:
+        """
+        GIVEN:
+            - A QueryError for a JSON subpath field (custom_fields.value)
+        WHEN:
+            - _map_emit_error processes it
+        THEN:
+            - The resulting error names the field using its canonical
+              dotted form, including the subpath (FieldRef.__str__ yields
+              this dotted name, so every user-reachable emit kind can name
+              it)
+        """
+        diagnostic = _diagnostic(
+            kind,
+            field=FieldRef("custom_fields", "value"),
+            field_kind=FieldKind.JSON,
+        )
+        error = _map_emit_error(QueryError(diagnostic))
+        assert "custom_fields.value" in str(error)
+
+
+class TestParseDiagnosticMessages:
+    """Parse-time diagnostics are host-worded too, off field_kind."""
+
+    def test_too_deep_is_a_400_without_library_prose(self) -> None:
+        """
+        GIVEN:
+            - A parse-time Diagnostic for TOO_DEEP with no field
+        WHEN:
+            - _single_diagnostic_to_error processes it
+        THEN:
+            - It becomes a SearchQueryError with no library prose in its
+              message
+        """
+        error = _single_diagnostic_to_error(
+            _diagnostic(DiagnosticKind.TOO_DEEP, field=None, field_kind=None),
+        )
+        assert isinstance(error, SearchQueryError)
+        assert _LIBRARY_PROSE not in str(error)
+
+    @pytest.mark.parametrize(
+        ("kind", "field_kind"),
+        [
+            (DiagnosticKind.PATTERN_ON_NUMERIC, FieldKind.U64),
+            (DiagnosticKind.PATTERN_ON_BOOLEAN_EXISTS, FieldKind.BOOLEAN_EXISTS),
+            (DiagnosticKind.PATTERN_ON_SUBPATH, FieldKind.JSON),
+        ],
+    )
+    def test_pattern_on_kinds_name_the_field_and_its_kind(
+        self,
+        kind: DiagnosticKind,
+        field_kind: FieldKind,
+    ) -> None:
+        """
+        GIVEN:
+            - A parse-time Diagnostic for a pattern used against a kind
+              that cannot take one
+              (PATTERN_ON_NUMERIC/PATTERN_ON_BOOLEAN_EXISTS/PATTERN_ON_SUBPATH)
+        WHEN:
+            - _single_diagnostic_to_error processes it
+        THEN:
+            - The message names both the field and its kind, with no
+              library prose
+        """
+        error = _single_diagnostic_to_error(
+            _diagnostic(kind, field=FieldRef("asn"), field_kind=field_kind),
+        )
+        message = str(error)
+        assert _LIBRARY_PROSE not in message
+        assert "asn" in message
+        assert field_kind.name.lower() in message
+
+    def test_single_char_bracket_range_names_the_field_and_the_value(self) -> None:
+        """
+        GIVEN:
+            - A SINGLE_CHAR_BRACKET_RANGE diagnostic for "title" with
+              raw_value "200[1-9]"
+        WHEN:
+            - _single_diagnostic_to_error processes it
+        THEN:
+            - The resulting SearchQueryError names both the field and the
+              offending value, with no library prose
+        """
+        diagnostic = Diagnostic(
+            kind=DiagnosticKind.SINGLE_CHAR_BRACKET_RANGE,
+            cause=cause_for(DiagnosticKind.SINGLE_CHAR_BRACKET_RANGE),
+            message=_LIBRARY_PROSE,
+            field=FieldRef("title"),
+            field_kind=FieldKind.TEXT,
+            raw_value="200[1-9]",
+        )
+        error = _single_diagnostic_to_error(diagnostic)
+        message = str(error)
+        assert isinstance(error, SearchQueryError)
+        assert _LIBRARY_PROSE not in message
+        assert "title" in message
+        assert "200[1-9]" in message
+
+
+class TestRealQueriesRouteCorrectly:
+    """The routing table against diagnostics emit() really produces."""
+
+    def test_text_range_is_a_400_naming_the_field(
+        self,
+        query_index: tantivy.Index,
+    ) -> None:
+        """
+        GIVEN:
+            - A real query index
+        WHEN:
+            - parse_user_query is called with a text-range query
+              ("title:[a to b]")
+        THEN:
+            - It raises SearchQueryError naming "title"
+        """
+        with pytest.raises(SearchQueryError) as excinfo:
+            parse_user_query(query_index, "title:[a to b]", UTC)
+        assert "title" in str(excinfo.value)
+
+    def test_wildcard_on_a_numeric_field_is_a_400_naming_the_field(
+        self,
+        query_index: tantivy.Index,
+    ) -> None:
+        """
+        GIVEN:
+            - A real query index
+        WHEN:
+            - parse_user_query is called with a wildcard on a numeric
+              field ("asn:12*")
+        THEN:
+            - It raises SearchQueryError naming "asn"
+        """
+        with pytest.raises(SearchQueryError) as excinfo:
+            parse_user_query(query_index, "asn:12*", UTC)
+        assert "asn" in str(excinfo.value)
+
+    def test_single_char_bracket_range_is_a_400_naming_field_and_value(
+        self,
+        query_index: tantivy.Index,
+    ) -> None:
+        """
+        GIVEN:
+            - A real query index
+        WHEN:
+            - parse_user_query is called with "title:200[1-9]"
+        THEN:
+            - It raises SearchQueryError naming both "title" and
+              "200[1-9]"
+        """
+        with pytest.raises(SearchQueryError) as excinfo:
+            parse_user_query(query_index, "title:200[1-9]", UTC)
+        message = str(excinfo.value)
+        assert "title" in message
+        assert "200[1-9]" in message
+
+    def test_internal_diagnostic_escapes_as_a_query_error(
+        self,
+        query_index: tantivy.Index,
+        monkeypatch: pytest.MonkeyPatch,
+    ) -> None:
+        """
+        GIVEN:
+            - tantivy_emit monkeypatched to raise a QueryError with an
+              INTERNAL-cause diagnostic (BACKEND_REJECTED), the one case
+              with no query text of its own involved
+        WHEN:
+            - parse_user_query runs a normal query ("invoice")
+        THEN:
+            - The QueryError propagates unconverted; emit() reporting a
+              defect in itself must not become a user-facing 400
+        """
+        import documents.search._query as query_mod
+
+        def raise_internal(*args: object, **kwargs: object) -> None:
+            raise QueryError(_diagnostic(DiagnosticKind.BACKEND_REJECTED))
+
+        monkeypatch.setattr(query_mod, "tantivy_emit", raise_internal)
+        with pytest.raises(QueryError):
+            parse_user_query(query_index, "invoice", UTC)

src/documents/tests/search/test_exists_on_json_fields.py

@@ -0,0 +1,119 @@
+"""``field:*`` on a JSON field is user error, not an operator alert.
+
+whoosh-compat classifies EXISTS_REQUIRES_FAST as MISCONFIGURED, and
+_map_emit_error used to route every MISCONFIGURED diagnostic to an ERROR log.
+But the kind is decided from the registry's own FieldSpec (kind plus fast)
+without consulting the index schema, and field_descriptors() builds the JSON
+fields non-fast deliberately, so nothing is misconfigured and no operator
+action can clear the condition. Any authenticated user could otherwise emit
+ERROR lines in a loop by repeating ``notes:*``.
+
+SCHEMA_FIELD_MISSING, the other MISCONFIGURED kind, does compare the registry
+against the live schema, so it stays an ERROR log. But it is not a 400
+either: the exact same query would succeed once the index is rebuilt, so it
+is a transient server-side condition, not a permanently bad request, and is
+re-raised the same way an INTERNAL cause is.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import UTC
+from typing import TYPE_CHECKING
+
+import pytest
+from whoosh_compat.errors import Diagnostic
+from whoosh_compat.errors import DiagnosticKind
+from whoosh_compat.errors import QueryError
+from whoosh_compat.errors import cause_for
+from whoosh_compat.fields import FieldKind
+from whoosh_compat.fields import FieldRef
+
+from documents.search._errors import SearchQueryError
+from documents.search._query import _map_emit_error
+from documents.search._query import parse_user_query
+
+if TYPE_CHECKING:
+    import tantivy
+
+
+pytestmark = pytest.mark.search
+
+# Every spelling of "does this JSON field have a value" a user can type.
+EXISTS_QUERIES = [
+    "notes:*",
+    "notes.note:*",
+    "notes.user:*",
+    "custom_fields:*",
+    "custom_fields.name:*",
+    "custom_fields.value:*",
+]
+
+
+class TestJsonExistsIsUserError:
+    @pytest.mark.parametrize("query", EXISTS_QUERIES)
+    def test_query_is_a_400_that_emits_no_error_log(
+        self,
+        query_index: tantivy.Index,
+        caplog: pytest.LogCaptureFixture,
+        query: str,
+    ) -> None:
+        """
+        GIVEN:
+            - A real query index, and every spelling of "does this JSON
+              field have a value" (notes:*, notes.note:*, custom_fields:*,
+              etc.)
+        WHEN:
+            - parse_user_query runs the query
+        THEN:
+            - It raises SearchQueryError naming the field, and no
+              ERROR-level log record is emitted; EXISTS_REQUIRES_FAST on a
+              JSON field is by design, not a misconfiguration an operator
+              could act on
+        """
+        with caplog.at_level(logging.WARNING, logger="paperless.search"):
+            with pytest.raises(SearchQueryError) as excinfo:
+                parse_user_query(query_index, query, UTC)
+        assert query.split(":", maxsplit=1)[0] in str(excinfo.value)
+        assert [r for r in caplog.records if r.levelno >= logging.ERROR] == []
+
+
+class TestGenuineMisconfigurationStillLogs:
+    def test_schema_field_missing_is_an_error_log_and_reraised(
+        self,
+        caplog: pytest.LogCaptureFixture,
+    ) -> None:
+        """
+        GIVEN:
+            - A QueryError for SCHEMA_FIELD_MISSING: the registry naming a
+              field the index schema does not have
+        WHEN:
+            - _map_emit_error processes it
+        THEN:
+            - It logs exactly one ERROR record naming the diagnostic kind
+              (a real mismatch an operator can fix, so it keeps the
+              alert), and the original QueryError propagates unchanged
+              rather than becoming a SearchQueryError: the exact same
+              query would succeed once the index is rebuilt, so this is a
+              transient server-side condition, not a permanently bad
+              request, and surfaces as a 500 rather than a 400
+        """
+        kind = DiagnosticKind.SCHEMA_FIELD_MISSING
+        error = QueryError(
+            Diagnostic(
+                kind=kind,
+                cause=cause_for(kind),
+                message="field 'asn' is not defined in the index schema",
+                field=FieldRef("asn"),
+                field_kind=FieldKind.U64,
+            ),
+        )
+        with (
+            caplog.at_level(logging.ERROR, logger="paperless.search"),
+            pytest.raises(QueryError) as excinfo,
+        ):
+            _map_emit_error(error)
+        assert excinfo.value is error
+        records = [r for r in caplog.records if r.levelno == logging.ERROR]
+        assert len(records) == 1
+        assert kind.name in records[0].getMessage()

src/documents/tests/search/test_fuzzy_alternative.py

@@ -0,0 +1,223 @@
+"""_fuzzy_alternative, the near-match side of the widening hook.
+
+A leaf's words each become a Fuzzy leaf on the leaf's own field, so a
+typo in one word of a term still finds the document while everything
+around the term keeps constraining it. A Term's words are OR'd for
+per-word recall; a Phrase's are AND-ed, because quoting must not widen a
+search. CJK words get no fuzzy side at all.
+"""
+
+from __future__ import annotations
+
+import pytest
+import whoosh_compat as wc
+import whoosh_compat.ast as wc_ast
+
+from documents.search._query import _fuzzy_alternative
+from documents.tests.search._ast_helpers import CONTENT
+from documents.tests.search._ast_helpers import TITLE
+from documents.tests.search._ast_helpers import content
+
+pytestmark = pytest.mark.search
+
+
+# 130 characters, past the analyzer's 129-byte remove_long limit (128 is
+# kept, 129 is dropped), so the index never holds it and neither side
+# should search for it.
+_TOO_LONG = "x" * 130
+
+
+def _fuzzy(field: wc.FieldRef, text: str) -> wc_ast.Fuzzy:
+    return wc_ast.Fuzzy(field=field, text=text, distance=1, prefix=True)
+
+
+class TestTheAlternative:
+    def test_a_single_word_becomes_one_fuzzy_leaf(self) -> None:
+        """
+        GIVEN:
+            - A one-word term
+        WHEN:
+            - The fuzzy alternative is built
+        THEN:
+            - It is a single Fuzzy leaf on the same field, distance 1 and
+              prefix matching, which is what the old clause used
+        """
+        assert _fuzzy_alternative(content("invoice")) == _fuzzy(CONTENT, "invoice")
+
+    def test_the_leafs_own_field_is_used(self) -> None:
+        """
+        GIVEN:
+            - A term fielded on title
+        WHEN:
+            - The fuzzy alternative is built
+        THEN:
+            - The Fuzzy leaf is on title, not spread across the default
+              fields. This is the fielding fix: today's clause searches a
+              fielded word everywhere
+        """
+        assert _fuzzy_alternative(wc_ast.Term(field=TITLE, text="invoice")) == _fuzzy(
+            TITLE,
+            "invoice",
+        )
+
+    def test_several_words_are_ored(self) -> None:
+        """
+        GIVEN:
+            - A hyphenated term the analyzer splits into two words
+        WHEN:
+            - The fuzzy alternative is built
+        THEN:
+            - The words are OR'd. AND would be stricter than the exact
+              side, which is OR'd for an unfielded term, and would lose
+              today's per-word recall inside a term
+        """
+        assert _fuzzy_alternative(content("COVID-19")) == wc_ast.Or(
+            children=(_fuzzy(CONTENT, "covid"), _fuzzy(CONTENT, "19")),
+        )
+
+    def test_words_are_split_where_the_index_splits_them(self) -> None:
+        """
+        GIVEN:
+            - A term the analyzer lowercases and folds
+        WHEN:
+            - The fuzzy alternative is built
+        THEN:
+            - The Fuzzy text is the analyzer's output, not the raw text,
+              so it is in the same shape as the index terms
+        """
+        assert _fuzzy_alternative(content("Éclair")) == _fuzzy(CONTENT, "eclair")
+
+    def test_the_splitter_does_not_stem(self) -> None:
+        """
+        GIVEN:
+            - A word the stemmer would change
+        WHEN:
+            - The fuzzy alternative is built
+        THEN:
+            - The word is unstemmed. Fuzzy text goes through the field's
+              pattern_normalizer, which stems it once; stemming here too
+              would search for a term the index does not hold
+        """
+        assert _fuzzy_alternative(content("universities")) == _fuzzy(
+            CONTENT,
+            "universities",
+        )
+
+    @pytest.mark.parametrize(
+        "text",
+        [
+            pytest.param("x", id="one_character"),
+            pytest.param("__", id="only_dropped_characters"),
+            pytest.param(_TOO_LONG, id="past_remove_long"),
+            pytest.param("", id="empty"),
+            pytest.param("   ", id="whitespace"),
+        ],
+    )
+    def test_text_with_no_usable_word_gets_no_alternative(self, text: str) -> None:
+        """
+        GIVEN:
+            - Text with no word the index could hold: a single character,
+              characters the tokenizer drops, a word past remove_long, or
+              nothing at all
+        WHEN:
+            - The fuzzy alternative is built
+        THEN:
+            - None, so the leaf is left alone. Returning an alternative
+              here would collapse Or(leaf, fuzzy) to a required clause
+              that can never match, making a query return LESS with fuzzy
+              on than off
+        """
+        assert _fuzzy_alternative(content(text)) is None
+
+    def test_a_one_character_word_is_dropped_from_a_longer_term(self) -> None:
+        """
+        GIVEN:
+            - A term mixing a one-character word with a real one
+        WHEN:
+            - The fuzzy alternative is built
+        THEN:
+            - Only the real word survives. A one-character prefix fuzzy
+              term matches every term in the field
+        """
+        assert _fuzzy_alternative(content("h52.1")) == _fuzzy(CONTENT, "h52")
+
+    def test_the_leaf_span_is_copied(self) -> None:
+        """
+        GIVEN:
+            - A leaf with a source span, whose words both survive the
+              one-character filter so there is an Or to inspect
+        WHEN:
+            - The fuzzy alternative is built
+        THEN:
+            - Every node it builds carries that span, so an emit-time
+              diagnostic still points into the query text
+        """
+        leaf = wc_ast.Term(field=CONTENT, text="ab-cd", startchar=4, endchar=9)
+
+        alternative = _fuzzy_alternative(leaf)
+
+        assert isinstance(alternative, wc_ast.Or)
+        spans = {
+            (node.startchar, node.endchar)
+            for node in (alternative, *alternative.children)
+        }
+        assert spans == {(4, 9)}
+
+
+class TestPhrasesAreNotWidened:
+    def test_a_multi_word_phrase_requires_every_word(self) -> None:
+        """
+        GIVEN:
+            - A quoted two-word phrase
+        WHEN:
+            - Its fuzzy alternative is built
+        THEN:
+            - The words are And-combined, where a Term's are Or-combined
+              (test_a_hyphenated_term_matches_on_one_word pins that). The
+              parser's default group is And, so an Or here would let the
+              quoted phrase match strictly more than the same two words
+              unquoted
+        """
+        leaf = wc_ast.Phrase(field=CONTENT, text="tax report")
+
+        assert _fuzzy_alternative(leaf) == wc_ast.And(
+            children=(_fuzzy(CONTENT, "tax"), _fuzzy(CONTENT, "report")),
+        )
+
+
+class TestCjkGetsNoFuzzySide:
+    @pytest.mark.parametrize(
+        "text",
+        [
+            pytest.param("東京", id="cjk_only"),
+            pytest.param("東京都の報告書", id="longer_run"),
+            pytest.param("서울", id="hangul"),
+        ],
+    )
+    def test_a_cjk_word_is_skipped(self, text: str) -> None:
+        """
+        GIVEN:
+            - A CJK term
+        WHEN:
+            - Its fuzzy alternative is built
+        THEN:
+            - There is none. The content analyzer keeps the run as one
+              token, so a prefix Fuzzy over it would match any run within
+              one edit of its start, which is the "東京都 matches 京都"
+              failure the bigram fields exist to avoid
+        """
+        assert _fuzzy_alternative(content(text)) is None
+
+    def test_latin_beside_cjk_still_gets_its_fuzzy_side(self) -> None:
+        """
+        GIVEN:
+            - A term mixing a CJK run and a latin word
+        WHEN:
+            - Its fuzzy alternative is built
+        THEN:
+            - Only the latin word is fuzzed. Skipping CJK words must not
+              cost the latin half its near-match
+        """
+        leaf = content("東京 report")
+
+        assert _fuzzy_alternative(leaf) == _fuzzy(CONTENT, "report")

src/documents/tests/search/test_fuzzy_scoring.py

@@ -0,0 +1,134 @@
+"""What survives ADVANCED_FUZZY_SEARCH_THRESHOLD.
+
+The rest of the fuzzy tests run at threshold 0.0 so they see matching
+behavior rather than the filter. These run at 0.5, the value the docs
+recommend, because the filter has properties of its own that nothing else
+covers: a near-miss must not be cut according to how well the query's
+correctly spelled words matched, and an exact match must outrank one that
+is only partly exact.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from pytest_django.fixtures import SettingsWrapper
+
+    from documents.models import Document
+    from documents.search._backend import TantivyBackend
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+# Long enough that BM25 scores it well below a short document holding the
+# same word, which is what makes the spread these tests are about.
+_PADDING = "lorem ipsum dolor sit amet " * 40
+
+
+@pytest.fixture(autouse=True)
+def _threshold(settings: SettingsWrapper) -> None:
+    settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5
+
+
+class TestNearMissesAreNotCutByTheSpread:
+    def test_two_equal_near_misses_both_survive(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - Two documents that both contain a rare correctly spelled
+              word and both near-match a misspelled one. One has the rare
+              word in its title, the other buried in a long body, so BM25
+              scores them very differently
+        WHEN:
+            - Both words are searched at threshold 0.5
+        THEN:
+            - Both survive. Neither is a better answer than the other:
+              they differ only in where the correctly spelled word sits.
+              If the widened tree were scored normally, the fuzzy side
+              would inherit the BM25 spread of the correct word and the
+              weaker document would be cut
+        """
+        titled = index_document(
+            title="Zarquon",
+            content=f"invoices {_PADDING}",
+        )
+        buried = index_document(
+            title="Nothing",
+            content=f"{_PADDING} zarquon invoicing",
+        )
+
+        assert matched_ids("zarquon invoce") == {titled.pk, buried.pk}
+
+    def test_near_misses_still_rank_among_themselves(
+        self,
+        backend: TantivyBackend,
+        index_document: Callable[..., Document],
+    ) -> None:
+        """
+        GIVEN:
+            - A document near-matching the query in several fields and one
+              near-matching it in a single field
+        WHEN:
+            - The misspelled word is searched
+        THEN:
+            - Both survive, and the multi-field one ranks first. Flat
+              scoring alone would tie every near-miss and lose relevance
+              ordering entirely, which is what the tiebreak clause exists
+              to prevent
+        """
+        strong = index_document(
+            title="Invoices",
+            content="invoices for invoicing",
+        )
+        weak = index_document(title="Nothing", content="invoices")
+
+        hits = backend.search_ids("invoce", user=None)
+
+        assert set(hits) == {strong.pk, weak.pk}
+        assert hits[0] == strong.pk
+
+
+class TestExactOutranksNearMiss:
+    def test_a_fully_exact_document_outranks_a_partly_exact_one(
+        self,
+        backend: TantivyBackend,
+        index_document: Callable[..., Document],
+    ) -> None:
+        """
+        GIVEN:
+            - A document matching both query words exactly but buried in a
+              long body, and one matching the first word exactly several
+              times while only near-matching the second
+        WHEN:
+            - Both words are searched, spelled correctly
+        THEN:
+            - The fully exact document ranks first. docs/configuration.md
+              promises fuzzy results rank below exact matches, and the
+              widened tree alone would break it: a partly exact document
+              collects real BM25 for what it did match plus a constant for
+              what it only near-matched
+
+        Only the ordering is asserted, not that the partly exact document
+        survives the threshold. Spec 4.2a accepts that a large enough BM25
+        spread still cuts a near-miss, so whether it survives depends on
+        the corpus and is not a property to pin.
+        """
+        index_document(
+            title="Invoice Invoice Invoice",
+            content="reportage weekly",
+        )
+        fully = index_document(
+            title="Nothing",
+            content=f"{_PADDING} invoice report",
+        )
+
+        hits = backend.search_ids("invoice report", user=None)
+
+        assert hits[0] == fully.pk

src/documents/tests/search/test_fuzzy_tokenization.py

@@ -0,0 +1,211 @@
+"""The words a leaf contributes to its fuzzy alternative.
+
+Each leaf is widened in the tree now, so nothing is re-parsed as a string
+and a boolean keyword can no longer be read as grammar. What still has to
+hold is that a word is stemmed exactly once (analysis is not idempotent)
+and that hyphenated, dotted and quoted terms keep contributing their
+words.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from documents.models import Document
+
+pytestmark = [
+    pytest.mark.search,
+    pytest.mark.django_db,
+    pytest.mark.usefixtures("fuzzy_enabled"),
+]
+
+
+class TestFuzzyClauseWords:
+    def test_a_stemmed_word_is_not_stemmed_a_second_time(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - Documents whose content contains "universities", a
+              one-transposition typo of it ("universties"), and two
+              unrelated words that share its stem prefix ("univalent",
+              "unicycle")
+        WHEN:
+            - Searching for "universities" with the fuzzy blend enabled
+        THEN:
+            - Only the correctly-spelled document and its typo match; the
+              clause does not widen far enough to reach the unrelated
+              words. 'universities' stems to 'univers'; feeding that back
+              to tantivy would stem it again to 'univ', whose fuzzy prefix
+              reaches unrelated words - the clause must stay wide enough
+              for a typo and no wider
+        """
+        wanted = index_document(
+            title="A",
+            content="universities of europe",
+        )
+        typo = index_document(
+            title="B",
+            content="universties of europe",
+        )
+        index_document(
+            title="C",
+            content="univalent chemical bonds",
+        )
+        index_document(
+            title="D",
+            content="unicycle repair manual",
+        )
+
+        assert matched_ids("universities") == {wanted.pk, typo.pk}
+
+    def test_a_hyphenated_term_still_reaches_the_clause(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document whose content contains a near-miss of "COVID-19"
+              ("covidx")
+        WHEN:
+            - Searching for "COVID-19" with the fuzzy blend enabled
+        THEN:
+            - The document matches; 'COVID-19' is one raw token, so unless
+              it is split into words, it carries characters the re-parse
+              would read as grammar, is dropped, and the whole query loses
+              its fuzzy clause
+        """
+        misspelled = index_document(
+            title="A",
+            content="covidx testing results",
+        )
+
+        assert matched_ids("COVID-19") == {misspelled.pk}
+
+    def test_a_phrase_still_reaches_the_clause(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document whose content near-misses a quoted phrase
+        WHEN:
+            - Searching for the quoted phrase '"tax reports"' with the
+              fuzzy blend enabled
+        THEN:
+            - The document matches; a phrase is one raw token carrying a
+              space, and is the whole query's only free text here, so it
+              must still reach the clause
+        """
+        near_miss = index_document(
+            title="A",
+            content="taxation reportage weekly",
+        )
+
+        assert matched_ids('"tax reports"') == {near_miss.pk}
+
+
+class TestBooleanKeywordsInRawText:
+    """Tantivy's boolean keywords used to reach its parser with their case
+    intact, through the word string the old clause was re-parsed from, so a
+    quoted phrase could smuggle grammar in. Leaves are built as AST nodes
+    now, which closes that off structurally; these pin it shut."""
+
+    @pytest.fixture
+    def corpus(self, index_document: Callable[..., Document]) -> dict[str, int]:
+        both = index_document(
+            title="A",
+            content="taxation reportage weekly",
+        )
+        tax_only = index_document(
+            title="B",
+            content="taxation only here",
+        )
+        report_only = index_document(
+            title="C",
+            content="reportage only here",
+        )
+        return {
+            "both": both.pk,
+            "tax_only": tax_only.pk,
+            "report_only": report_only.pk,
+        }
+
+    @pytest.mark.parametrize(
+        ("keyword_spelling", "ordinary_spelling"),
+        [
+            pytest.param('"tax AND reports"', '"tax and reports"', id="and"),
+            pytest.param('"tax OR reports"', '"tax or reports"', id="or"),
+            pytest.param('"tax NOT reports"', '"tax not reports"', id="not"),
+            pytest.param('"tax IN reports"', '"tax in reports"', id="in"),
+        ],
+    )
+    def test_a_keyword_inside_a_phrase_stays_an_ordinary_word(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        corpus: dict[str, int],
+        keyword_spelling: str,
+        ordinary_spelling: str,
+    ) -> None:
+        """
+        GIVEN:
+            - Three documents: one with both "taxation" and "reportage",
+              one with only "taxation", one with only "reportage"
+        WHEN:
+            - A quoted phrase carries a tantivy boolean keyword as one of
+              its words, spelled in upper case and in lower case
+        THEN:
+            - Both spellings match the same documents, so the keyword is
+              an ordinary word of the phrase rather than grammar: AND does
+              not make it a conjunction, NOT does not give it its own
+              exclusion, IN does not fail the parse. Only the upper-case
+              spelling was ever grammar
+        """
+        assert matched_ids(keyword_spelling) == matched_ids(ordinary_spelling)
+
+    def test_a_phrase_needs_a_near_match_for_every_word(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        corpus: dict[str, int],
+    ) -> None:
+        """
+        GIVEN:
+            - Three documents: one with both "taxation" and "reportage",
+              one with only "taxation", one with only "reportage"
+        WHEN:
+            - '"tax reports"' is searched, both words misspelled
+        THEN:
+            - Only the document near-matching both words comes back. A
+              quoted phrase asks for more than the bare words, so its
+              fuzzy side requires every one of them
+        """
+        assert matched_ids('"tax reports"') == {corpus["both"]}
+
+    def test_a_trailing_keyword_is_just_a_word(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        corpus: dict[str, int],
+    ) -> None:
+        """
+        GIVEN:
+            - Three documents: one with both "taxation" and "reportage",
+              one with only "taxation", one with only "reportage"
+        WHEN:
+            - '"tax AND"' is searched, a phrase that used to be a tantivy
+              syntax error once the clause was re-parsed as a string
+            - The same phrase is searched with the keyword in lower case
+        THEN:
+            - Both match the same documents, and neither raises. Nothing
+              is re-parsed any more, so a trailing keyword cannot cost the
+              query its fuzzy side
+        """
+        assert matched_ids('"tax AND"') == matched_ids('"tax and"')

src/documents/tests/search/test_fuzzy_widening.py

@@ -0,0 +1,467 @@
+"""Fuzzy matching applied inside the parsed query rather than beside it.
+
+With a threshold set, every leaf gains a near-match alternative where it
+sits, so fielding, negation, AND and positive filters all constrain the
+fuzzy match exactly as they constrain the exact one. The old clause was a
+flat bag of words OR'd in at the top level, which none of them reached.
+
+Threshold is 0.0 here so these tests see the matching behavior, not the
+score filter. The filter has its own file, test_fuzzy_scoring.py.
+"""
+
+from __future__ import annotations
+
+import datetime
+from typing import TYPE_CHECKING
+
+import pytest
+
+from documents.models import Document
+from documents.models import StoragePath
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from pytest_django.fixtures import SettingsWrapper
+
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+
+@pytest.fixture(autouse=True)
+def _fuzzy_on(settings: SettingsWrapper) -> None:
+    settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.0
+
+
+class TestStructureIsHonoured:
+    @pytest.fixture
+    def near_matches_both(
+        self,
+        index_document: Callable[..., Document],
+    ) -> Document:
+        """One document near-matching both query words, one only the first."""
+        both = index_document(title="A", content="invoices report")
+        index_document(title="B", content="invoices only")
+        return both
+
+    def test_a_fielded_term_fuzzes_only_that_field(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - One document with a near-miss of the word in its title, one
+              with a near-miss in its content only
+        WHEN:
+            - "title:invoce" is searched
+        THEN:
+            - Only the title document matches. The old clause searched a
+              fielded word across every default field
+        """
+        titled = index_document(title="Invoces", content="nothing")
+        index_document(title="Nothing", content="invoces here")
+
+        assert matched_ids("title:invoce") == {titled.pk}
+
+    def test_every_word_needs_a_near_match(
+        self,
+        near_matches_both: Document,
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document near-matching both words, and one near-matching
+              only the first
+        WHEN:
+            - "invoce reprot" is searched (both words misspelled)
+        THEN:
+            - Only the document near-matching both survives. The old
+              clause OR'd the words, so anything near one of them matched
+        """
+        assert matched_ids("invoce reprot") == {near_matches_both.pk}
+
+    def test_a_negated_word_is_not_fuzzed(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document containing "taxi" and one containing "tax"
+        WHEN:
+            - "invoice NOT tax" is searched
+        THEN:
+            - The "taxi" document survives and the "tax" one does not. A
+              negated leaf keeps its exact side only: fuzzing it with
+              prefix matching would exclude every word starting near it
+        """
+        taxi = index_document(title="A", content="invoice taxi fare")
+        index_document(title="B", content="invoice tax return")
+
+        assert matched_ids("invoice NOT tax") == {taxi.pk}
+
+    def test_a_negation_inside_a_branch_still_binds(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - An "invoice" document containing "secret", one without, and
+              a "bill" document
+        WHEN:
+            - "(invoce AND NOT secret) OR bill" is searched
+        THEN:
+            - The secret document stays out. The old clause restated only
+              top-level conjunctive exclusions, so an Or branch's NOT was
+              never applied to the fuzzy side
+        """
+        index_document(title="A", content="invoices secret")
+        clean = index_document(title="B", content="invoices only")
+        bill = index_document(title="C", content="bill")
+
+        assert matched_ids("(invoce AND NOT secret) OR bill") == {
+            clean.pk,
+            bill.pk,
+        }
+
+    def test_a_structured_filter_constrains_the_fuzzy_match(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - Two near-miss documents created in different years
+        WHEN:
+            - "created:2024 invoce" is searched
+        THEN:
+            - Only the 2024 one matches. The old blend restated negations
+              above the fuzzy clause but never positive constraints, so a
+              filter did not reach the fuzzy side at all.
+
+              created is the right field to test with: it is a DATE field,
+              so it has no fuzzy side of its own and cannot be widened.
+              type: would not test this, because it is an alias for
+              document_type, which is one of the five default search
+              fields and so gets widened like any other leaf
+        """
+        matching = index_document(
+            title="A",
+            content="invoices",
+            created=datetime.date(2024, 6, 1),
+        )
+        index_document(
+            title="B",
+            content="invoices",
+            created=datetime.date(2023, 6, 1),
+        )
+
+        assert matched_ids("created:2024 invoce") == {matching.pk}
+
+    def test_a_filter_on_a_non_default_field_constrains_the_fuzzy_match(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - Two near-miss documents with different storage paths
+        WHEN:
+            - "path:archive invoce" is searched
+        THEN:
+            - Only the matching one comes back. storage_path is a TEXT
+              field that is NOT one of the default search fields, so it
+              gets no fuzzy side; the release note promises this case and
+              the date test above does not cover it
+        """
+        archive = StoragePath.objects.create(name="archive", path="archive/{title}")
+        other = StoragePath.objects.create(name="misc", path="misc/{title}")
+        matching = index_document(
+            title="A",
+            content="invoices",
+            storage_path=archive,
+        )
+        index_document(
+            title="B",
+            content="invoices",
+            storage_path=other,
+        )
+
+        assert matched_ids("path:archive invoce") == {matching.pk}
+
+    def test_a_require_filters_on_the_widened_side_too(
+        self,
+        near_matches_both: Document,
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document near-matching both words and one near-matching
+              only the scored side
+        WHEN:
+            - "invoce REQUIRE reprot" is searched
+        THEN:
+            - Only the document near-matching both comes back. The
+              filter-only side is unscored but still filters, and it is
+              widened like any other leaf, so a near miss satisfies it
+        """
+        assert matched_ids("invoce REQUIRE reprot") == {near_matches_both.pk}
+
+
+class TestRecallIsKept:
+    def test_a_typo_still_finds_its_document(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A correctly spelled document
+        WHEN:
+            - A one-edit misspelling is searched
+        THEN:
+            - It matches. This is what the whole feature is for
+        """
+        doc = index_document(title="A", content="invoice total")
+
+        assert matched_ids("invoce") == {doc.pk}
+
+    def test_a_hyphenated_term_matches_on_one_word(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document holding a near-miss of one half of a hyphenated
+              term
+        WHEN:
+            - "COVID-19" is searched
+        THEN:
+            - It matches. The words inside a term are OR'd on the fuzzy
+              side, which is the per-word recall the old clause had
+        """
+        doc = index_document(title="A", content="covidx cases")
+
+        assert matched_ids("COVID-19") == {doc.pk}
+
+    def test_a_word_the_index_cannot_hold_does_not_narrow_the_query(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - An "invoice" document
+        WHEN:
+            - "invoice __" is searched, where __ is characters the
+              tokenizer discards entirely
+        THEN:
+            - It still matches. If such a word got a fuzzy alternative,
+              the leaf would collapse to a required clause that can never
+              match and the query would return less with fuzzy on than off
+        """
+        doc = index_document(title="A", content="invoice total")
+
+        assert matched_ids("invoice __") == {doc.pk}
+
+    def test_a_one_character_word_no_longer_matches_everything(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - Two unrelated documents
+        WHEN:
+            - "x invoice" is searched, where x is a one-character word no
+              document holds
+        THEN:
+            - Nothing matches. One-character words get no fuzzy side, and
+              the exact side requires a term no document has. Today this
+              query matches the whole corpus, because a one-character
+              prefix fuzzy term matches every term in the field
+        """
+        index_document(title="A", content="invoice total")
+        index_document(title="B", content="unrelated")
+
+        assert matched_ids("x invoice") == set()
+
+
+class TestCjkAndFuzzyTogether:
+    def test_a_cjk_term_keeps_its_bigram_side_with_fuzzy_on(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document with a CJK term inside an unspaced run
+        WHEN:
+            - That term is searched with fuzzy on
+        THEN:
+            - It matches through the bigram side, which sits in the same
+              Or as the fuzzy side and is unaffected by it
+        """
+        doc = index_document(
+            title="A",
+            content="東京都の公共文書について",
+        )
+
+        assert matched_ids("東京") == {doc.pk}
+
+    def test_a_negated_cjk_term_keeps_its_bigram_side(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - An "invoice" document whose only 東京 is inside a run, and
+              one with no 東京 at all
+        WHEN:
+            - "invoice NOT 東京" is searched with fuzzy on
+        THEN:
+            - The 東京 document is excluded. A negated leaf loses its
+              fuzzy side but keeps its CJK one, so the exclusion still
+              reaches inside the run
+        """
+        index_document(title="A", content="invoice 東京都の報告書")
+        clean = index_document(title="B", content="invoice only")
+
+        assert matched_ids("invoice NOT 東京") == {clean.pk}
+
+
+class TestTheHookIsSkipped:
+    def test_a_plain_query_passes_no_hook(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+        settings: SettingsWrapper,
+        monkeypatch: pytest.MonkeyPatch,
+    ) -> None:
+        """
+        GIVEN:
+            - Fuzzy off and a query with no CJK in it
+        WHEN:
+            - It is parsed
+        THEN:
+            - emit() is called once, with rewrite_leaf=None. Nothing is
+              widened, so the query is exactly what it was before any of
+              this work
+        """
+        settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = None
+        from documents.search import _query
+
+        calls: list[object] = []
+        real = _query.tantivy_emit
+
+        def spy(*args: object, **kwargs: object) -> object:
+            calls.append(kwargs.get("rewrite_leaf"))
+            return real(*args, **kwargs)
+
+        monkeypatch.setattr(_query, "tantivy_emit", spy)
+        index_document(title="A", content="invoice")
+
+        matched_ids("invoice")
+
+        assert calls == [None]
+
+
+class TestTheEmitRegistryIsInvisible:
+    def test_a_non_cjk_query_gives_the_same_result_under_either_registry(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+        monkeypatch: pytest.MonkeyPatch,
+    ) -> None:
+        """
+        GIVEN:
+            - A plain latin document and a non-CJK query with fuzzy on,
+              which now selects the registry carrying the internal bigram
+              fields even though nothing in the query is CJK
+        WHEN:
+            - The same query runs with that selection forced back to the
+              public registry
+        THEN:
+            - The same documents come back. The bigram fields are absent
+              from PUBLIC_FIELDS and the two registries agree on every
+              field a query can name, so adding them changes nothing for
+              a query that never reaches them
+        """
+        from documents.search import _query
+        from documents.search._registry import get_field_registry
+
+        index_document(title="A", content="invoice total")
+        index_document(title="B", content="unrelated")
+
+        with_bigram_fields = matched_ids("invoce")
+
+        monkeypatch.setattr(
+            _query,
+            "_get_emit_field_registry",
+            lambda language: get_field_registry(language),
+        )
+
+        assert matched_ids("invoce") == with_bigram_fields
+
+
+class TestQuotingDoesNotWiden:
+    @pytest.fixture
+    def pks(self, index_document: Callable[..., Document]) -> dict[str, int]:
+        """One document per word, and one holding both."""
+        return {
+            "tax": index_document(title="A", content="tax invoice").pk,
+            "report": index_document(title="B", content="report invoice").pk,
+            "both": index_document(title="C", content="tax report").pk,
+        }
+
+    @pytest.mark.parametrize(
+        "query",
+        [
+            pytest.param("tax report", id="unquoted"),
+            pytest.param('"tax report"', id="quoted"),
+        ],
+    )
+    def test_a_quoted_phrase_needs_every_word_like_the_bare_words(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        pks: dict[str, int],
+        query: str,
+    ) -> None:
+        """
+        GIVEN:
+            - A document per word, and one holding both
+        WHEN:
+            - The words are searched unquoted and as a quoted phrase
+        THEN:
+            - Only the document holding both matches, either way. With
+              the phrase's words Or'd on the fuzzy side, the quoted form
+              matched every document, including near-misses of one word
+        """
+        assert matched_ids(query) == {pks["both"]}
+
+
+class TestCjkIsNotFuzzed:
+    def test_a_cjk_term_does_not_match_a_run_sharing_its_start(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document holding 東京都 and one holding only 京都, each
+              inside a longer unspaced run
+        WHEN:
+            - "東京" is searched with fuzzy on
+        THEN:
+            - Only the 東京都 document matches. A prefix Fuzzy over the
+              whole run would match the 京都 document too, undoing what
+              the bigram fields' multitoken=AND guarantees with fuzzy off
+        """
+        tokyo = index_document(title="A", content="東京都の報告書")
+        index_document(title="B", content="京都の観光案内について")
+
+        assert matched_ids("東京") == {tokyo.pk}

src/documents/tests/search/test_highlight_query_guard.py

@@ -0,0 +1,238 @@
+"""Regression coverage for the unguarded TEXT-mode highlight query.
+
+parse_simple_text_highlight_query re-parses simple-search tokens through
+Tantivy's query-string parser to build a SnippetGenerator-compatible query.
+Simple-search tokens keep arbitrary punctuation (quotes, colons, brackets,
+slashes), so any token carrying Tantivy query grammar raised an unguarded
+ValueError. The search itself had already succeeded by the time this ran:
+only the highlight step failed, and with the DocumentViewSet.list
+exception handler narrowed elsewhere on this branch, that ValueError now
+reaches the client as a bare 500 rather than a 400.
+
+Covers three angles:
+  - the query builder itself: quoting each token as its own escaped phrase
+    should let it parse instead of raising, for every failure mode a plain-
+    text query can trigger (syntax error, unknown field, unsupported regex).
+  - highlight_hits: even when a token still can't be expressed as a
+    highlight query, the guard must fall back to a query that still
+    produces usable highlight HTML, not silently empty ones.
+  - the real API endpoint: pinning the previously-500 status to 200.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+import tantivy
+from rest_framework import status
+
+from documents.search._backend import SearchMode
+from documents.search._query import parse_simple_text_highlight_query
+from paperless_testing.factories import DocumentFactory
+
+if TYPE_CHECKING:
+    from rest_framework.test import APIClient
+
+    from documents.search._backend import TantivyBackend
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+# Each spelling below trips a different Tantivy parser failure mode:
+#   'a"b'     -> Syntax Error (unterminated quote)
+#   foo:bar   -> unknown field
+#   (a        -> Syntax Error (unbalanced group)
+#   [a        -> Syntax Error (unbalanced range)
+#   /a/       -> Unsupported query (regex queries disallowed)
+_MALFORMED_QUERIES = [
+    pytest.param('a"b', id="unterminated_quote"),
+    pytest.param("foo:bar", id="unknown_field"),
+    pytest.param("(a", id="unbalanced_group"),
+    pytest.param("[a", id="unbalanced_range"),
+    pytest.param("/a/", id="unsupported_regex"),
+]
+
+
+class TestParseSimpleTextHighlightQueryDoesNotRaise:
+    """The query builder itself must tolerate Tantivy syntax in its tokens."""
+
+    @pytest.mark.parametrize("raw_query", _MALFORMED_QUERIES)
+    def test_malformed_token_does_not_raise(
+        self,
+        query_index: tantivy.Index,
+        raw_query: str,
+    ) -> None:
+        """
+        GIVEN:
+            - A simple-search query token carrying Tantivy query grammar
+              (unterminated quote, unknown field, unbalanced group/range,
+              or unsupported regex)
+        WHEN:
+            - parse_simple_text_highlight_query builds a highlight query
+              from it
+        THEN:
+            - It returns a tantivy.Query instead of raising, since each
+              token is quoted as its own escaped phrase rather than fed
+              to the parser raw
+        """
+        assert isinstance(
+            parse_simple_text_highlight_query(query_index, raw_query),
+            tantivy.Query,
+        )
+
+
+class TestHighlightHitsProducesUsableHighlights:
+    """highlight_hits must keep producing real <b>-wrapped snippet HTML for
+    these queries, not merely avoid raising."""
+
+    @pytest.mark.parametrize(
+        "raw_query",
+        [*_MALFORMED_QUERIES, pytest.param("plain text", id="plain_text_sanity")],
+    )
+    def test_highlight_still_contains_matched_text(
+        self,
+        backend: TantivyBackend,
+        raw_query: str,
+    ) -> None:
+        """
+        GIVEN:
+            - A document whose content contains the raw query text
+              verbatim
+        WHEN:
+            - backend.highlight_hits builds highlights for a TEXT-mode
+              search using that same (possibly Tantivy-grammar-carrying)
+              query text
+        THEN:
+            - The hit still carries a content highlight with real
+              <b>-wrapped matched-term markup, not an empty fallback
+        """
+        doc = DocumentFactory.create(
+            title="probe",
+            content=f"needle content containing {raw_query} literally here",
+        )
+        backend.add_or_update(doc)
+
+        hits = backend.highlight_hits(
+            raw_query,
+            [doc.pk],
+            search_mode=SearchMode.TEXT,
+        )
+
+        assert len(hits) == 1
+        highlights = hits[0]["highlights"]
+        assert "content" in highlights, (
+            f"Expected a content highlight for {raw_query!r}, got: {highlights!r}"
+        )
+        assert "<b>" in highlights["content"], (
+            f"Highlight for {raw_query!r} carries no matched-term markup: "
+            f"{highlights['content']!r}"
+        )
+
+
+class TestHighlightGuardDiscriminatesOnValueError:
+    """The guard added to highlight_hits must catch exactly ValueError, the
+    same shape as the sibling notes_text guard, and let anything else
+    through -- so a real library defect is never mistaken for a harmless
+    syntax error."""
+
+    def test_non_value_error_is_not_swallowed(
+        self,
+        backend: TantivyBackend,
+        monkeypatch: pytest.MonkeyPatch,
+    ) -> None:
+        """
+        GIVEN:
+            - parse_simple_text_highlight_query patched to raise
+              RuntimeError instead of a syntax-related ValueError
+        WHEN:
+            - backend.highlight_hits is called
+        THEN:
+            - The RuntimeError propagates unguarded; the highlight guard
+              must catch exactly ValueError, the same shape as the
+              sibling notes_text guard, never mistaking a real library
+              defect for a harmless syntax error
+        """
+        import documents.search._backend as backend_mod
+
+        def raise_runtime_error(*args: object, **kwargs: object) -> object:
+            raise RuntimeError("synthetic bug, unrelated to query syntax")
+
+        monkeypatch.setattr(
+            backend_mod,
+            "parse_simple_text_highlight_query",
+            raise_runtime_error,
+        )
+
+        doc = DocumentFactory.create(title="probe", content="anything here")
+        backend.add_or_update(doc)
+
+        with pytest.raises(RuntimeError):
+            backend.highlight_hits(
+                "anything",
+                [doc.pk],
+                search_mode=SearchMode.TEXT,
+            )
+
+
+@pytest.mark.usefixtures("_search_index")
+class TestApiNoLongerReturns500:
+    """Pins the actual regression: a matching TEXT-mode search whose query
+    string carries Tantivy syntax must return results, not a server error."""
+
+    @pytest.mark.parametrize("raw_query", _MALFORMED_QUERIES)
+    def test_malformed_text_query_returns_200(
+        self,
+        admin_client: APIClient,
+        raw_query: str,
+    ) -> None:
+        """
+        GIVEN:
+            - A matching document whose content contains the raw query
+              text, indexed via the real search index fixture
+        WHEN:
+            - A TEXT-mode search is issued through the real API with a
+              query string carrying Tantivy syntax
+        THEN:
+            - The response is 200 with the expected result count, not a
+              500 (the regression this file exists to pin)
+        """
+        from documents.search import get_backend
+
+        doc = DocumentFactory.create(
+            title="probe",
+            content=f"needle content containing {raw_query} literally here",
+        )
+        get_backend().add_or_update(doc)
+
+        response = admin_client.get(f"/api/documents/?text={raw_query}")
+
+        assert response.status_code == status.HTTP_200_OK
+        assert response.data["count"] == 1
+
+    def test_plain_text_query_still_returns_200(
+        self,
+        admin_client: APIClient,
+    ) -> None:
+        """
+        GIVEN:
+            - A matching document indexed via the real search index
+              fixture
+        WHEN:
+            - An ordinary TEXT-mode search (no Tantivy syntax) is issued
+        THEN:
+            - The response is 200 with the expected result count; sanity
+              check that the guard does not mask a total failure of the
+              ordinary highlight path
+        """
+        from documents.search import get_backend
+
+        doc = DocumentFactory.create(
+            title="probe",
+            content="needle content containing plain text literally here",
+        )
+        get_backend().add_or_update(doc)
+
+        response = admin_client.get("/api/documents/?text=plain text")
+
+        assert response.status_code == status.HTTP_200_OK
+        assert response.data["count"] == 1

src/documents/tests/search/test_json_field_prefixes.py

@@ -0,0 +1,180 @@
+"""Bare notes:/custom_fields: prefix resolution.
+
+"notes:foo"/"custom_fields:foo" were valid fielded searches before the
+whoosh-compat migration. The registry only exposes them as JSON subpaths, so
+each JSON FieldSpec declares a default subpath (SubpathSpec(default=True)):
+notes: resolves to notes.note:, custom_fields: resolves to
+custom_fields.value:. This replaced an earlier regex-based rewrite
+(_rewrite_bare_json_field_prefixes) that ran on the raw query string before
+parsing and was blind to quoting, so a phrase like
+content:"payment notes: none" was silently corrupted into a notes-field
+search and matched nothing. Resolving the default subpath inside the parser
+instead means quoting is already understood by the time it happens.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from documents.models import CustomField
+from documents.models import CustomFieldInstance
+from documents.models import Note
+from paperless_testing.factories import DocumentFactory
+from paperless_testing.factories import UserFactory
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from documents.models import Document
+    from documents.search._backend import TantivyBackend
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+
+class TestBareJsonFieldPrefixes:
+    def test_bare_notes_prefix_searches_note_text(
+        self,
+        backend: TantivyBackend,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document with a note whose text contains a word, and a
+              decoy document whose content (not notes) contains the same
+              word
+        WHEN:
+            - A bare "notes:" prefix query is run (notes declares "note"
+              as its default subpath)
+        THEN:
+            - Only the document whose note matches is returned; the
+              decoy's content match does not resurface through a demoted
+              text search
+        """
+        alice = UserFactory(username="alice")
+        with_note = DocumentFactory(title="Has note", content="x")
+        Note.objects.create(document=with_note, user=alice, note="crocodile")
+        backend.add_or_update(with_note)
+        # This document's CONTENT contains the words a demoted text search
+        # would match; it must NOT match once the prefix addresses notes.
+        index_document(
+            title="Notes about things",
+            content="notes crocodile mention",
+        )
+        assert matched_ids("notes:crocodile") == {with_note.pk}
+
+    def test_bare_custom_fields_prefix_searches_values(
+        self,
+        backend: TantivyBackend,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document with a custom field instance whose value
+              contains a word, and a decoy document whose content (not a
+              custom field value) contains the same word
+        WHEN:
+            - A bare "custom_fields:" prefix query is run (custom_fields
+              declares "value" as its default subpath)
+        THEN:
+            - Only the document whose custom field value matches is
+              returned
+        """
+        field = CustomField.objects.create(
+            name="Policy Number",
+            data_type=CustomField.FieldDataType.STRING,
+        )
+        with_value = DocumentFactory(title="Has field", content="x")
+        CustomFieldInstance.objects.create(
+            document=with_value,
+            field=field,
+            value_text="crocodile",
+        )
+        backend.add_or_update(with_value)
+        index_document(
+            title="Custom things",
+            content="custom fields crocodile",
+        )
+        assert matched_ids("custom_fields:crocodile") == {with_value.pk}
+
+    def test_subpath_spellings_are_untouched(
+        self,
+        backend: TantivyBackend,
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document with a note carrying both an author and note text
+        WHEN:
+            - The explicit subpath spellings "notes.user:" and
+              "notes.note:" are queried
+        THEN:
+            - Both resolve to their intended subpath and match the
+              document; the default-subpath resolution for the bare
+              prefix does not interfere with explicit subpath addressing
+        """
+        bob = UserFactory(username="bob")
+        doc = DocumentFactory(title="Bob note", content="x")
+        Note.objects.create(document=doc, user=bob, note="remark")
+        backend.add_or_update(doc)
+        assert matched_ids("notes.user:bob") == {doc.pk}
+        assert matched_ids("notes.note:remark") == {doc.pk}
+
+
+class TestQuotedPhraseContainingNotesColonIsNotCorrupted:
+    """The regex rewrite this migration removes was blind to quoting: it
+    matched "notes:" anywhere in the raw query string, including inside an
+    already-quoted phrase on an unrelated field, silently turning
+    content:"payment notes: none" into a notes-field search that matched
+    nothing. Resolving the default subpath during parsing (which is
+    quote-aware) fixes this."""
+
+    def test_quoted_phrase_with_notes_colon_matches_by_content(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document whose content literally contains the text
+              "payment notes: none" inside a quoted phrase
+        WHEN:
+            - A query quoting that exact phrase against the content
+              field is run
+        THEN:
+            - It matches by content, rather than the "notes:" substring
+              inside the quotes being corrupted into a notes-field search
+              that matches nothing (the bug the deleted regex rewrite
+              caused, since it was blind to quoting)
+        """
+        target = index_document(
+            title="Statement",
+            content="payment notes: none",
+        )
+        assert matched_ids('content:"payment notes: none"') == {target.pk}
+
+    def test_quoted_phrase_matches_the_same_document_unquoted(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document whose content contains the same words as the
+              previous test's phrase, but without the colon
+        WHEN:
+            - A query quoting that phrase against the content field is
+              run
+        THEN:
+            - It matches by content, proving the earlier fix is about
+              quote-awareness specifically, not about the words
+              themselves being unsearchable
+        """
+        target = index_document(
+            title="Statement",
+            content="payment notes none",
+        )
+        assert matched_ids('content:"payment notes none"') == {target.pk}

src/documents/tests/search/test_json_subpath_completeness.py

@@ -0,0 +1,95 @@
+"""Every declared JSON subpath must actually be written to the index.
+
+PUBLIC_FIELDS declares each JSON field's subpaths (e.g. ``notes`` ->
+{"user", "note"}), but nothing coupled that declaration to what
+``_backend.py``'s document builder actually writes into the JSON blob at
+index time. A subpath declared but never written would be
+queryable-but-always-empty -- syntactically valid, silently matching
+nothing -- with no test failure anywhere.
+
+This indexes one real document carrying values for every JSON field
+(a Note, a CustomFieldInstance, a DocumentBarcode) and inspects the document's own stored
+JSON payload, rather than running field-specific queries: that way a
+future JSON field's subpaths are covered automatically, without a new
+per-subpath query having to be added by hand each time.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+import tantivy
+from whoosh_compat import FieldKind
+
+from documents.models import CustomField
+from documents.models import CustomFieldInstance
+from documents.models import Document
+from documents.models import Note
+from documents.search._fields import PUBLIC_FIELDS
+from paperless_testing.factories import DocumentBarcodeFactory
+from paperless_testing.factories import UserFactory
+
+if TYPE_CHECKING:
+    from documents.search._backend import TantivyBackend
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+
+class TestJsonSubpathsAreWrittenAtIndexTime:
+    def test_every_declared_json_subpath_appears_in_the_stored_document(
+        self,
+        backend: TantivyBackend,
+    ) -> None:
+        """
+        GIVEN:
+            - A document with a Note, a CustomFieldInstance and a
+              DocumentBarcode attached
+        WHEN:
+            - The document is indexed via TantivyBackend.add_or_update
+        THEN:
+            - Every subpath PUBLIC_FIELDS declares for notes/custom_fields/barcodes
+              is present as a key in the document's stored JSON payload
+        """
+        user = UserFactory(username="completeness-user")
+        field = CustomField.objects.create(
+            name="Completeness Field",
+            data_type=CustomField.FieldDataType.STRING,
+        )
+        doc = Document.objects.create(
+            title="Completeness doc",
+            content="x",
+            checksum="json-subpath-completeness",
+        )
+        Note.objects.create(document=doc, user=user, note="a note")
+        CustomFieldInstance.objects.create(
+            document=doc,
+            field=field,
+            value_text="a value",
+        )
+        DocumentBarcodeFactory(document=doc, value="a barcode")
+        backend.add_or_update(doc)
+
+        index = backend._index
+        searcher = index.searcher()
+        hits = searcher.search(
+            tantivy.Query.term_query(index.schema, "id", doc.pk),
+            limit=1,
+        ).hits
+        assert hits, "the document was not indexed"
+        stored = searcher.doc(hits[0][1]).to_dict()
+
+        json_fields = [f for f in PUBLIC_FIELDS if f.kind is FieldKind.JSON]
+        assert json_fields, "no JSON fields declared - fixture is stale"
+        for field_spec in json_fields:
+            stored_values = stored.get(field_spec.name)
+            assert stored_values, (
+                f"{field_spec.name} was not written to the index at all"
+            )
+            written_keys = stored_values[0].keys()
+            for subpath in field_spec.subpaths:
+                assert subpath in written_keys, (
+                    f"{field_spec.name}.{subpath} is declared in PUBLIC_FIELDS "
+                    "but _backend.py's document builder never writes it - it "
+                    "would be queryable but always empty"
+                )

src/documents/tests/search/test_keyword_pattern_literal.py

@@ -0,0 +1,62 @@
+"""Wildcard patterns on KEYWORD fields must stay literal.
+
+``checksum`` is the only KEYWORD field: it is indexed with the raw tokenizer,
+so its terms are never lowercased, folded or stemmed. Running its wildcard
+patterns through the stemming normalizer rewrote hex prefixes ("ceded" ->
+"cede") and returned documents whose checksum did not start with what the user
+typed, which for an identity field is a wrong answer.
+
+This covers only the registry-level normalizer, which is all that exists to
+prove at this point in the stack: user queries are not yet routed through
+whoosh-compat (that lands with the query-layer PR), so the same fact proven
+end to end against real indexed documents lives in
+``test_checksum_prefix_queries.py``.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from documents.search._registry import get_field_registry
+
+if TYPE_CHECKING:
+    from whoosh_compat import FieldRegistry
+    from whoosh_compat import PatternNormalizer
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+
+def _normalizer(registry: FieldRegistry, name: str) -> PatternNormalizer:
+    ref = registry.make_ref(name)
+    assert ref is not None
+    resolved = registry.resolve(ref)
+    assert resolved is not None
+    assert resolved.spec.pattern_normalizer is not None
+    return resolved.spec.pattern_normalizer
+
+
+class TestKeywordPatternNormalizer:
+    @pytest.mark.parametrize(
+        "run",
+        [
+            pytest.param("ceded", id="stems_to_cede"),
+            pytest.param("added", id="stems_to_ad"),
+            pytest.param("cafed", id="stems_to_cafe"),
+        ],
+    )
+    def test_keyword_runs_are_folded_not_stemmed(self, run: str) -> None:
+        """
+        GIVEN:
+            - The "checksum" field's registered pattern normalizer
+              (KEYWORD kind, "en" registry)
+        WHEN:
+            - A wildcard pattern run is normalized
+        THEN:
+            - The run is returned unchanged, never widened to a stem (which
+              would return checksums that do not start with what the user
+              typed)
+        """
+        normalize = _normalizer(get_field_registry("en"), "checksum")
+        assert normalize(run) == run

src/documents/tests/search/test_lock_backoff.py

@@ -15,7 +15,7 @@ from documents.search._backend import SearchIndexLockError
 from documents.search._backend import TantivyBackend
 from documents.tasks import index_document
 from documents.tasks import remove_document_from_index
-from documents.tests.factories import DocumentFactory
+from paperless_testing.factories import DocumentFactory
 
 if TYPE_CHECKING:
     from collections.abc import Generator

src/documents/tests/search/test_migration_fulltext_query_field_prefixes.py

@@ -1,6 +1,6 @@
 import pytest
 
-from documents.tests.utils import TestMigrations
+from paperless_testing.migrations import TestMigrations
 
 pytestmark = pytest.mark.search
 

src/documents/tests/search/test_negated_leaf_ids.py

@@ -0,0 +1,203 @@
+"""_negated_leaf_ids, the pre-scan that tells the widening hook which
+leaves sit under a negation.
+
+A negated leaf keeps its CJK alternative but gets no fuzzy one: with
+prefix matching, NOT tax would otherwise exclude "taxi" and "taxonomy",
+which is not what excluding a word means. The hook cannot see a leaf's
+context, so the tree is walked once up front and the hook compares by
+identity.
+"""
+
+from __future__ import annotations
+
+import pytest
+import whoosh_compat.ast as wc_ast
+
+from documents.search._query import _negated_leaf_ids
+from documents.tests.search._ast_helpers import CONTENT
+from documents.tests.search._ast_helpers import content
+
+pytestmark = pytest.mark.search
+
+
+class TestCollection:
+    def test_a_bare_tree_has_no_negated_leaves(self) -> None:
+        """
+        GIVEN:
+            - A tree with no negation in it
+        WHEN:
+            - It is pre-scanned
+        THEN:
+            - The set is empty
+        """
+        tree = wc_ast.And(children=(content("invoice"), content("report")))
+
+        assert _negated_leaf_ids(tree) == frozenset()
+
+    def test_a_leaf_under_not_is_collected(self) -> None:
+        """
+        GIVEN:
+            - A term under a Not
+        WHEN:
+            - The tree is pre-scanned
+        THEN:
+            - That leaf's id is collected, and the positive one is not
+        """
+        positive = content("invoice")
+        negated = content("secret")
+        tree = wc_ast.And(children=(positive, wc_ast.Not(child=negated)))
+
+        ids = _negated_leaf_ids(tree)
+
+        assert id(negated) in ids
+        assert id(positive) not in ids
+
+    def test_the_negative_side_of_andnot_is_collected(self) -> None:
+        """
+        GIVEN:
+            - An AndNot, whose positive and negative sides are both terms
+        WHEN:
+            - The tree is pre-scanned
+        THEN:
+            - Only the negative side is collected
+        """
+        positive = content("invoice")
+        negated = content("secret")
+        tree = wc_ast.AndNot(positive=positive, negative=negated)
+
+        ids = _negated_leaf_ids(tree)
+
+        assert id(negated) in ids
+        assert id(positive) not in ids
+
+    def test_every_leaf_beneath_a_negation_is_collected(self) -> None:
+        """
+        GIVEN:
+            - A Not whose child is a group of several leaves, one of them
+              a Phrase
+        WHEN:
+            - The tree is pre-scanned
+        THEN:
+            - All of them are collected, at any depth and whatever the
+              leaf type
+        """
+        a = content("alpha")
+        b = content("beta")
+        phrase = wc_ast.Phrase(field=CONTENT, text="gamma delta")
+        tree = wc_ast.Not(
+            child=wc_ast.Or(
+                children=(a, wc_ast.And(children=(b, phrase))),
+            ),
+        )
+
+        assert _negated_leaf_ids(tree) == frozenset({id(a), id(b), id(phrase)})
+
+    def test_a_double_negation_stays_collected(self) -> None:
+        """
+        GIVEN:
+            - A term under two nested Nots
+        WHEN:
+            - The tree is pre-scanned
+        THEN:
+            - It is still collected. The rule is parity-blind on purpose:
+              the cost of over-collecting is a lost widening, while
+              under-collecting would make a negation exclude far more than
+              the user asked
+        """
+        leaf = content("tax")
+        tree = wc_ast.Not(child=wc_ast.Not(child=leaf))
+
+        assert _negated_leaf_ids(tree) == frozenset({id(leaf)})
+
+    @pytest.mark.parametrize(
+        "build",
+        [
+            pytest.param(
+                lambda leaf: wc_ast.Boosted(child=leaf, boost=2.0),
+                id="boosted",
+            ),
+            pytest.param(
+                lambda leaf: wc_ast.Require(scored=leaf, filter_only=leaf),
+                id="require",
+            ),
+            pytest.param(
+                lambda leaf: wc_ast.AndMaybe(required=leaf, optional=leaf),
+                id="andmaybe",
+            ),
+        ],
+    )
+    def test_containers_that_are_not_negations_collect_nothing(
+        self,
+        build,
+    ) -> None:
+        """
+        GIVEN:
+            - A leaf inside a container that is not a negation
+        WHEN:
+            - The tree is pre-scanned
+        THEN:
+            - Nothing is collected. Only Not.child and AndNot.negative are
+              negative positions
+        """
+        leaf = content("invoice")
+
+        assert _negated_leaf_ids(build(leaf)) == frozenset()
+
+    def test_a_bare_leaf_is_accepted(self) -> None:
+        """
+        GIVEN:
+            - A tree that is a single leaf, with no container at all
+        WHEN:
+            - It is pre-scanned
+        THEN:
+            - The set is empty and nothing raises
+        """
+        assert _negated_leaf_ids(content("invoice")) == frozenset()
+
+
+class TestTotality:
+    @pytest.mark.parametrize(
+        "node",
+        [
+            pytest.param(wc_ast.Every(), id="every"),
+            pytest.param(wc_ast.Nothing(), id="nothing"),
+            pytest.param(
+                wc_ast.Wildcard(field=CONTENT, pattern="inv*"),
+                id="wildcard",
+            ),
+            pytest.param(
+                wc_ast.Fuzzy(field=CONTENT, text="invoce", distance=1, prefix=True),
+                id="fuzzy",
+            ),
+        ],
+    )
+    def test_an_unrecognised_node_does_not_raise(self, node: wc_ast.Node) -> None:
+        """
+        GIVEN:
+            - A node type the walk does not collect from, including one
+              (Fuzzy) that this code will later build itself
+        WHEN:
+            - It is pre-scanned, bare and under a Not
+        THEN:
+            - Nothing raises. This function runs outside emit()'s error
+              conversion, so an exception here is an unconverted 500
+        """
+        assert _negated_leaf_ids(node) == frozenset()
+        assert _negated_leaf_ids(wc_ast.Not(child=node)) == frozenset()
+
+    def test_a_deep_tree_does_not_exhaust_the_stack(self) -> None:
+        """
+        GIVEN:
+            - A tree nested far deeper than the parser's 200-group cap
+        WHEN:
+            - It is pre-scanned
+        THEN:
+            - It completes. The walk is iterative, so depth costs heap
+              rather than Python stack frames
+        """
+        leaf = content("invoice")
+        node: wc_ast.Node = leaf
+        for _ in range(5000):
+            node = wc_ast.Not(child=node)
+
+        assert _negated_leaf_ids(node) == frozenset({id(leaf)})

src/documents/tests/search/test_pattern_normalizer.py

@@ -0,0 +1,156 @@
+"""The pattern normalizer's stem-alternates contract, and its consistency
+with the index-side analyzer.
+
+Query patterns are normalized but were not stemmed, while index terms are
+stemmed, so the natural spelling of a prefix search matched nothing:
+``invoice*`` found no document although ``invoic*`` did. v2's index was
+UNSTEMMED (whoosh ``TEXT()`` defaults to ``StandardAnalyzer``), so this
+regressed against both baselines.
+
+These are pure unit tests against ``_make_pattern_normalizer`` and
+``stem_pattern_text`` directly, no query routing involved. The end-to-end
+proof that a real wildcard query actually reaches a stemmed index term
+lives in ``test_pattern_stemming.py``.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from documents.search._registry import _make_pattern_normalizer
+from documents.search._tokenizer import ascii_fold
+from documents.search._tokenizer import paperless_text_analyzer
+from documents.search._tokenizer import stem_pattern_text
+
+if TYPE_CHECKING:
+    from whoosh_compat import PatternNormalizer
+
+
+class TestStemsMatchTheIndexAnalyzer:
+    """stem_pattern_text rebuilds paperless_text_analyzer's stemming tail rather
+    than sharing it, so a filter added to the index analyzer alone would silently
+    stop patterns from reaching the terms it produces.
+    """
+
+    @pytest.mark.parametrize(
+        "language",
+        ["en", "de", "fr", "es", "sv", None, "klingon"],
+    )
+    @pytest.mark.parametrize(
+        "word",
+        ["Copies", "copyright", "Companies", "Invoices", "laufen", "casas", "Straße"],
+    )
+    def test_stem_equals_the_index_term(self, word: str, language: str | None) -> None:
+        """
+        GIVEN:
+            - A word, across several representative index languages
+              ("en", "de", "fr", "es", "sv"), no language, and an
+              unsupported language ("klingon")
+        WHEN:
+            - `stem_pattern_text` (the pattern-side stemmer) processes the
+              folded word, and `paperless_text_analyzer` (the index-side
+              analyzer) independently processes the same word
+        THEN:
+            - The two produce the identical term. `stem_pattern_text`
+              rebuilds `paperless_text_analyzer`'s stemming tail rather
+              than sharing it, so a filter added to the index analyzer
+              alone would silently stop patterns from reaching the terms
+              it produces; this pins the two staying in sync
+        """
+        indexed = paperless_text_analyzer(language).analyze(word)[0]
+        assert stem_pattern_text(ascii_fold(word.lower()), language) == indexed
+
+
+def _forms(normalize: PatternNormalizer, text: str) -> tuple[str, ...]:
+    """The distinct forms a term may match, in order, the way the emitter reads
+    the normalizer's answer (see whoosh_compat.PatternNormalizer)."""
+    result = normalize(text)
+    if isinstance(result, str):
+        return (result,)
+    return tuple(dict.fromkeys(result))
+
+
+class TestPatternNormalizer:
+    @pytest.mark.parametrize(
+        ("text", "expected"),
+        [
+            ("Invoice", ("invoice", "invoic")),
+            ("companies", ("companies", "compani")),
+            # y -> i is a substitution, so both forms are needed: the index
+            # holds "librari" for "library" and "library" for "librarian".
+            ("library", ("library", "librari")),
+            # A run the stemmer leaves alone collapses back to one form, so it
+            # costs exactly the one regex branch it did before.
+            ("invoic", ("invoic",)),
+            ("Universit", ("universit",)),
+            ("Café", ("cafe",)),
+        ],
+    )
+    def test_offers_the_typed_run_and_its_stem(
+        self,
+        text: str,
+        expected: tuple[str, ...],
+    ) -> None:
+        """
+        GIVEN:
+            - The "en" pattern normalizer
+        WHEN:
+            - It processes a literal run (e.g. "Invoice", "library",
+              "Café")
+        THEN:
+            - It returns the folded run and, where it differs, the
+              stemmed form, as distinct alternatives; a run the stemmer
+              leaves alone (e.g. "invoic") collapses back to the single
+              folded form. "library" needs both forms since y -> i is a
+              substitution: the index holds "librari" for "library" and
+              "library" for "librarian"
+        """
+        assert _forms(_make_pattern_normalizer("en"), text) == expected
+
+    def test_run_that_yields_no_token_falls_back_to_the_typed_run(self) -> None:
+        """
+        GIVEN:
+            - The "en" pattern normalizer
+        WHEN:
+            - It processes a run past the analyzer's remove_long limit
+        THEN:
+            - The run analyzes to zero tokens, so there is no stem to
+              offer, and only the folded run remains
+        """
+        over_long = "invoices" * 20
+        assert _forms(_make_pattern_normalizer("en"), over_long) == (over_long,)
+
+    @pytest.mark.parametrize("language", [None, "klingon"])
+    def test_unstemmed_language_folds_only(self, language: str | None) -> None:
+        """
+        GIVEN:
+            - A pattern normalizer with no language configured, or one
+              this build has no stemmer for ("klingon")
+        WHEN:
+            - It processes "Invoices"
+        THEN:
+            - Only the folded form ("invoices") is offered, since with no
+              stemmer configured the index holds surface forms and the
+              pattern must keep them too
+        """
+        assert _forms(_make_pattern_normalizer(language), "Invoices") == ("invoices",)
+
+    @pytest.mark.parametrize("char", ["a", "Z", "é"])
+    def test_a_single_character_collapses_to_one_folded_form(self, char: str) -> None:
+        """
+        GIVEN:
+            - The "en" pattern normalizer
+        WHEN:
+            - It processes a single character
+        THEN:
+            - Exactly one, one-character form is returned. A bracket
+              class body is normalized one character at a time and the
+              answer is used only when it is a single one-character
+              form, so a stemmer that changed a lone character would
+              silently disable folding inside classes
+        """
+        forms = _forms(_make_pattern_normalizer("en"), char)
+        assert len(forms) == 1
+        assert len(forms[0]) == 1

src/documents/tests/search/test_pattern_stemming.py

@@ -0,0 +1,213 @@
+"""Wildcard patterns must match a stemmed index, end to end.
+
+Query patterns are normalized but were not stemmed, while index terms are
+stemmed, so the natural spelling of a prefix search matched nothing:
+``invoice*`` found no document although ``invoic*`` did. v2's index was
+UNSTEMMED (whoosh ``TEXT()`` defaults to ``StandardAnalyzer``), so this
+regressed against both baselines.
+
+These are end-to-end tests against a real indexed document and a real
+query, proving the pattern normalizer's stem-alternates contract actually
+reaches a stemmed index term. The pure unit tests against the normalizer
+function itself live in ``test_pattern_normalizer.py``.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from documents.models import Document
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+CONTENT = (
+    "invoice total due for electricity from both companies, "
+    "payments made to the university library, copies attached"
+)
+
+
+@pytest.fixture
+def indexed_doc(index_document: Callable[..., Document]) -> Document:
+    return index_document(
+        title="Invoice 2020 productname",
+        content=CONTENT,
+        archive_serial_number=900,
+    )
+
+
+class TestPrefixStemming:
+    @pytest.mark.parametrize(
+        "query",
+        [
+            "invoice*",
+            "electricity*",
+            "companies*",
+            "payments*",
+            "library*",
+            "title:Invoice*",
+        ],
+    )
+    def test_full_word_prefix_matches_its_stem(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        indexed_doc: Document,
+        query: str,
+    ) -> None:
+        """
+        GIVEN:
+            - A document indexed with content containing "invoice",
+              "electricity", "companies", "payments", "library" and title
+              "Invoice 2020 productname"
+        WHEN:
+            - A prefix wildcard on the full, unstemmed word is queried
+              (e.g. "invoice*", "title:Invoice*")
+        THEN:
+            - The document matches, since the pattern normalizer offers
+              the word's stem as an alternative alongside the typed run,
+              reaching the stemmed index term
+        """
+        assert matched_ids(query) == {indexed_doc.id}
+
+    @pytest.mark.parametrize("query", ["invoic*", "electr*", "payment*"])
+    def test_already_stemmed_prefix_still_matches(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        indexed_doc: Document,
+        query: str,
+    ) -> None:
+        """
+        GIVEN:
+            - The same indexed document
+        WHEN:
+            - A prefix wildcard is typed already in its stemmed spelling
+              (e.g. "invoic*")
+        THEN:
+            - The document still matches, since the typed-run alternative
+              is itself a prefix of the stored stemmed term
+        """
+        assert matched_ids(query) == {indexed_doc.id}
+
+    @pytest.mark.parametrize("query", ["univers*", "librar*"])
+    def test_partial_prefix_reaches_the_stemmed_term(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        indexed_doc: Document,
+        query: str,
+    ) -> None:
+        """
+        GIVEN:
+            - The same indexed document
+        WHEN:
+            - A prefix shorter than a whole word is queried ("univers*",
+              "librar*")
+        THEN:
+            - It still matches, and neither case needs the two-alternative
+              path to do it: measured under "en", the stemmer leaves
+              "librar" alone, so it has one form, and that form is a
+              prefix of the "librari" the index holds for "library";
+              "univers" stems to the *shorter* "univ", and the run as
+              typed and its stem are both prefixes of the "univers" the
+              index holds for "university". The case where the two forms
+              genuinely diverge, and only one of them matches, is
+              test_stem_substitution_reaches_both_the_inflection_and_the_compound
+        """
+        assert matched_ids(query) == {indexed_doc.id}
+
+    def test_full_word_reaches_the_stem_but_a_fragment_of_it_does_not(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        indexed_doc: Document,
+    ) -> None:
+        """
+        GIVEN:
+            - The same indexed document, storing "university" as "univers"
+        WHEN:
+            - "universities*", "universit*" and "univers*" are each queried
+        THEN:
+            - "universities*" matches, since the stem of "universities" is
+              that same "univers"; "universit*" matches nothing, since
+              "universit" is a prefix of neither its own stem nor the
+              stored term. The alternatives widen recall without turning
+              a wildcard into a prefix search over the original text.
+              usage.md tells a reader whose `universit*` finds nothing to
+              shorten it to `univers*`, which matches
+        """
+        assert matched_ids("universities*") == {indexed_doc.id}
+        assert matched_ids("universit*") == set()
+        assert matched_ids("univers*") == {indexed_doc.id}
+
+    def test_pattern_past_the_stem_boundary_is_documented_not_fixed(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        indexed_doc: Document,
+    ) -> None:
+        """
+        GIVEN:
+            - The same indexed document, with "productname" indexed as
+              "productnam"
+        WHEN:
+            - "produ*name" (a pattern straddling the stem boundary) is
+              queried
+        THEN:
+            - It matches nothing; produ*name cannot match a stemmed
+              index, and usage.md must not advertise it. Pinned so the
+              limitation is deliberate, not accidental
+        """
+        assert matched_ids("produ*name") == set()
+
+    def test_stem_substitution_reaches_both_the_inflection_and_the_compound(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+        indexed_doc: Document,
+    ) -> None:
+        """
+        GIVEN:
+            - The indexed document (containing "copies") plus a second
+              document titled "Copyright notice" with content "copyright
+              notice for the work"
+        WHEN:
+            - "copy*" and "copyright*" are each queried
+        THEN:
+            - "copy*" matches both documents, and "copyright*" matches
+              only the compound one. English stemming substitutes as well
+              as truncates: "copy" and "copies" both index as "copi",
+              while "copyright" keeps its literal "y". Neither form is a
+              prefix of the other, so no single normalized string reaches
+              both; the run is therefore emitted as a disjunction of the
+              folded and stemmed forms, and "copy*" reaches the base
+              word, its inflections and the compound alike
+        """
+        compound = index_document(
+            title="Copyright notice",
+            content="copyright notice for the work",
+            archive_serial_number=901,
+        )
+
+        assert matched_ids("copy*") == {indexed_doc.id, compound.id}
+        assert matched_ids("copyright*") == {compound.id}
+
+
+class TestBracketClassStillFolds:
+    def test_class_body_matches_case_insensitively(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        indexed_doc: Document,
+    ) -> None:
+        """
+        GIVEN:
+            - The indexed document, titled "Invoice 2020 productname"
+        WHEN:
+            - A bracket-class pattern mixing case is queried
+              ("title:[IP]nvoice*")
+        THEN:
+            - It matches: the class body is folded per character, which
+              the alternatives contract preserves only because a lone
+              character stems to itself
+        """
+        assert matched_ids("title:[IP]nvoice*") == {indexed_doc.id}

src/documents/tests/search/test_permission_field_isolation.py

@@ -0,0 +1,199 @@
+"""Permission filtering must hold against the real indexed document shape.
+
+Only three of the index's unsigned ``*_id`` columns are load-bearing:
+``owner_id``, ``viewer_id`` and ``viewer_group_id``, all read by
+build_permission_filter. The rest (correspondent/document_type/storage_path/tag
+ids) were written on every document and read by nothing, and were dropped.
+
+These tests index real Documents through the backend's own document builder and
+assert result-level visibility per user, so a mistake about which columns are
+load-bearing shows up as documents leaking across users rather than as a passing
+unit test over a hand-built index.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+from django.contrib.auth.models import Group
+from django.contrib.auth.models import User
+
+from documents.models import Correspondent
+from documents.models import Document
+from documents.models import DocumentType
+from documents.models import StoragePath
+from documents.models import Tag
+from paperless_testing.factories import UserFactory
+from paperless_testing.permissions import grant_object
+
+if TYPE_CHECKING:
+    from documents.search._backend import TantivyBackend
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+
+@pytest.fixture
+def owner() -> User:
+    return UserFactory(username="owner")
+
+
+@pytest.fixture
+def stranger() -> User:
+    return UserFactory(username="stranger")
+
+
+@pytest.fixture
+def viewer() -> User:
+    return UserFactory(username="viewer")
+
+
+@pytest.fixture
+def group_member() -> User:
+    user = UserFactory(username="group_member")
+    user.groups.add(Group.objects.create(name="accounting"))
+    return user
+
+
+class TestPermissionFilteringOnIndexedDocuments:
+    def test_unowned_document_is_visible_to_everyone(
+        self,
+        backend: TantivyBackend,
+        stranger: User,
+    ) -> None:
+        """
+        GIVEN:
+            - A document with no owner, indexed via the backend's real
+              document builder
+        WHEN:
+            - A stranger (no relation to the document) searches
+        THEN:
+            - The document is visible to them
+        """
+        doc = Document.objects.create(
+            title="Public Invoice",
+            content="invoice total due",
+            checksum="perm-unowned",
+        )
+        backend.add_or_update(doc)
+
+        assert backend.search_ids("invoice", user=stranger) == [doc.pk]
+
+    def test_owned_document_is_visible_only_to_its_owner(
+        self,
+        backend: TantivyBackend,
+        owner: User,
+        stranger: User,
+    ) -> None:
+        """
+        GIVEN:
+            - A document owned by one user, indexed via the backend's
+              real document builder
+        WHEN:
+            - The owner and an unrelated stranger each search
+        THEN:
+            - The owner sees the document; the stranger does not
+        """
+        doc = Document.objects.create(
+            title="Private Invoice",
+            content="invoice total due",
+            checksum="perm-owned",
+            owner=owner,
+        )
+        backend.add_or_update(doc)
+
+        assert backend.search_ids("invoice", user=owner) == [doc.pk]
+        assert backend.search_ids("invoice", user=stranger) == []
+
+    def test_explicitly_shared_document_is_visible_to_the_viewer(
+        self,
+        backend: TantivyBackend,
+        owner: User,
+        viewer: User,
+        stranger: User,
+    ) -> None:
+        """
+        GIVEN:
+            - A document owned by one user and explicitly shared with a
+              second user via guardian's view_document permission,
+              indexed via the backend's real document builder
+        WHEN:
+            - The shared viewer and an unrelated stranger each search
+        THEN:
+            - The viewer sees the document; the stranger does not
+        """
+        doc = Document.objects.create(
+            title="Shared Invoice",
+            content="invoice total due",
+            checksum="perm-shared-user",
+            owner=owner,
+        )
+        grant_object(viewer, doc, "view_document")
+        backend.add_or_update(doc)
+
+        assert backend.search_ids("invoice", user=viewer) == [doc.pk]
+        assert backend.search_ids("invoice", user=stranger) == []
+
+    def test_group_shared_document_is_visible_to_group_members(
+        self,
+        backend: TantivyBackend,
+        owner: User,
+        group_member: User,
+        stranger: User,
+    ) -> None:
+        """
+        GIVEN:
+            - A document owned by one user and shared with a group via
+              guardian's view_document permission, indexed via the
+              backend's real document builder
+        WHEN:
+            - A member of that group and an unrelated stranger each
+              search
+        THEN:
+            - The group member sees the document; the stranger does not
+        """
+        doc = Document.objects.create(
+            title="Group Invoice",
+            content="invoice total due",
+            checksum="perm-shared-group",
+            owner=owner,
+        )
+        grant_object(group_member.groups.first(), doc, "view_document")
+        backend.add_or_update(doc)
+
+        assert backend.search_ids("invoice", user=group_member) == [doc.pk]
+        assert backend.search_ids("invoice", user=stranger) == []
+
+    def test_metadata_does_not_widen_visibility(
+        self,
+        backend: TantivyBackend,
+        owner: User,
+        stranger: User,
+    ) -> None:
+        """
+        GIVEN:
+            - A document owned by one user and carrying
+              correspondent/document_type/storage_path/tag metadata,
+              indexed via the backend's real document builder
+        WHEN:
+            - The owner and an unrelated stranger each search
+        THEN:
+            - The owner sees the document; the stranger does not, since
+              the dropped, non-load-bearing metadata *_id columns must
+              not widen visibility beyond the owner_id/viewer_id/
+              viewer_group_id filter
+        """
+        doc = Document.objects.create(
+            title="Tagged Invoice",
+            content="invoice total due",
+            checksum="perm-metadata",
+            owner=owner,
+            correspondent=Correspondent.objects.create(name="ACME"),
+            document_type=DocumentType.objects.create(name="Bill"),
+            storage_path=StoragePath.objects.create(name="Archive", path="archive/"),
+        )
+        doc.tags.add(Tag.objects.create(name="paid"))
+        backend.add_or_update(doc)
+
+        assert backend.search_ids("invoice", user=owner) == [doc.pk]
+        assert backend.search_ids("invoice", user=stranger) == []

src/documents/tests/search/test_query_negation.py

@@ -0,0 +1,154 @@
+"""Negation must survive the blended query.
+
+parse_user_query ORs an exact clause with an optional fuzzy clause. That
+clause is built from positive terms only, so unless the query's exclusions
+are applied to the blend as a whole, a document the exact clause excluded
+is re-admitted by it. CJK terms are widened inside the exact clause itself,
+so the query's own structure constrains them.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from documents.models import Document
+
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+
+class TestNegationConstrainsEveryClause:
+    @pytest.mark.usefixtures("fuzzy_enabled")
+    def test_fuzzy_clause_does_not_readmit_an_excluded_document(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - Two documents both matching a positive term, one of which
+              also contains a word the query excludes, with the fuzzy
+              blend clause enabled
+        WHEN:
+            - A query combining the positive term with a NOT exclusion is
+              run
+        THEN:
+            - Only the document without the excluded word is returned;
+              the fuzzy clause (built from positive terms only) does not
+              readmit the document the exact clause excluded
+        """
+        secret = index_document(
+            title="Invoice A",
+            content="invoice total secret",
+        )
+        public = index_document(
+            title="Invoice B",
+            content="invoice total public",
+        )
+
+        assert matched_ids("invoice") == {secret.pk, public.pk}
+        assert matched_ids("invoice NOT secret") == {public.pk}
+
+    def test_an_excluded_document_stays_out_of_a_cjk_match(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - Two documents both containing a CJK run, one of which also
+              contains a word the query excludes
+        WHEN:
+            - A query combining the CJK term with a NOT exclusion is run
+        THEN:
+            - Only the document without the excluded word is returned;
+              the CJK term's bigram match sits beside the NOT inside the
+              same query, so the exclusion applies to it
+        """
+        secret = index_document(
+            title="Tokyo A",
+            content="東京都の秘密です secret",
+        )
+        public = index_document(
+            title="Tokyo B",
+            content="東京都の報告書です public",
+        )
+
+        assert matched_ids("東京") == {secret.pk, public.pk}
+        assert matched_ids("東京 NOT secret") == {public.pk}
+
+    @pytest.mark.usefixtures("fuzzy_enabled")
+    def test_disjunctive_negation_still_admits_the_other_branch(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document matching a positive term and also containing a
+              word a disjunctive NOT branch excludes, plus an unrelated
+              document
+        WHEN:
+            - A query of the shape "term OR NOT excluded_word" is run
+        THEN:
+            - Both documents are returned; "invoice OR NOT secret"
+              excludes nothing on its own, so a document matching the
+              left branch stays in even though it contains the excluded
+              word
+        """
+        secret_invoice = index_document(
+            title="Invoice A",
+            content="invoice total secret",
+        )
+        unrelated = index_document(
+            title="Recipe",
+            content="flour and water",
+        )
+
+        assert matched_ids("invoice OR NOT secret") == {
+            secret_invoice.pk,
+            unrelated.pk,
+        }
+
+    def test_a_negation_under_or_constrains_its_own_cjk_term(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - Two CJK documents, one of which also contains a word an OR
+              branch's own NOT excludes, plus an unrelated latin document
+        WHEN:
+            - The exclusion is under a disjunctive OR branch, and in
+              conjunctive position
+        THEN:
+            - Either way the excluded document is left out. The CJK
+              term's bigram match is widened in place inside its own OR
+              branch, so that branch's NOT applies to it; the other
+              branch still admits the latin document
+        """
+        secret = index_document(
+            title="Tokyo A",
+            content="東京都の秘密です secret",
+        )
+        public = index_document(
+            title="Tokyo B",
+            content="東京都の報告書です public",
+        )
+        bill = index_document(
+            title="Bill",
+            content="bill payment received",
+        )
+
+        assert matched_ids("(東京 AND NOT secret) OR bill") == {
+            bill.pk,
+            public.pk,
+        }
+        assert matched_ids("東京 AND NOT secret") == {public.pk}
+        assert secret.pk in matched_ids("東京")

src/documents/tests/search/test_registry.py

@@ -0,0 +1,224 @@
+from collections.abc import Sequence
+
+import pytest
+from whoosh_compat import FieldKind
+from whoosh_compat import FieldRegistry
+from whoosh_compat.fields import ResolvedField
+
+from documents.search._fields import PUBLIC_FIELDS
+from documents.search._registry import get_field_registry
+
+
+@pytest.fixture
+def registry() -> FieldRegistry:
+    return get_field_registry(None)
+
+
+def _resolve(registry: FieldRegistry, name: str) -> ResolvedField:
+    ref = registry.make_ref(name)
+    assert ref is not None, f"{name} is not a valid field ref"
+    resolved = registry.resolve(ref)
+    assert resolved is not None, f"{name} did not resolve"
+    return resolved
+
+
+def _distinct_forms(result: str | Sequence[str]) -> tuple[str, ...]:
+    """The forms a term may match, in order, the way whoosh-compat's emitter
+    reads a pattern_normalizer's answer: a bare str is one form, a sequence is
+    several, deduplicated."""
+    if isinstance(result, str):
+        return (result,)
+    return tuple(dict.fromkeys(result))
+
+
+class TestFieldRegistry:
+    def test_no_queryable_field_name_ends_in_id(self) -> None:
+        """
+        GIVEN:
+            - PUBLIC_FIELDS, the canonical query-syntax field table
+        WHEN:
+            - Every declared field name is inspected
+        THEN:
+            - None of them end in "_id" (internal id columns, written for
+              permission filtering and joins, must never reach the query
+              surface; checked against PUBLIC_FIELDS rather than the
+              registry so a leak is caught where it is declared)
+        """
+        leaked = [f.name for f in PUBLIC_FIELDS if f.name.endswith("_id")]
+        assert not leaked, f"internal id fields reached the query surface: {leaked}"
+
+    def test_type_alias_resolves_to_document_type(
+        self,
+        registry: FieldRegistry,
+    ) -> None:
+        """
+        GIVEN:
+            - The field registry
+        WHEN:
+            - The alias "type" is resolved
+        THEN:
+            - It resolves to the canonical "document_type" field
+        """
+        assert _resolve(registry, "type").spec.name == "document_type"
+
+    def test_path_alias_resolves_to_storage_path(self, registry: FieldRegistry) -> None:
+        """
+        GIVEN:
+            - The field registry
+        WHEN:
+            - The alias "path" is resolved
+        THEN:
+            - It resolves to the canonical "storage_path" field
+        """
+        assert _resolve(registry, "path").spec.name == "storage_path"
+
+    def test_notes_json_subpaths_resolve(self, registry: FieldRegistry) -> None:
+        """
+        GIVEN:
+            - The field registry
+        WHEN:
+            - "notes.user" is resolved
+        THEN:
+            - It resolves to the "notes" field with json_path "user"
+        """
+        resolved = _resolve(registry, "notes.user")
+        assert resolved.spec.name == "notes"
+        assert resolved.json_path == "user"
+        assert resolved.is_subpath is True
+
+    def test_custom_fields_json_subpaths_resolve(self, registry: FieldRegistry) -> None:
+        """
+        GIVEN:
+            - The field registry
+        WHEN:
+            - "custom_fields.name" and "custom_fields.value" are resolved
+        THEN:
+            - Both resolve without error
+        """
+        for raw in ("custom_fields.name", "custom_fields.value"):
+            _resolve(registry, raw)
+
+    def test_tag_is_comma_values(self, registry: FieldRegistry) -> None:
+        """
+        GIVEN:
+            - The field registry
+        WHEN:
+            - The "tag" field is resolved
+        THEN:
+            - It is marked comma_values=True
+        """
+        assert _resolve(registry, "tag").spec.comma_values is True
+
+    def test_correspondent_is_not_comma_values(self, registry: FieldRegistry) -> None:
+        """
+        GIVEN:
+            - The field registry
+        WHEN:
+            - The "correspondent" field is resolved
+        THEN:
+            - It is not marked comma_values ("tag" is the only field that
+              opts in; end to end the two readings of
+              "correspondent:foo,bar" agree anyway, since the analyzer
+              splits the literal value on the comma regardless, so this is
+              only observable at the registry level)
+        """
+        assert _resolve(registry, "correspondent").spec.comma_values is False
+
+    def test_created_is_date_kind(self, registry: FieldRegistry) -> None:
+        """
+        GIVEN:
+            - The field registry
+        WHEN:
+            - The "created" field is resolved
+        THEN:
+            - Its kind is DATE and date_only is True
+        """
+        resolved = _resolve(registry, "created")
+        assert resolved.spec.kind is FieldKind.DATE
+        assert resolved.spec.date_only is True
+
+    def test_analyzer_lowercases_and_ascii_folds(self, registry: FieldRegistry) -> None:
+        """
+        GIVEN:
+            - The field registry with no language configured (no stemmer
+              in the analyzer chain)
+        WHEN:
+            - The "title" field's analyzer processes "Café"
+        THEN:
+            - It is lowercased and ASCII-folded to the single token "cafe"
+        """
+        resolved = _resolve(registry, "title")
+        assert resolved.spec.analyzer is not None
+        assert resolved.spec.analyzer("Café") == ["cafe"]
+
+    def test_checksum_analyzer_is_identity_single_token(
+        self,
+        registry: FieldRegistry,
+    ) -> None:
+        """
+        GIVEN:
+            - The field registry
+        WHEN:
+            - The "checksum" field's analyzer (raw tokenizer, no
+              splitting) processes "ABC-123"
+        THEN:
+            - It is returned unchanged as a single token
+        """
+        resolved = _resolve(registry, "checksum")
+        assert resolved.spec.analyzer is not None
+        assert resolved.spec.analyzer("ABC-123") == ["ABC-123"]
+
+    def test_pattern_normalizer_follows_the_registry_language(
+        self,
+        registry: FieldRegistry,
+    ) -> None:
+        """
+        GIVEN:
+            - A registry with no language, and a registry built for "en"
+        WHEN:
+            - The "title" field's pattern normalizer processes "Running"
+        THEN:
+            - With no language, only the folded run is offered
+              ("running"), since the index holds surface forms
+            - With "en", the stem is offered too ("run"), since indexed
+              terms are stemmed and the pattern has to reach them
+        """
+        resolved = _resolve(registry, "title")
+        assert resolved.spec.pattern_normalizer is not None
+        assert _distinct_forms(resolved.spec.pattern_normalizer("Running")) == (
+            "running",
+        )
+
+        resolved_en = _resolve(get_field_registry("en"), "title")
+        assert resolved_en.spec.pattern_normalizer is not None
+        assert _distinct_forms(resolved_en.spec.pattern_normalizer("Running")) == (
+            "running",
+            "run",
+        )
+
+    def test_registry_is_cached_per_language(self) -> None:
+        """
+        GIVEN:
+            - Two calls to get_field_registry("en")
+        WHEN:
+            - Both calls are made
+        THEN:
+            - They return the same registry instance
+        """
+        a = get_field_registry("en")
+        b = get_field_registry("en")
+        assert a is b
+
+    def test_registry_rebuilds_on_language_change(self) -> None:
+        """
+        GIVEN:
+            - A call to get_field_registry("en") and a call to
+              get_field_registry("de")
+        WHEN:
+            - Both calls are made
+        THEN:
+            - They return different registry instances
+        """
+        a = get_field_registry("en")
+        b = get_field_registry("de")
+        assert a is not b

src/documents/tests/search/test_schema.py

@@ -5,14 +5,19 @@ from typing import TYPE_CHECKING
 
 import pytest
 
+from documents.search._fields import PUBLIC_FIELDS
 from documents.search._schema import SCHEMA_VERSION
+from documents.search._schema import build_schema
+from documents.search._schema import field_descriptors
 from documents.search._schema import needs_rebuild
+from documents.search._schema import schema_fingerprint
 
 if TYPE_CHECKING:
-    from pathlib import Path
-
+    import tantivy
     from pytest_django.fixtures import Settings
 
+    from paperless_testing.dirs import PaperlessDirs
+
 
 pytestmark = pytest.mark.search
 
@@ -20,60 +25,127 @@ pytestmark = pytest.mark.search
 class TestNeedsRebuild:
     """needs_rebuild covers all sentinel-file states that require a full reindex."""
 
-    def test_returns_true_when_settings_file_missing(self, index_dir: Path) -> None:
-        assert needs_rebuild(index_dir) is True
+    def test_returns_true_when_settings_file_missing(
+        self,
+        paperless_dirs: PaperlessDirs,
+    ) -> None:
+        assert needs_rebuild(paperless_dirs.index_dir) is True
 
     def test_returns_false_when_version_and_language_match(
         self,
-        index_dir: Path,
+        paperless_dirs: PaperlessDirs,
         settings: Settings,
     ) -> None:
         settings.SEARCH_LANGUAGE = "en"
-        (index_dir / ".index_settings.json").write_text(
-            json.dumps({"schema_version": SCHEMA_VERSION, "language": "en"}),
+        (paperless_dirs.index_dir / ".index_settings.json").write_text(
+            json.dumps(
+                {
+                    "schema_version": SCHEMA_VERSION,
+                    "language": "en",
+                    "schema_fingerprint": schema_fingerprint(),
+                },
+            ),
         )
-        assert needs_rebuild(index_dir) is False
+        assert needs_rebuild(paperless_dirs.index_dir) is False
 
     def test_returns_true_on_schema_version_mismatch(
         self,
-        index_dir: Path,
+        paperless_dirs: PaperlessDirs,
         settings: Settings,
     ) -> None:
         settings.SEARCH_LANGUAGE = None
-        (index_dir / ".index_settings.json").write_text(
+        (paperless_dirs.index_dir / ".index_settings.json").write_text(
             json.dumps({"schema_version": SCHEMA_VERSION - 1, "language": None}),
         )
-        assert needs_rebuild(index_dir) is True
+        assert needs_rebuild(paperless_dirs.index_dir) is True
 
     def test_returns_true_when_version_is_not_an_integer(
         self,
-        index_dir: Path,
+        paperless_dirs: PaperlessDirs,
         settings: Settings,
     ) -> None:
         settings.SEARCH_LANGUAGE = None
-        (index_dir / ".index_settings.json").write_text(
+        (paperless_dirs.index_dir / ".index_settings.json").write_text(
             json.dumps({"schema_version": "not-a-number", "language": None}),
         )
-        assert needs_rebuild(index_dir) is True
+        assert needs_rebuild(paperless_dirs.index_dir) is True
 
     def test_returns_true_when_language_key_missing(
         self,
-        index_dir: Path,
+        paperless_dirs: PaperlessDirs,
         settings: Settings,
     ) -> None:
         settings.SEARCH_LANGUAGE = "en"
-        (index_dir / ".index_settings.json").write_text(
+        (paperless_dirs.index_dir / ".index_settings.json").write_text(
             json.dumps({"schema_version": SCHEMA_VERSION}),
         )
-        assert needs_rebuild(index_dir) is True
+        assert needs_rebuild(paperless_dirs.index_dir) is True
 
     def test_returns_true_when_language_differs(
         self,
-        index_dir: Path,
+        paperless_dirs: PaperlessDirs,
         settings: Settings,
     ) -> None:
         settings.SEARCH_LANGUAGE = "de"
-        (index_dir / ".index_settings.json").write_text(
+        (paperless_dirs.index_dir / ".index_settings.json").write_text(
             json.dumps({"schema_version": SCHEMA_VERSION, "language": "en"}),
         )
-        assert needs_rebuild(index_dir) is True
+        assert needs_rebuild(paperless_dirs.index_dir) is True
+
+
+def _schema_fields(schema: tantivy.Schema) -> dict[str, dict]:
+    """{name: field-state} for every field declared on a tantivy Schema.
+
+    tantivy-py 0.26 exposes no public introspection API on Schema (no
+    __iter__, get_field, to_json, etc.) -- __reduce__() (used internally for
+    pickling) is the only way to recover the field list, so we lean on it
+    here for test assertions only.
+    """
+    state = schema.__reduce__()[1][0]
+    return {field["name"]: field for field in state["inner"]}
+
+
+class TestSchemaMatchesPublicFields:
+    def test_every_public_field_is_in_the_schema(self) -> None:
+        """
+        GIVEN:
+            - PUBLIC_FIELDS and the tantivy schema built by build_schema()
+        WHEN:
+            - Every field declared in PUBLIC_FIELDS is checked against the
+              schema
+        THEN:
+            - Each one is present as a field in the built schema
+        """
+        schema = build_schema()
+        schema_field_names = set(_schema_fields(schema))
+        for field in PUBLIC_FIELDS:
+            assert field.name in schema_field_names, (
+                f"{field.name} is in PUBLIC_FIELDS but missing from build_schema()"
+            )
+
+
+class TestFastFlagAgreement:
+    def test_every_public_field_fast_flag_matches_the_built_schema(self) -> None:
+        """
+        GIVEN:
+            - PUBLIC_FIELDS and field_descriptors() (the latter is exactly
+              the input build_schema()'s SchemaBuilder consumes for the
+              `fast` kwarg on every field kind, so it pins the agreement
+              without depending on a private tantivy-py pickled
+              representation)
+        WHEN:
+            - Every PUBLIC_FIELDS entry's fast flag is compared against
+              field_descriptors()' fast flag for the same field
+        THEN:
+            - They agree for every field, catching a fast=True
+              PUBLIC_FIELDS entry the builder silently ignores here
+              instead of at a user's field:* existence query, which
+              whoosh-compat's registry trusts PUBLIC_FIELDS' fast flag to
+              resolve
+        """
+        descriptor_fast = {d.name: d.fast for d in field_descriptors()}
+        for public_field in PUBLIC_FIELDS:
+            assert descriptor_fast[public_field.name] == public_field.fast, (
+                f"{public_field.name}: PUBLIC_FIELDS says fast={public_field.fast} but"
+                f" field_descriptors() says fast={descriptor_fast[public_field.name]}"
+            )

src/documents/tests/search/test_schema_fingerprint.py

@@ -0,0 +1,597 @@
+"""The schema fingerprint stamped into .index_settings.json.
+
+tantivy compares schemas by *ordered* field list, and `tantivy.Index(schema,
+path=...)` (what every write path does) raises on any difference. SCHEMA_VERSION
+is the manual guard against that, but build_schema() is edited for *parser*
+reasons - adding an alias, flipping fast=True, adding a subpath - by people not
+thinking about the on-disk index, and forgetting the bump is exactly how this
+branch's bug happened.
+
+The fingerprint is the automatic guard: it hashes the field descriptor list that
+build_schema() itself iterates, so any change to a field's name, kind, options
+or *position* forces a rebuild on its own.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import TYPE_CHECKING
+
+import pytest
+import tantivy
+
+from documents.search import _schema
+from documents.search._schema import SCHEMA_VERSION
+from documents.search._schema import FieldDescriptor
+from documents.search._schema import _write_sentinels
+from documents.search._schema import build_schema
+from documents.search._schema import field_descriptors
+from documents.search._schema import needs_rebuild
+from documents.search._schema import schema_fingerprint
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from pytest_django.fixtures import SettingsWrapper
+
+    from paperless_testing.dirs import PaperlessDirs
+
+pytestmark = pytest.mark.search
+
+# The on-disk field layout of a v2 index, pinned as data. Any edit here is an
+# index-format change: it must come with a rebuild, which the fingerprint now
+# forces automatically. Reproduced from build_schema()'s output as it stood
+# before the descriptor refactor, so it also pins that the refactor changed
+# nothing.
+PINNED_DESCRIPTORS: tuple[FieldDescriptor, ...] = (
+    FieldDescriptor("id", "u64", stored=True, indexed=True, fast=True, tokenizer=None),
+    FieldDescriptor(
+        "title",
+        "text",
+        stored=True,
+        indexed=True,
+        fast=False,
+        tokenizer="paperless_text",
+    ),
+    FieldDescriptor(
+        "content",
+        "text",
+        stored=True,
+        indexed=True,
+        fast=False,
+        tokenizer="paperless_text",
+    ),
+    FieldDescriptor(
+        "correspondent",
+        "text",
+        stored=True,
+        indexed=True,
+        fast=False,
+        tokenizer="paperless_text",
+    ),
+    FieldDescriptor(
+        "document_type",
+        "text",
+        stored=True,
+        indexed=True,
+        fast=False,
+        tokenizer="paperless_text",
+    ),
+    FieldDescriptor(
+        "storage_path",
+        "text",
+        stored=True,
+        indexed=True,
+        fast=False,
+        tokenizer="paperless_text",
+    ),
+    FieldDescriptor(
+        "original_filename",
+        "text",
+        stored=True,
+        indexed=True,
+        fast=False,
+        tokenizer="paperless_text",
+    ),
+    FieldDescriptor(
+        "tag",
+        "text",
+        stored=True,
+        indexed=True,
+        fast=False,
+        tokenizer="paperless_text",
+    ),
+    FieldDescriptor(
+        "checksum",
+        "text",
+        stored=True,
+        indexed=True,
+        fast=False,
+        tokenizer="raw",
+    ),
+    FieldDescriptor("asn", "u64", stored=True, indexed=True, fast=True, tokenizer=None),
+    FieldDescriptor(
+        "page_count",
+        "u64",
+        stored=True,
+        indexed=True,
+        fast=True,
+        tokenizer=None,
+    ),
+    FieldDescriptor(
+        "num_notes",
+        "u64",
+        stored=True,
+        indexed=True,
+        fast=True,
+        tokenizer=None,
+    ),
+    FieldDescriptor(
+        "created",
+        "date",
+        stored=True,
+        indexed=True,
+        fast=True,
+        tokenizer=None,
+    ),
+    FieldDescriptor(
+        "modified",
+        "date",
+        stored=True,
+        indexed=True,
+        fast=True,
+        tokenizer=None,
+    ),
+    FieldDescriptor(
+        "added",
+        "date",
+        stored=True,
+        indexed=True,
+        fast=True,
+        tokenizer=None,
+    ),
+    FieldDescriptor(
+        "notes",
+        "json",
+        stored=True,
+        indexed=True,
+        fast=False,
+        tokenizer="paperless_text",
+    ),
+    FieldDescriptor(
+        "notes_text",
+        "text",
+        stored=True,
+        indexed=True,
+        fast=False,
+        tokenizer="paperless_text",
+    ),
+    FieldDescriptor(
+        "custom_fields",
+        "json",
+        stored=True,
+        indexed=True,
+        fast=False,
+        tokenizer="paperless_text",
+    ),
+    FieldDescriptor(
+        "barcodes",
+        "json",
+        stored=True,
+        indexed=True,
+        fast=False,
+        tokenizer="paperless_text",
+    ),
+    FieldDescriptor(
+        "title_sort",
+        "text",
+        stored=False,
+        indexed=True,
+        fast=True,
+        tokenizer="simple_analyzer",
+    ),
+    FieldDescriptor(
+        "correspondent_sort",
+        "text",
+        stored=False,
+        indexed=True,
+        fast=True,
+        tokenizer="simple_analyzer",
+    ),
+    FieldDescriptor(
+        "type_sort",
+        "text",
+        stored=False,
+        indexed=True,
+        fast=True,
+        tokenizer="simple_analyzer",
+    ),
+    FieldDescriptor(
+        "bigram_content",
+        "text",
+        stored=False,
+        indexed=True,
+        fast=False,
+        tokenizer="bigram_analyzer",
+    ),
+    FieldDescriptor(
+        "bigram_title",
+        "text",
+        stored=False,
+        indexed=True,
+        fast=False,
+        tokenizer="bigram_analyzer",
+    ),
+    FieldDescriptor(
+        "bigram_correspondent",
+        "text",
+        stored=False,
+        indexed=True,
+        fast=False,
+        tokenizer="bigram_analyzer",
+    ),
+    FieldDescriptor(
+        "bigram_document_type",
+        "text",
+        stored=False,
+        indexed=True,
+        fast=False,
+        tokenizer="bigram_analyzer",
+    ),
+    FieldDescriptor(
+        "bigram_tag",
+        "text",
+        stored=False,
+        indexed=True,
+        fast=False,
+        tokenizer="bigram_analyzer",
+    ),
+    FieldDescriptor(
+        "simple_title",
+        "text",
+        stored=False,
+        indexed=True,
+        fast=False,
+        tokenizer="simple_search_analyzer",
+    ),
+    FieldDescriptor(
+        "simple_content",
+        "text",
+        stored=False,
+        indexed=True,
+        fast=False,
+        tokenizer="simple_search_analyzer",
+    ),
+    FieldDescriptor(
+        "autocomplete_word",
+        "text",
+        stored=False,
+        indexed=True,
+        fast=False,
+        tokenizer="raw",
+    ),
+    FieldDescriptor(
+        "owner_id",
+        "u64",
+        stored=False,
+        indexed=True,
+        fast=True,
+        tokenizer=None,
+    ),
+    FieldDescriptor(
+        "viewer_id",
+        "u64",
+        stored=False,
+        indexed=True,
+        fast=True,
+        tokenizer=None,
+    ),
+    FieldDescriptor(
+        "viewer_group_id",
+        "u64",
+        stored=False,
+        indexed=True,
+        fast=True,
+        tokenizer=None,
+    ),
+)
+
+
+def _schema_fields(schema: tantivy.Schema) -> list[dict]:
+    """The tantivy-level field list, in declaration order.
+
+    tantivy-py 0.26 exposes no public introspection API on Schema, so
+    __reduce__() (its pickling hook) is the only way to recover the field list.
+    It is used here, in a test, precisely because it is the representation the
+    persisted fingerprint must NOT depend on.
+    """
+    return schema.__reduce__()[1][0]["inner"]
+
+
+def _sentinels(index_dir: Path, **overrides: object) -> None:
+    data = {
+        "schema_version": SCHEMA_VERSION,
+        "language": None,
+        "schema_fingerprint": schema_fingerprint(),
+    }
+    data.update(overrides)
+    (index_dir / ".index_settings.json").write_text(json.dumps(data))
+
+
+class TestDescriptorsDescribeTheBuiltSchema:
+    def test_descriptors_match_the_pinned_field_layout(self) -> None:
+        """
+        GIVEN:
+            - PINNED_DESCRIPTORS, a frozen snapshot of the v2 on-disk field
+              layout, reproduced from build_schema()'s output as it stood
+              before the descriptor refactor
+        WHEN:
+            - field_descriptors() is called
+        THEN:
+            - It matches the pinned layout exactly, in the same order,
+              pinning that the refactor changed nothing
+        """
+        assert tuple(field_descriptors()) == PINNED_DESCRIPTORS
+
+    def test_built_schema_matches_the_descriptors(self) -> None:
+        """
+        GIVEN:
+            - The schema built by build_schema()
+        WHEN:
+            - Its fields are read back via __reduce__() (schema.__reduce__(),
+              tantivy-py's pickling hook)
+        THEN:
+            - Every field's name, kind, stored/fast flags and tokenizer
+              match what field_descriptors() declared as input; the
+              descriptors are not a parallel description, they are the
+              input, so a descriptor edit cannot claim a shape the
+              SchemaBuilder did not actually build
+        """
+        kinds = {"text": "text", "json": "json_object", "u64": "u64", "date": "date"}
+        built = [
+            (
+                field["name"],
+                field["type"],
+                field["options"]["stored"],
+                bool(field["options"].get("fast")),
+                (field["options"].get("indexing") or {}).get("tokenizer"),
+            )
+            for field in _schema_fields(build_schema())
+        ]
+        expected = [
+            (
+                descriptor.name,
+                kinds[descriptor.kind],
+                descriptor.stored,
+                descriptor.fast,
+                descriptor.tokenizer,
+            )
+            for descriptor in field_descriptors()
+        ]
+        assert built == expected
+
+
+class TestFingerprintSensitivity:
+    def test_a_field_option_change_moves_the_fingerprint(
+        self,
+        monkeypatch: pytest.MonkeyPatch,
+    ) -> None:
+        """
+        GIVEN:
+            - The current schema fingerprint
+        WHEN:
+            - A single field descriptor's "fast" option is changed, with
+              no other change
+        THEN:
+            - The fingerprint changes
+        """
+        before = schema_fingerprint()
+        changed = field_descriptors()
+        changed[1] = changed[1]._replace(fast=True)
+        monkeypatch.setattr(_schema, "field_descriptors", lambda: changed)
+
+        assert schema_fingerprint() != before
+
+    def test_reordering_alone_moves_the_fingerprint(
+        self,
+        monkeypatch: pytest.MonkeyPatch,
+    ) -> None:
+        """
+        GIVEN:
+            - The current schema fingerprint
+        WHEN:
+            - Two field descriptors are swapped, with no other change (the
+              original bug: same fields, different declaration order)
+        THEN:
+            - The fingerprint changes; a set- or dict-based fingerprint
+              would be blind to this, and tantivy would reject every write
+              against the existing index
+        """
+        before = schema_fingerprint()
+        swapped = field_descriptors()
+        swapped[1], swapped[2] = swapped[2], swapped[1]
+        monkeypatch.setattr(_schema, "field_descriptors", lambda: swapped)
+
+        assert schema_fingerprint() != before
+
+
+class TestFingerprintIsIndependentOfTantivy:
+    def test_a_tantivy_option_key_addition_would_not_move_it(self) -> None:
+        """
+        GIVEN:
+            - The built schema's raw field list, and the same list with a
+              new tantivy-internal option key added (simulating a
+              tantivy-py upgrade)
+        WHEN:
+            - Both raw lists are hashed directly, and schema_fingerprint()
+              is compared against a hash of field_descriptors()
+        THEN:
+            - The raw hashes differ (hashing schema.__reduce__() would
+              force a global reindex on every tantivy-py upgrade), but
+              schema_fingerprint() is unaffected, since it hashes
+              field_descriptors(), never tantivy's own representation
+        """
+        fields = _schema_fields(build_schema())
+        upgraded = [
+            {**field, "options": {**field["options"], "coerce": True}}
+            for field in fields
+        ]
+        assert _hash(upgraded) != _hash(fields)
+        assert schema_fingerprint() == _fingerprint_of(field_descriptors())
+
+    def test_fingerprint_never_touches_the_schema_builder(
+        self,
+        monkeypatch: pytest.MonkeyPatch,
+    ) -> None:
+        """
+        GIVEN:
+            - tantivy.SchemaBuilder replaced with a stand-in that raises if
+              constructed
+        WHEN:
+            - build_schema() is called (and raises), then
+              schema_fingerprint() is called again
+        THEN:
+            - schema_fingerprint() still matches its earlier value,
+              proving it never consults SchemaBuilder
+        """
+        before = schema_fingerprint()
+
+        class _RemovedSchemaBuilder:
+            def __init__(self) -> None:
+                raise AssertionError("tantivy.SchemaBuilder was consulted")
+
+        monkeypatch.setattr(tantivy, "SchemaBuilder", _RemovedSchemaBuilder)
+        with pytest.raises(AssertionError):
+            build_schema()
+
+        assert schema_fingerprint() == before
+
+
+def _hash(payload: object) -> str:
+    return hashlib.blake2b(json.dumps(payload).encode()).hexdigest()
+
+
+def _fingerprint_of(descriptors: list[FieldDescriptor]) -> str:
+    return _hash([list(descriptor) for descriptor in descriptors])
+
+
+class TestNeedsRebuildOnFingerprint:
+    def test_matching_fingerprint_does_not_rebuild(
+        self,
+        paperless_dirs: PaperlessDirs,
+        settings: SettingsWrapper,
+    ) -> None:
+        """
+        GIVEN:
+            - An index directory whose sentinel file records the current
+              schema_fingerprint()
+        WHEN:
+            - needs_rebuild() is called
+        THEN:
+            - It returns False
+        """
+        settings.SEARCH_LANGUAGE = None
+        _sentinels(paperless_dirs.index_dir)
+
+        assert needs_rebuild(paperless_dirs.index_dir) is False
+
+    def test_stale_fingerprint_rebuilds_despite_a_matching_version(
+        self,
+        paperless_dirs: PaperlessDirs,
+        settings: SettingsWrapper,
+        monkeypatch: pytest.MonkeyPatch,
+    ) -> None:
+        """
+        GIVEN:
+            - An index directory whose sentinel matches SCHEMA_VERSION,
+              but field_descriptors() is patched to add a field the
+              fingerprint never saw (schema edited, version not bumped)
+        WHEN:
+            - needs_rebuild() is called
+        THEN:
+            - It returns True; without the fingerprint check,
+              `reindex --if-needed` would report the index up to date and
+              every subsequent write would raise
+        """
+        settings.SEARCH_LANGUAGE = None
+        _sentinels(paperless_dirs.index_dir)
+        extended = [
+            *field_descriptors(),
+            FieldDescriptor(
+                "new_field",
+                "u64",
+                stored=False,
+                indexed=True,
+                fast=True,
+                tokenizer=None,
+            ),
+        ]
+        monkeypatch.setattr(_schema, "field_descriptors", lambda: extended)
+
+        assert needs_rebuild(paperless_dirs.index_dir) is True
+
+    def test_reordered_schema_rebuilds(
+        self,
+        paperless_dirs: PaperlessDirs,
+        settings: SettingsWrapper,
+        monkeypatch: pytest.MonkeyPatch,
+    ) -> None:
+        """
+        GIVEN:
+            - An index directory whose sentinel matches the current
+              fingerprint, but field_descriptors() is patched to swap two
+              fields' order
+        WHEN:
+            - needs_rebuild() is called
+        THEN:
+            - It returns True
+        """
+        settings.SEARCH_LANGUAGE = None
+        _sentinels(paperless_dirs.index_dir)
+        reordered = field_descriptors()
+        reordered[1], reordered[2] = reordered[2], reordered[1]
+        monkeypatch.setattr(_schema, "field_descriptors", lambda: reordered)
+
+        assert needs_rebuild(paperless_dirs.index_dir) is True
+
+    def test_missing_fingerprint_rebuilds(
+        self,
+        paperless_dirs: PaperlessDirs,
+        settings: SettingsWrapper,
+    ) -> None:
+        """
+        GIVEN:
+            - An index directory whose sentinel has no "schema_fingerprint"
+              key at all
+        WHEN:
+            - needs_rebuild() is called
+        THEN:
+            - It returns True; an index whose schema shape nobody recorded
+              is rebuilt rather than trusted
+        """
+        settings.SEARCH_LANGUAGE = None
+        (paperless_dirs.index_dir / ".index_settings.json").write_text(
+            json.dumps({"schema_version": SCHEMA_VERSION, "language": None}),
+        )
+
+        assert needs_rebuild(paperless_dirs.index_dir) is True
+
+    def test_written_sentinels_satisfy_the_check(
+        self,
+        paperless_dirs: PaperlessDirs,
+        settings: SettingsWrapper,
+    ) -> None:
+        """
+        GIVEN:
+            - An index directory whose sentinels are written by
+              _write_sentinels() itself
+        WHEN:
+            - needs_rebuild() is called
+        THEN:
+            - It returns False
+        """
+        settings.SEARCH_LANGUAGE = "en"
+        _write_sentinels(paperless_dirs.index_dir)
+
+        assert needs_rebuild(paperless_dirs.index_dir) is False

src/documents/tests/search/test_schema_version.py

@@ -0,0 +1,227 @@
+"""SCHEMA_VERSION must change whenever build_schema()'s field list or order does.
+
+tantivy compares schemas by *ordered* field list. ``Index.open()`` loads the
+schema from the index's own ``meta.json``, so reads against an index built by an
+older release keep working after a field reorder. Writes do not:
+``WriteBatch.__enter__`` calls ``tantivy.Index(build_schema(), path=...)``, an
+open-or-create that raises ``ValueError`` on any schema difference. Nothing
+catches that ValueError, so consumption, index_document and bulk edit all
+hard-fail while ``/api/status/`` still reports the index healthy.
+
+The only thing that saves such an install is ``needs_rebuild()`` noticing the
+version stamped in ``.index_settings.json`` is stale.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING
+
+import pytest
+import tantivy
+from django.conf import settings as django_settings
+
+from documents.search._schema import SCHEMA_VERSION
+from documents.search._schema import build_schema
+from documents.search._schema import needs_rebuild
+from documents.search._schema import open_or_rebuild_index
+from documents.search._schema import schema_fingerprint
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+pytestmark = [pytest.mark.search]
+
+RELEASED_V1_SCHEMA_VERSION = 1
+
+
+def _build_released_v1_schema() -> tantivy.Schema:
+    """Frozen copy of build_schema() as shipped in v3.0.x (schema version 1).
+
+    Deliberately duplicated rather than imported: it must keep describing the
+    on-disk layout of already-deployed indexes even as build_schema() evolves.
+    """
+    sb = tantivy.SchemaBuilder()
+
+    sb.add_unsigned_field("id", stored=True, indexed=True, fast=True)
+    sb.add_text_field("checksum", stored=True, tokenizer_name="raw")
+
+    for field in (
+        "title",
+        "correspondent",
+        "document_type",
+        "storage_path",
+        "original_filename",
+        "content",
+    ):
+        sb.add_text_field(field, stored=True, tokenizer_name="paperless_text")
+
+    for field in ("title_sort", "correspondent_sort", "type_sort"):
+        sb.add_text_field(
+            field,
+            stored=False,
+            tokenizer_name="simple_analyzer",
+            fast=True,
+        )
+
+    for field in (
+        "bigram_content",
+        "bigram_title",
+        "bigram_correspondent",
+        "bigram_document_type",
+        "bigram_tag",
+    ):
+        sb.add_text_field(field, stored=False, tokenizer_name="bigram_analyzer")
+
+    for field in ("simple_title", "simple_content"):
+        sb.add_text_field(field, stored=False, tokenizer_name="simple_search_analyzer")
+
+    sb.add_text_field("autocomplete_word", stored=False, tokenizer_name="raw")
+    sb.add_text_field("tag", stored=True, tokenizer_name="paperless_text")
+
+    sb.add_json_field("notes", stored=True, tokenizer_name="paperless_text")
+    sb.add_text_field("notes_text", stored=True, tokenizer_name="paperless_text")
+    sb.add_json_field("custom_fields", stored=True, tokenizer_name="paperless_text")
+
+    for field in (
+        "correspondent_id",
+        "document_type_id",
+        "storage_path_id",
+        "tag_id",
+        "owner_id",
+        "viewer_id",
+        "viewer_group_id",
+    ):
+        sb.add_unsigned_field(field, stored=False, indexed=True, fast=True)
+
+    for field in ("created", "modified", "added"):
+        sb.add_date_field(field, stored=True, indexed=True, fast=True)
+
+    for field in ("asn", "page_count", "num_notes"):
+        sb.add_unsigned_field(field, stored=True, indexed=True, fast=True)
+
+    return sb.build()
+
+
+@pytest.fixture
+def released_v1_index(tmp_path: Path) -> Path:
+    """An index directory as a v3.0.x install would leave it on disk."""
+    index_dir = tmp_path / "index"
+    index_dir.mkdir()
+    tantivy.Index(_build_released_v1_schema(), path=str(index_dir))
+    (index_dir / ".index_settings.json").write_text(
+        json.dumps(
+            {
+                "schema_version": RELEASED_V1_SCHEMA_VERSION,
+                "language": django_settings.SEARCH_LANGUAGE,
+            },
+        ),
+    )
+    return index_dir
+
+
+class TestUpgradeFromReleasedV1Index:
+    def test_released_v1_index_is_flagged_for_rebuild(
+        self,
+        released_v1_index: Path,
+    ) -> None:
+        """
+        GIVEN:
+            - An index directory laid out exactly as a v3.0.x (schema
+              version 1) install would leave it
+        WHEN:
+            - needs_rebuild() is called
+        THEN:
+            - It returns True; if this fails,
+              `document_index reindex --if-needed` prints "Search index is
+              up to date" and skips, leaving the mismatched index in place
+        """
+        assert needs_rebuild(released_v1_index) is True
+
+    def test_opening_a_v1_index_leaves_it_writable(
+        self,
+        released_v1_index: Path,
+    ) -> None:
+        """
+        GIVEN:
+            - A v1 index directory
+        WHEN:
+            - open_or_rebuild_index() is called against it
+        THEN:
+            - The directory can be reopened with the current schema
+              without raising; end to end, open_or_rebuild_index must
+              hand back an index the write path can reopen. Before the
+              version bump, needs_rebuild() returned False here, and the
+              stale directory survived untouched, so every subsequent
+              write against it raised tantivy's own schema-mismatch
+              ValueError
+        """
+        open_or_rebuild_index(released_v1_index)
+
+        tantivy.Index(build_schema(), path=str(released_v1_index))
+
+    def test_rebuilt_index_is_not_rebuilt_again(
+        self,
+        released_v1_index: Path,
+    ) -> None:
+        """
+        GIVEN:
+            - A v1 index directory that has just been rebuilt by
+              open_or_rebuild_index()
+        WHEN:
+            - needs_rebuild() is called again
+        THEN:
+            - It returns False; the rebuild must stamp the version it
+              actually wrote, otherwise every startup wipes and reindexes
+              the whole corpus
+        """
+        open_or_rebuild_index(released_v1_index)
+
+        assert needs_rebuild(released_v1_index) is False
+
+
+class TestCorruptedIndex:
+    @pytest.fixture
+    def current_index_missing_meta_json(self, tmp_path: Path) -> Path:
+        """A current-schema index directory whose meta.json has gone missing.
+
+        The sentinel matches the current schema, so needs_rebuild() reports
+        False and open_or_rebuild_index() takes the plain Index.open() path.
+        """
+        index_dir = tmp_path / "index"
+        index_dir.mkdir()
+        tantivy.Index(build_schema(), path=str(index_dir))
+        (index_dir / ".index_settings.json").write_text(
+            json.dumps(
+                {
+                    "schema_version": SCHEMA_VERSION,
+                    "language": django_settings.SEARCH_LANGUAGE,
+                    "schema_fingerprint": schema_fingerprint(),
+                },
+            ),
+        )
+        (index_dir / "meta.json").unlink()
+        return index_dir
+
+    def test_missing_meta_json_is_rebuilt_instead_of_raising(
+        self,
+        current_index_missing_meta_json: Path,
+    ) -> None:
+        """
+        GIVEN:
+            - An index directory with a sentinel matching the current
+              schema, but a missing/corrupted meta.json (e.g. an
+              interrupted write left the directory in a torn state)
+        WHEN:
+            - open_or_rebuild_index() is called against it
+        THEN:
+            - It rebuilds the index instead of letting tantivy's
+              ValueError propagate; before this, a corrupted index left
+              every read, write, and `document_index reindex --if-needed`
+              hard-failing indefinitely, since the sentinel already
+              matched and reported the index as up to date
+        """
+        open_or_rebuild_index(current_index_missing_meta_json)
+
+        assert (current_index_missing_meta_json / "meta.json").exists()
+        tantivy.Index.open(str(current_index_missing_meta_json))

src/documents/tests/search/test_stem_pattern_text.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+import pytest
+
+from documents.search._tokenizer import stem_pattern_text
+
+pytestmark = pytest.mark.search
+
+
+class TestStemPatternText:
+    def test_unsupported_language_returns_text_unchanged(self) -> None:
+        """
+        GIVEN:
+            - A language code with no Snowball stemmer mapping
+        WHEN:
+            - A pattern run is stemmed for that language
+        THEN:
+            - The run is returned unchanged, since the stemming gate that
+              disables stemming for an unsupported language also disables
+              the pattern-side stemmer
+        """
+        assert stem_pattern_text("running", "klingon") == "running"
+
+    def test_run_past_remove_long_limit_returns_text_unchanged(self) -> None:
+        """
+        GIVEN:
+            - A supported language and a run longer than the remove_long
+              filter's limit (129 characters, matching Document.title's
+              max_length)
+        WHEN:
+            - The over-long run is stemmed
+        THEN:
+            - The remove_long filter drops the token entirely, leaving no
+              stem to substitute, so the run is returned unchanged
+        """
+        long_run = "a" * 130
+        assert stem_pattern_text(long_run, "en") == long_run

src/documents/tests/search/test_tokenizer.py

@@ -7,8 +7,8 @@ import pytest
 import tantivy
 
 from documents.search._tokenizer import _bigram_analyzer
-from documents.search._tokenizer import _paperless_text
 from documents.search._tokenizer import _simple_search_analyzer
+from documents.search._tokenizer import paperless_text_analyzer
 from documents.search._tokenizer import register_tokenizers
 
 if TYPE_CHECKING:
@@ -25,7 +25,7 @@ class TestTokenizers:
         sb.add_text_field("content", stored=True, tokenizer_name="paperless_text")
         schema = sb.build()
         idx = tantivy.Index(schema, path=None)
-        idx.register_tokenizer("paperless_text", _paperless_text(""))
+        idx.register_tokenizer("paperless_text", paperless_text_analyzer(""))
         return idx
 
     @pytest.fixture

src/documents/tests/search/test_translate.py

@@ -1,810 +0,0 @@
-from __future__ import annotations
-
-from datetime import UTC
-from datetime import datetime
-from typing import TYPE_CHECKING
-from zoneinfo import ZoneInfo
-
-import pytest
-import time_machine
-
-from documents.search._dates import _precision_bounds
-
-if TYPE_CHECKING:
-    import tantivy
-from documents.search._query import _FIELD_BOOSTS
-from documents.search._query import DEFAULT_SEARCH_FIELDS
-from documents.search._translate import OPEN_HI
-from documents.search._translate import OPEN_LO
-from documents.search._translate import Comma
-from documents.search._translate import FieldRange
-from documents.search._translate import FieldValue
-from documents.search._translate import FieldValueList
-from documents.search._translate import InvalidDateQuery
-from documents.search._translate import Passthrough
-from documents.search._translate import resolve_commas
-from documents.search._translate import scan
-from documents.search._translate import translate_query
-from documents.search._translate import translate_range
-from documents.search._translate import translate_scalar
-
-
-@pytest.mark.search
-class TestPrecisionBounds:
-    @pytest.mark.parametrize(
-        ("digits", "expected"),
-        [
-            ("2020", ((2020, 1, 1), (2021, 1, 1))),
-            ("202003", ((2020, 3, 1), (2020, 4, 1))),
-            ("202012", ((2020, 12, 1), (2021, 1, 1))),
-            ("20200115", ((2020, 1, 15), (2020, 1, 16))),
-            ("20201231", ((2020, 12, 31), (2021, 1, 1))),
-        ],
-    )
-    def test_valid(self, digits, expected):
-        lo, hi = _precision_bounds(digits)
-        assert (lo.year, lo.month, lo.day) == expected[0]
-        assert (hi.year, hi.month, hi.day) == expected[1]
-
-    @pytest.mark.parametrize("digits", ["202023", "20200230", "20201301", "20", "abcd"])
-    def test_invalid_returns_none(self, digits):
-        assert _precision_bounds(digits) is None
-
-
-@pytest.mark.search
-class TestScan:
-    def test_plain_words_are_passthrough(self):
-        assert scan("bank statement") == [Passthrough("bank statement")]
-
-    def test_field_value(self):
-        assert scan("created:2020") == [FieldValue("created", "2020")]
-
-    def test_field_value_in_boolean(self):
-        toks = scan("created:2020 OR foo")
-        assert toks == [
-            FieldValue("created", "2020"),
-            Passthrough(" OR foo"),
-        ]
-
-    def test_field_value_in_parens(self):
-        toks = scan("(created:2020 OR foo)")
-        assert toks == [
-            Passthrough("("),
-            FieldValue("created", "2020"),
-            Passthrough(" OR foo)"),
-        ]
-
-    def test_quoted_value(self):
-        assert scan('correspondent:"A B"') == [FieldValue("correspondent", '"A B"')]
-
-    def test_field_range(self):
-        assert scan("created:[2020 TO 2021]") == [
-            FieldRange("created", "[", "2020", "2021", "]"),
-        ]
-
-    @pytest.mark.parametrize(
-        ("query", "expected"),
-        [
-            pytest.param(
-                "created:[2020 to]",
-                FieldRange("created", "[", "2020", "", "]"),
-                id="open_upper",
-            ),
-            pytest.param(
-                "created:[to 2020]",
-                FieldRange("created", "[", "", "2020", "]"),
-                id="open_lower",
-            ),
-        ],
-    )
-    def test_open_range(self, query, expected):
-        assert scan(query) == [expected]
-
-    def test_comma_inside_range_not_split(self):
-        # No depth-0 comma here; the whole thing is one range token.
-        toks = scan("created:[2020 TO 2021]")
-        assert len(toks) == 1
-
-    # --- Edge-case / regression tests (scan must never raise) ---
-
-    def test_url_is_passthrough(self):
-        # "http" is not a known field; the whole URL must pass through verbatim.
-        assert scan("http://example.com") == [Passthrough("http://example.com")]
-
-    def test_unterminated_quote_is_passthrough(self):
-        # title is a known field but the quoted value has no closing quote;
-        # _consume_value returns None so the whole string falls into passthrough.
-        assert scan('title:"abc') == [Passthrough('title:"abc')]
-
-    def test_unterminated_bracket_is_passthrough(self):
-        # created is a known field but the range bracket is never closed;
-        # _consume_range returns None so the whole string falls into passthrough.
-        assert scan("created:[2020") == [Passthrough("created:[2020")]
-
-    def test_empty_value_at_end_is_passthrough(self):
-        # created is a known field but there is no value after the colon
-        # (_consume_value returns None for start >= n), so passthrough.
-        assert scan("created:") == [Passthrough("created:")]
-
-    def test_value_containing_colon(self):
-        # The bare-word value reader stops at whitespace/paren, not at colon,
-        # so "2020:30" is consumed as a single value token.
-        assert scan("created:2020:30") == [FieldValue("created", "2020:30")]
-
-    def test_comma_followed_by_unconsumable_value_stops(self):
-        # A comma followed by whitespace is neither a value-list continuation nor a
-        # clause separator: the value stops and the comma stays as passthrough.
-        assert scan("tag:foo, bar") == [
-            FieldValue("tag", "foo"),
-            Passthrough(", bar"),
-        ]
-
-    def test_bracket_without_to_is_open_upper_bound(self):
-        # A bracketed value with no TO falls back to (value, "") -> open upper bound.
-        assert scan("created:[2020]") == [
-            FieldRange("created", "[", "2020", "", "]"),
-        ]
-
-    def test_known_field_name_midword_is_passthrough(self):
-        # A known field name embedded mid-word is not a field token (the
-        # word-boundary guard); the whole run stays passthrough.
-        assert scan("xtag:foo") == [Passthrough("xtag:foo")]
-
-
-@pytest.mark.search
-class TestCommaResolution:
-    def test_value_list_multi_value_field(self):
-        toks = resolve_commas(scan("tag:foo,bar"))
-        assert toks == [FieldValueList("tag", ("foo", "bar"))]
-
-    def test_value_list_three(self):
-        toks = resolve_commas(scan("tag_id:1,2,3"))
-        assert toks == [FieldValueList("tag_id", ("1", "2", "3"))]
-
-    def test_text_field_comma_is_literal(self):
-        # correspondent is not multi-value: comma stays inside the value.
-        toks = resolve_commas(scan("correspondent:foo,bar"))
-        assert toks == [FieldValue("correspondent", "foo,bar")]
-
-    def test_clause_separator_before_known_field(self):
-        toks = resolve_commas(scan("tag:foo,type:bar"))
-        assert toks == [FieldValue("tag", "foo"), Comma(), FieldValue("type", "bar")]
-
-    def test_clause_separator_after_range(self):
-        toks = resolve_commas(scan("created:[2020 TO 2021],added:[2022 TO 2023]"))
-        assert toks == [
-            FieldRange("created", "[", "2020", "2021", "]"),
-            Comma(),
-            FieldRange("added", "[", "2022", "2023", "]"),
-        ]
-
-    def test_clause_separator_after_quote(self):
-        toks = resolve_commas(scan('correspondent:"A B",created:[2020 TO 2021]'))
-        assert toks == [
-            FieldValue("correspondent", '"A B"'),
-            Comma(),
-            FieldRange("created", "[", "2020", "2021", "]"),
-        ]
-
-    def test_url_comma_is_literal_passthrough(self):
-        toks = resolve_commas(scan("http://example.com/a,b"))
-        assert toks == [Passthrough("http://example.com/a,b")]
-
-    def test_non_multi_value_comma_is_literal(self):
-        # title is not in MULTI_VALUE_FIELDS: comma stays inside the value.
-        toks = resolve_commas(scan("title:10,20"))
-        assert toks == [FieldValue("title", "10,20")]
-
-    def test_clause_separator_before_known_date_field(self):
-        # The comma between a bare value and a known date field acts as a
-        # clause separator; both sides survive as distinct tokens.
-        toks = resolve_commas(scan("correspondent:foo,created:[2020 TO 2021]"))
-        assert toks == [
-            FieldValue("correspondent", "foo"),
-            Comma(),
-            FieldRange("created", "[", "2020", "2021", "]"),
-        ]
-
-
-@pytest.mark.search
-class TestTranslateScalar:
-    @pytest.mark.parametrize(
-        ("field", "value", "expected"),
-        [
-            (
-                "created",
-                "2020",
-                "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",
-            ),
-            (
-                "created",
-                "202003",
-                "created:[2020-03-01T00:00:00Z TO 2020-04-01T00:00:00Z}",
-            ),
-            (
-                "created",
-                "20200115",
-                "created:[2020-01-15T00:00:00Z TO 2020-01-16T00:00:00Z}",
-            ),
-            (
-                "created",
-                "2020-01-15",
-                "created:[2020-01-15T00:00:00Z TO 2020-01-16T00:00:00Z}",
-            ),
-            (
-                "created",
-                "2020-03",
-                "created:[2020-03-01T00:00:00Z TO 2020-04-01T00:00:00Z}",
-            ),
-        ],
-    )
-    def test_partial_and_iso_dates(self, field: str, value: str, expected: str) -> None:
-        assert translate_scalar(field, value, UTC) == expected
-
-    def test_invalid_date_raises(self) -> None:
-        with pytest.raises(InvalidDateQuery) as exc_info:
-            translate_scalar("created", "202023", UTC)
-        assert exc_info.value.field == "created"
-        assert exc_info.value.value == "202023"
-
-    def test_keyword_delegates(self) -> None:
-        # keyword path produces a half-open range; just assert it is a created range
-        out = translate_scalar("created", "today", UTC)
-        assert out.startswith("created:[") and out.endswith("}")
-
-    def test_14digit_compact_datetime(self) -> None:
-        out = translate_scalar("created", "20240115120000", UTC)
-        assert "20240115120000" not in out
-        assert out.startswith("created:")
-        assert out == "created:[2024-01-15T12:00:00Z TO 2024-01-15T12:00:00Z]"
-
-    def test_14digit_invalid_month_raises(self) -> None:
-        with pytest.raises(InvalidDateQuery) as exc_info:
-            translate_scalar("created", "20231300120000", UTC)
-        assert exc_info.value.field == "created"
-        assert exc_info.value.value == "20231300120000"
-
-    def test_unrecognized_value_raises(self) -> None:
-        # A value that is not a keyword, digits, ISO date, or compact timestamp
-        # raises rather than producing invalid Tantivy syntax or silently matching
-        # nothing.
-        with pytest.raises(InvalidDateQuery) as exc_info:
-            translate_scalar("created", "garbage", UTC)
-        assert exc_info.value.field == "created"
-        assert exc_info.value.value == "garbage"
-
-
-@pytest.mark.search
-class TestTranslateRange:
-    @pytest.mark.parametrize(
-        ("lo", "hi", "expected"),
-        [
-            ("2005", "2009", "created:[2005-01-01T00:00:00Z TO 2010-01-01T00:00:00Z}"),
-            (
-                "202001",
-                "202006",
-                "created:[2020-01-01T00:00:00Z TO 2020-07-01T00:00:00Z}",
-            ),
-            (
-                "20200101",
-                "20201231",
-                "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",
-            ),
-            (
-                "2020-01-01",
-                "2020-12-31",
-                "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",
-            ),
-        ],
-    )
-    def test_absolute_ranges(self, lo, hi, expected):
-        assert translate_range("created", lo, hi, UTC) == expected
-
-    def test_reversed_swaps(self):
-        assert translate_range("created", "2009", "2005", UTC) == (
-            "created:[2005-01-01T00:00:00Z TO 2010-01-01T00:00:00Z}"
-        )
-
-    def test_open_upper(self):
-        out = translate_range("created", "2020", "", UTC)
-        assert out == f"created:[2020-01-01T00:00:00Z TO {OPEN_HI}]"
-
-    def test_open_lower(self):
-        out = translate_range("created", "", "2020", UTC)
-        assert out == f"created:[{OPEN_LO} TO 2021-01-01T00:00:00Z}}"
-
-    def test_invalid_bound_raises(self):
-        with pytest.raises(InvalidDateQuery) as exc_info:
-            translate_range("created", "202023", "2025", UTC)
-        assert exc_info.value.field == "created"
-        assert exc_info.value.value == "202023"
-
-    def test_invalid_high_bound_raises(self):
-        # Low bound parses, high bound does not -> raise on the high bound.
-        with pytest.raises(InvalidDateQuery) as exc_info:
-            translate_range("created", "2020", "garbage", UTC)
-        assert exc_info.value.field == "created"
-        assert exc_info.value.value == "garbage"
-
-
-@pytest.mark.search
-class TestTranslateQuery:
-    @pytest.mark.parametrize(
-        ("raw", "expected"),
-        [
-            (
-                "created:2020",
-                "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",
-            ),
-            ("tag:foo,bar", "tag:foo AND tag:bar"),
-            # 'type' is a user-facing alias rewritten to 'document_type' (the real schema field)
-            ("tag:foo,type:bar", "tag:foo AND document_type:bar"),
-            (
-                "created:[2020 TO 2021],added:[2022 TO 2023]",
-                (
-                    "created:[2020-01-01T00:00:00Z TO 2022-01-01T00:00:00Z}"
-                    " AND "
-                    "added:[2022-01-01T00:00:00Z TO 2024-01-01T00:00:00Z}"
-                ),
-            ),
-            # correspondent is not multi-value: comma stays literal inside the value
-            ("correspondent:foo,bar", "correspondent:foo,bar"),
-        ],
-    )
-    def test_golden(self, raw: str, expected: str) -> None:
-        assert translate_query(raw, UTC) == expected
-
-    @pytest.mark.parametrize(
-        "raw",
-        [
-            "created:2020",
-            "created:202003",
-            "created:[20200101 TO 20201231]",
-            "created:[2020-01-01 TO 2020-12-31]",
-            "created:[2020 to]",
-            "created:[to 2020]",
-            "title:x,created:[2020 TO 2021]",
-            "created:2020 OR foo",
-            "(created:2020 OR invoice)",
-            "tag:foo,type:bar",
-            "bank statement",
-        ],
-    )
-    def test_parse_acceptance(self, index: tantivy.Index, raw: str) -> None:
-        translated = translate_query(raw, UTC)
-        # Must not raise:
-        index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
-
-
-@pytest.mark.search
-class TestFieldAliasing:
-    """Whoosh->Tantivy field-name aliasing (type/path -> document_type/storage_path)."""
-
-    def test_type_alias(self) -> None:
-        assert translate_query("type:invoice", UTC) == "document_type:invoice"
-
-    def test_path_alias(self) -> None:
-        assert translate_query("path:/foo/bar", UTC) == "storage_path:/foo/bar"
-
-    def test_type_id_alias(self) -> None:
-        assert translate_query("type_id:5", UTC) == "document_type_id:5"
-
-    def test_path_id_alias(self) -> None:
-        assert translate_query("path_id:7", UTC) == "storage_path_id:7"
-
-    def test_clause_separator_plus_alias(self) -> None:
-        # Comma between known fields acts as AND separator; alias still applied.
-        assert (
-            translate_query("tag:foo,type:bar", UTC) == "tag:foo AND document_type:bar"
-        )
-
-    def test_type_range_alias(self) -> None:
-        # type is not a date field; range passes through verbatim with alias applied.
-        assert (
-            translate_query("type:[2020 TO 2021]", UTC)
-            == "document_type:[2020 TO 2021]"
-        )
-
-    def test_parse_acceptance_type(self, index: tantivy.Index) -> None:
-        # Translated output must be accepted by the real Tantivy parser.
-        translated = translate_query("type:invoice", UTC)
-        index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
-
-    def test_parse_acceptance_path(self, index: tantivy.Index) -> None:
-        translated = translate_query("path:foo", UTC)
-        index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
-
-
-# Freeze time so relative-date tests are deterministic.
-_FROZEN_NOW = datetime(2026, 3, 28, 12, 0, 0, tzinfo=UTC)
-
-
-@pytest.mark.search
-class TestRelativeRanges:
-    """Relative date-range tokens resolved against a frozen clock."""
-
-    @time_machine.travel(_FROZEN_NOW, tick=False)
-    def test_minus_7_days_to_now(self) -> None:
-        assert translate_query("added:[-7 days to now]", UTC) == (
-            "added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]"
-        )
-
-    @time_machine.travel(_FROZEN_NOW, tick=False)
-    def test_minus_1_week_to_now(self) -> None:
-        assert translate_query("added:[-1 week to now]", UTC) == (
-            "added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]"
-        )
-
-    @time_machine.travel(_FROZEN_NOW, tick=False)
-    def test_minus_1_month_to_now(self) -> None:
-        assert translate_query("created:[-1 month to now]", UTC) == (
-            "created:[2026-02-28T12:00:00Z TO 2026-03-28T12:00:00Z]"
-        )
-
-    @time_machine.travel(_FROZEN_NOW, tick=False)
-    def test_minus_1_year_to_now(self) -> None:
-        assert translate_query("modified:[-1 year to now]", UTC) == (
-            "modified:[2025-03-28T12:00:00Z TO 2026-03-28T12:00:00Z]"
-        )
-
-    @time_machine.travel(_FROZEN_NOW, tick=False)
-    def test_minus_3_hours_to_now(self) -> None:
-        assert translate_query("added:[-3 hours to now]", UTC) == (
-            "added:[2026-03-28T09:00:00Z TO 2026-03-28T12:00:00Z]"
-        )
-
-    @time_machine.travel(_FROZEN_NOW, tick=False)
-    def test_uppercase_units(self) -> None:
-        assert translate_query("added:[-1 WEEK TO NOW]", UTC) == (
-            "added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]"
-        )
-
-    @time_machine.travel(_FROZEN_NOW, tick=False)
-    def test_now_minus_7d_compact(self) -> None:
-        assert translate_query("added:[now-7d TO now]", UTC) == (
-            "added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]"
-        )
-
-    @time_machine.travel(_FROZEN_NOW, tick=False)
-    def test_reversed_range_swapped(self) -> None:
-        # now+1h TO now-1h is reversed; translate_range swaps -> lo=now-1h, hi=now+1h
-        assert translate_query("added:[now+1h TO now-1h]", UTC) == (
-            "added:[2026-03-28T11:00:00Z TO 2026-03-28T13:00:00Z]"
-        )
-
-    @pytest.mark.parametrize(
-        "raw",
-        [
-            "added:[-7 days to now]",
-            "added:[-1 week to now]",
-            "created:[-1 month to now]",
-            "modified:[-1 year to now]",
-            "added:[-3 hours to now]",
-            "added:[now-7d TO now]",
-            "added:[now+1h TO now-1h]",
-        ],
-    )
-    @time_machine.travel(_FROZEN_NOW, tick=False)
-    def test_parse_acceptance(self, index: tantivy.Index, raw: str) -> None:
-        translated = translate_query(raw, UTC)
-        index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
-
-
-@pytest.mark.search
-class TestWhooshUnitAbbreviations:
-    """
-    Whoosh's PlusMinus date grammar accepted abbreviated unit spellings
-    (e.g. "yrs", "mos", "wks", "hrs", "mins", "secs"); saved views/searches
-    created under the old Whoosh backend can contain those tokens (see
-    https://github.com/paperless-ngx/paperless-ngx/issues/13482), so the
-    Tantivy translator must still accept them.
-    """
-
-    @time_machine.travel(_FROZEN_NOW, tick=False)
-    def test_minus_999_yrs(self) -> None:
-        assert translate_query("created:[-999yrs to now]", UTC) == (
-            "created:[1027-03-28T12:00:00Z TO 2026-03-28T12:00:00Z]"
-        )
-
-    @pytest.mark.parametrize(
-        ("token", "expected_lo"),
-        [
-            ("-1y", "2025-03-28T12:00:00Z"),
-            ("-1yr", "2025-03-28T12:00:00Z"),
-            ("-3mos", "2025-12-28T12:00:00Z"),
-            ("-3mo", "2025-12-28T12:00:00Z"),
-            ("-2wks", "2026-03-14T12:00:00Z"),
-            ("-2wk", "2026-03-14T12:00:00Z"),
-            ("-5dys", "2026-03-23T12:00:00Z"),
-            ("-5dy", "2026-03-23T12:00:00Z"),
-            ("-1hrs", "2026-03-28T11:00:00Z"),
-            ("-1hr", "2026-03-28T11:00:00Z"),
-            ("-10mins", "2026-03-28T11:50:00Z"),
-            ("-10min", "2026-03-28T11:50:00Z"),
-            ("-30secs", "2026-03-28T11:59:30Z"),
-            ("-30sec", "2026-03-28T11:59:30Z"),
-        ],
-    )
-    @time_machine.travel(_FROZEN_NOW, tick=False)
-    def test_abbreviated_units(self, token: str, expected_lo: str) -> None:
-        assert translate_query(f"added:[{token} to now]", UTC) == (
-            f"added:[{expected_lo} TO 2026-03-28T12:00:00Z]"
-        )
-
-    @pytest.mark.parametrize(
-        "raw",
-        [
-            "created:[-999yrs to now]",
-            "added:[-1y to now]",
-            "created:[-3mos to now]",
-            "added:[-2wks to now]",
-            "added:[-5dys to now]",
-            "added:[-1hrs to now]",
-            "added:[-10mins to now]",
-            "added:[-30secs to now]",
-        ],
-    )
-    @time_machine.travel(_FROZEN_NOW, tick=False)
-    def test_parse_acceptance(self, index: tantivy.Index, raw: str) -> None:
-        translated = translate_query(raw, UTC)
-        index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
-
-
-@pytest.mark.search
-class TestOperatorNormalization:
-    """Post-render operator normalization in translate_query."""
-
-    def test_spaced_dash_removed(self) -> None:
-        assert (
-            translate_query("H52.1 - Kurzsichtigkeit", UTC) == "H52.1 Kurzsichtigkeit"
-        )
-
-    def test_spaced_dash_simple(self) -> None:
-        assert translate_query("bar - baz", UTC) == "bar baz"
-
-    def test_trailing_operator_stripped(self) -> None:
-        assert translate_query("foo -", UTC) == "foo"
-
-    def test_date_range_preserved(self) -> None:
-        out = translate_query("created:[2020 TO 2021]", UTC)
-        # Must not corrupt the ISO range
-        assert out == "created:[2020-01-01T00:00:00Z TO 2022-01-01T00:00:00Z}"
-
-    def test_date_scalar_with_or(self) -> None:
-        out = translate_query("created:2020 OR foo", UTC)
-        # The created scalar becomes a range; " OR foo" passes through verbatim.
-        assert out.startswith("created:[")
-        assert "OR foo" in out
-
-    def test_parse_acceptance_spaced_dash(self, index: tantivy.Index) -> None:
-        translated = translate_query("H52.1 - Kurzsichtigkeit", UTC)
-        index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
-
-    def test_parse_acceptance_trailing_op(self, index: tantivy.Index) -> None:
-        translated = translate_query("foo -", UTC)
-        index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
-
-
-@pytest.mark.search
-class TestMultiWordDateKeywords:
-    """scan() must consume multi-word date keywords as a single value."""
-
-    def test_scan_previous_week_as_single_token(self) -> None:
-        # "created:previous week" must produce one FieldValue with value "previous week",
-        # not FieldValue("created","previous") + Passthrough(" week").
-        toks = scan("created:previous week")
-        assert toks == [FieldValue("created", "previous week")]
-
-    def test_scan_this_month_as_single_token(self) -> None:
-        toks = scan("added:this month")
-        assert toks == [FieldValue("added", "this month")]
-
-    def test_scan_previous_month_as_single_token(self) -> None:
-        toks = scan("created:previous month")
-        assert toks == [FieldValue("created", "previous month")]
-
-    def test_scan_this_year_as_single_token(self) -> None:
-        toks = scan("added:this year")
-        assert toks == [FieldValue("added", "this year")]
-
-    def test_scan_previous_year_as_single_token(self) -> None:
-        toks = scan("created:previous year")
-        assert toks == [FieldValue("created", "previous year")]
-
-    def test_scan_previous_quarter_as_single_token(self) -> None:
-        toks = scan("created:previous quarter")
-        assert toks == [FieldValue("created", "previous quarter")]
-
-    def test_quoted_multi_word_keyword_still_works(self) -> None:
-        # The quoted form must continue to work as before.
-        toks = scan('created:"previous week"')
-        assert toks == [FieldValue("created", '"previous week"')]
-
-    def test_non_date_field_not_affected(self) -> None:
-        # "previous" stops at the space for non-date fields; " week" passes through.
-        toks = scan("correspondent:previous week")
-        assert toks == [
-            FieldValue("correspondent", "previous"),
-            Passthrough(" week"),
-        ]
-
-
-@pytest.mark.search
-class TestKeywordDateResolution:
-    """Relative date keywords resolve to exact ISO ranges against a frozen clock.
-
-    Frozen at 2026-03-28 12:00 UTC (a Saturday in Q1) so the week, month,
-    quarter and year rollovers are all exercised by a single anchor.
-    """
-
-    # created is a DateField: bounds are UTC midnight, no timezone offset.
-    @pytest.mark.parametrize(
-        ("keyword", "expected"),
-        [
-            pytest.param(
-                "today",
-                "created:[2026-03-28T00:00:00Z TO 2026-03-29T00:00:00Z}",
-                id="today",
-            ),
-            pytest.param(
-                "yesterday",
-                "created:[2026-03-27T00:00:00Z TO 2026-03-28T00:00:00Z}",
-                id="yesterday",
-            ),
-            pytest.param(
-                "previous week",
-                "created:[2026-03-16T00:00:00Z TO 2026-03-23T00:00:00Z}",
-                id="previous-week",
-            ),
-            pytest.param(
-                "this month",
-                "created:[2026-03-01T00:00:00Z TO 2026-04-01T00:00:00Z}",
-                id="this-month",
-            ),
-            pytest.param(
-                "previous month",
-                "created:[2026-02-01T00:00:00Z TO 2026-03-01T00:00:00Z}",
-                id="previous-month",
-            ),
-            pytest.param(
-                "this year",
-                "created:[2026-01-01T00:00:00Z TO 2027-01-01T00:00:00Z}",
-                id="this-year",
-            ),
-            pytest.param(
-                "previous year",
-                "created:[2025-01-01T00:00:00Z TO 2026-01-01T00:00:00Z}",
-                id="previous-year",
-            ),
-            pytest.param(
-                "previous quarter",
-                "created:[2025-10-01T00:00:00Z TO 2026-01-01T00:00:00Z}",
-                id="previous-quarter",
-            ),
-        ],
-    )
-    @time_machine.travel(_FROZEN_NOW, tick=False)
-    def test_date_only_field_keyword_ranges(
-        self,
-        keyword: str,
-        expected: str,
-    ) -> None:
-        assert translate_query(f"created:{keyword}", UTC) == expected
-
-    # added is a DateTimeField: local-tz midnight converted to UTC. Tokyo
-    # (+09:00, no DST) shifts each midnight boundary back to 15:00Z the day
-    # before, so this also exercises the local-midnight offset path.
-    @pytest.mark.parametrize(
-        ("keyword", "expected"),
-        [
-            pytest.param(
-                "today",
-                "added:[2026-03-27T15:00:00Z TO 2026-03-28T15:00:00Z}",
-                id="today",
-            ),
-            pytest.param(
-                "yesterday",
-                "added:[2026-03-26T15:00:00Z TO 2026-03-27T15:00:00Z}",
-                id="yesterday",
-            ),
-            pytest.param(
-                "previous week",
-                "added:[2026-03-15T15:00:00Z TO 2026-03-22T15:00:00Z}",
-                id="previous-week",
-            ),
-            pytest.param(
-                "this month",
-                "added:[2026-02-28T15:00:00Z TO 2026-03-31T15:00:00Z}",
-                id="this-month",
-            ),
-            pytest.param(
-                "previous month",
-                "added:[2026-01-31T15:00:00Z TO 2026-02-28T15:00:00Z}",
-                id="previous-month",
-            ),
-            pytest.param(
-                "this year",
-                "added:[2025-12-31T15:00:00Z TO 2026-12-31T15:00:00Z}",
-                id="this-year",
-            ),
-            pytest.param(
-                "previous year",
-                "added:[2024-12-31T15:00:00Z TO 2025-12-31T15:00:00Z}",
-                id="previous-year",
-            ),
-            pytest.param(
-                "previous quarter",
-                "added:[2025-09-30T15:00:00Z TO 2025-12-31T15:00:00Z}",
-                id="previous-quarter",
-            ),
-        ],
-    )
-    @time_machine.travel(_FROZEN_NOW, tick=False)
-    def test_datetime_field_keyword_ranges_local_tz(
-        self,
-        keyword: str,
-        expected: str,
-    ) -> None:
-        assert translate_query(f"added:{keyword}", ZoneInfo("Asia/Tokyo")) == expected
-
-
-@pytest.mark.search
-class TestISODatetimeBounds:
-    """Full ISO datetime tokens in range bounds must be parsed directly."""
-
-    def test_translate_range_iso_bounds_passthrough(self) -> None:
-        # Already-ISO datetime bounds must pass through as-is (exact instant).
-        result = translate_range(
-            "created",
-            "2020-01-01T00:00:00Z",
-            "2021-01-01T00:00:00Z",
-            UTC,
-        )
-        assert result == "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z]"
-
-    def test_translate_query_iso_range_preserved(self) -> None:
-        q = "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
-        assert translate_query(q, UTC) == q
-
-    def test_translate_query_comma_separated_iso_ranges(self) -> None:
-        q = (
-            "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z],"
-            "added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
-        )
-        result = translate_query(q, UTC)
-        assert result == (
-            "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
-            " AND "
-            "added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
-        )
-
-    def test_translate_query_text_before_comma_separated_date_clause(self) -> None:
-        result = translate_query("schäfersee,created:previous year", UTC)
-        assert result == (
-            "schäfersee AND created:[2025-01-01T00:00:00Z TO 2026-01-01T00:00:00Z}"
-        )
-
-    def test_invalid_iso_datetime_raises(self) -> None:
-        # A token with "T" that is not valid ISO datetime -> raise.
-        with pytest.raises(InvalidDateQuery) as exc_info:
-            translate_range(
-                "created",
-                "2020-01-01T99:00:00Z",
-                "2021-01-01T00:00:00Z",
-                UTC,
-            )
-        assert exc_info.value.field == "created"
-        assert exc_info.value.value == "2020-01-01T99:00:00Z"
-
-    def test_parse_acceptance_iso_bounds(self, index: tantivy.Index) -> None:
-        q = "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
-        translated = translate_query(q, UTC)
-        index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
-
-    def test_parse_acceptance_comma_iso_ranges(self, index: tantivy.Index) -> None:
-        q = (
-            "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z],"
-            "added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
-        )
-        translated = translate_query(q, UTC)
-        index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)

src/documents/tests/search/test_unicode_normalization.py

@@ -0,0 +1,184 @@
+"""Search text is held in one Unicode normal form on both sides.
+
+A bigram is a pair of codepoints, so decomposed and composed spellings of
+the same Japanese word produce different bigrams. Unless the indexed text
+and the query string are normalized the same way, a document written one
+way is invisible to a query written the other.
+"""
+
+from __future__ import annotations
+
+import unicodedata
+from typing import TYPE_CHECKING
+
+import pytest
+
+from documents.models import Correspondent
+from documents.models import CustomField
+from documents.models import CustomFieldInstance
+from documents.models import DocumentType
+from documents.models import StoragePath
+from documents.models import Tag
+from documents.search._query import normalize_search_text
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from documents.models import Document
+    from documents.search._backend import TantivyBackend
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+# がっこうの書類 ("school documents"). The が is precomposed in NFC and
+# か + U+3099 in NFD, so the two spellings differ by one codepoint.
+_NFC = "がっこうの書類"
+_NFD = unicodedata.normalize("NFD", _NFC)
+# The bare word, for fielded queries against a name or filename.
+_NFC_WORD = "がっこう"
+_NFD_WORD = unicodedata.normalize("NFD", _NFC_WORD)
+
+
+class TestTheNormalizer:
+    def test_it_composes_decomposed_kana(self) -> None:
+        """
+        GIVEN:
+            - The same word spelled decomposed and composed
+        WHEN:
+            - Each is normalized
+        THEN:
+            - Both become the composed spelling. The inputs really do
+              differ, so the fixture is not vacuous
+        """
+        assert _NFD != _NFC
+        assert normalize_search_text(_NFD) == _NFC
+        assert normalize_search_text(_NFC) == _NFC
+
+    def test_it_leaves_halfwidth_katakana_alone(self) -> None:
+        """
+        GIVEN:
+            - A halfwidth katakana word carrying a voiced sound mark
+        WHEN:
+            - It is normalized
+        THEN:
+            - It is unchanged. Halfwidth katakana has no precomposed
+              voiced form, which is why _CJK_RE still has to list the
+              marks rather than rely on this
+        """
+        assert normalize_search_text("ﾊﾟﾝ") == "ﾊﾟﾝ"
+
+
+class TestEitherSpellingFindsEither:
+    @pytest.mark.parametrize(
+        "content",
+        [
+            pytest.param(_NFC, id="composed_document"),
+            pytest.param(_NFD, id="decomposed_document"),
+        ],
+    )
+    @pytest.mark.parametrize(
+        "query",
+        [
+            pytest.param("がっこう", id="composed_query"),
+            pytest.param(
+                unicodedata.normalize("NFD", "がっこう"),
+                id="decomposed_query",
+            ),
+        ],
+    )
+    def test_a_document_is_found_whichever_way_each_side_is_spelled(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+        content: str,
+        query: str,
+    ) -> None:
+        """
+        GIVEN:
+            - A document holding the word inside a longer unspaced run,
+              spelled composed or decomposed, and an unrelated document
+        WHEN:
+            - The word is searched, spelled composed or decomposed
+        THEN:
+            - It matches in all four combinations. Without normalizing
+              both sides, the decomposed run splits at the combining mark
+              and the two spellings produce different bigrams
+        """
+        match = index_document(title="A", content=content)
+        index_document(title="B", content="invoice only")
+
+        assert matched_ids(query) == {match.pk}
+
+
+class TestEverySearchableFieldIsNormalized:
+    """The query side is normalized in _parse_query, so every searchable
+    field has to be normalized on the way in as well.
+
+    A field left out is worse than normalizing nothing: both sides used to
+    be decomposed and matched each other, so normalizing only the query
+    turns a working search into no results. original_filename is the one
+    most likely to hold NFD in practice, since macOS stores filenames
+    decomposed.
+    """
+
+    @pytest.mark.parametrize(
+        "field",
+        [
+            pytest.param("title", id="title"),
+            pytest.param("content", id="content"),
+            pytest.param("original_filename", id="original_filename"),
+            pytest.param("correspondent", id="correspondent"),
+            pytest.param("document_type", id="document_type"),
+            pytest.param("storage_path", id="storage_path"),
+            pytest.param("tag", id="tag"),
+            pytest.param("custom_fields.value", id="custom_field"),
+        ],
+    )
+    def test_a_composed_query_finds_a_decomposed_value(
+        self,
+        backend: TantivyBackend,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+        field: str,
+    ) -> None:
+        """
+        GIVEN:
+            - A document carrying a decomposed Japanese word in one
+              searchable field, and an unrelated document
+        WHEN:
+            - The composed spelling is searched, fielded to that field
+        THEN:
+            - The document matches. The query is normalized either way, so
+              a field left unnormalized on the way in can never be found
+        """
+        kwargs: dict[str, object] = {"title": "A", "content": "invoice"}
+        if field == "correspondent":
+            kwargs["correspondent"] = Correspondent.objects.create(name=_NFD_WORD)
+        elif field == "document_type":
+            kwargs["document_type"] = DocumentType.objects.create(name=_NFD_WORD)
+        elif field == "storage_path":
+            kwargs["storage_path"] = StoragePath.objects.create(
+                name=_NFD_WORD,
+                path="archive/",
+            )
+        elif field in {"title", "content", "original_filename"}:
+            kwargs[field] = _NFD_WORD
+
+        doc = index_document(**kwargs)
+
+        if field == "tag":
+            doc.tags.add(Tag.objects.create(name=_NFD_WORD))
+        elif field == "custom_fields.value":
+            CustomFieldInstance.objects.create(
+                document=doc,
+                field=CustomField.objects.create(
+                    name="Note",
+                    data_type=CustomField.FieldDataType.STRING,
+                ),
+                value_text=_NFD_WORD,
+            )
+        # The relations above are attached after the factory built the
+        # document, so the index needs the newer state.
+        backend.add_or_update(doc)
+        index_document(title="B", content="invoice only")
+
+        assert matched_ids(f"{field}:{_NFC_WORD}") == {doc.pk}

src/documents/tests/search/test_widen_leaf.py

@@ -0,0 +1,127 @@
+"""_widen_leaf, the hook that adds a CJK alternative, a fuzzy one, or both.
+
+The alternatives are independent: a leaf can qualify for either, and a
+negated leaf keeps its CJK side while losing its fuzzy one.
+"""
+
+from __future__ import annotations
+
+import pytest
+import whoosh_compat.ast as wc_ast
+
+from documents.search._query import _cjk_alternative
+from documents.search._query import _fuzzy_alternative
+from documents.search._query import _widen_leaf
+from documents.tests.search._ast_helpers import NOTES
+from documents.tests.search._ast_helpers import content
+
+pytestmark = pytest.mark.search
+
+
+def _widen(
+    leaf: wc_ast.Term,
+    *,
+    fuzzy: bool = True,
+    negated: frozenset[int] = frozenset(),
+) -> wc_ast.Node:
+    return _widen_leaf(leaf, fuzzy=fuzzy, negated=negated)
+
+
+class TestWhichAlternativesAreAdded:
+    def test_a_latin_leaf_gains_only_the_fuzzy_side(self) -> None:
+        """
+        GIVEN:
+            - A latin term, which has no CJK in it
+        WHEN:
+            - The hook runs with fuzzy on
+        THEN:
+            - The Or holds the leaf and the boosted fuzzy alternative,
+              and nothing else
+        """
+        leaf = content("invoice")
+
+        assert _widen(leaf) == wc_ast.Or(
+            children=(
+                leaf,
+                wc_ast.Boosted(child=_fuzzy_alternative(leaf), boost=0.1),
+            ),
+        )
+
+    def test_a_cjk_leaf_gains_only_the_cjk_side_when_fuzzy_is_off(self) -> None:
+        """
+        GIVEN:
+            - A CJK term
+        WHEN:
+            - The hook runs with fuzzy off
+        THEN:
+            - Only the bigram alternative is added, which is exactly what
+              the CJK work shipped
+        """
+        leaf = content("東京")
+
+        assert _widen(leaf, fuzzy=False) == wc_ast.Or(
+            children=(leaf, _cjk_alternative(leaf)),
+        )
+
+    def test_a_cjk_leaf_gains_only_its_bigram_side(self) -> None:
+        """
+        GIVEN:
+            - A CJK term
+        WHEN:
+            - The hook runs with fuzzy on
+        THEN:
+            - The Or holds the leaf and the bigram alternative, and no
+              fuzzy one. A prefix Fuzzy over a whole unspaced run matches
+              any run within one edit of its start, and _cjk_alternative
+              already supplies the in-run recall
+        """
+        leaf = content("東京")
+
+        assert _fuzzy_alternative(leaf) is None
+        assert _widen(leaf) == wc_ast.Or(children=(leaf, _cjk_alternative(leaf)))
+
+    def test_a_negated_leaf_keeps_cjk_and_loses_fuzzy(self) -> None:
+        """
+        GIVEN:
+            - A CJK term whose id is in the negated set
+        WHEN:
+            - The hook runs with fuzzy on
+        THEN:
+            - The bigram alternative survives and the fuzzy one does not.
+              NOT X should exclude what X matches, which needs the bigram
+              side, but prefix fuzzy matching would exclude far more
+        """
+        leaf = content("東京")
+
+        assert _widen(leaf, negated=frozenset({id(leaf)})) == wc_ast.Or(
+            children=(leaf, _cjk_alternative(leaf)),
+        )
+
+    def test_a_negated_latin_leaf_is_returned_unchanged(self) -> None:
+        """
+        GIVEN:
+            - A latin term whose id is in the negated set
+        WHEN:
+            - The hook runs with fuzzy on
+        THEN:
+            - The leaf itself comes back. It qualifies for no alternative
+              at all, so there is no Or to build
+        """
+        leaf = content("tax")
+
+        assert _widen(leaf, negated=frozenset({id(leaf)})) is leaf
+
+    def test_a_leaf_outside_the_default_fields_is_returned_unchanged(self) -> None:
+        """
+        GIVEN:
+            - A term on a JSON subpath field, which is not one of the five
+              default search fields
+        WHEN:
+            - The hook runs with fuzzy on
+        THEN:
+            - The leaf comes back untouched. Widening is scoped to the
+              default search fields, as it was for CJK
+        """
+        leaf = wc_ast.Term(field=NOTES, text="invoice")
+
+        assert _widen(leaf) is leaf

src/documents/tests/test_admin.py

@@ -16,11 +16,11 @@ from documents.models import Document
 from documents.models import Tag
 from documents.search import get_backend
 from documents.search import reset_backend
-from documents.tests.factories import DocumentFactory
-from documents.tests.factories import TagFactory
-from documents.tests.factories import UserFactory
-from documents.tests.utils import DirectoriesMixin
 from paperless.admin import PaperlessUserAdmin
+from paperless_testing.dirs import DirectoriesMixin
+from paperless_testing.factories import DocumentFactory
+from paperless_testing.factories import TagFactory
+from paperless_testing.factories import UserFactory
 
 
 @pytest.fixture

src/documents/tests/test_api_app_config.py

@@ -3,7 +3,6 @@ from io import BytesIO
 from pathlib import Path
 from unittest.mock import patch
 
-from django.contrib.auth.models import User
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.test import override_settings
 from PIL import Image
@@ -11,10 +10,11 @@ from PIL.PngImagePlugin import PngInfo
 from rest_framework import status
 from rest_framework.test import APITestCase
 
-from documents.tests.utils import DirectoriesMixin
-from documents.tests.utils import read_streaming_response
 from paperless.models import ApplicationConfiguration
 from paperless.models import ColorConvertChoices
+from paperless_testing.dirs import DirectoriesMixin
+from paperless_testing.factories import UserFactory
+from paperless_testing.http import read_streaming_response
 
 
 class TestApiAppConfig(DirectoriesMixin, APITestCase):
@@ -23,7 +23,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        user = User.objects.create_superuser(username="temp_admin")
+        user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=user)
 
     def test_api_get_config(self) -> None:
@@ -74,6 +74,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
                 "barcode_enable_tag": None,
                 "barcode_tag_mapping": None,
                 "barcode_tag_split": None,
+                "barcode_store_values": None,
                 "remote_ocr_engine": None,
                 "remote_ocr_api_key": None,
                 "remote_ocr_endpoint": None,
@@ -81,6 +82,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
                 "ai_enabled": None,
                 "llm_embedding_backend": None,
                 "llm_embedding_model": None,
+                "llm_embedding_api_key": None,
                 "llm_embedding_endpoint": None,
                 "llm_embedding_chunk_size": None,
                 "llm_context_size": None,
@@ -194,6 +196,70 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
         self.assertEqual(config.language, None)
         self.assertEqual(config.barcode_tag_mapping, None)
 
+    def test_api_update_config_json_objects(self) -> None:
+        """
+        GIVEN:
+            - API request to update app config with JSON objects for the
+              user_args and barcode_tag_mapping JSONFields
+        WHEN:
+            - API is called
+        THEN:
+            - Correct HTTP response
+            - Both objects are stored as sent
+        """
+        user_args = {"unpaper_args": "--pre-rotate 90", "jobs": 2}
+        barcode_tag_mapping = {"TAG:(.*)": "\\g<1>", "ASN12.*": ""}
+        response = self.client.patch(
+            f"{self.ENDPOINT}1/",
+            json.dumps(
+                {
+                    "user_args": json.dumps(user_args),
+                    "barcode_tag_mapping": json.dumps(barcode_tag_mapping),
+                },
+            ),
+            content_type="application/json",
+        )
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        config = ApplicationConfiguration.objects.first()
+        assert config is not None
+        self.assertEqual(config.user_args, user_args)
+        self.assertEqual(config.barcode_tag_mapping, barcode_tag_mapping)
+
+    def test_api_update_config_rejects_invalid_json_objects(self) -> None:
+        """
+        GIVEN:
+            - API request to update app config with a JSON-encoded value that
+              is not an object for user_args or barcode_tag_mapping, or a
+              barcode_tag_mapping with a non-string substitute
+        WHEN:
+            - API is called
+        THEN:
+            - Request is rejected with a 400 naming the problem
+            - Config is not updated
+        """
+        not_objects = (True, 1, [1, 2, 3], "not a dict")
+        cases = [
+            (field, value, b"must be a JSON object")
+            for field in ("user_args", "barcode_tag_mapping")
+            for value in not_objects
+        ]
+        cases += [
+            ("barcode_tag_mapping", {"TAG:(.*)": 5}, b"values must be strings"),
+            ("barcode_tag_mapping", {"TAG:(.*)": None}, b"values must be strings"),
+        ]
+        for field, value, expected_message in cases:
+            with self.subTest(field=field, value=value):
+                response = self.client.patch(
+                    f"{self.ENDPOINT}1/",
+                    json.dumps({field: json.dumps(value)}),
+                    content_type="application/json",
+                )
+                self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+                self.assertIn(expected_message, response.content)
+                config = ApplicationConfiguration.objects.first()
+                assert config is not None
+                self.assertIsNone(getattr(config, field))
+
     def test_api_replace_app_logo(self) -> None:
         """
         GIVEN:
@@ -203,7 +269,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
         THEN:
             - old app_logo file is deleted
         """
-        admin = User.objects.create_superuser(username="admin")
+        admin = UserFactory(username="admin", superuser=True)
         self.client.force_login(user=admin)
         response = self.client.get("/logo/")
         self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
@@ -858,6 +924,49 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_405_METHOD_NOT_ALLOWED)
         self.assertEqual(ApplicationConfiguration.objects.count(), 1)
 
+    def test_update_llm_embedding_api_key(self) -> None:
+        """
+        GIVEN:
+            - Existing config with llm_embedding_api_key specified
+        WHEN:
+            - API to update llm_embedding_api_key is called with all *s
+            - API to update llm_embedding_api_key is called with empty string
+        THEN:
+            - llm_embedding_api_key is unchanged
+            - llm_embedding_api_key is set to None
+        """
+        config = ApplicationConfiguration.objects.first()
+        assert config is not None
+        config.llm_embedding_api_key = "1234567890"
+        config.save()
+
+        # Test with all *
+        response = self.client.patch(
+            f"{self.ENDPOINT}1/",
+            json.dumps(
+                {
+                    "llm_embedding_api_key": "*" * 32,
+                },
+            ),
+            content_type="application/json",
+        )
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        config.refresh_from_db()
+        self.assertEqual(config.llm_embedding_api_key, "1234567890")
+        # Test with empty string
+        response = self.client.patch(
+            f"{self.ENDPOINT}1/",
+            json.dumps(
+                {
+                    "llm_embedding_api_key": "",
+                },
+            ),
+            content_type="application/json",
+        )
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        config.refresh_from_db()
+        self.assertEqual(config.llm_embedding_api_key, None)
+
     def test_update_llm_api_key(self) -> None:
         """
         GIVEN:

src/documents/tests/test_api_bulk_download.py

@@ -4,7 +4,6 @@ import json
 import shutil
 import zipfile
 
-from django.contrib.auth.models import User
 from django.test import override_settings
 from django.utils import timezone
 from rest_framework import status
@@ -13,9 +12,11 @@ from rest_framework.test import APITestCase
 from documents.models import Correspondent
 from documents.models import Document
 from documents.models import DocumentType
-from documents.tests.utils import DirectoriesMixin
 from documents.tests.utils import SampleDirMixin
-from documents.tests.utils import read_streaming_response
+from paperless_testing.dirs import DirectoriesMixin
+from paperless_testing.factories import UserFactory
+from paperless_testing.http import read_streaming_response
+from paperless_testing.permissions import grant_global
 
 
 class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
@@ -24,7 +25,7 @@ class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.user = User.objects.create_superuser(username="temp_admin")
+        self.user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.user)
 
         self.doc1 = Document.objects.create(title="unrelated", checksum="A")
@@ -165,7 +166,15 @@ class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
             ),
             content_type="application/json",
         )
-        response.close()
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(response["Content-Type"], "application/zip")
+
+        with zipfile.ZipFile(io.BytesIO(read_streaming_response(response))) as zipf:
+            self.assertEqual(zipf.infolist()[0].compress_type, zipfile.ZIP_LZMA)
+
+            with self.doc2.source_file as f:
+                self.assertEqual(f.read(), zipf.read("2021-01-01 document A.pdf"))
 
     @override_settings(FILENAME_FORMAT="{correspondent}/{title}")
     def test_formatted_download_originals(self) -> None:
@@ -325,7 +334,8 @@ class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
                 )
 
     def test_download_insufficient_permissions(self) -> None:
-        user = User.objects.create_user(username="temp_user")
+        user = UserFactory(username="temp_user")
+        grant_global(user, "view_document")
         self.client.force_authenticate(user=user)
 
         self.doc2.owner = self.user
@@ -339,3 +349,29 @@ class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
 
         self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
         self.assertEqual(response.content, b"Insufficient permissions")
+
+    def test_bad_search_query_returns_400(self) -> None:
+        """
+        GIVEN:
+            - Bulk download request selects documents via a saved-search
+              query filter
+        WHEN:
+            - The query contains a malformed field value (an invalid date)
+        THEN:
+            - The response is a 400 naming the bad value, exactly like the
+              search list endpoint, never a 500
+        """
+        response = self.client.post(
+            self.ENDPOINT,
+            json.dumps(
+                {
+                    "all": True,
+                    "filters": {"query": "added:notadate"},
+                    "content": "originals",
+                },
+            ),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+        self.assertIn(b"notadate", response.content)

src/documents/tests/test_api_bulk_edit.py

@@ -2,27 +2,30 @@ import json
 from unittest import mock
 
 from auditlog.models import LogEntry
-from django.contrib.auth.models import Permission
 from django.contrib.auth.models import User
 from django.test import override_settings
-from guardian.shortcuts import assign_perm
 from rest_framework import status
 from rest_framework.test import APITestCase
 
 from documents.models import Correspondent
 from documents.models import CustomField
+from documents.models import CustomFieldInstance
 from documents.models import Document
 from documents.models import DocumentType
 from documents.models import StoragePath
 from documents.models import Tag
-from documents.tests.utils import DirectoriesMixin
+from paperless_testing.dirs import DirectoriesMixin
+from paperless_testing.factories import UserFactory
+from paperless_testing.permissions import grant_all_global
+from paperless_testing.permissions import grant_global
+from paperless_testing.permissions import grant_object
 
 
 class TestBulkEditAPI(DirectoriesMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        user = User.objects.create_superuser(username="temp_admin")
+        user = UserFactory(username="temp_admin", superuser=True)
         self.user = user
         self.client.force_authenticate(user=user)
 
@@ -284,9 +287,9 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         m,
     ) -> None:
         self.setup_mock(m, "modify_custom_fields")
-        user = User.objects.create_user(username="doc-owner")
-        user.user_permissions.add(Permission.objects.get(codename="change_document"))
-        other_user = User.objects.create_user(username="other-user")
+        user = UserFactory(username="doc-owner")
+        grant_global(user, "change_document")
+        other_user = UserFactory(username="other-user")
         source_doc = Document.objects.create(
             checksum="source",
             title="Source",
@@ -520,6 +523,30 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(args[0], [self.doc1.id])
         self.assertEqual(len(kwargs), 0)
 
+    @mock.patch("documents.views.bulk_edit.delete")
+    def test_delete_documents_endpoint_with_excluded_documents(self, m) -> None:
+        self.setup_mock(m, "delete")
+        response = self.client.post(
+            "/api/documents/delete/",
+            json.dumps(
+                {
+                    "all": True,
+                    "excluded_documents": [
+                        self.doc2.id,
+                        self.doc3.id,
+                        self.doc4.id,
+                        self.doc5.id,
+                    ],
+                },
+            ),
+            content_type="application/json",
+        )
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        m.assert_called_once()
+        args, kwargs = m.call_args
+        self.assertEqual(args[0], [self.doc1.id])
+        self.assertEqual(len(kwargs), 0)
+
     @mock.patch("documents.views.bulk_edit.reprocess")
     def test_reprocess_documents_endpoint(self, m) -> None:
         self.setup_mock(m, "reprocess")
@@ -691,6 +718,26 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
         self.assertIn(b"documents is required unless all is true", response.content)
 
+    def test_api_rejects_excluded_documents_unless_all_is_true(self) -> None:
+        response = self.client.post(
+            "/api/documents/bulk_edit/",
+            json.dumps(
+                {
+                    "documents": [self.doc1.id],
+                    "excluded_documents": [self.doc2.id],
+                    "method": "set_storage_path",
+                    "parameters": {"storage_path": self.sp1.id},
+                },
+            ),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+        self.assertIn(
+            b"excluded_documents is only supported when all is true",
+            response.content,
+        )
+
     @mock.patch("documents.serialisers.bulk_edit.set_storage_path")
     def test_api_bulk_edit_with_all_true_resolves_documents_from_filters(
         self,
@@ -717,13 +764,34 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(args[0], [self.doc2.id])
         self.assertEqual(kwargs["storage_path"], self.sp1.id)
 
+    @mock.patch("documents.serialisers.bulk_edit.set_storage_path")
+    def test_api_bulk_edit_with_all_true_excludes_documents(self, m) -> None:
+        self.setup_mock(m, "set_storage_path")
+
+        response = self.client.post(
+            "/api/documents/bulk_edit/",
+            json.dumps(
+                {
+                    "all": True,
+                    "excluded_documents": [self.doc2.id, self.doc4.id],
+                    "method": "set_storage_path",
+                    "parameters": {"storage_path": self.sp1.id},
+                },
+            ),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        m.assert_called_once()
+        args, kwargs = m.call_args
+        self.assertCountEqual(args[0], [self.doc1.id, self.doc3.id, self.doc5.id])
+        self.assertEqual(kwargs["storage_path"], self.sp1.id)
+
     @mock.patch("documents.serialisers.bulk_edit.set_storage_path")
     def test_api_bulk_edit_with_all_true_resolves_owned_duplicates(self, m) -> None:
         self.setup_mock(m, "set_storage_path")
-        user = User.objects.create_user(username="duplicate-owner")
-        user.user_permissions.add(
-            Permission.objects.get(codename="change_document"),
-        )
+        user = UserFactory(username="duplicate-owner")
+        grant_global(user, "change_document")
         first_duplicate = Document.objects.create(
             checksum="owned-duplicate",
             title="First duplicate",
@@ -767,6 +835,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         for filters in (
             {"text": "new doc 2017-03-16"},
             {"title_search": "apple"},
+            {"title_search": "apple", "query": "created:[-1 year to now]"},
         ):
             with self.subTest(filters=filters):
                 get_backend.return_value.search_ids.return_value = [self.doc2.id]
@@ -818,7 +887,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
                     "all": True,
                     "filters": {
                         "text": "B",
-                        "query": "c1",
+                        "title_search": "c1",
                     },
                     "method": "set_storage_path",
                     "parameters": {"storage_path": self.sp1.id},
@@ -1077,6 +1146,33 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
             ],
         )
 
+    def test_api_selection_data_with_excluded_documents(self) -> None:
+        response = self.client.post(
+            "/api/documents/selection_data/",
+            json.dumps(
+                {
+                    "all": True,
+                    "excluded_documents": [self.doc2.id],
+                },
+            ),
+            content_type="application/json",
+        )
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertCountEqual(
+            response.data["selected_correspondents"],
+            [
+                {"id": self.c1.id, "document_count": 0},
+                {"id": self.c2.id, "document_count": 1},
+            ],
+        )
+        self.assertCountEqual(
+            response.data["selected_tags"],
+            [
+                {"id": self.t1.id, "document_count": 1},
+                {"id": self.t2.id, "document_count": 2},
+            ],
+        )
+
     def test_api_selection_data_requires_view_permission(self) -> None:
         self.doc2.owner = self.user
         self.doc2.save()
@@ -1084,6 +1180,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         user1 = User.objects.create(username="user1")
         self.client.force_authenticate(user=user1)
 
+        grant_object(user1, self.doc2, "view_document")
+
         response = self.client.post(
             "/api/documents/selection_data/",
             json.dumps({"documents": [self.doc2.id]}),
@@ -1091,7 +1189,16 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         )
 
         self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
-        self.assertEqual(response.content, b"Insufficient permissions")
+
+        grant_global(user1, "view_document")
+        user1 = User.objects.get(pk=user1.pk)
+        self.client.force_authenticate(user=user1)
+        response = self.client.post(
+            "/api/documents/selection_data/",
+            json.dumps({"documents": [self.doc2.id]}),
+            content_type="application/json",
+        )
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
 
     @mock.patch("documents.serialisers.bulk_edit.set_permissions")
     def test_set_permissions(self, m) -> None:
@@ -1152,6 +1259,150 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertIn(b"set_permissions not specified", response.content)
         m.assert_not_called()
 
+    @mock.patch("documents.serialisers.bulk_edit.set_permissions")
+    def test_set_permissions_rejects_malformed_set_permissions(self, m) -> None:
+        """
+        GIVEN:
+            - A set_permissions bulk edit where set_permissions is not an
+              object, has a non-list users value, or has an unknown action
+        WHEN:
+            - API to bulk edit is called
+        THEN:
+            - API returns HTTP 400 describing the problem
+            - set_permissions is not called
+        """
+        self.setup_mock(m, "set_permissions")
+
+        for set_permissions, expected_message in (
+            (False, b"Expected a dictionary"),
+            ({"view": {"users": False}}, b"Expected a list"),
+            ({"not_a_real_action": {"users": [1]}}, b"Unknown permission action"),
+        ):
+            with self.subTest(set_permissions=set_permissions):
+                response = self.client.post(
+                    "/api/documents/bulk_edit/",
+                    json.dumps(
+                        {
+                            "documents": [self.doc2.id],
+                            "method": "set_permissions",
+                            "parameters": {"set_permissions": set_permissions},
+                        },
+                    ),
+                    content_type="application/json",
+                )
+                self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+                self.assertIn(expected_message, response.content)
+        m.assert_not_called()
+
+    @mock.patch("documents.serialisers.bulk_edit.set_permissions")
+    def test_set_permissions_rejects_invalid_owner(self, m) -> None:
+        """
+        GIVEN:
+            - A set_permissions bulk edit with an owner that is a nonexistent
+              id, a boolean, a list, a dict, or a non-numeric string
+        WHEN:
+            - API to bulk edit is called
+        THEN:
+            - API returns HTTP 400
+            - set_permissions is not called
+        """
+        self.setup_mock(m, "set_permissions")
+
+        for bad_owner in (
+            999999,
+            True,
+            ["not", "an", "id"],
+            {"nested": "dict"},
+            "not-a-number",
+        ):
+            with self.subTest(owner=bad_owner):
+                response = self.client.post(
+                    "/api/documents/bulk_edit/",
+                    json.dumps(
+                        {
+                            "documents": [self.doc2.id],
+                            "method": "set_permissions",
+                            "parameters": {
+                                "set_permissions": {
+                                    "view": {"users": [self.user.id]},
+                                },
+                                "owner": bad_owner,
+                            },
+                        },
+                    ),
+                    content_type="application/json",
+                )
+                self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+                self.assertIn(b"Specified owner cannot be found", response.content)
+        m.assert_not_called()
+
+    @mock.patch("documents.serialisers.bulk_edit.set_permissions")
+    def test_set_permissions_null_is_a_noop(self, m) -> None:
+        """
+        GIVEN:
+            - A set_permissions bulk edit with set_permissions null and an
+              owner, i.e. a request that only changes the owner
+        WHEN:
+            - API to bulk edit is called
+        THEN:
+            - Request succeeds
+            - set_permissions receives no users or groups for any action
+        """
+        self.setup_mock(m, "set_permissions")
+
+        response = self.client.post(
+            "/api/documents/bulk_edit/",
+            json.dumps(
+                {
+                    "documents": [self.doc2.id],
+                    "method": "set_permissions",
+                    "parameters": {
+                        "set_permissions": None,
+                        "owner": self.user.id,
+                    },
+                },
+            ),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        m.assert_called_once()
+        self.assertEqual(
+            m.call_args.kwargs["set_permissions"],
+            {"view": {}, "change": {}},
+        )
+
+    @mock.patch("documents.serialisers.bulk_edit.set_permissions")
+    def test_set_permissions_passes_validated_owner_id(self, m) -> None:
+        """
+        GIVEN:
+            - A set_permissions bulk edit with the owner id given as a string
+        WHEN:
+            - API to bulk edit is called
+        THEN:
+            - set_permissions receives the owner as an integer id
+        """
+        self.setup_mock(m, "set_permissions")
+
+        response = self.client.post(
+            "/api/documents/bulk_edit/",
+            json.dumps(
+                {
+                    "documents": [self.doc2.id],
+                    "method": "set_permissions",
+                    "parameters": {
+                        "set_permissions": {"view": {"users": [self.user.id]}},
+                        "owner": str(self.user.id),
+                    },
+                },
+            ),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        m.assert_called_once()
+        self.assertEqual(m.call_args.kwargs["owner"], self.user.id)
+
     @mock.patch("documents.serialisers.bulk_edit.set_permissions")
     def test_set_permissions_merge(self, m) -> None:
         self.setup_mock(m, "set_permissions")
@@ -1282,12 +1533,12 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.doc1.owner = User.objects.get(username="temp_admin")
         self.doc1.save()
         user1 = User.objects.create(username="user1")
-        user1.user_permissions.add(*Permission.objects.all())
+        grant_all_global(user1)
         user1.save()
         self.client.force_authenticate(user=user1)
 
         permissions = {
-            "owner": user1.id,
+            "view": {"users": [user1.id]},
         }
 
         response = self.client.post(
@@ -1336,8 +1587,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.doc1.owner = User.objects.get(username="temp_admin")
         self.doc1.save()
         user1 = User.objects.create(username="user1")
-        assign_perm("view_document", user1, self.doc1)
-        user1.user_permissions.add(*Permission.objects.all())
+        grant_object(user1, self.doc1, "view_document")
+        grant_all_global(user1)
         user1.save()
         self.client.force_authenticate(user=user1)
 
@@ -1358,7 +1609,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         m.assert_not_called()
         self.assertEqual(response.content, b"Insufficient permissions")
 
-        assign_perm("change_document", user1, self.doc1)
+        grant_object(user1, self.doc1, "change_document")
 
         response = self.client.post(
             "/api/documents/bulk_edit/",
@@ -1423,14 +1674,182 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
             content_type="application/json",
         )
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+
+        response = self.client.post(
+            "/api/documents/rotate/",
+            json.dumps(
+                {
+                    "documents": [self.doc2.id, self.doc3.id],
+                    "degrees": 45,
+                },
+            ),
+            content_type="application/json",
+        )
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+        self.assertIn(b"degrees must be a multiple of 90", response.content)
         m.assert_not_called()
 
+    @mock.patch("documents.serialisers.bulk_edit.rotate")
+    def test_bulk_edit_rotate_rejects_invalid_degrees(self, m) -> None:
+        """
+        GIVEN:
+            - A legacy rotate bulk edit with degrees that are null, not an
+              integer, or not a multiple of 90
+        WHEN:
+            - API to bulk edit is called
+        THEN:
+            - API returns HTTP 400
+            - rotate is not called
+        """
+        self.setup_mock(m, "rotate")
+
+        for degrees, expected_message in (
+            (None, b"invalid rotation degrees"),
+            (True, b"invalid rotation degrees"),
+            ("foo", b"invalid rotation degrees"),
+            (90.5, b"invalid rotation degrees"),
+            (45, b"degrees must be a multiple of 90"),
+        ):
+            with self.subTest(degrees=degrees):
+                response = self.client.post(
+                    "/api/documents/bulk_edit/",
+                    json.dumps(
+                        {
+                            "documents": [self.doc2.id],
+                            "method": "rotate",
+                            "parameters": {"degrees": degrees},
+                        },
+                    ),
+                    content_type="application/json",
+                )
+                self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+                self.assertIn(expected_message, response.content)
+        m.assert_not_called()
+
+    @mock.patch("documents.serialisers.bulk_edit.rotate")
+    def test_bulk_edit_rotate_passes_integer_degrees(self, m) -> None:
+        """
+        GIVEN:
+            - A legacy rotate bulk edit with degrees given as a string
+        WHEN:
+            - API to bulk edit is called
+        THEN:
+            - rotate receives the degrees as an integer
+        """
+        self.setup_mock(m, "rotate")
+
+        response = self.client.post(
+            "/api/documents/bulk_edit/",
+            json.dumps(
+                {
+                    "documents": [self.doc2.id],
+                    "method": "rotate",
+                    "parameters": {"degrees": "-90"},
+                },
+            ),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        m.assert_called_once()
+        self.assertEqual(m.call_args.kwargs["degrees"], -90)
+
+    @mock.patch("documents.serialisers.bulk_edit.split")
+    def test_bulk_edit_split_rejects_invalid_pages(self, m) -> None:
+        """
+        GIVEN:
+            - A legacy split bulk edit of a 5 page document with pages that
+              are not a string, not numeric, zero, a reversed range, or past
+              the last page
+        WHEN:
+            - API to bulk edit is called
+        THEN:
+            - API returns HTTP 400
+            - split is not called
+        """
+        self.setup_mock(m, "split")
+
+        for pages in (None, "", "a", "1-2-3", "0", "3-1", "1-6", "1-5000000"):
+            with self.subTest(pages=pages):
+                response = self.client.post(
+                    "/api/documents/bulk_edit/",
+                    json.dumps(
+                        {
+                            "documents": [self.doc2.id],
+                            "method": "split",
+                            "parameters": {"pages": pages},
+                        },
+                    ),
+                    content_type="application/json",
+                )
+                self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+                self.assertIn(b"invalid pages specified", response.content)
+        m.assert_not_called()
+
+    @mock.patch("documents.serialisers.bulk_edit.split")
+    def test_bulk_edit_split_rejects_unknown_page_count(self, m) -> None:
+        """
+        GIVEN:
+            - A legacy split bulk edit of a document without a page count
+        WHEN:
+            - API to bulk edit is called
+        THEN:
+            - API returns HTTP 400
+            - split is not called
+        """
+        self.setup_mock(m, "split")
+
+        for pages in ("1", "1-5000000"):
+            with self.subTest(pages=pages):
+                response = self.client.post(
+                    "/api/documents/bulk_edit/",
+                    json.dumps(
+                        {
+                            "documents": [self.doc1.id],
+                            "method": "split",
+                            "parameters": {"pages": pages},
+                        },
+                    ),
+                    content_type="application/json",
+                )
+                self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+                self.assertIn(b"document page count is unknown", response.content)
+        m.assert_not_called()
+
+    @mock.patch("documents.serialisers.bulk_edit.split")
+    def test_bulk_edit_split_parses_pages(self, m) -> None:
+        """
+        GIVEN:
+            - A legacy split bulk edit with single pages and ranges
+        WHEN:
+            - API to bulk edit is called
+        THEN:
+            - split receives one page list per comma separated group
+        """
+        self.setup_mock(m, "split")
+
+        response = self.client.post(
+            "/api/documents/bulk_edit/",
+            json.dumps(
+                {
+                    "documents": [self.doc2.id],
+                    "method": "split",
+                    "parameters": {"pages": "1,2-4,5"},
+                },
+            ),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        m.assert_called_once()
+        self.assertEqual(m.call_args.kwargs["pages"], [[1], [2, 3, 4], [5]])
+
     @mock.patch("documents.views.bulk_edit.rotate")
     def test_rotate_insufficient_permissions(self, m) -> None:
         self.doc1.owner = User.objects.get(username="temp_admin")
         self.doc1.save()
         user1 = User.objects.create(username="user1")
-        user1.user_permissions.add(*Permission.objects.all())
+        grant_all_global(user1)
         user1.save()
         self.client.force_authenticate(user=user1)
 
@@ -1491,7 +1910,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.doc1.owner = User.objects.get(username="temp_admin")
         self.doc1.save()
         user1 = User.objects.create(username="user1")
-        user1.user_permissions.add(*Permission.objects.all())
+        grant_all_global(user1)
         user1.save()
         self.client.force_authenticate(user=user1)
 
@@ -1530,11 +1949,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
     @mock.patch("documents.views.bulk_edit.merge")
     def test_merge_and_delete_requires_change_permission(self, m) -> None:
         self.setup_mock(m, "merge")
-        user = User.objects.create_user(username="no-change")
-        user.user_permissions.add(
-            Permission.objects.get(codename="add_document"),
-            Permission.objects.get(codename="delete_document"),
-        )
+        user = UserFactory(username="no-change")
+        grant_global(user, "add_document", "delete_document")
         self.client.force_authenticate(user=user)
 
         response = self.client.post(
@@ -1636,6 +2052,43 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
 
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
 
+    def test_legacy_bulk_edit_rejects_out_of_bounds_pdf_doc_index(self) -> None:
+        response = self.client.post(
+            "/api/documents/bulk_edit/",
+            json.dumps(
+                {
+                    "documents": [self.doc2.id],
+                    "method": "edit_pdf",
+                    "parameters": {
+                        "operations": [{"page": 1, "doc": 2**32}],
+                    },
+                },
+            ),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+        self.assertIn(b"doc index is out of bounds", response.content)
+
+    def test_legacy_bulk_edit_rejects_empty_pdf_operations(self) -> None:
+        response = self.client.post(
+            "/api/documents/bulk_edit/",
+            json.dumps(
+                {
+                    "documents": [self.doc2.id],
+                    "method": "edit_pdf",
+                    "parameters": {"operations": []},
+                },
+            ),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+        self.assertEqual(
+            response.json(),
+            {"operations": ["This list may not be empty."]},
+        )
+
     @mock.patch("documents.views.bulk_edit.edit_pdf")
     def test_edit_pdf(self, m) -> None:
         self.setup_mock(m, "edit_pdf")
@@ -1686,6 +2139,13 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
         self.assertIn(b"Expected a list of items", response.content)
 
+        response = self.client.post(
+            "/api/documents/edit_pdf/",
+            {"documents": [self.doc2.id], "operations": []},
+            format="json",
+        )
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+
         response = self.client.post(
             "/api/documents/edit_pdf/",
             json.dumps(
@@ -1697,7 +2157,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
             content_type="application/json",
         )
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-        self.assertIn(b"invalid operation entry", response.content)
+        self.assertIn(b"Expected a dictionary", response.content)
 
         response = self.client.post(
             "/api/documents/edit_pdf/",
@@ -1710,7 +2170,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
             content_type="application/json",
         )
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-        self.assertIn(b"page must be an integer", response.content)
+        self.assertIn(b"valid integer is required", response.content)
 
         response = self.client.post(
             "/api/documents/edit_pdf/",
@@ -1723,7 +2183,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
             content_type="application/json",
         )
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-        self.assertIn(b"rotate must be an integer", response.content)
+        self.assertIn(b"valid integer is required", response.content)
 
         response = self.client.post(
             "/api/documents/edit_pdf/",
@@ -1736,7 +2196,27 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
             content_type="application/json",
         )
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-        self.assertIn(b"doc must be an integer", response.content)
+        self.assertIn(b"valid integer is required", response.content)
+
+        # A negative index fails the field's min_value before the
+        # object-level bound against len(operations) is checked
+        for doc_index, expected_message in (
+            (-1, b"greater than or equal to 0"),
+            (2**32, b"doc index is out of bounds"),
+        ):
+            with self.subTest(doc_index=doc_index):
+                response = self.client.post(
+                    "/api/documents/edit_pdf/",
+                    json.dumps(
+                        {
+                            "documents": [self.doc2.id],
+                            "operations": [{"page": 1, "doc": doc_index}],
+                        },
+                    ),
+                    content_type="application/json",
+                )
+                self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+                self.assertIn(expected_message, response.content)
 
         response = self.client.post(
             "/api/documents/edit_pdf/",
@@ -1744,7 +2224,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
                 {
                     "documents": [self.doc2.id],
                     "update_document": True,
-                    "operations": [{"page": 1, "doc": 1}, {"page": 2, "doc": 2}],
+                    "operations": [{"page": 1, "doc": 0}, {"page": 2, "doc": 1}],
                 },
             ),
             content_type="application/json",
@@ -1769,6 +2249,72 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
         self.assertIn(b"Invalid source_mode", response.content)
 
+    @mock.patch("documents.views.bulk_edit.edit_pdf")
+    def test_edit_pdf_rejects_invalid_operation_values(self, m) -> None:
+        """
+        GIVEN:
+            - An edit_pdf operation with a non-positive page, a boolean in
+              place of an integer, or a rotation that is not a multiple of 90
+        WHEN:
+            - API to edit the PDF is called
+        THEN:
+            - API returns HTTP 400 with the field error
+            - edit_pdf is not called
+        """
+        self.setup_mock(m, "edit_pdf")
+        for operation, expected_message in (
+            ({"page": 0}, b"greater than or equal to 1"),
+            ({"page": True}, b"valid integer is required"),
+            ({"page": 1, "rotate": True}, b"valid integer is required"),
+            ({"page": 1, "doc": True}, b"valid integer is required"),
+            ({"page": 1, "rotate": 45}, b"rotate must be a multiple of 90"),
+        ):
+            with self.subTest(operation=operation):
+                response = self.client.post(
+                    "/api/documents/edit_pdf/",
+                    json.dumps(
+                        {
+                            "documents": [self.doc2.id],
+                            "operations": [operation],
+                        },
+                    ),
+                    content_type="application/json",
+                )
+                self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+                self.assertIn(expected_message, response.content)
+        m.assert_not_called()
+
+    def test_legacy_bulk_edit_keys_pdf_operation_errors_under_operations(
+        self,
+    ) -> None:
+        """
+        GIVEN:
+            - A legacy bulk_edit edit_pdf request with an invalid operation
+        WHEN:
+            - API to bulk edit is called
+        THEN:
+            - API returns HTTP 400
+            - The error is keyed under operations and the operation index,
+              matching the edit_pdf endpoint
+        """
+        response = self.client.post(
+            "/api/documents/bulk_edit/",
+            json.dumps(
+                {
+                    "documents": [self.doc2.id],
+                    "method": "edit_pdf",
+                    "parameters": {"operations": [{"page": 1, "rotate": 45}]},
+                },
+            ),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+        self.assertEqual(
+            response.json(),
+            {"operations": {"0": {"rotate": ["rotate must be a multiple of 90"]}}},
+        )
+
     @mock.patch("documents.views.bulk_edit.edit_pdf")
     def test_edit_pdf_page_out_of_bounds(self, m) -> None:
         self.setup_mock(m, "edit_pdf")
@@ -1791,7 +2337,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.doc1.owner = User.objects.get(username="temp_admin")
         self.doc1.save()
         user1 = User.objects.create(username="user1")
-        user1.user_permissions.add(*Permission.objects.all())
+        grant_all_global(user1)
         user1.save()
         self.client.force_authenticate(user=user1)
 
@@ -1826,7 +2372,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
     @mock.patch("documents.views.bulk_edit.edit_pdf")
     def test_edit_pdf_update_requires_change_permission(self, m) -> None:
         self.setup_mock(m, "edit_pdf")
-        user = User.objects.create_user(username="no-change")
+        user = UserFactory(username="no-change")
         self.client.force_authenticate(user=user)
 
         response = self.client.post(
@@ -1853,11 +2399,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
     ) -> None:
         self.setup_mock(edit_pdf_mock, "edit_pdf")
         self.setup_mock(remove_password_mock, "remove_password")
-        user = User.objects.create_user(username="no-delete")
-        user.user_permissions.add(
-            Permission.objects.get(codename="add_document"),
-            Permission.objects.get(codename="change_document"),
-        )
+        user = UserFactory(username="no-delete")
+        grant_global(user, "add_document", "change_document")
         self.client.force_authenticate(user=user)
 
         cases = [
@@ -1944,7 +2487,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.doc1.owner = User.objects.get(username="temp_admin")
         self.doc1.save()
         user1 = User.objects.create(username="user1")
-        user1.user_permissions.add(*Permission.objects.all())
+        grant_all_global(user1)
         user1.save()
         self.client.force_authenticate(user=user1)
 
@@ -1984,7 +2527,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         WHEN:
             - API to bulk edit documents is called
         THEN:
-            - Audit log is created
+            - Audit log is created with the old and new correspondent
         """
         LogEntry.objects.all().delete()
         response = self.client.post(
@@ -2000,7 +2543,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         )
 
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertEqual(LogEntry.objects.filter(object_pk=self.doc1.id).count(), 1)
+        entry = LogEntry.objects.get_for_object(self.doc1).get()
+        self.assertEqual(entry.changes, {"correspondent": [None, self.c2.id]})
 
     @override_settings(AUDIT_LOG_ENABLED=True)
     def test_bulk_edit_audit_log_enabled_tags(self) -> None:
@@ -2008,16 +2552,18 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         GIVEN:
             - Audit log is enabled
         WHEN:
-            - API to bulk edit tags is called
+            - API to bulk edit tags is called on an untagged document and a
+              document with several tags
         THEN:
-            - Audit log is created
+            - Audit log is created for each document with its full tag list
+              before and after the edit
         """
         LogEntry.objects.all().delete()
         response = self.client.post(
             "/api/documents/bulk_edit/",
             json.dumps(
                 {
-                    "documents": [self.doc1.id],
+                    "documents": [self.doc1.id, self.doc4.id],
                     "method": "modify_tags",
                     "parameters": {
                         "add_tags": [self.t1.id],
@@ -2029,18 +2575,32 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         )
 
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertEqual(LogEntry.objects.filter(object_pk=self.doc1.id).count(), 1)
+        entry = LogEntry.objects.get_for_object(self.doc1).get()
+        self.assertEqual(entry.changes, {"tags": [[], [self.t1.id]]})
+        entry = LogEntry.objects.get_for_object(self.doc4).get()
+        self.assertEqual(
+            entry.changes,
+            {"tags": [[self.t1.id, self.t2.id], [self.t1.id]]},
+        )
 
     @override_settings(AUDIT_LOG_ENABLED=True)
     def test_bulk_edit_audit_log_enabled_custom_fields(self) -> None:
         """
         GIVEN:
             - Audit log is enabled
+            - A document with two custom fields
         WHEN:
-            - API to bulk edit custom fields is called
+            - API to bulk edit custom fields is called to add a third
         THEN:
-            - Audit log is created
+            - Audit log is created with every custom field instance before and
+              after the edit
+            - Audit log is created for the new custom field instance
         """
+        cf3 = CustomField.objects.create(name="cf3", data_type="string")
+        existing = [
+            CustomFieldInstance.objects.create(document=self.doc1, field=field)
+            for field in (self.cf2, cf3)
+        ]
         LogEntry.objects.all().delete()
         response = self.client.post(
             "/api/documents/bulk_edit/",
@@ -2058,4 +2618,38 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         )
 
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertEqual(LogEntry.objects.filter(object_pk=self.doc1.id).count(), 2)
+        added = CustomFieldInstance.objects.get(document=self.doc1, field=self.cf1)
+        existing_ids = [instance.id for instance in existing]
+        entry = LogEntry.objects.get_for_object(self.doc1).get()
+        self.assertEqual(
+            entry.changes,
+            {"custom_fields": [existing_ids, [*existing_ids, added.id]]},
+        )
+        self.assertEqual(LogEntry.objects.get_for_object(added).count(), 1)
+
+    def test_api_bulk_edit_with_bad_search_query_returns_400(self) -> None:
+        """
+        GIVEN:
+            - Bulk edit request selects documents via a saved-search query
+              filter
+        WHEN:
+            - The query contains a malformed field value (an invalid date)
+        THEN:
+            - The response is a 400 naming the bad value, exactly like the
+              search list endpoint, never a 500
+        """
+        response = self.client.post(
+            "/api/documents/bulk_edit/",
+            json.dumps(
+                {
+                    "all": True,
+                    "filters": {"query": "added:notadate"},
+                    "method": "set_storage_path",
+                    "parameters": {"storage_path": self.sp1.id},
+                },
+            ),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+        self.assertIn(b"notadate", response.content)

src/documents/tests/test_api_chat.py

@@ -4,20 +4,22 @@ from typing import TYPE_CHECKING
 from unittest import mock
 
 import pytest
-from django.contrib.auth.models import Permission
-from django.contrib.auth.models import User
 from rest_framework import status
 from rest_framework.test import APIClient
 from rest_framework.test import APITestCase
 
+from paperless_testing.factories import UserFactory
+from paperless_testing.permissions import grant_global
+
 if TYPE_CHECKING:
+    from django.contrib.auth.models import User
     from pytest_mock import MockerFixture
 
 
 class TestChatStreamingViewInputValidation(APITestCase):
     def setUp(self) -> None:
         super().setUp()
-        self.user = User.objects.create_superuser(username="temp_admin")
+        self.user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.user)
 
     def _mock_ai_enabled(self) -> mock.MagicMock:
@@ -38,6 +40,42 @@ class TestChatStreamingViewInputValidation(APITestCase):
             )
         assert resp.status_code == status.HTTP_400_BAD_REQUEST
 
+    def test_answer_is_not_compressed(self) -> None:
+        """
+        GIVEN:
+            - A client that accepts compressed responses
+        WHEN:
+            - It asks the chat endpoint a question
+        THEN:
+            - The answer is streamed unencoded, chunk for chunk
+
+        The stream compressors buffer, so a compressed answer arrives in one
+        piece. The view cannot opt out by flagging the request: DRF's request
+        wrapper proxies reads but keeps writes to itself, so the flag never
+        reaches the Django request the middleware sees.
+        """
+        chunks = [f"token{i} " for i in range(40)]
+        with (
+            mock.patch(
+                "documents.views.AIConfig",
+                return_value=self._mock_ai_enabled(),
+            ),
+            mock.patch(
+                "documents.views.stream_chat_with_documents",
+                return_value=iter(chunks),
+            ),
+        ):
+            resp = self.client.post(
+                "/api/documents/chat/",
+                {"q": "What is in my archive?"},
+                format="json",
+                HTTP_ACCEPT_ENCODING="gzip, deflate, br, zstd",
+            )
+
+        assert resp.status_code == status.HTTP_200_OK
+        assert not resp.has_header("Content-Encoding")
+        assert list(resp.streaming_content) == [c.encode() for c in chunks]
+
     def test_missing_question_is_rejected(self) -> None:
         with mock.patch(
             "documents.views.AIConfig",
@@ -77,9 +115,7 @@ class TestChatStreamingViewUnrestrictedFlag:
         needs to reach the view at all. Model-level only: says nothing
         about which documents (if any) this user can actually see.
         """
-        regular_user.user_permissions.add(
-            *Permission.objects.filter(codename="view_document"),
-        )
+        grant_global(regular_user, "view_document")
         return user_client
 
     @pytest.mark.parametrize(

src/documents/tests/test_api_custom_fields.py

@@ -4,23 +4,24 @@ from unittest import mock
 from unittest.mock import ANY
 
 from django.contrib.auth.models import Permission
-from django.contrib.auth.models import User
 from django.test import override_settings
-from guardian.shortcuts import assign_perm
 from rest_framework import status
 from rest_framework.test import APITestCase
 
 from documents.models import CustomField
 from documents.models import CustomFieldInstance
 from documents.models import Document
-from documents.tests.utils import DirectoriesMixin
+from paperless_testing.dirs import DirectoriesMixin
+from paperless_testing.factories import UserFactory
+from paperless_testing.permissions import grant_global
+from paperless_testing.permissions import grant_object
 
 
 class TestCustomFieldsAPI(DirectoriesMixin, APITestCase):
     ENDPOINT = "/api/custom_fields/"
 
     def setUp(self) -> None:
-        self.user = User.objects.create_superuser(username="temp_admin")
+        self.user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.user)
         return super().setUp()
 
@@ -1174,11 +1175,9 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase):
     def test_documentlink_patch_requires_change_permission_on_target_documents(
         self,
     ) -> None:
-        source_owner = User.objects.create_user(username="source-owner")
-        source_owner.user_permissions.add(
-            Permission.objects.get(codename="change_document"),
-        )
-        other_user = User.objects.create_user(username="other-user")
+        source_owner = UserFactory(username="source-owner")
+        grant_global(source_owner, "change_document")
+        other_user = UserFactory(username="other-user")
 
         source_doc = Document.objects.create(
             title="Source",
@@ -1221,11 +1220,9 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase):
     def test_documentlink_patch_allowed_with_change_permission_on_target_documents(
         self,
     ) -> None:
-        source_owner = User.objects.create_user(username="source-owner")
-        source_owner.user_permissions.add(
-            Permission.objects.get(codename="change_document"),
-        )
-        other_user = User.objects.create_user(username="other-user")
+        source_owner = UserFactory(username="source-owner")
+        grant_global(source_owner, "change_document")
+        other_user = UserFactory(username="other-user")
 
         source_doc = Document.objects.create(
             title="Source",
@@ -1244,7 +1241,7 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase):
             data_type=CustomField.FieldDataType.DOCUMENTLINK,
         )
 
-        assign_perm("change_document", source_owner, target_doc)
+        grant_object(source_owner, target_doc, "change_document")
         self.client.force_authenticate(user=source_owner)
 
         resp = self.client.patch(
@@ -1337,7 +1334,7 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(results[0]["document_count"], 1)
 
         # Test as user without access to the document
-        non_superuser = User.objects.create_user(username="non_superuser")
+        non_superuser = UserFactory(username="non_superuser")
         non_superuser.user_permissions.add(
             *Permission.objects.exclude(codename="view_global_statistics"),
         )

src/documents/tests/test_api_document_versions.py

@@ -5,11 +5,10 @@ from typing import TYPE_CHECKING
 from unittest import mock
 
 from auditlog.models import LogEntry  # type: ignore[import-untyped]
-from django.contrib.auth.models import Permission
-from django.contrib.auth.models import User
 from django.contrib.contenttypes.models import ContentType
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.test import TestCase as DjangoTestCase
+from django.test import override_settings
 from django.utils import timezone
 from rest_framework import status
 from rest_framework.test import APITestCase
@@ -18,10 +17,17 @@ from documents.data_models import DocumentSource
 from documents.filters import EffectiveContentFilter
 from documents.filters import TitleContentFilter
 from documents.models import Document
-from documents.tests.utils import DirectoriesMixin
-from documents.tests.utils import read_streaming_response
+from documents.models import Note
+from documents.models import ShareLink
 from documents.versioning import annotate_effective_content
 from documents.views import DocumentSelectionMixin
+from paperless_testing.dirs import DirectoriesMixin
+from paperless_testing.factories import DocumentFactory
+from paperless_testing.factories import UserFactory
+from paperless_testing.http import read_streaming_response
+from paperless_testing.permissions import grant_all_global
+from paperless_testing.permissions import grant_global
+from paperless_testing.permissions import grant_object
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -31,7 +37,7 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.user = User.objects.create_superuser(username="temp_admin")
+        self.user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.user)
 
     def _make_pdf_upload(self, name: str = "version.pdf") -> SimpleUploadedFile:
@@ -89,11 +95,9 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
         self.assertEqual(resp.status_code, status.HTTP_404_NOT_FOUND)
 
     def test_root_endpoint_returns_403_when_user_lacks_permission(self) -> None:
-        owner = User.objects.create_user(username="owner")
-        viewer = User.objects.create_user(username="viewer")
-        viewer.user_permissions.add(
-            Permission.objects.get(codename="view_document"),
-        )
+        owner = UserFactory(username="owner")
+        viewer = UserFactory(username="viewer")
+        grant_global(viewer, "view_document")
         root = Document.objects.create(
             title="root",
             checksum="root",
@@ -283,11 +287,9 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
         self.assertEqual(mock_backend.add_or_update.call_args[0][0].id, root.id)
 
     def test_delete_version_returns_403_without_permission(self) -> None:
-        owner = User.objects.create_user(username="owner")
-        other = User.objects.create_user(username="other")
-        other.user_permissions.add(
-            Permission.objects.get(codename="delete_document"),
-        )
+        owner = UserFactory(username="owner")
+        other = UserFactory(username="other")
+        grant_global(other, "delete_document")
         root = Document.objects.create(
             title="root",
             checksum="root",
@@ -371,11 +373,9 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
         self.assertTrue(resp.data["is_root"])
 
     def test_update_version_label_returns_403_without_permission(self) -> None:
-        owner = User.objects.create_user(username="owner")
-        other = User.objects.create_user(username="other")
-        other.user_permissions.add(
-            Permission.objects.get(codename="change_document"),
-        )
+        owner = UserFactory(username="owner")
+        other = UserFactory(username="other")
+        grant_global(other, "change_document")
         root = Document.objects.create(
             title="root",
             checksum="root",
@@ -553,11 +553,9 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
         self.assertEqual(resp.status_code, status.HTTP_404_NOT_FOUND)
 
     def test_metadata_returns_403_when_user_lacks_permission(self) -> None:
-        owner = User.objects.create_user(username="owner")
-        other = User.objects.create_user(username="other")
-        other.user_permissions.add(
-            Permission.objects.get(codename="view_document"),
-        )
+        owner = UserFactory(username="owner")
+        other = UserFactory(username="other")
+        grant_global(other, "view_document")
         doc = Document.objects.create(
             title="root",
             checksum="root",
@@ -653,8 +651,8 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
         self.assertEqual(resp.status_code, status.HTTP_500_INTERNAL_SERVER_ERROR)
 
     def test_update_version_returns_403_without_permission(self) -> None:
-        owner = User.objects.create_user(username="owner")
-        other = User.objects.create_user(username="other")
+        owner = UserFactory(username="owner")
+        other = UserFactory(username="other")
         root = Document.objects.create(
             title="root",
             checksum="root",
@@ -672,8 +670,8 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
         self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
 
     def test_update_version_requires_global_change_permission(self) -> None:
-        user = User.objects.create_user(username="add-only")
-        user.user_permissions.add(Permission.objects.get(codename="add_document"))
+        user = UserFactory(username="add-only")
+        grant_global(user, "add_document")
         root = Document.objects.create(
             title="root",
             checksum="root",
@@ -829,6 +827,26 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
         self.assertEqual(resp.status_code, status.HTTP_200_OK)
         self.assertEqual(resp.data["content"], "v1-content")
 
+    def test_page_count_resolves_to_latest_version(self) -> None:
+        root = DocumentFactory(page_count=2)
+        DocumentFactory(root_document=root, version_index=1, page_count=1)
+        unversioned = DocumentFactory(page_count=5)
+
+        resp = self.client.get("/api/documents/?fields=id,page_count")
+        self.assertEqual(resp.status_code, status.HTTP_200_OK)
+        self.assertEqual(
+            {doc["id"]: doc["page_count"] for doc in resp.data["results"]},
+            {root.id: 1, unversioned.id: 5},
+        )
+
+        resp = self.client.get(f"/api/documents/{root.id}/")
+        self.assertEqual(resp.status_code, status.HTTP_200_OK)
+        self.assertEqual(resp.data["page_count"], 1)
+
+        resp = self.client.get(f"/api/documents/{root.id}/?version={root.id}")
+        self.assertEqual(resp.status_code, status.HTTP_200_OK)
+        self.assertEqual(resp.data["page_count"], 2)
+
     def _make_root_with_out_of_order_versions(self) -> tuple[Document, ...]:
         """
         A root whose newest version has a *lower* id than an older one, which is
@@ -978,7 +996,7 @@ class TestVersionAwareFilters(DjangoTestCase):
         superseded content -- selecting documents the list view, filtered by
         the same term, does not show.
         """
-        user = User.objects.create_superuser(username="bulk_selection")
+        user = UserFactory(username="bulk_selection", superuser=True)
 
         selected = DocumentSelectionMixin()._resolve_document_ids(
             user=user,
@@ -1005,7 +1023,7 @@ class TestBulkSelectionExcludesVersions(DjangoTestCase):
         "Select all matching" reconstructs the document list, which never
         contains version documents as rows of their own.
         """
-        user = User.objects.create_superuser(username="bulk_versions")
+        user = UserFactory(username="bulk_versions", superuser=True)
         root = Document.objects.create(
             title="shared-title root",
             checksum="bulk-root",
@@ -1030,3 +1048,152 @@ class TestBulkSelectionExcludesVersions(DjangoTestCase):
         )
 
         self.assertEqual(selected, [root.id])
+
+
+class TestVersionActionPermissions(DirectoriesMixin, APITestCase):
+    def setUp(self):
+        super().setUp()
+        self.user = UserFactory()
+        grant_all_global(self.user)
+        self.client.force_authenticate(self.user)
+        self.root = DocumentFactory(owner=UserFactory())
+        self.version = DocumentFactory(root_document=self.root, owner=None)
+
+    @override_settings(AUDIT_LOG_ENABLED=True)
+    def test_actions_reject_stale_version_ownership(self):
+        note = Note.objects.create(document=self.version, note="Version note")
+        for owner in (None, self.user):
+            self.version.owner = owner
+            self.version.save(update_fields=["owner"])
+            for action in (
+                "notes",
+                "suggestions",
+                "ai_suggestions",
+                "history",
+                "share_links",
+            ):
+                with self.subTest(owner=owner, action=action):
+                    response = self.client.get(
+                        f"/api/documents/{self.version.pk}/{action}/",
+                    )
+                    self.assertEqual(response.status_code, 403)
+            response = self.client.post(
+                f"/api/documents/{self.version.pk}/notes/",
+                {"note": "New note"},
+            )
+            self.assertEqual(response.status_code, 403)
+            response = self.client.delete(
+                f"/api/documents/{self.version.pk}/notes/?id={note.pk}",
+            )
+            self.assertEqual(response.status_code, 403)
+            response = self.client.post(
+                "/api/share_links/",
+                {"document": self.version.pk, "file_version": "original"},
+            )
+            self.assertEqual(response.status_code, 403)
+            response = self.client.post(
+                "/api/share_link_bundles/",
+                {"document_ids": [self.version.pk], "file_version": "original"},
+                format="json",
+            )
+            self.assertEqual(response.status_code, 400)
+            response = self.client.post(
+                "/api/documents/email/",
+                {
+                    "documents": [self.version.pk],
+                    "addresses": "recipient@example.com",
+                    "subject": "Version",
+                    "message": "Version",
+                },
+                format="json",
+            )
+            self.assertEqual(response.status_code, 403)
+            with (
+                mock.patch("documents.views.AIConfig") as ai_config,
+                mock.patch("documents.views.stream_chat_with_documents") as chat,
+            ):
+                ai_config.return_value.ai_enabled = True
+                response = self.client.post(
+                    "/api/documents/chat/",
+                    {"q": "Version?", "document_id": self.version.pk},
+                    format="json",
+                )
+            self.assertEqual(response.status_code, 403)
+            chat.assert_not_called()
+        self.assertTrue(Note.objects.filter(pk=note.pk).exists())
+        self.assertFalse(ShareLink.objects.exists())
+
+    @mock.patch("documents.views.build_share_link_bundle.apply_async")
+    def test_root_permissions_allow_sharing_a_private_version(self, build_mock):
+        self.version.owner = UserFactory()
+        self.version.save(update_fields=["owner"])
+        grant_object(self.user, self.root, "view_document", "change_document")
+        note = Note.objects.create(document=self.version, note="Version note")
+        response = self.client.get(f"/api/documents/{self.version.pk}/notes/")
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data[0]["id"], note.pk)
+        response = self.client.post(
+            "/api/share_links/",
+            {"document": self.version.pk, "file_version": "original"},
+        )
+        self.assertEqual(response.status_code, 201)
+        self.assertEqual(ShareLink.objects.get().document_id, self.version.pk)
+        response = self.client.get(f"/api/documents/{self.version.pk}/share_links/")
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(len(response.data), 1)
+        response = self.client.post(
+            "/api/share_link_bundles/",
+            {"document_ids": [self.version.pk], "file_version": "original"},
+            format="json",
+        )
+        self.assertEqual(response.status_code, 201)
+        build_mock.assert_called_once()
+
+    def test_root_view_permission_does_not_allow_note_changes(self):
+        grant_object(self.user, self.root, "view_document")
+        note = Note.objects.create(document=self.version, note="Version note")
+        response = self.client.get(f"/api/documents/{self.version.pk}/notes/")
+        self.assertEqual(response.status_code, 200)
+        response = self.client.post(
+            f"/api/documents/{self.version.pk}/notes/",
+            {"note": "New note"},
+        )
+        self.assertEqual(response.status_code, 403)
+        response = self.client.delete(
+            f"/api/documents/{self.version.pk}/notes/?id={note.pk}",
+        )
+        self.assertEqual(response.status_code, 403)
+        self.assertTrue(Note.objects.filter(pk=note.pk).exists())
+
+    @override_settings(AUDIT_LOG_ENABLED=True)
+    def test_history_uses_root_ownership(self):
+        self.root.owner = self.user
+        self.root.save(update_fields=["owner"])
+        self.version.owner = UserFactory()
+        self.version.save(update_fields=["owner"])
+        response = self.client.get(f"/api/documents/{self.version.pk}/history/")
+        self.assertEqual(response.status_code, 200)
+
+    def test_selection_data_rejects_stale_version_ownership(self):
+        for owner in (None, self.user):
+            self.version.owner = owner
+            self.version.save(update_fields=["owner"])
+            with self.subTest(owner=owner):
+                response = self.client.post(
+                    "/api/documents/selection_data/",
+                    {"documents": [self.version.pk]},
+                    format="json",
+                )
+                self.assertEqual(response.status_code, 403)
+
+    def test_selection_data_allows_private_version_of_permitted_root(self):
+        self.version.owner = UserFactory()
+        self.version.save(update_fields=["owner"])
+        grant_object(self.user, self.root, "view_document")
+        other = DocumentFactory(owner=self.user)
+        response = self.client.post(
+            "/api/documents/selection_data/",
+            {"documents": [self.version.pk, other.pk]},
+            format="json",
+        )
+        self.assertEqual(response.status_code, 200)

src/documents/tests/test_api_documents.py

@@ -23,7 +23,6 @@ from django.core.files.uploadedfile import SimpleUploadedFile
 from django.db import DataError
 from django.test import override_settings
 from django.utils import timezone
-from guardian.shortcuts import assign_perm
 from rest_framework import status
 from rest_framework.test import APITestCase
 
@@ -48,18 +47,22 @@ from documents.models import Workflow
 from documents.models import WorkflowAction
 from documents.models import WorkflowTrigger
 from documents.signals.handlers import run_workflows
-from documents.tests.factories import DocumentFactory
-from documents.tests.factories import TagFactory
 from documents.tests.utils import ConsumeTaskMixin
-from documents.tests.utils import DirectoriesMixin
-from documents.tests.utils import read_streaming_response
+from paperless_testing.dirs import DirectoriesMixin
+from paperless_testing.factories import DocumentFactory
+from paperless_testing.factories import TagFactory
+from paperless_testing.factories import UserFactory
+from paperless_testing.http import read_streaming_response
+from paperless_testing.permissions import grant_all_global
+from paperless_testing.permissions import grant_global
+from paperless_testing.permissions import grant_object
 
 
 class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.user = User.objects.create_superuser(username="temp_admin")
+        self.user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.user)
         cache.clear()
 
@@ -357,10 +360,10 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         with Path(filename).open("wb") as f:
             f.write(content)
 
-        user1 = User.objects.create_user(username="test1")
-        user2 = User.objects.create_user(username="test2")
-        user1.user_permissions.add(*Permission.objects.filter(codename="view_document"))
-        user2.user_permissions.add(*Permission.objects.filter(codename="view_document"))
+        user1 = UserFactory(username="test1")
+        user2 = UserFactory(username="test2")
+        grant_global(user1, "view_document")
+        grant_global(user2, "view_document")
 
         self.client.force_authenticate(user2)
 
@@ -383,7 +386,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         response = self.client.get(f"/api/documents/{doc.pk}/thumb/")
         self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
 
-        assign_perm("view_document", user2, doc)
+        grant_object(user2, doc, "view_document")
 
         response = self.client.get(f"/api/documents/{doc.pk}/download/")
         self.assertEqual(response.status_code, status.HTTP_200_OK)
@@ -760,8 +763,8 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
             - History is returned
         """
         # No auditlog permissions
-        user = User.objects.create_user(username="test")
-        user.user_permissions.add(*Permission.objects.filter(codename="view_document"))
+        user = UserFactory(username="test")
+        grant_global(user, "view_document")
         self.client.force_authenticate(user=user)
         doc = Document.objects.create(
             title="First title",
@@ -776,7 +779,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         # superuser
         user.is_superuser = True
         user.save()
-        user2 = User.objects.create_user(username="test2")
+        user2 = UserFactory(username="test2")
         doc2 = Document.objects.create(
             title="Second title",
             checksum="456",
@@ -1073,11 +1076,9 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         self.assertEqual(response.data["duplicate_documents"], [])
 
     def test_has_duplicates_filter_respects_document_permissions(self) -> None:
-        owner = User.objects.create_user(username="duplicate-owner")
-        requester = User.objects.create_user(username="duplicate-requester")
-        requester.user_permissions.add(
-            Permission.objects.get(codename="view_document"),
-        )
+        owner = UserFactory(username="duplicate-owner")
+        requester = UserFactory(username="duplicate-requester")
+        grant_global(requester, "view_document")
         visible_document = Document.objects.create(
             title="visible document",
             checksum="permission-match",
@@ -1096,7 +1097,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
             [document["id"] for document in response.data["results"]],
         )
 
-        assign_perm("view_document", requester, hidden_duplicate)
+        grant_object(requester, hidden_duplicate, "view_document")
         response = self.client.get("/api/documents/?has_duplicates=true")
         self.assertIn(
             visible_document.id,
@@ -1317,10 +1318,10 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         THEN:
             - Owner filters work correctly but still respect permissions
         """
-        u1 = User.objects.create_user("user1")
-        u2 = User.objects.create_user("user2")
-        u1.user_permissions.add(*Permission.objects.filter(codename="view_document"))
-        u2.user_permissions.add(*Permission.objects.filter(codename="view_document"))
+        u1 = UserFactory(username="user1")
+        u2 = UserFactory(username="user2")
+        grant_global(u1, "view_document")
+        grant_global(u2, "view_document")
 
         u1_doc1 = Document.objects.create(
             title="none1",
@@ -1353,7 +1354,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         )
 
         self.client.force_authenticate(user=u1)
-        assign_perm("view_document", u1, u2_doc2)
+        grant_object(u1, u2_doc2, "view_document")
 
         # Will not show any u1 docs or u2_doc1 which isn't shared
         response = self.client.get(f"/api/documents/?owner__id__none={u1.id}")
@@ -1400,7 +1401,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
             [u1_doc1.id, u1_doc2.id, u2_doc2.id],
         )
 
-        assign_perm("view_document", u2, u1_doc1)
+        grant_object(u2, u1_doc1, "view_document")
 
         # Will show only documents shared by user
         response = self.client.get(f"/api/documents/?shared_by__id={u1.id}")
@@ -1424,8 +1425,8 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
             - The document is returned exactly once, not once per permission path
             (regression test for https://github.com/paperless-ngx/paperless-ngx/issues/13331)
         """
-        user = User.objects.create_user("user1")
-        user.user_permissions.add(*Permission.objects.filter(codename="view_document"))
+        user = UserFactory(username="user1")
+        grant_global(user, "view_document")
         group = Group.objects.create(name="group1")
         user.groups.add(group)
 
@@ -1433,7 +1434,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         tag2 = TagFactory()
         doc = DocumentFactory(title="shared", owner=user)
         doc.tags.add(tag1, tag2)
-        assign_perm("view_document", group, doc)
+        grant_object(group, doc, "view_document")
 
         self.client.force_authenticate(user=user)
         response = self.client.get(
@@ -1452,11 +1453,9 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         THEN:
             - The document does not appear in their results
         """
-        owner = User.objects.create_user("owner1")
-        stranger = User.objects.create_user("stranger1")
-        stranger.user_permissions.add(
-            *Permission.objects.filter(codename="view_document"),
-        )
+        owner = UserFactory(username="owner1")
+        stranger = UserFactory(username="stranger1")
+        grant_global(stranger, "view_document")
 
         DocumentFactory(title="private", owner=owner)
 
@@ -1474,17 +1473,17 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         THEN:
             - Only the group member sees the document
         """
-        owner = User.objects.create_user("owner2")
-        member = User.objects.create_user("member1")
-        non_member = User.objects.create_user("nonmember1")
+        owner = UserFactory(username="owner2")
+        member = UserFactory(username="member1")
+        non_member = UserFactory(username="nonmember1")
         for u in (member, non_member):
-            u.user_permissions.add(*Permission.objects.filter(codename="view_document"))
+            grant_global(u, "view_document")
 
         group = Group.objects.create(name="group2")
         member.groups.add(group)
 
         doc = DocumentFactory(title="shared2", owner=owner)
-        assign_perm("view_document", group, doc)
+        grant_object(group, doc, "view_document")
 
         self.client.force_authenticate(user=member)
         response = self.client.get("/api/documents/")
@@ -1785,8 +1784,8 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         THEN:
             - Statistics only include inbox counts for tags accessible by the user
         """
-        u1 = User.objects.create_user("user1")
-        u2 = User.objects.create_user("user2")
+        u1 = UserFactory(username="user1")
+        u2 = UserFactory(username="user2")
         inbox_tag_u1 = Tag.objects.create(name="inbox_u1", is_inbox_tag=True, owner=u1)
         Tag.objects.create(name="inbox_u2", is_inbox_tag=True, owner=u2)
         doc_u1 = Document.objects.create(
@@ -1816,11 +1815,9 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         self.assertEqual(response.data["documents_inbox"], 0)
 
     def test_statistics_with_statistics_permission(self) -> None:
-        owner = User.objects.create_user("owner")
-        stats_user = User.objects.create_user("stats-user")
-        stats_user.user_permissions.add(
-            Permission.objects.get(codename="view_global_statistics"),
-        )
+        owner = UserFactory(username="owner")
+        stats_user = UserFactory(username="stats-user")
+        grant_global(stats_user, "view_global_statistics")
 
         inbox_tag = Tag.objects.create(
             name="stats_inbox",
@@ -1986,7 +1983,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_405_METHOD_NOT_ALLOWED)
 
     def test_upload_insufficient_permissions(self) -> None:
-        self.client.force_authenticate(user=User.objects.create_user("testuser2"))
+        self.client.force_authenticate(user=UserFactory(username="testuser2"))
 
         with (Path(__file__).parent / "samples" / "simple.pdf").open("rb") as f:
             response = self.client.post(
@@ -2782,9 +2779,9 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         mock_get_date_parser.assert_not_called()
 
     def test_saved_views(self) -> None:
-        u1 = User.objects.create_user("user1")
-        u2 = User.objects.create_user("user2")
-        u3 = User.objects.create_user("user3")
+        u1 = UserFactory(username="user1")
+        u2 = UserFactory(username="user2")
+        u3 = UserFactory(username="user3")
 
         view_perm = Permission.objects.get(codename="view_savedview")
         change_perm = Permission.objects.get(codename="change_savedview")
@@ -2807,9 +2804,9 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
             sort_field="",
         )
 
-        assign_perm("view_savedview", u1, v2)
-        assign_perm("change_savedview", u1, v2)
-        assign_perm("view_savedview", u1, v3)
+        grant_object(u1, v2, "view_savedview")
+        grant_object(u1, v2, "change_savedview")
+        grant_object(u1, v3, "view_savedview")
 
         self.client.force_authenticate(user=u1)
 
@@ -3064,7 +3061,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         self.assertListEqual(saved_view_settings["sidebar_views_visible_ids"], [v2.id])
 
     def test_saved_view_create_update_patch(self) -> None:
-        User.objects.create_user("user1")
+        UserFactory(username="user1")
 
         view = {
             "name": "test",
@@ -3127,7 +3124,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
             - Display options are updated
             - Display fields are validated
         """
-        User.objects.create_user("user1")
+        UserFactory(username="user1")
 
         view = {
             "name": "test",
@@ -3568,11 +3565,11 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         THEN:
             - Notes are neither created nor deleted
         """
-        user1 = User.objects.create_user(username="test1")
-        user1.user_permissions.add(*Permission.objects.all())
+        user1 = UserFactory(username="test1")
+        grant_all_global(user1)
         user1.save()
 
-        user2 = User.objects.create_user(username="test2")
+        user2 = UserFactory(username="test2")
         user2.save()
 
         doc = Document.objects.create(
@@ -3592,7 +3589,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         self.assertEqual(resp.content, b"Insufficient permissions to view notes")
         self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
 
-        assign_perm("view_document", user1, doc)
+        grant_object(user1, doc, "view_document")
 
         resp = self.client.post(
             f"/api/documents/{doc.pk}/notes/",
@@ -3615,6 +3612,47 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         self.assertEqual(response.content, b"Insufficient permissions to delete notes")
         self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
 
+    def test_notes_require_global_document_permissions(self) -> None:
+        user = UserFactory(username="note_editor")
+        grant_global(user, "view_note", "add_note", "delete_note")
+        doc = Document.objects.create(
+            title="test",
+            mime_type="application/pdf",
+            content="notes",
+            owner=user,
+        )
+        note = Note.objects.create(note="Existing", document=doc, user=user)
+        self.client.force_authenticate(user)
+
+        response = self.client.get(f"/api/documents/{doc.pk}/notes/")
+        self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
+
+        grant_global(user, "view_document")
+        user = User.objects.get(pk=user.pk)
+        self.client.force_authenticate(user)
+        response = self.client.get(f"/api/documents/{doc.pk}/notes/")
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+        response = self.client.post(
+            f"/api/documents/{doc.pk}/notes/",
+            data={"note": "New"},
+        )
+        self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
+
+        grant_global(user, "change_document")
+        user = User.objects.get(pk=user.pk)
+        self.client.force_authenticate(user)
+        response = self.client.post(
+            f"/api/documents/{doc.pk}/notes/",
+            data={"note": "New"},
+        )
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+        response = self.client.delete(
+            f"/api/documents/{doc.pk}/notes/?id={note.pk}",
+        )
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+
     def test_delete_note(self) -> None:
         """
         GIVEN:
@@ -3748,12 +3786,12 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
             - Unique items are created
             - Non-unique items are not allowed
         """
-        user1 = User.objects.create_user(username="test1")
-        user1.user_permissions.add(*Permission.objects.filter(codename="add_tag"))
+        user1 = UserFactory(username="test1")
+        grant_global(user1, "add_tag")
         user1.save()
 
-        user2 = User.objects.create_user(username="test2")
-        user2.user_permissions.add(*Permission.objects.filter(codename="add_tag"))
+        user2 = UserFactory(username="test2")
+        grant_global(user2, "add_tag")
         user2.save()
 
         # User 1 creates tag 1 owned by user 1 by default
@@ -3808,12 +3846,12 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
             - Unique items are created
             - Non-unique items are not allowed on update
         """
-        user1 = User.objects.create_user(username="test1")
-        user1.user_permissions.add(*Permission.objects.filter(codename="change_tag"))
+        user1 = UserFactory(username="test1")
+        grant_global(user1, "change_tag")
         user1.save()
 
-        user2 = User.objects.create_user(username="test2")
-        user2.user_permissions.add(*Permission.objects.filter(codename="change_tag"))
+        user2 = UserFactory(username="test2")
+        grant_global(user2, "change_tag")
         user2.save()
 
         # Create name tag 1 owned by user 1
@@ -3857,6 +3895,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
             },
         )
         self.assertEqual(resp.status_code, status.HTTP_201_CREATED)
+        self.assertEqual(resp.data["document_title"], doc.title)
 
         resp = self.client.post(
             "/api/share_links/",
@@ -3867,6 +3906,17 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
             },
         )
         self.assertEqual(resp.status_code, status.HTTP_201_CREATED)
+        self.assertEqual(resp.data["document_title"], doc.title)
+
+        response = self.client.get("/api/share_links/", format="json")
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(response.data["count"], 2)
+        self.assertTrue(
+            all(
+                link["document_title"] == doc.title for link in response.data["results"]
+            ),
+        )
 
         response = self.client.get(
             f"/api/documents/{doc.pk}/share_links/",
@@ -3878,6 +3928,9 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         resp_data = response.json()
 
         self.assertEqual(len(resp_data), 2)
+        self.assertTrue(
+            all(link["document_title"] == doc.title for link in resp_data),
+        )
 
         self.assertGreater(len(resp_data[1]["slug"]), 0)
         self.assertIsNone(resp_data[1]["expiration"])
@@ -3903,6 +3956,23 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
 
         self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
 
+    def test_order_share_links_by_document_title(self) -> None:
+        document_zulu = Document.objects.create(title="Zulu")
+        document_alpha = Document.objects.create(title="Alpha")
+        ShareLink.objects.create(document=document_zulu, slug="zulu-link")
+        ShareLink.objects.create(document=document_alpha, slug="alpha-link")
+
+        response = self.client.get(
+            "/api/share_links/?ordering=document__title",
+            format="json",
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(
+            [link["document_title"] for link in response.data["results"]],
+            ["Alpha", "Zulu"],
+        )
+
     def test_share_links_permissions_aware(self) -> None:
         """
         GIVEN:
@@ -3912,11 +3982,11 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         THEN:
             - Links only shown if user has permissions
         """
-        user1 = User.objects.create_user(username="test1")
-        user1.user_permissions.add(*Permission.objects.all())
+        user1 = UserFactory(username="test1")
+        grant_all_global(user1)
         user1.save()
 
-        user2 = User.objects.create_user(username="test2")
+        user2 = UserFactory(username="test2")
         user2.save()
 
         doc = Document.objects.create(
@@ -3936,7 +4006,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         self.assertEqual(resp.content, b"Insufficient permissions to add share link")
         self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
 
-        assign_perm("change_document", user1, doc)
+        grant_object(user1, doc, "change_document")
 
         resp = self.client.get(
             f"/api/documents/{doc.pk}/share_links/",
@@ -3953,11 +4023,11 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         THEN:
             - Share link creation is denied until view permission is granted
         """
-        user1 = User.objects.create_user(username="test1")
-        user1.user_permissions.add(*Permission.objects.filter(codename="add_sharelink"))
+        user1 = UserFactory(username="test1")
+        grant_global(user1, "add_sharelink")
         user1.save()
 
-        user2 = User.objects.create_user(username="test2")
+        user2 = UserFactory(username="test2")
         user2.save()
 
         doc = Document.objects.create(
@@ -3979,8 +4049,21 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         )
         self.assertEqual(create_resp.status_code, status.HTTP_403_FORBIDDEN)
 
-        assign_perm("view_document", user1, doc)
+        grant_object(user1, doc, "view_document")
 
+        create_resp = self.client.post(
+            "/api/share_links/",
+            data={
+                "document": doc.pk,
+                "file_version": "original",
+            },
+            format="json",
+        )
+        self.assertEqual(create_resp.status_code, status.HTTP_403_FORBIDDEN)
+
+        grant_global(user1, "view_document")
+        user1 = User.objects.get(pk=user1.pk)
+        self.client.force_authenticate(user1)
         create_resp = self.client.post(
             "/api/share_links/",
             data={
@@ -4001,11 +4084,11 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         THEN:
             - ASN +1 from user2's doc is returned for user1
         """
-        user1 = User.objects.create_user(username="test1")
-        user1.user_permissions.add(*Permission.objects.all())
+        user1 = UserFactory(username="test1")
+        grant_all_global(user1)
         user1.save()
 
-        user2 = User.objects.create_user(username="test2")
+        user2 = UserFactory(username="test2")
         user2.save()
 
         doc1 = Document.objects.create(
@@ -4045,8 +4128,8 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         THEN:
             - ASN 1 is returned
         """
-        user1 = User.objects.create_user(username="test1")
-        user1.user_permissions.add(*Permission.objects.all())
+        user1 = UserFactory(username="test1")
+        grant_all_global(user1)
         user1.save()
 
         doc1 = Document.objects.create(
@@ -4074,7 +4157,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         THEN:
             - Explicit error is returned
         """
-        user1 = User.objects.create_superuser(username="test1")
+        user1 = UserFactory(username="test1", superuser=True)
 
         self.client.force_authenticate(user1)
 
@@ -4252,8 +4335,8 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         THEN:
             - Error response is returned
         """
-        user1 = User.objects.create_user(username="test1")
-        user1.user_permissions.add(*Permission.objects.all())
+        user1 = UserFactory(username="test1")
+        grant_all_global(user1)
         user1.save()
 
         doc = Document.objects.create(
@@ -4358,7 +4441,7 @@ class TestDocumentApiTagColors(DirectoriesMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.user = User.objects.create_superuser(username="temp_admin")
+        self.user = UserFactory(username="temp_admin", superuser=True)
 
         self.client.force_authenticate(user=self.user)
 
@@ -4438,7 +4521,7 @@ class TestDocumentApiCustomFieldsSorting(DirectoriesMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.user = User.objects.create_superuser(username="temp_admin")
+        self.user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.user)
 
         self.doc1 = Document.objects.create(

src/documents/tests/test_api_email.py

@@ -2,16 +2,16 @@ import json
 import shutil
 from unittest import mock
 
-from django.contrib.auth.models import Permission
-from django.contrib.auth.models import User
 from django.core import mail
 from django.test import override_settings
 from rest_framework import status
 from rest_framework.test import APITestCase
 
 from documents.models import Document
-from documents.tests.utils import DirectoriesMixin
 from documents.tests.utils import SampleDirMixin
+from paperless_testing.dirs import DirectoriesMixin
+from paperless_testing.factories import UserFactory
+from paperless_testing.permissions import grant_global
 
 
 class TestEmail(DirectoriesMixin, SampleDirMixin, APITestCase):
@@ -20,7 +20,7 @@ class TestEmail(DirectoriesMixin, SampleDirMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.user = User.objects.create_superuser(username="temp_admin")
+        self.user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.user)
 
         self.doc1 = Document.objects.create(
@@ -303,8 +303,8 @@ class TestEmail(DirectoriesMixin, SampleDirMixin, APITestCase):
         THEN:
             - Forbidden response is returned
         """
-        user1 = User.objects.create_user(username="test1")
-        user1.user_permissions.add(*Permission.objects.filter(codename="view_document"))
+        user1 = UserFactory(username="test1")
+        grant_global(user1, "view_document")
 
         doc_owned = Document.objects.create(
             title="owned_doc",
@@ -338,8 +338,8 @@ class TestEmail(DirectoriesMixin, SampleDirMixin, APITestCase):
         THEN:
             - Request succeeds
         """
-        user1 = User.objects.create_user(username="test1")
-        user1.user_permissions.add(*Permission.objects.filter(codename="view_document"))
+        user1 = UserFactory(username="test1")
+        grant_global(user1, "view_document")
 
         self.client.force_authenticate(user1)
 

src/documents/tests/test_api_filter_by_custom_fields.py

@@ -4,7 +4,6 @@ from collections.abc import Callable
 from datetime import date
 from urllib.parse import quote
 
-from django.contrib.auth.models import User
 from rest_framework.test import APITestCase
 
 from documents.models import CustomField
@@ -13,7 +12,8 @@ from documents.models import Document
 from documents.models import SavedView
 from documents.models import SavedViewFilterRule
 from documents.serialisers import DocumentSerializer
-from documents.tests.utils import DirectoriesMixin
+from paperless_testing.dirs import DirectoriesMixin
+from paperless_testing.factories import UserFactory
 
 
 class DocumentWrapper:
@@ -35,7 +35,7 @@ class TestCustomFieldsSearch(DirectoriesMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.user = User.objects.create_superuser(username="temp_admin")
+        self.user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.user)
 
         # Create one custom field per type. The fields are called f"{type}_field".

src/documents/tests/test_api_objects.py

@@ -2,10 +2,13 @@ import datetime
 import json
 from unittest import mock
 
-from django.contrib.auth.models import Permission
+from django.contrib.auth.models import Group
 from django.contrib.auth.models import User
+from django.db import connection
 from django.test import override_settings
-from guardian.shortcuts import assign_perm
+from django.test.utils import CaptureQueriesContext
+from guardian.shortcuts import get_groups_with_perms
+from guardian.shortcuts import get_users_with_perms
 from rest_framework import status
 from rest_framework.test import APITestCase
 
@@ -16,14 +19,17 @@ from documents.models import Document
 from documents.models import DocumentType
 from documents.models import StoragePath
 from documents.models import Tag
-from documents.tests.utils import DirectoriesMixin
+from paperless_testing.dirs import DirectoriesMixin
+from paperless_testing.factories import UserFactory
+from paperless_testing.permissions import grant_global
+from paperless_testing.permissions import grant_object
 
 
 class TestApiObjects(DirectoriesMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        user = User.objects.create_superuser(username="temp_admin")
+        user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=user)
 
         self.tag1 = Tag.objects.create(name="t1", is_inbox_tag=True)
@@ -148,7 +154,7 @@ class TestApiObjects(DirectoriesMixin, APITestCase):
 
         # A newer document owned by another user must not leak through the
         # aggregate for a non-superuser who cannot view it
-        other = User.objects.create_user(username="other")
+        other = UserFactory(username="other")
         Document.objects.create(
             mime_type="application/pdf",
             correspondent=self.c1,
@@ -157,10 +163,8 @@ class TestApiObjects(DirectoriesMixin, APITestCase):
             owner=other,
         )
 
-        user = User.objects.create_user(username="regular")
-        user.user_permissions.add(
-            Permission.objects.get(codename="view_correspondent"),
-        )
+        user = UserFactory(username="regular")
+        grant_global(user, "view_correspondent")
         self.client.force_authenticate(user=user)
 
         response = self.client.get("/api/correspondents/?last_correspondence=true")
@@ -195,7 +199,7 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        user = User.objects.create_superuser(username="temp_admin")
+        user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=user)
 
         self.sp1 = StoragePath.objects.create(name="sp1", path="Something/{checksum}")
@@ -450,8 +454,9 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
         self.assertEqual(response.data, "folder/Something.pdf")
 
     def test_test_storage_path_requires_document_view_permission(self) -> None:
-        owner = User.objects.create_user(username="owner")
-        unprivileged = User.objects.create_user(username="unprivileged")
+        owner = UserFactory(username="owner")
+        unprivileged = UserFactory(username="unprivileged")
+        grant_global(unprivileged, "view_document")
         document = Document.objects.create(
             mime_type="application/pdf",
             owner=owner,
@@ -473,16 +478,31 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
         self.assertIn("document", response.data)
 
     def test_test_storage_path_allows_shared_document_view_permission(self) -> None:
-        owner = User.objects.create_user(username="owner")
-        viewer = User.objects.create_user(username="viewer")
+        owner = UserFactory(username="owner")
+        viewer = UserFactory(username="viewer")
         document = Document.objects.create(
             mime_type="application/pdf",
             owner=owner,
             title="Shared",
             checksum="123",
         )
-        assign_perm("view_document", viewer, document)
+        grant_object(viewer, document, "view_document")
 
+        self.client.force_authenticate(user=viewer)
+        response = self.client.post(
+            f"{self.ENDPOINT}test/",
+            json.dumps(
+                {
+                    "document": document.id,
+                    "path": "path/{{ title }}",
+                },
+            ),
+            content_type="application/json",
+        )
+        self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
+
+        grant_global(viewer, "view_document")
+        viewer = User.objects.get(pk=viewer.pk)
         self.client.force_authenticate(user=viewer)
         response = self.client.post(
             f"{self.ENDPOINT}test/",
@@ -520,11 +540,12 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
     def test_test_storage_path_exposes_basic_document_context_but_not_sensitive_owner_data(
         self,
     ) -> None:
-        owner = User.objects.create_user(
+        owner = UserFactory(
             username="owner",
             password="password",
             email="owner@example.com",
         )
+        grant_global(owner, "view_document")
         document = Document.objects.create(
             mime_type="application/pdf",
             owner=owner,
@@ -586,8 +607,8 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
     def test_test_storage_path_includes_related_objects_for_visible_document(
         self,
     ) -> None:
-        owner = User.objects.create_user(username="owner")
-        viewer = User.objects.create_user(username="viewer")
+        owner = UserFactory(username="owner")
+        viewer = UserFactory(username="viewer")
         private_correspondent = Correspondent.objects.create(
             name="Private Correspondent",
             owner=owner,
@@ -599,7 +620,8 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
             title="Document",
             checksum="123",
         )
-        assign_perm("view_document", viewer, document)
+        grant_object(viewer, document, "view_document")
+        grant_global(viewer, "view_document")
 
         self.client.force_authenticate(user=viewer)
         response = self.client.post(
@@ -631,7 +653,7 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
         self.assertEqual(response.data, "Private Correspondent.pdf")
 
     def test_test_storage_path_superuser_can_view_private_related_objects(self) -> None:
-        owner = User.objects.create_user(username="owner")
+        owner = UserFactory(username="owner")
         private_correspondent = Correspondent.objects.create(
             name="Private Correspondent",
             owner=owner,
@@ -662,8 +684,8 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
     def test_test_storage_path_includes_doc_type_storage_path_and_tags(
         self,
     ) -> None:
-        owner = User.objects.create_user(username="owner")
-        viewer = User.objects.create_user(username="viewer")
+        owner = UserFactory(username="owner")
+        viewer = UserFactory(username="viewer")
         private_document_type = DocumentType.objects.create(
             name="Private Type",
             owner=owner,
@@ -686,7 +708,8 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
             checksum="123",
         )
         document.tags.add(private_tag)
-        assign_perm("view_document", viewer, document)
+        grant_object(viewer, document, "view_document")
+        grant_global(viewer, "view_document")
 
         self.client.force_authenticate(user=viewer)
         response = self.client.post(
@@ -722,8 +745,8 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
     def test_test_storage_path_includes_custom_fields_for_visible_document(
         self,
     ) -> None:
-        owner = User.objects.create_user(username="owner")
-        viewer = User.objects.create_user(username="viewer")
+        owner = UserFactory(username="owner")
+        viewer = UserFactory(username="viewer")
         document = Document.objects.create(
             mime_type="application/pdf",
             owner=owner,
@@ -739,7 +762,8 @@ class TestApiStoragePaths(DirectoriesMixin, APITestCase):
             field=custom_field,
             value_int=42,
         )
-        assign_perm("view_document", viewer, document)
+        grant_object(viewer, document, "view_document")
+        grant_global(viewer, "view_document")
 
         self.client.force_authenticate(user=viewer)
         response = self.client.post(
@@ -761,7 +785,7 @@ class TestBulkEditObjects(APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.temp_admin = User.objects.create_superuser(username="temp_admin")
+        self.temp_admin = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.temp_admin)
 
         self.t1 = Tag.objects.create(name="t1")
@@ -842,6 +866,66 @@ class TestBulkEditObjects(APITestCase):
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(StoragePath.objects.count(), 0)
 
+    def test_bulk_objects_set_permissions_batched_across_object_count(
+        self,
+    ) -> None:
+        """
+        GIVEN:
+            - Many tags are being bulk-edited to set permissions at once
+        WHEN:
+            - bulk_edit_objects API endpoint is called with set_permissions
+              operation over a small batch vs. a much larger one
+        THEN:
+            - Permissions are applied correctly at both scales
+            - Query count does not grow with the number of tags, i.e. each
+              user/group is applied across all tags with one batched call
+              rather than one call per (tag, identity) pair
+        """
+        group1 = Group.objects.create(name="perm-group")
+        permissions = {
+            "view": {"users": [self.user1.id, self.user2.id], "groups": [group1.id]},
+            "change": {"users": [self.user1.id], "groups": [group1.id]},
+        }
+
+        def run_with_n_tags(n: int) -> int:
+            tags = [Tag.objects.create(name=f"perm-tag-{n}-{i}") for i in range(n)]
+            with CaptureQueriesContext(connection) as ctx:
+                response = self.client.post(
+                    "/api/bulk_edit_objects/",
+                    json.dumps(
+                        {
+                            "objects": [t.id for t in tags],
+                            "object_type": "tags",
+                            "operation": "set_permissions",
+                            "permissions": permissions,
+                            "merge": False,
+                        },
+                    ),
+                    content_type="application/json",
+                )
+            self.assertEqual(response.status_code, status.HTTP_200_OK)
+            for tag in tags:
+                self.assertEqual(get_users_with_perms(tag).count(), 2)
+                self.assertEqual(get_groups_with_perms(tag).count(), 1)
+            return len(ctx.captured_queries)
+
+        small_batch_queries = run_with_n_tags(5)
+        large_batch_queries = run_with_n_tags(50)
+
+        # A tolerance rather than equality, matching the N+1 check in
+        # test_views.py: bulk_create's batch_size caps rows per INSERT, so a
+        # large enough selection does legitimately add statements, and the
+        # per-process ContentType cache makes the first run carry an extra
+        # query. Neither can hide a regression to per-object assignment,
+        # which would be ~10x the small-batch count here.
+        self.assertLessEqual(
+            large_batch_queries,
+            small_batch_queries + 5,
+            "Permission assignment appears to scale with object count: "
+            f"{small_batch_queries} queries for 5 tags vs. "
+            f"{large_batch_queries} for 50",
+        )
+
     def test_bulk_objects_delete_all_filtered(self) -> None:
         """
         GIVEN:
@@ -933,9 +1017,7 @@ class TestBulkEditObjects(APITestCase):
         THEN:
             - User is able to delete objects
         """
-        self.user1.user_permissions.add(
-            *Permission.objects.filter(codename="delete_tag"),
-        )
+        grant_global(self.user1, "delete_tag")
         self.user1.save()
         self.client.force_authenticate(user=self.user1)
 
@@ -965,9 +1047,7 @@ class TestBulkEditObjects(APITestCase):
         self.t2.owner = User.objects.get(username="temp_admin")
         self.t2.save()
 
-        self.user1.user_permissions.add(
-            *Permission.objects.filter(codename="delete_tag"),
-        )
+        grant_global(self.user1, "delete_tag")
         self.user1.save()
         self.client.force_authenticate(user=self.user1)
 
@@ -1000,9 +1080,7 @@ class TestBulkEditObjects(APITestCase):
         self.t2.owner = User.objects.get(username="temp_admin")
         self.t2.save()
 
-        self.user1.user_permissions.add(
-            *Permission.objects.filter(codename="delete_tag"),
-        )
+        grant_global(self.user1, "delete_tag")
         self.user1.save()
         self.client.force_authenticate(user=self.user1)
 

src/documents/tests/test_api_ocr_templates.py

@@ -10,7 +10,7 @@ from documents.models import CustomField
 from documents.models import DocumentType
 from documents.models import OcrTemplate
 from documents.models import OcrTemplateZone
-from documents.tests.utils import DirectoriesMixin
+from paperless_testing.dirs import DirectoriesMixin
 
 
 class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):

src/documents/tests/test_api_permissions.py

@@ -5,9 +5,7 @@ from unittest import mock
 from allauth.mfa.models import Authenticator
 from allauth.mfa.totp.internal import auth as totp_auth
 from django.contrib.auth.models import Group
-from django.contrib.auth.models import Permission
 from django.contrib.auth.models import User
-from guardian.shortcuts import assign_perm
 from guardian.shortcuts import get_perms
 from guardian.shortcuts import get_users_with_perms
 from rest_framework import status
@@ -19,7 +17,11 @@ from documents.models import DocumentType
 from documents.models import MatchingModel
 from documents.models import StoragePath
 from documents.models import Tag
-from documents.tests.utils import DirectoriesMixin
+from paperless_testing.dirs import DirectoriesMixin
+from paperless_testing.factories import UserFactory
+from paperless_testing.permissions import grant_all_global
+from paperless_testing.permissions import grant_global
+from paperless_testing.permissions import grant_object
 
 
 class TestApiAuth(DirectoriesMixin, APITestCase):
@@ -93,14 +95,14 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
         self.assertNotIn("X-Version", response)
 
     def test_api_version_with_auth(self) -> None:
-        user = User.objects.create_superuser(username="test")
+        user = UserFactory(username="test", superuser=True)
         self.client.force_authenticate(user)
         response = self.client.get("/api/documents/")
         self.assertIn("X-Api-Version", response)
         self.assertIn("X-Version", response)
 
     def test_api_insufficient_permissions(self) -> None:
-        user = User.objects.create_user(username="test")
+        user = UserFactory(username="test")
         self.client.force_authenticate(user)
 
         Document.objects.create(title="Test")
@@ -137,8 +139,8 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
         )
 
     def test_api_sufficient_permissions(self) -> None:
-        user = User.objects.create_user(username="test")
-        user.user_permissions.add(*Permission.objects.all())
+        user = UserFactory(username="test")
+        grant_all_global(user)
         user.is_staff = True
         self.client.force_authenticate(user)
 
@@ -166,9 +168,9 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
         )
 
     def test_api_get_object_permissions(self) -> None:
-        user1 = User.objects.create_user(username="test1")
-        user2 = User.objects.create_user(username="test2")
-        user1.user_permissions.add(*Permission.objects.filter(codename="view_document"))
+        user1 = UserFactory(username="test1")
+        user2 = UserFactory(username="test2")
+        grant_global(user1, "view_document")
         self.client.force_authenticate(user1)
 
         self.assertEqual(
@@ -205,7 +207,7 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
         THEN:
             - Object created with current user as owner
         """
-        user1 = User.objects.create_superuser(username="user1")
+        user1 = UserFactory(username="user1", superuser=True)
 
         self.client.force_authenticate(user1)
 
@@ -234,7 +236,7 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
         THEN:
             - Object created with no owner
         """
-        user1 = User.objects.create_superuser(username="user1")
+        user1 = UserFactory(username="user1", superuser=True)
 
         self.client.force_authenticate(user1)
 
@@ -265,7 +267,7 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
         THEN:
             - Object permissions are set appropriately
         """
-        user1 = User.objects.create_superuser(username="user1")
+        user1 = UserFactory(username="user1", superuser=True)
         user2 = User.objects.create(username="user2")
         group1 = Group.objects.create(name="group1")
 
@@ -313,7 +315,7 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
         THEN:
             - Object permissions are set appropriately
         """
-        user1 = User.objects.create_superuser(username="user1")
+        user1 = UserFactory(username="user1", superuser=True)
         user2 = User.objects.create(username="user2")
         group1 = Group.objects.create(name="group1")
 
@@ -363,7 +365,7 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
             mime_type="application/pdf",
             content="this is a document",
         )
-        user1 = User.objects.create_superuser(username="user1")
+        user1 = UserFactory(username="user1", superuser=True)
         user2 = User.objects.create(username="user2")
         group1 = Group.objects.create(name="group1")
 
@@ -413,16 +415,16 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
             mime_type="application/pdf",
             content="this is a document",
         )
-        user1 = User.objects.create_superuser(username="user1")
+        user1 = UserFactory(username="user1", superuser=True)
         user2 = User.objects.create(username="user2")
         group1 = Group.objects.create(name="group1")
         doc.owner = user1
         doc.save()
 
-        assign_perm("view_document", user2, doc)
-        assign_perm("change_document", user2, doc)
-        assign_perm("view_document", group1, doc)
-        assign_perm("change_document", group1, doc)
+        grant_object(user2, doc, "view_document")
+        grant_object(user2, doc, "change_document")
+        grant_object(group1, doc, "view_document")
+        grant_object(group1, doc, "change_document")
 
         self.client.force_authenticate(user1)
 
@@ -446,11 +448,9 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
         self.assertIn("change_document", get_perms(group1, doc))
 
     def test_document_permissions_change_requires_owner(self) -> None:
-        owner = User.objects.create_user(username="owner")
-        editor = User.objects.create_user(username="editor")
-        editor.user_permissions.add(
-            *Permission.objects.all(),
-        )
+        owner = UserFactory(username="owner")
+        editor = UserFactory(username="editor")
+        grant_all_global(editor)
 
         doc = Document.objects.create(
             title="Ownered doc",
@@ -460,8 +460,8 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
             owner=owner,
         )
 
-        assign_perm("view_document", editor, doc)
-        assign_perm("change_document", editor, doc)
+        grant_object(editor, doc, "view_document")
+        grant_object(editor, doc, "change_document")
 
         self.client.force_authenticate(editor)
         response = self.client.patch(
@@ -499,9 +499,9 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
 
     def test_dynamic_permissions_fields(self) -> None:
-        user1 = User.objects.create_user(username="user1")
-        user1.user_permissions.add(*Permission.objects.filter(codename="view_document"))
-        user2 = User.objects.create_user(username="user2")
+        user1 = UserFactory(username="user1")
+        grant_global(user1, "view_document")
+        user2 = UserFactory(username="user2")
 
         Document.objects.create(title="Test", content="content 1", checksum="1")
         doc2 = Document.objects.create(
@@ -523,10 +523,10 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
             owner=user1,
         )
 
-        assign_perm("view_document", user1, doc2)
-        assign_perm("view_document", user1, doc3)
-        assign_perm("change_document", user1, doc3)
-        assign_perm("view_document", user2, doc4)
+        grant_object(user1, doc2, "view_document")
+        grant_object(user1, doc3, "view_document")
+        grant_object(user1, doc3, "change_document")
+        grant_object(user2, doc4, "view_document")
 
         self.client.force_authenticate(user1)
 
@@ -574,8 +574,8 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
         owned by someone else with no explicit guardian grant -- mirrors
         guardian's own ObjectPermissionChecker.has_perm() superuser shortcut.
         """
-        superuser = User.objects.create_superuser(username="admin")
-        other_user = User.objects.create_user(username="user2")
+        superuser = UserFactory(username="admin", superuser=True)
+        other_user = UserFactory(username="user2")
         Document.objects.create(
             title="Test",
             content="content",
@@ -602,7 +602,7 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
         THEN:
             - MFA required error is returned
         """
-        user1 = User.objects.create_user(username="user1")
+        user1 = UserFactory(username="user1")
         user1.set_password("password")
         user1.save()
 
@@ -626,7 +626,7 @@ class TestApiAuth(DirectoriesMixin, APITestCase):
         THEN:
             - MFA code is required
         """
-        user1 = User.objects.create_user(username="user1")
+        user1 = UserFactory(username="user1")
         user1.set_password("password")
         user1.save()
 
@@ -688,7 +688,7 @@ class TestApiUser(DirectoriesMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.user = User.objects.create_superuser(username="temp_admin")
+        self.user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.user)
 
     def test_get_users(self) -> None:
@@ -858,10 +858,8 @@ class TestApiUser(DirectoriesMixin, APITestCase):
         )
         self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
 
-        regular_user = User.objects.create_user(username="regular_user")
-        regular_user.user_permissions.add(
-            *Permission.objects.all(),
-        )
+        regular_user = UserFactory(username="regular_user")
+        grant_all_global(regular_user)
         self.client.force_authenticate(regular_user)
         Authenticator.objects.create(
             user=user1,
@@ -885,9 +883,9 @@ class TestApiUser(DirectoriesMixin, APITestCase):
             - Only superusers can change superuser status
         """
 
-        user1 = User.objects.create_user(username="user1")
-        user1.user_permissions.add(*Permission.objects.all())
-        user2 = User.objects.create_superuser(username="user2")
+        user1 = UserFactory(username="user1")
+        grant_all_global(user1)
+        user2 = UserFactory(username="user2", superuser=True)
 
         self.client.force_authenticate(user1)
 
@@ -972,9 +970,9 @@ class TestApiUser(DirectoriesMixin, APITestCase):
             - Only superusers can change staff status
         """
 
-        user1 = User.objects.create_user(username="user1")
-        user1.user_permissions.add(*Permission.objects.all())
-        user2 = User.objects.create_superuser(username="user2")
+        user1 = UserFactory(username="user1")
+        grant_all_global(user1)
+        user2 = UserFactory(username="user2", superuser=True)
 
         self.client.force_authenticate(user1)
 
@@ -1027,7 +1025,7 @@ class TestApiGroup(DirectoriesMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.user = User.objects.create_superuser(username="temp_admin")
+        self.user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.user)
 
     def test_get_groups(self) -> None:
@@ -1128,7 +1126,7 @@ class TestBulkEditObjectPermissions(APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.temp_admin = User.objects.create_superuser(username="temp_admin")
+        self.temp_admin = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.temp_admin)
 
         self.t1 = Tag.objects.create(name="t1")
@@ -1276,7 +1274,7 @@ class TestBulkEditObjectPermissions(APITestCase):
             },
         }
 
-        assign_perm("view_tag", self.user3, self.t1)
+        grant_object(self.user3, self.t1, "view_tag")
         self.t1.owner = self.user3
         self.t1.save()
 
@@ -1373,13 +1371,9 @@ class TestBulkEditObjectPermissions(APITestCase):
         """
         self.t1.owner = self.user2
         self.t1.save()
-        assign_perm("view_tag", self.user1, self.t1)
-        assign_perm("change_tag", self.user1, self.t1)
-        self.user1.user_permissions.add(
-            *Permission.objects.filter(
-                codename__in=["view_tag", "change_tag"],
-            ),
-        )
+        grant_object(self.user1, self.t1, "view_tag")
+        grant_object(self.user1, self.t1, "change_tag")
+        grant_global(self.user1, "view_tag", "change_tag")
         user1 = User.objects.get(pk=self.user1.pk)
         self.client.force_authenticate(user=user1)
 
@@ -1426,13 +1420,9 @@ class TestBulkEditObjectPermissions(APITestCase):
         """
         owned = Tag.objects.create(name="owned", owner=self.user1)
         shared = Tag.objects.create(name="shared", owner=self.user2)
-        assign_perm("view_tag", self.user1, shared)
-        assign_perm("change_tag", self.user1, shared)
-        self.user1.user_permissions.add(
-            *Permission.objects.filter(
-                codename__in=["view_tag", "change_tag"],
-            ),
-        )
+        grant_object(self.user1, shared, "view_tag")
+        grant_object(self.user1, shared, "change_tag")
+        grant_global(self.user1, "view_tag", "change_tag")
         user1 = User.objects.get(pk=self.user1.pk)
         self.client.force_authenticate(user=user1)
 
@@ -1473,14 +1463,10 @@ class TestBulkEditObjectPermissions(APITestCase):
         """
         self.t1.owner = self.user2
         self.t1.save()
-        assign_perm("view_tag", self.user1, self.t1)
-        assign_perm("change_tag", self.user1, self.t1)
-        assign_perm("delete_tag", self.user1, self.t1)
-        self.user1.user_permissions.add(
-            *Permission.objects.filter(
-                codename__in=["view_tag", "change_tag", "delete_tag"],
-            ),
-        )
+        grant_object(self.user1, self.t1, "view_tag")
+        grant_object(self.user1, self.t1, "change_tag")
+        grant_object(self.user1, self.t1, "delete_tag")
+        grant_global(self.user1, "view_tag", "change_tag", "delete_tag")
         user1 = User.objects.get(pk=self.user1.pk)
         self.client.force_authenticate(user=user1)
 
@@ -1499,6 +1485,110 @@ class TestBulkEditObjectPermissions(APITestCase):
         self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
         self.assertTrue(Tag.objects.filter(pk=self.t1.id).exists())
 
+    def test_bulk_object_set_permissions_rejects_empty_permissions(self) -> None:
+        """
+        GIVEN:
+            - Existing objects
+        WHEN:
+            - bulk_edit_objects API endpoint is called with set_permissions
+              operation and an empty permissions dict
+        THEN:
+            - Validation fails rather than silently applying a no-op
+        """
+        response = self.client.post(
+            "/api/bulk_edit_objects/",
+            json.dumps(
+                {
+                    "objects": [self.t1.id],
+                    "object_type": "tags",
+                    "operation": "set_permissions",
+                    "permissions": {},
+                },
+            ),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+
+    def test_bulk_object_set_permissions_rejects_non_dict_permissions(self) -> None:
+        """
+        GIVEN:
+            - Existing objects
+        WHEN:
+            - bulk_edit_objects API endpoint is called with set_permissions
+              operation and a non-dict permissions value
+        THEN:
+            - Validation fails rather than crashing
+        """
+        response = self.client.post(
+            "/api/bulk_edit_objects/",
+            json.dumps(
+                {
+                    "objects": [self.t1.id],
+                    "object_type": "tags",
+                    "operation": "set_permissions",
+                    "permissions": False,
+                },
+            ),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+
+    def test_bulk_object_set_permissions_rejects_unknown_action(self) -> None:
+        """
+        GIVEN:
+            - Existing objects
+        WHEN:
+            - bulk_edit_objects API endpoint is called with set_permissions
+              operation and an unrecognized permission action name
+        THEN:
+            - Validation fails rather than silently no-oping
+        """
+        response = self.client.post(
+            "/api/bulk_edit_objects/",
+            json.dumps(
+                {
+                    "objects": [self.t1.id],
+                    "object_type": "tags",
+                    "operation": "set_permissions",
+                    "permissions": {"not_a_real_action": {"users": [self.user1.id]}},
+                },
+            ),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+
+    def test_bulk_object_set_permissions_null_users_clears_users(self) -> None:
+        """
+        GIVEN:
+            - An object a user has view permission on
+        WHEN:
+            - bulk_edit_objects API endpoint is called with set_permissions
+              operation, merge off, and an explicit null for the view users
+        THEN:
+            - Request succeeds and null is treated as an empty user list,
+              so the existing view permission is removed
+        """
+        grant_object(self.user1, self.t1, "view_tag")
+
+        response = self.client.post(
+            "/api/bulk_edit_objects/",
+            json.dumps(
+                {
+                    "objects": [self.t1.id],
+                    "object_type": "tags",
+                    "operation": "set_permissions",
+                    "permissions": {"view": {"users": None}},
+                },
+            ),
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertNotIn(self.user1, get_users_with_perms(self.t1))
+
     def test_bulk_edit_object_permissions_validation(self) -> None:
         """
         GIVEN:
@@ -1576,7 +1666,7 @@ class TestFullPermissionsFlag(APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.admin = User.objects.create_superuser(username="admin")
+        self.admin = UserFactory(username="admin", superuser=True)
 
     def test_full_perms_flag(self) -> None:
         """

src/documents/tests/test_api_post_document_nfc.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import unicodedata
 from typing import TYPE_CHECKING
 from unittest import mock
@@ -7,8 +9,11 @@ import pytest
 from django.core.files.uploadedfile import SimpleUploadedFile
 
 if TYPE_CHECKING:
+    from rest_framework.test import APIClient
+
     from documents.data_models import ConsumableDocument
     from documents.data_models import DocumentMetadataOverrides
+    from paperless_testing.dirs import PaperlessDirs
 
 
 @pytest.fixture()
@@ -18,22 +23,14 @@ def consume_file_mock():
         yield m
 
 
-@pytest.fixture()
-def directories(tmp_path, settings, _media_settings):
-    scratch = tmp_path / "scratch"
-    scratch.mkdir()
-    settings.SCRATCH_DIR = scratch
-    return scratch
-
-
 @pytest.mark.django_db
 class TestPostDocumentNFCNormalization:
     def test_nfd_filename_normalized_to_nfc(
         self,
-        admin_client,
+        admin_client: APIClient,
         consume_file_mock: mock.MagicMock,
-        directories,
-    ):
+        paperless_dirs: PaperlessDirs,
+    ) -> None:
         """Uploaded file with NFD filename must have its name stored as NFC."""
         nfd = unicodedata.normalize("NFD", "Rechnung März.pdf")
         nfc = unicodedata.normalize("NFC", "Rechnung März.pdf")
@@ -69,10 +66,10 @@ class TestPostDocumentNFCNormalization:
 
     def test_already_nfc_filename_unchanged(
         self,
-        admin_client,
+        admin_client: APIClient,
         consume_file_mock: mock.MagicMock,
-        directories,
-    ):
+        paperless_dirs: PaperlessDirs,
+    ) -> None:
         """Uploaded file with already-NFC filename must pass through unchanged."""
         nfc = unicodedata.normalize("NFC", "Invoice_2024.pdf")
 

src/documents/tests/test_api_profile.py

@@ -8,7 +8,8 @@ from rest_framework import status
 from rest_framework.authtoken.models import Token
 from rest_framework.test import APITestCase
 
-from documents.tests.utils import DirectoriesMixin
+from paperless_testing.dirs import DirectoriesMixin
+from paperless_testing.factories import UserFactory
 
 
 # see allauth.socialaccount.providers.openid.provider.OpenIDProvider
@@ -55,23 +56,23 @@ class TestApiProfile(DirectoriesMixin, APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.user = User.objects.create_superuser(
+        self.user = UserFactory(
             username="temp_admin",
             first_name="firstname",
             last_name="surname",
+            superuser=True,
         )
         self.client.force_authenticate(user=self.user)
 
-    def setupSocialAccount(self) -> None:
+    def setupSocialAccount(self) -> SocialAccount:
         SocialApp.objects.create(
             name="Keycloak",
             provider="openid_connect",
             provider_id="keycloak-test",
         )
-        self.user.socialaccount_set.add(
-            SocialAccount(uid="123456789", provider="keycloak-test"),
-            bulk=False,
-        )
+        social_account = SocialAccount(uid="123456789", provider="keycloak-test")
+        self.user.socialaccount_set.add(social_account, bulk=False)
+        return social_account
 
     def test_get_profile(self) -> None:
         """
@@ -109,19 +110,17 @@ class TestApiProfile(DirectoriesMixin, APITestCase):
         THEN:
             - Profile is returned with social accounts
         """
-        self.setupSocialAccount()
+        social_account = self.setupSocialAccount()
 
-        openid_provider = (
-            MockOpenIDConnectProvider(
-                app=SocialApp.objects.get(provider_id="keycloak-test"),
-            ),
+        openid_provider = MockOpenIDConnectProvider(
+            app=SocialApp.objects.get(provider_id="keycloak-test"),
         )
         mock_list_providers.return_value = [
             openid_provider,
         ]
         mock_get_provider_account.return_value = MockOpenIDConnectProviderAccount(
             mock_social_account_dict={
-                "name": openid_provider[0].name,
+                "name": openid_provider.name,
             },
         )
 
@@ -133,7 +132,7 @@ class TestApiProfile(DirectoriesMixin, APITestCase):
             response.data["social_accounts"],
             [
                 {
-                    "id": 1,
+                    "id": social_account.pk,
                     "provider": "keycloak-test",
                     "name": "Keycloak",
                 },
@@ -150,7 +149,7 @@ class TestApiProfile(DirectoriesMixin, APITestCase):
         THEN:
             - Profile is returned with "Unknown App" as name
         """
-        self.setupSocialAccount()
+        social_account = self.setupSocialAccount()
 
         # Remove the social app
         SocialApp.objects.get(provider_id="keycloak-test").delete()
@@ -163,7 +162,7 @@ class TestApiProfile(DirectoriesMixin, APITestCase):
             response.data["social_accounts"],
             [
                 {
-                    "id": 1,
+                    "id": social_account.pk,
                     "provider": "keycloak-test",
                     "name": "Unknown App",
                 },
@@ -401,7 +400,7 @@ class TestApiTOTPViews(APITestCase):
     def setUp(self) -> None:
         super().setUp()
 
-        self.user = User.objects.create_superuser(username="temp_admin")
+        self.user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.user)
 
     def test_get_totp(self) -> None:

src/documents/tests/test_api_search.py

@@ -7,10 +7,8 @@ import time_machine
 from dateutil.relativedelta import relativedelta
 from django.contrib.auth.models import Group
 from django.contrib.auth.models import Permission
-from django.contrib.auth.models import User
 from django.test import override_settings
 from django.utils import timezone
-from guardian.shortcuts import assign_perm
 from rest_framework import status
 from rest_framework.test import APITestCase
 
@@ -27,10 +25,13 @@ from documents.models import Tag
 from documents.models import Workflow
 from documents.search import get_backend
 from documents.search import reset_backend
-from documents.tests.factories import DocumentFactory
-from documents.tests.utils import DirectoriesMixin
 from paperless_mail.models import MailAccount
 from paperless_mail.models import MailRule
+from paperless_testing.dirs import DirectoriesMixin
+from paperless_testing.factories import DocumentFactory
+from paperless_testing.factories import UserFactory
+from paperless_testing.permissions import grant_global
+from paperless_testing.permissions import grant_object
 
 pytestmark = pytest.mark.search
 
@@ -40,7 +41,7 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         super().setUp()
         reset_backend()
 
-        self.user = User.objects.create_superuser(username="temp_admin")
+        self.user = UserFactory(username="temp_admin", superuser=True)
         self.client.force_authenticate(user=self.user)
 
     def tearDown(self) -> None:
@@ -244,8 +245,68 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         self.assertEqual(response.data["count"], 1)
         self.assertEqual(response.data["results"][0]["id"], title_match.id)
 
+    def test_simple_search_combined_with_query_filter(self) -> None:
+        """
+        GIVEN:
+            - A document with the term in its title, created this year
+            - A document with the term only in its content, created this year
+            - A document with the term in its title, created years ago
+        WHEN:
+            - A title or text search is combined with a relative created
+              date query, as the filter editor sends it
+        THEN:
+            - The simple search keeps its own field scope and the query only
+              narrows the results (issue #14436)
+        """
+        this_year = timezone.localdate().replace(month=1, day=1)
+        title_match = Document.objects.create(
+            title="foo report",
+            content="nothing relevant",
+            checksum="F1",
+            created=this_year,
+            pk=21,
+        )
+        content_match = Document.objects.create(
+            title="something else",
+            content="foo appears here",
+            checksum="F2",
+            created=this_year,
+            pk=22,
+        )
+        old_title_match = Document.objects.create(
+            title="foo archive",
+            content="nothing relevant",
+            checksum="F3",
+            created=datetime.date(2001, 1, 1),
+            pk=23,
+        )
+        backend = get_backend()
+        for doc in (title_match, content_match, old_title_match):
+            backend.add_or_update(doc)
+
+        query = "created:[-1 year to now]"
+        response = self.client.get(
+            "/api/documents/",
+            {"title_search": "foo", "query": query},
+        )
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(
+            [r["id"] for r in response.data["results"]],
+            [title_match.id],
+        )
+
+        response = self.client.get(
+            "/api/documents/",
+            {"text": "foo", "query": query},
+        )
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertCountEqual(
+            [r["id"] for r in response.data["results"]],
+            [title_match.id, content_match.id],
+        )
+
     def test_search_rejects_multiple_search_modes(self) -> None:
-        response = self.client.get("/api/documents/?text=bank&query=bank")
+        response = self.client.get("/api/documents/?text=bank&title_search=bank")
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
         self.assertEqual(
             response.data["detail"],
@@ -786,6 +847,10 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
             tick=False,
         ):
             response = self.client.get("/api/documents/?query=added:previous month")
+        assert response.status_code == 200, (
+            f"expected a successful search response, got {response.status_code}: "
+            f"{response.data!r}"
+        )
         results = response.data["results"]
 
         self.assertEqual(len(results), 1)
@@ -818,6 +883,26 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
         self.assertIn("invalid-date", str(response.data["query"]))
 
+    def test_search_multiple_bad_fields_returns_all_messages(self) -> None:
+        """
+        GIVEN:
+            - One document added
+        WHEN:
+            - Query with multiple bad fields (e.g. invalid date and invalid number)
+        THEN:
+            - 400 Bad Request with error messages for every bad field,
+              so the user can fix them all in one round-trip
+        """
+        response = self.client.get(
+            "/api/documents/",
+            {"query": "created:notadate AND asn:notanumber"},
+        )
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+        messages = response.data["query"]
+        self.assertEqual(len(messages), 2)
+        self.assertTrue(any("created" in m for m in messages))
+        self.assertTrue(any("asn" in m for m in messages))
+
     @override_settings(
         TIME_ZONE="UTC",
     )
@@ -861,6 +946,29 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         results = response.data["results"]
         self.assertEqual({r["id"] for r in results}, {1, 2})
 
+    @mock.patch("documents.search._backend.parse_user_query")
+    def test_search_parser_bug_surfaces_as_500_not_400(self, m) -> None:
+        """
+        GIVEN:
+            - The query parser itself fails (a whoosh-compat bug, per
+              QueryParserError's own contract: not user-fixable input)
+        WHEN:
+            - Any search request runs
+        THEN:
+            - The error surfaces as a 500 monitoring can see, never a 400
+              blaming the user for a library defect
+        """
+        from whoosh_compat.errors import QueryParserError
+
+        m.side_effect = QueryParserError("synthetic parser bug")
+
+        self.client.raise_request_exception = False
+        response = self.client.get("/api/documents/?query=anything")
+        self.assertEqual(
+            response.status_code,
+            status.HTTP_500_INTERNAL_SERVER_ERROR,
+        )
+
     @mock.patch("documents.search._backend.TantivyBackend.autocomplete")
     def test_search_autocomplete_limits(self, m) -> None:
         """
@@ -902,9 +1010,9 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         THEN:
             - Terms only within docs user has access to are returned
         """
-        u1 = User.objects.create_user("user1")
-        u2 = User.objects.create_user("user2")
-        u1.user_permissions.add(Permission.objects.get(codename="view_document"))
+        u1 = UserFactory(username="user1")
+        u2 = UserFactory(username="user2")
+        grant_global(u1, "view_document")
 
         self.client.force_authenticate(user=u1)
 
@@ -944,7 +1052,7 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(response.data, ["applebaum", "apples"])
 
-        assign_perm("view_document", u1, d3)
+        grant_object(u1, d3, "view_document")
         backend.add_or_update(d3)
 
         response = self.client.get("/api/search/autocomplete/?term=app")
@@ -952,10 +1060,10 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         self.assertEqual(response.data, ["applebaum", "apples", "appletini"])
 
     def test_search_autocomplete_group_revocation_is_immediate(self) -> None:
-        user = User.objects.create_user("group-user")
-        owner = User.objects.create_user("document-owner")
+        user = UserFactory(username="group-user")
+        owner = UserFactory(username="document-owner")
         group = Group.objects.create(name="temporary-viewers")
-        user.user_permissions.add(Permission.objects.get(codename="view_document"))
+        grant_global(user, "view_document")
         user.groups.add(group)
 
         document = Document.objects.create(
@@ -964,7 +1072,7 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
             checksum="group-revocation",
             owner=owner,
         )
-        assign_perm("view_document", group, document)
+        grant_object(group, document, "view_document")
         get_backend().add_or_update(document)
         self.client.force_authenticate(user=user)
 
@@ -1044,11 +1152,9 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         self.assertIsNone(response.data["corrected_query"])
 
     def test_search_spelling_suggestion_suppressed_for_private_terms(self) -> None:
-        owner = User.objects.create_user("owner")
-        attacker = User.objects.create_user("attacker")
-        attacker.user_permissions.add(
-            Permission.objects.get(codename="view_document"),
-        )
+        owner = UserFactory(username="owner")
+        attacker = UserFactory(username="attacker")
+        grant_global(attacker, "view_document")
 
         backend = get_backend()
         for i in range(5):
@@ -1150,6 +1256,59 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         self.assertIn(d3.id, result_ids)
         self.assertNotIn(d4.id, result_ids)
 
+    def test_search_more_like_version_uses_its_root(self) -> None:
+        """
+        GIVEN:
+            - A document similar in content to a root document, and one that is not
+            - A version of the root document, which is never indexed
+        WHEN:
+            - API request for more like the version
+        THEN:
+            - The documents similar to the root are returned, not the version
+        """
+        indexed = {}
+        for name, title, content, day in (
+            ("root", "bank statement 1", "things i paid for in august", (2019, 3, 4)),
+            (
+                "similar",
+                "bank statement 3",
+                "things i paid for in september",
+                (2020, 7, 9),
+            ),
+            (
+                "other",
+                "Quarterly Report",
+                "quarterly revenue profit margin",
+                (2021, 11, 30),
+            ),
+        ):
+            with time_machine.travel(
+                timezone.make_aware(datetime.datetime(*day)),
+                tick=False,
+            ):
+                indexed[name] = DocumentFactory(
+                    title=title,
+                    content=content,
+                    created=datetime.date(*day),
+                    added=timezone.make_aware(datetime.datetime(*day)),
+                )
+        version = DocumentFactory(
+            root_document=indexed["root"],
+            version_index=1,
+            content="things i paid for in august",
+        )
+        backend = get_backend()
+        for document in indexed.values():
+            backend.add_or_update(document)
+
+        response = self.client.get(f"/api/documents/?more_like_id={version.id}")
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        result_ids = [r["id"] for r in response.data["results"]]
+        self.assertIn(indexed["similar"].id, result_ids)
+        self.assertNotIn(indexed["other"].id, result_ids)
+        self.assertNotIn(version.id, result_ids)
+
     def test_more_like_requires_id_of_existing_document(self) -> None:
         """
         GIVEN:
@@ -1175,11 +1334,9 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         THEN:
             - The request is rejected
         """
-        owner = User.objects.create_user("owner")
-        attacker = User.objects.create_user("attacker")
-        attacker.user_permissions.add(
-            Permission.objects.get(codename="view_document"),
-        )
+        owner = UserFactory(username="owner")
+        attacker = UserFactory(username="attacker")
+        grant_global(attacker, "view_document")
 
         private_seed = Document.objects.create(
             title="private bank statement",
@@ -1217,6 +1374,39 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
         self.assertEqual(response.content, b"Insufficient permissions.")
 
+    def test_search_more_like_version_requires_view_permission_on_its_root(
+        self,
+    ) -> None:
+        """
+        GIVEN:
+            - Another user's private root document with an unowned version
+        WHEN:
+            - A user without access requests more-like-this for the version
+        THEN:
+            - The request is rejected, the version's missing owner grants nothing
+        """
+        owner = UserFactory(username="owner")
+        attacker = UserFactory(username="attacker")
+        grant_global(attacker, "view_document")
+        private_root = DocumentFactory(
+            title="private bank statement",
+            content="quarterly treasury bank statement wire transfer",
+            owner=owner,
+        )
+        version = DocumentFactory(
+            root_document=private_root,
+            version_index=1,
+            owner=None,
+        )
+        get_backend().add_or_update(private_root)
+
+        self.client.force_authenticate(user=attacker)
+
+        response = self.client.get(f"/api/documents/?more_like_id={version.id}")
+
+        self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
+        self.assertEqual(response.content, b"Insufficient permissions.")
+
     def test_search_filtering(self) -> None:
         t = Tag.objects.create(name="tag")
         t2 = Tag.objects.create(name="tag2")
@@ -1487,11 +1677,11 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
             - Only owned docs are returned for regular users
             - All docs are returned for superuser
         """
-        superuser = User.objects.create_superuser("superuser")
-        u1 = User.objects.create_user("user1")
-        u2 = User.objects.create_user("user2")
-        u1.user_permissions.add(*Permission.objects.filter(codename="view_document"))
-        u2.user_permissions.add(*Permission.objects.filter(codename="view_document"))
+        superuser = UserFactory(username="superuser", superuser=True)
+        u1 = UserFactory(username="user1")
+        u2 = UserFactory(username="user2")
+        grant_global(u1, "view_document")
+        grant_global(u2, "view_document")
 
         Document.objects.create(checksum="1", content="test 1", owner=u1)
         Document.objects.create(checksum="2", content="test 2", owner=u2)
@@ -1541,10 +1731,10 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         THEN:
             - Only docs with granted view permissions are returned
         """
-        u1 = User.objects.create_user("user1")
-        u2 = User.objects.create_user("user2")
-        u1.user_permissions.add(*Permission.objects.filter(codename="view_document"))
-        u2.user_permissions.add(*Permission.objects.filter(codename="view_document"))
+        u1 = UserFactory(username="user1")
+        u2 = UserFactory(username="user2")
+        grant_global(u1, "view_document")
+        grant_global(u2, "view_document")
 
         d1 = Document.objects.create(checksum="1", content="test 1", owner=u1)
         d2 = Document.objects.create(checksum="2", content="test 2", owner=u2)
@@ -1569,9 +1759,9 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         r = self.client.get("/api/documents/?query=test&owner__isnull=true")
         self.assertEqual(r.data["count"], 1)
 
-        assign_perm("view_document", u1, d2)
-        assign_perm("view_document", u1, d3)
-        assign_perm("view_document", u2, d1)
+        grant_object(u1, d2, "view_document")
+        grant_object(u1, d3, "view_document")
+        grant_object(u2, d1, "view_document")
 
         backend.add_or_update(d1)
         backend.add_or_update(d2)
@@ -1594,8 +1784,8 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         self.assertEqual(r.data["count"], 1)
 
     def test_search_sorting(self) -> None:
-        u1 = User.objects.create_user("user1")
-        u2 = User.objects.create_user("user2")
+        u1 = UserFactory(username="user1")
+        u2 = UserFactory(username="user2")
         c1 = Correspondent.objects.create(name="corres Ax")
         c2 = Correspondent.objects.create(name="corres Cx")
         c3 = Correspondent.objects.create(name="corres Bx")
@@ -1845,8 +2035,8 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
             pk=5,
         )
 
-        user1 = User.objects.create_user("bank user1")
-        user2 = User.objects.create_superuser("user2")
+        user1 = UserFactory(username="bank user1")
+        user2 = UserFactory(username="user2", superuser=True)
         group1 = Group.objects.create(name="bank group1")
         Group.objects.create(name="group2")
 
@@ -1878,7 +2068,7 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
             sort_field="",
             owner=user2,
         )
-        assign_perm("view_savedview", user1, shared_view)
+        grant_object(user1, shared_view, "view_savedview")
         mail_account1 = MailAccount.objects.create(name="bank mail account 1")
         mail_account2 = MailAccount.objects.create(name="mail account 2")
         mail_rule1 = MailRule.objects.create(
@@ -1947,36 +2137,10 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         self.assertEqual(len(response.data["documents"]), 1)
         self.assertEqual(response.data["documents"][0]["id"], title_match.id)
 
-    def test_global_search_returns_latest_version_content(self) -> None:
-        root = Document.objects.create(
-            title="bank statement",
-            content="superseded content",
-            checksum="GSV1",
-            pk=23,
-        )
-        Document.objects.create(
-            title="bank statement v2",
-            content="latest content",
-            checksum="GSV2",
-            pk=24,
-            root_document=root,
-            version_index=1,
-        )
-
-        self.client.force_authenticate(self.user)
-
-        response = self.client.get("/api/search/?query=bank&db_only=true")
-        self.assertEqual(response.status_code, status.HTTP_200_OK)
-        returned = {doc["id"]: doc["content"] for doc in response.data["documents"]}
-        self.assertEqual(returned.get(root.id), "latest content")
-
     def test_global_search_filters_owned_mail_objects(self) -> None:
-        user1 = User.objects.create_user("mail-search-user")
-        user2 = User.objects.create_user("other-mail-search-user")
-        user1.user_permissions.add(
-            Permission.objects.get(codename="view_mailaccount"),
-            Permission.objects.get(codename="view_mailrule"),
-        )
+        user1 = UserFactory(username="mail-search-user")
+        user2 = UserFactory(username="other-mail-search-user")
+        grant_global(user1, "view_mailaccount", "view_mailrule")
 
         own_account = MailAccount.objects.create(
             name="bank owned account",
@@ -2058,3 +2222,77 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
         response = self.client.get("/api/search/?query=no")
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+
+    def _assert_query_finds(self, doc: Document, query: str) -> None:
+        get_backend().add_or_update(doc)
+        response = self.client.get("/api/documents/", {"query": query})
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        ids = [r["id"] for r in response.data["results"]]
+        self.assertIn(doc.id, ids)
+
+    def test_search_by_asn(self) -> None:
+        """
+        GIVEN:
+            - A document with an archive serial number, indexed
+        WHEN:
+            - A query filters by "asn:<value>"
+        THEN:
+            - The document is found
+        """
+        doc = Document.objects.create(
+            title="Has ASN",
+            content="content",
+            checksum="asn-checksum",
+            archive_serial_number=555,
+        )
+        self._assert_query_finds(doc, "asn:555")
+
+    def test_search_by_page_count(self) -> None:
+        """
+        GIVEN:
+            - A document with a page count, indexed
+        WHEN:
+            - A query filters by "page_count:<value>"
+        THEN:
+            - The document is found
+        """
+        doc = Document.objects.create(
+            title="Multi-page",
+            content="content",
+            checksum="page-count-checksum",
+            page_count=42,
+        )
+        self._assert_query_finds(doc, "page_count:42")
+
+    def test_search_by_original_filename(self) -> None:
+        """
+        GIVEN:
+            - A document with an original filename, indexed
+        WHEN:
+            - A query filters by "original_filename:<value>"
+        THEN:
+            - The document is found
+        """
+        doc = Document.objects.create(
+            title="Named file",
+            content="content",
+            checksum="filename-checksum",
+            original_filename="quarterly-report.pdf",
+        )
+        self._assert_query_finds(doc, "original_filename:quarterly-report.pdf")
+
+    def test_search_by_checksum(self) -> None:
+        """
+        GIVEN:
+            - A document with a checksum, indexed
+        WHEN:
+            - A query filters by "checksum:<value>"
+        THEN:
+            - The document is found
+        """
+        doc = Document.objects.create(
+            title="Checksum doc",
+            content="content",
+            checksum="deadbeef1234",
+        )
+        self._assert_query_finds(doc, "checksum:deadbeef1234")

src/documents/tests/test_api_search_errors.py

@@ -0,0 +1,334 @@
+"""The search list endpoint's exception handling: what becomes a 400 and
+what a library defect surfaces as instead.
+
+Companion to documents/tests/search/test_error_routing.py, which pins the
+Cause -> SearchQueryError/QueryError routing inside documents/search/_query.py.
+These tests pin the layer above it: DocumentViewSet.list's own except clauses,
+which decide what an already-routed error becomes on the wire.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+from rest_framework import status
+from whoosh_compat.errors import Cause
+from whoosh_compat.errors import Diagnostic
+from whoosh_compat.errors import DiagnosticKind
+from whoosh_compat.errors import QueryError
+
+from documents.search import SearchQueryError
+
+if TYPE_CHECKING:
+    from rest_framework.test import APIClient
+
+    from documents.models import Document
+
+pytestmark = [pytest.mark.django_db, pytest.mark.usefixtures("_search_index")]
+
+
+class TestSearchQueryErrorStillBecomesA400:
+    def test_search_query_error_becomes_a_400_naming_the_field(
+        self,
+        admin_client: APIClient,
+        monkeypatch: pytest.MonkeyPatch,
+        searchable_document: Document,
+    ) -> None:
+        """
+        GIVEN:
+            - parse_user_query() raising a SearchQueryError naming a field
+        WHEN:
+            - The document list endpoint is queried
+        THEN:
+            - The response is a 400 whose body names the field
+        """
+        import documents.search._backend as backend_mod
+
+        def raise_search_query_error(*args: object, **kwargs: object) -> object:
+            raise SearchQueryError("bad value for field 'added'")
+
+        monkeypatch.setattr(
+            backend_mod,
+            "parse_user_query",
+            raise_search_query_error,
+        )
+
+        response = admin_client.get("/api/documents/?query=anything")
+
+        assert response.status_code == status.HTTP_400_BAD_REQUEST
+        assert "added" in str(response.data["query"])
+
+
+class TestLibraryDefectsPropagate:
+    """The exact regression this task exists to fix: an unexpected or
+    INTERNAL-cause library error must not be relabeled a 400."""
+
+    def test_unexpected_exception_is_not_converted_to_a_400(
+        self,
+        admin_client: APIClient,
+        monkeypatch: pytest.MonkeyPatch,
+        searchable_document: Document,
+    ) -> None:
+        """
+        GIVEN:
+            - parse_user_query() raising an unrelated exception
+              (ZeroDivisionError), not a SearchQueryError
+        WHEN:
+            - The document list endpoint is queried
+        THEN:
+            - The exception propagates unconverted, rather than being
+              relabeled a 400
+        """
+        import documents.search._backend as backend_mod
+
+        def raise_zero_division(*args: object, **kwargs: object) -> object:
+            raise ZeroDivisionError("synthetic bug, unrelated to search grammar")
+
+        monkeypatch.setattr(
+            backend_mod,
+            "parse_user_query",
+            raise_zero_division,
+        )
+
+        with pytest.raises(ZeroDivisionError):
+            admin_client.get("/api/documents/?query=anything")
+
+    def test_internal_cause_query_error_is_not_converted_to_a_400(
+        self,
+        admin_client: APIClient,
+        monkeypatch: pytest.MonkeyPatch,
+        searchable_document: Document,
+    ) -> None:
+        """
+        GIVEN:
+            - A real query string running through the real parse and
+              routing pipeline (pre-parse rewrites, wc.parse(), and
+              _map_emit_error's own Cause routing all run for real), except
+              the final emit call (tantivy_emit) is forced to report a
+              library-internal defect (Cause.INTERNAL) - the one
+              library-internal failure mode reachable from a real query
+        WHEN:
+            - The document list endpoint is queried
+        THEN:
+            - The QueryError propagates unconverted, rather than being
+              relabeled a 400
+        """
+        import documents.search._query as query_mod
+
+        def raise_internal(*args: object, **kwargs: object) -> object:
+            raise QueryError(
+                Diagnostic(
+                    kind=DiagnosticKind.BACKEND_REJECTED,
+                    cause=Cause.INTERNAL,
+                    message="synthetic whoosh-compat emitter defect",
+                ),
+            )
+
+        monkeypatch.setattr(query_mod, "tantivy_emit", raise_internal)
+
+        with pytest.raises(QueryError):
+            admin_client.get("/api/documents/?query=invoice")
+
+
+class TestSelectionPathsAgreeWithSearch:
+    """DocumentSelectionMixin backs bulk edit, bulk download, and a
+    more_like_id selection filter. It catches only SearchQueryError -- the
+    same contract the search list endpoint enforces above -- so all three
+    must map SearchQueryError to a 400 and let anything else surface."""
+
+    def test_bulk_edit_maps_search_query_error_to_a_400(
+        self,
+        admin_client: APIClient,
+        monkeypatch: pytest.MonkeyPatch,
+        searchable_document: Document,
+    ) -> None:
+        """
+        GIVEN:
+            - parse_user_query() raising a SearchQueryError naming a field
+        WHEN:
+            - The bulk_edit endpoint is called with a query filter
+        THEN:
+            - The response is a 400 whose body names the field
+        """
+        import documents.search._backend as backend_mod
+
+        def raise_search_query_error(*args: object, **kwargs: object) -> object:
+            raise SearchQueryError("bad value for field 'added'")
+
+        monkeypatch.setattr(
+            backend_mod,
+            "parse_user_query",
+            raise_search_query_error,
+        )
+
+        response = admin_client.post(
+            "/api/documents/bulk_edit/",
+            {
+                "documents": [],
+                "all": True,
+                "filters": {"query": "anything"},
+                "method": "set_document_type",
+                "parameters": {"document_type": None},
+            },
+            format="json",
+        )
+
+        assert response.status_code == status.HTTP_400_BAD_REQUEST
+        assert "added" in str(response.data["query"])
+
+    def test_bulk_edit_lets_an_unexpected_exception_surface(
+        self,
+        admin_client: APIClient,
+        monkeypatch: pytest.MonkeyPatch,
+        searchable_document: Document,
+    ) -> None:
+        """
+        GIVEN:
+            - parse_user_query() raising an unrelated exception
+              (ZeroDivisionError), not a SearchQueryError
+        WHEN:
+            - The bulk_edit endpoint is called with a query filter
+        THEN:
+            - The exception propagates unconverted, rather than being
+              relabeled a 400
+        """
+        import documents.search._backend as backend_mod
+
+        def raise_zero_division(*args: object, **kwargs: object) -> object:
+            raise ZeroDivisionError("synthetic bug, unrelated to search grammar")
+
+        monkeypatch.setattr(
+            backend_mod,
+            "parse_user_query",
+            raise_zero_division,
+        )
+
+        with pytest.raises(ZeroDivisionError):
+            admin_client.post(
+                "/api/documents/bulk_edit/",
+                {
+                    "documents": [],
+                    "all": True,
+                    "filters": {"query": "anything"},
+                    "method": "set_document_type",
+                    "parameters": {"document_type": None},
+                },
+                format="json",
+            )
+
+    def test_bulk_download_maps_search_query_error_to_a_400(
+        self,
+        admin_client: APIClient,
+        monkeypatch: pytest.MonkeyPatch,
+        searchable_document: Document,
+    ) -> None:
+        """
+        GIVEN:
+            - parse_user_query() raising a SearchQueryError naming a field
+        WHEN:
+            - The bulk_download endpoint is called with a query filter
+        THEN:
+            - The response is a 400 whose body names the field
+        """
+        import documents.search._backend as backend_mod
+
+        def raise_search_query_error(*args: object, **kwargs: object) -> object:
+            raise SearchQueryError("bad value for field 'added'")
+
+        monkeypatch.setattr(
+            backend_mod,
+            "parse_user_query",
+            raise_search_query_error,
+        )
+
+        response = admin_client.post(
+            "/api/documents/bulk_download/",
+            {
+                "documents": [],
+                "all": True,
+                "filters": {"query": "anything"},
+            },
+            format="json",
+        )
+
+        assert response.status_code == status.HTTP_400_BAD_REQUEST
+        assert "added" in str(response.data["query"])
+
+    def test_more_like_id_selection_filter_maps_search_query_error_to_a_400(
+        self,
+        admin_client: APIClient,
+        monkeypatch: pytest.MonkeyPatch,
+        searchable_document: Document,
+    ) -> None:
+        """
+        GIVEN:
+            - TantivyBackend.more_like_this_ids() raising a
+              SearchQueryError
+        WHEN:
+            - The bulk_download endpoint is called with a more_like_id
+              filter
+        THEN:
+            - The response is a 400
+        """
+        import documents.search._backend as backend_mod
+
+        def raise_search_query_error(*args: object, **kwargs: object) -> object:
+            raise SearchQueryError("similar-document lookup is unavailable")
+
+        monkeypatch.setattr(
+            backend_mod.TantivyBackend,
+            "more_like_this_ids",
+            raise_search_query_error,
+        )
+
+        response = admin_client.post(
+            "/api/documents/bulk_download/",
+            {
+                "documents": [],
+                "all": True,
+                "filters": {"more_like_id": searchable_document.pk},
+            },
+            format="json",
+        )
+
+        assert response.status_code == status.HTTP_400_BAD_REQUEST
+
+    def test_more_like_id_selection_filter_lets_an_unexpected_exception_surface(
+        self,
+        admin_client: APIClient,
+        monkeypatch: pytest.MonkeyPatch,
+        searchable_document: Document,
+    ) -> None:
+        """
+        GIVEN:
+            - TantivyBackend.more_like_this_ids() raising an unrelated
+              exception (ZeroDivisionError), not a SearchQueryError
+        WHEN:
+            - The bulk_download endpoint is called with a more_like_id
+              filter
+        THEN:
+            - The exception propagates unconverted, rather than being
+              relabeled a 400
+        """
+        import documents.search._backend as backend_mod
+
+        def raise_zero_division(*args: object, **kwargs: object) -> object:
+            raise ZeroDivisionError("synthetic bug, unrelated to similarity lookup")
+
+        monkeypatch.setattr(
+            backend_mod.TantivyBackend,
+            "more_like_this_ids",
+            raise_zero_division,
+        )
+
+        with pytest.raises(ZeroDivisionError):
+            admin_client.post(
+                "/api/documents/bulk_download/",
+                {
+                    "documents": [],
+                    "all": True,
+                    "filters": {"more_like_id": searchable_document.pk},
+                },
+                format="json",
+            )

src/documents/tests/test_api_search_query_length.py

@@ -0,0 +1,277 @@
+"""The query-length cap in ``_get_tantivy_query_and_mode``.
+
+whoosh-compat's fieldname tagger is O(n^2) in plain word characters, so an
+unbounded ``query`` (SearchMode.QUERY) string is a CPU-exhaustion vector
+against a single request handler. The GET search endpoint is incidentally
+bounded by the web server's header limit, but the POST selection-filter
+path (bulk edit, bulk download) is not -- that is the real vector, so it
+must be pinned here too, not just the GET path.
+
+The cap is enforced once, in the shared helper both entry points call, so
+these tests exercise the real endpoints rather than the helper directly:
+a construct that looks right in isolation has repeatedly behaved
+differently end to end on this branch.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+from unittest import mock
+
+import pytest
+from rest_framework import status
+
+import documents.search._backend
+from documents.views import _MAX_QUERY_LENGTH
+
+if TYPE_CHECKING:
+    from rest_framework.test import APIClient
+
+    from documents.models import Document
+
+pytestmark = [pytest.mark.django_db, pytest.mark.usefixtures("_search_index")]
+
+
+class TestGetSearchEndpointEnforcesTheCap:
+    def test_query_one_over_the_cap_is_a_400(
+        self,
+        admin_client: APIClient,
+        searchable_document: Document,
+    ) -> None:
+        """
+        GIVEN:
+            - The GET search endpoint
+        WHEN:
+            - A query one character over `_MAX_QUERY_LENGTH` is submitted
+        THEN:
+            - The response is a 400 naming both the actual length and the
+              cap, and the query is rejected before it ever reaches the
+              parser -- the 400 alone doesn't prove that, since the parser
+              could run first and the view could discard the result
+        """
+        query = "a" * (_MAX_QUERY_LENGTH + 1)
+
+        with mock.patch(
+            "documents.search._backend.parse_user_query",
+            wraps=documents.search._backend.parse_user_query,
+        ) as parse_spy:
+            response = admin_client.get("/api/documents/", {"query": query})
+
+        assert response.status_code == status.HTTP_400_BAD_REQUEST
+        message = str(response.data["query"])
+        assert str(_MAX_QUERY_LENGTH) in message
+        assert str(_MAX_QUERY_LENGTH + 1) in message
+        parse_spy.assert_not_called()
+
+    def test_query_at_exactly_the_cap_is_accepted(
+        self,
+        admin_client: APIClient,
+        searchable_document: Document,
+    ) -> None:
+        """
+        GIVEN:
+            - The GET search endpoint
+        WHEN:
+            - A query exactly `_MAX_QUERY_LENGTH` characters long is
+              submitted
+        THEN:
+            - The response is a 200 (the cap is inclusive, not exclusive)
+        """
+        query = "a" * _MAX_QUERY_LENGTH
+
+        response = admin_client.get("/api/documents/", {"query": query})
+
+        assert response.status_code == status.HTTP_200_OK
+
+    def test_an_ordinary_query_is_unaffected(
+        self,
+        admin_client: APIClient,
+        searchable_document: Document,
+    ) -> None:
+        """
+        GIVEN:
+            - The GET search endpoint and an indexed document
+        WHEN:
+            - An ordinary, well-under-the-cap query is submitted
+        THEN:
+            - The cap has no effect on a normal search: the matching
+              document is returned
+        """
+        response = admin_client.get("/api/documents/", {"query": "invoice"})
+
+        assert response.status_code == status.HTTP_200_OK
+        assert response.data["count"] == 1
+
+
+class TestPostSelectionPathsEnforceTheCap:
+    """The bulk-edit and bulk-download selection filters share the same
+    helper the GET search path uses. This is the path that actually
+    matters: it is not bounded by a web server's header-length limit the
+    way the GET path incidentally is."""
+
+    def test_bulk_edit_query_one_over_the_cap_is_a_400(
+        self,
+        admin_client: APIClient,
+        searchable_document: Document,
+    ) -> None:
+        """
+        GIVEN:
+            - The bulk-edit selection-filter endpoint
+        WHEN:
+            - Its `filters.query` is one character over `_MAX_QUERY_LENGTH`
+        THEN:
+            - The response is a 400 naming both the actual length and the
+              cap, and the query is rejected before it ever reaches the
+              parser -- this is the path with no web-server header-length
+              limit to fall back on, so this is the invariant that matters
+        """
+        query = "a" * (_MAX_QUERY_LENGTH + 1)
+
+        with mock.patch(
+            "documents.search._backend.parse_user_query",
+            wraps=documents.search._backend.parse_user_query,
+        ) as parse_spy:
+            response = admin_client.post(
+                "/api/documents/bulk_edit/",
+                {
+                    "documents": [],
+                    "all": True,
+                    "filters": {"query": query},
+                    "method": "set_document_type",
+                    "parameters": {"document_type": None},
+                },
+                format="json",
+            )
+
+        assert response.status_code == status.HTTP_400_BAD_REQUEST
+        message = str(response.data["query"])
+        assert str(_MAX_QUERY_LENGTH) in message
+        assert str(_MAX_QUERY_LENGTH + 1) in message
+        parse_spy.assert_not_called()
+
+    @mock.patch("documents.bulk_edit.bulk_update_documents.apply_async")
+    def test_bulk_edit_query_at_exactly_the_cap_is_accepted(
+        self,
+        bulk_update_task_mock: mock.MagicMock,
+        admin_client: APIClient,
+        searchable_document: Document,
+    ) -> None:
+        """
+        GIVEN:
+            - The bulk-edit selection-filter endpoint
+        WHEN:
+            - Its `filters.query` is exactly `_MAX_QUERY_LENGTH` characters
+              long
+        THEN:
+            - The cap check accepts it and the request reaches the real
+              bulk-edit method (its Celery dispatch is mocked out here,
+              same as every other bulk-edit test, since nothing here is
+              testing that method itself)
+        """
+        query = "a" * _MAX_QUERY_LENGTH
+
+        response = admin_client.post(
+            "/api/documents/bulk_edit/",
+            {
+                "documents": [],
+                "all": True,
+                "filters": {"query": query},
+                "method": "set_document_type",
+                "parameters": {"document_type": None},
+            },
+            format="json",
+        )
+
+        assert response.status_code == status.HTTP_200_OK
+
+    def test_bulk_download_query_one_over_the_cap_is_a_400(
+        self,
+        admin_client: APIClient,
+        searchable_document: Document,
+    ) -> None:
+        """
+        GIVEN:
+            - The bulk-download selection-filter endpoint
+        WHEN:
+            - Its `filters.query` is one character over `_MAX_QUERY_LENGTH`
+        THEN:
+            - The response is a 400 naming both the actual length and the
+              cap, and the query is rejected before it ever reaches the
+              parser
+        """
+        query = "a" * (_MAX_QUERY_LENGTH + 1)
+
+        with mock.patch(
+            "documents.search._backend.parse_user_query",
+            wraps=documents.search._backend.parse_user_query,
+        ) as parse_spy:
+            response = admin_client.post(
+                "/api/documents/bulk_download/",
+                {
+                    "documents": [],
+                    "all": True,
+                    "filters": {"query": query},
+                },
+                format="json",
+            )
+
+        assert response.status_code == status.HTTP_400_BAD_REQUEST
+        message = str(response.data["query"])
+        assert str(_MAX_QUERY_LENGTH) in message
+        assert str(_MAX_QUERY_LENGTH + 1) in message
+        parse_spy.assert_not_called()
+
+
+class TestGlobalSearchEnforcesTheCapToo:
+    """GlobalSearchView calls the backend directly, not through the shared helper.
+
+    It hardcodes SearchMode.TEXT, which is linear rather than quadratic, so it
+    was never the CPU-exhaustion vector. It is capped anyway so that "every
+    user query string reaching the backend passes a length check" is an
+    invariant rather than a claim with an exception: the view already bounds
+    the query from below, and a later change letting it select a mode would
+    otherwise reopen the hole silently.
+    """
+
+    def test_query_one_over_the_cap_is_a_400(
+        self,
+        admin_client: APIClient,
+        searchable_document: Document,
+    ) -> None:
+        """
+        GIVEN:
+            - GlobalSearchView, which calls the backend directly in
+              SearchMode.TEXT rather than through the shared cap-checking
+              helper
+        WHEN:
+            - Its query is one character over `_MAX_QUERY_LENGTH`
+        THEN:
+            - The response is still a 400, keeping "every user query
+              string reaching the backend passes a length check" an
+              invariant with no exception, even though TEXT mode is
+              linear and was never itself the CPU-exhaustion vector
+        """
+        response = admin_client.get(
+            "/api/search/",
+            {"query": "a" * (_MAX_QUERY_LENGTH + 1)},
+        )
+        assert response.status_code == status.HTTP_400_BAD_REQUEST
+
+    def test_query_at_exactly_the_cap_is_accepted(
+        self,
+        admin_client: APIClient,
+        searchable_document: Document,
+    ) -> None:
+        """
+        GIVEN:
+            - GlobalSearchView
+        WHEN:
+            - Its query is exactly `_MAX_QUERY_LENGTH` characters long
+        THEN:
+            - The response is a 200 (the cap is inclusive, not exclusive)
+        """
+        response = admin_client.get(
+            "/api/search/",
+            {"query": "a" * _MAX_QUERY_LENGTH},
+        )
+        assert response.status_code == status.HTTP_200_OK