Chore: make the backend test suite pass on MariaDB

Make the reference predicates for the string custom field filters follow
the column collation, which is case insensitive on MariaDB, and quote
identifiers in the effective content query matchers with the backend's
own quoting.

src-ui/src/app/components/common/custom-fields-dropdown/custom-fields-dropdown.component.scss

@@ -1,10 +1,3 @@
 .custom-fields-dropdown {
     min-width: 300px;
-
-    // correct position on mobile
-    @media (max-width: 575.98px) {
-        &.show {
-            margin-left: -245px !important;
-        }
-    }
 }

src-ui/src/app/components/document-detail/document-detail.component.html

@@ -308,7 +308,7 @@
                   </tr>
                   <tr class="align-middle">
                     <td i18n>Original SHA256 checksum</td>
-                    <td><span class="badge bg-light font-monospace text-wrap text-break text-start">{{metadata()?.original_checksum}}</span></td>
+                    <td><span class="badge bg-light text-dark font-monospace text-wrap text-break text-start">{{metadata()?.original_checksum}}</span></td>
                   </tr>
                   <tr>
                     <td i18n>Original file size</td>

src/conftest.py

@@ -77,6 +77,30 @@ def _clear_django_caches() -> None:
         cache.clear()
 
 
+@pytest.fixture(autouse=True)
+def _keep_connection_open_in_atomic(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Make ``close_old_connections()`` a no-op inside the test's transaction.
+
+    The Celery signal handlers call it, as a worker would between tasks. Tests
+    run inside an open atomic block, where Postgres really closes the
+    connection, taking the test transaction and every later test on the
+    worker with it. SQLite in-memory ignores the close, which hid this.
+    """
+    from django.db.backends.base.base import BaseDatabaseWrapper
+
+    original = BaseDatabaseWrapper.close_if_unusable_or_obsolete
+
+    def close_unless_atomic(self: BaseDatabaseWrapper) -> None:
+        if not self.in_atomic_block:
+            original(self)
+
+    monkeypatch.setattr(
+        BaseDatabaseWrapper,
+        "close_if_unusable_or_obsolete",
+        close_unless_atomic,
+    )
+
+
 @pytest.fixture
 def paperless_dirs(
     tmp_path: Path,

src/documents/matching.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 import logging
 import re
 from fnmatch import fnmatch
-from fnmatch import translate as fnmatch_translate
 from typing import TYPE_CHECKING
 
 from rest_framework import serializers
@@ -548,6 +547,48 @@ def existing_document_matches_workflow(
     return (True, None)
 
 
+def glob_to_database_regex(pattern: str) -> str:
+    """
+    Translates an fnmatch style glob into a regular expression which the
+    database can evaluate, matching the whole value.
+
+    fnmatch.translate is not usable here: it emits syntax such as (?s:...),
+    atomic groups and named group backreferences, which PostgreSQL rejects.
+    Only constructs common to Python, PostgreSQL and MariaDB are produced.
+    """
+    parts: list[str] = []
+    i = 0
+    n = len(pattern)
+    while i < n:
+        char = pattern[i]
+        i += 1
+        if char == "*":
+            parts.append(".*")
+        elif char == "?":
+            parts.append(".")
+        elif char == "[":
+            j = i
+            if j < n and pattern[j] == "!":
+                j += 1
+            if j < n and pattern[j] == "]":
+                j += 1
+            while j < n and pattern[j] != "]":
+                j += 1
+            if j >= n:
+                parts.append(r"\[")
+            else:
+                stuff = pattern[i:j].replace("\\", r"\\").replace("[", r"\[")
+                i = j + 1
+                if stuff.startswith("!"):
+                    stuff = "^" + stuff[1:]
+                elif stuff.startswith("^"):
+                    stuff = "\\" + stuff
+                parts.append(f"[{stuff}]")
+        else:
+            parts.append(re.escape(char))
+    return "^" + "".join(parts) + "$"
+
+
 def prefilter_documents_by_workflowtrigger(
     documents: QuerySet[Document],
     trigger: WorkflowTrigger,
@@ -628,8 +669,9 @@ def prefilter_documents_by_workflowtrigger(
         documents = documents.annotate(**annotations).filter(custom_field_q)
 
     if trigger.filter_filename:
-        regex = fnmatch_translate(trigger.filter_filename).lstrip("^").rstrip("$")
+        documents = documents.filter(
-        documents = documents.filter(original_filename__iregex=regex)
+            original_filename__iregex=glob_to_database_regex(trigger.filter_filename),
+        )
 
     return documents
 

src/documents/permissions.py

@@ -576,9 +576,12 @@ def annotate_document_count_by_ids(
                        or CustomFieldInstance)
         related_object_field: field on the relation pointing back to queryset pk
         document_ids: the document ids to count against -- a concrete list/set,
-                       or a simple (already resolved) queryset of ids. Keep
+                       or a simple (already resolved) queryset of ids. Callers
-                       the list small: it is embedded in the statement as
+                       that need this filtered by a complex condition (e.g. a
-                       literal parameters.
+                       permission check) should resolve it to a concrete list
+                       first if the same ids will be reused across multiple
+                       calls, rather than passing the complex queryset itself
+                       into each -- see ``_get_selection_data_for_queryset``.
         target_field: field on the relation pointing to Document id
     """
 

src/documents/tests/test_api_bulk_edit.py

@@ -1141,8 +1141,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertCountEqual(
             response.data["selected_document_types"],
             [
-                {"id": self.dt1.id, "document_count": 1},
+                {"id": self.c1.id, "document_count": 1},
-                {"id": self.dt2.id, "document_count": 0},
+                {"id": self.c2.id, "document_count": 0},
             ],
         )
 

src/documents/tests/test_api_custom_fields.py

@@ -1062,7 +1062,7 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase):
                 "custom_fields": [
                     {
                         "field": custom_field_doclink.id,
-                        "value": [2, 3, 4],
+                        "value": [doc2.id, doc3.id, doc4.id],
                     },
                 ],
             },
@@ -1073,13 +1073,13 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(CustomFieldInstance.objects.count(), 4)
         _cf_5 = doc2.custom_fields.first()
         assert _cf_5 is not None
-        self.assertEqual(_cf_5.value, [1])
+        self.assertEqual(_cf_5.value, [doc1.id])
         _cf_6 = doc3.custom_fields.first()
         assert _cf_6 is not None
-        self.assertEqual(_cf_6.value, [1])
+        self.assertEqual(_cf_6.value, [doc1.id])
         _cf_7 = doc4.custom_fields.first()
         assert _cf_7 is not None
-        self.assertEqual(_cf_7.value, [1])
+        self.assertEqual(_cf_7.value, [doc1.id])
 
         # Add links appends if necessary
         resp = self.client.patch(
@@ -1088,7 +1088,7 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase):
                 "custom_fields": [
                     {
                         "field": custom_field_doclink.id,
-                        "value": [1, 4],
+                        "value": [doc1.id, doc4.id],
                     },
                 ],
             },
@@ -1098,7 +1098,7 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(resp.status_code, status.HTTP_200_OK)
         _cf_8 = doc4.custom_fields.first()
         assert _cf_8 is not None
-        self.assertEqual(_cf_8.value, [1, 3])
+        self.assertEqual(_cf_8.value, [doc1.id, doc3.id])
 
         # Remove one of the links, removed on other doc
         resp = self.client.patch(
@@ -1107,7 +1107,7 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase):
                 "custom_fields": [
                     {
                         "field": custom_field_doclink.id,
-                        "value": [2, 3],
+                        "value": [doc2.id, doc3.id],
                     },
                 ],
             },
@@ -1117,13 +1117,13 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(resp.status_code, status.HTTP_200_OK)
         _cf_9 = doc2.custom_fields.first()
         assert _cf_9 is not None
-        self.assertEqual(_cf_9.value, [1])
+        self.assertEqual(_cf_9.value, [doc1.id])
         _cf_10 = doc3.custom_fields.first()
         assert _cf_10 is not None
-        self.assertEqual(_cf_10.value, [1, 4])
+        self.assertEqual(_cf_10.value, [doc1.id, doc4.id])
         _cf_11 = doc4.custom_fields.first()
         assert _cf_11 is not None
-        self.assertEqual(_cf_11.value, [3])
+        self.assertEqual(_cf_11.value, [doc3.id])
 
         # Removes the field entirely
         resp = self.client.patch(
@@ -1140,10 +1140,10 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(_cf_12.value, [])
         _cf_13 = doc3.custom_fields.first()
         assert _cf_13 is not None
-        self.assertEqual(_cf_13.value, [4])
+        self.assertEqual(_cf_13.value, [doc4.id])
         _cf_14 = doc4.custom_fields.first()
         assert _cf_14 is not None
-        self.assertEqual(_cf_14.value, [3])
+        self.assertEqual(_cf_14.value, [doc3.id])
 
         # If field exists on target doc but value is None
         doc5 = Document.objects.create(
@@ -1170,7 +1170,7 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase):
         self.assertEqual(resp.status_code, status.HTTP_200_OK)
         _cf_15 = doc5.custom_fields.first()
         assert _cf_15 is not None
-        self.assertEqual(_cf_15.value, [1])
+        self.assertEqual(_cf_15.value, [doc1.id])
 
     def test_documentlink_patch_requires_change_permission_on_target_documents(
         self,

src/documents/tests/test_api_documents.py

@@ -49,7 +49,6 @@ from documents.models import WorkflowTrigger
 from documents.search import get_backend
 from documents.signals.handlers import run_workflows
 from documents.tests.utils import ConsumeTaskMixin
-from documents.views import DocumentViewSet
 from paperless_testing.dirs import DirectoriesMixin
 from paperless_testing.factories import DocumentFactory
 from paperless_testing.factories import TagFactory
@@ -1756,106 +1755,6 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         }
         self.assertEqual(counts_by_field[custom_field.id], 1)
 
-    def test_selection_data_counts_with_shared_by_filter(self) -> None:
-        """
-        GIVEN:
-            - Several documents owned by one user with the same correspondent
-              and document type, only one of which is shared
-        WHEN:
-            - Requesting the document list filtered by shared_by__id with
-              include_selection_data=true
-        THEN:
-            - Counts reflect only the shared document, not every document the
-              owner has with that correspondent or document type, however the
-              link-table lookups are batched
-        """
-        owner = User.objects.create_user(username="owner")
-        other = User.objects.create_user(username="other")
-        correspondent = Correspondent.objects.create(name="c1")
-        doc_type = DocumentType.objects.create(name="dt1")
-        tag = Tag.objects.create(name="tag")
-        docs = [
-            Document.objects.create(
-                checksum=str(i),
-                owner=owner,
-                correspondent=correspondent,
-                document_type=doc_type,
-            )
-            for i in range(3)
-        ]
-        for doc in docs:
-            doc.tags.add(tag)
-        grant_object(other, docs[0], "view_document")
-
-        for batch_size in (10_000, 1):
-            with (
-                self.subTest(link_batch_size=batch_size),
-                mock.patch.object(
-                    DocumentViewSet,
-                    "selection_data_link_batch_size",
-                    batch_size,
-                ),
-            ):
-                response = self.client.get(
-                    f"/api/documents/?shared_by__id={owner.id}"
-                    "&include_selection_data=true",
-                )
-
-                self.assertEqual(response.status_code, status.HTTP_200_OK)
-                self.assertEqual(response.data["count"], 1)
-                selection_data = response.data["selection_data"]
-                for key, obj in (
-                    ("selected_correspondents", correspondent),
-                    ("selected_document_types", doc_type),
-                    ("selected_tags", tag),
-                ):
-                    counts = {
-                        item["id"]: item["document_count"]
-                        for item in selection_data[key]
-                    }
-                    self.assertEqual(counts[obj.id], 1, key)
-
-    def test_selection_data_counts_documents_once_when_filter_joins(self) -> None:
-        """
-        GIVEN:
-            - A document matching a multi-valued filter through two tags
-        WHEN:
-            - Requesting the document list with include_selection_data=true
-        THEN:
-            - The document is counted once per correspondent, not once per
-              matching tag, however the link-table lookups are batched
-        """
-        correspondent = Correspondent.objects.create(name="c1")
-        tag_a = Tag.objects.create(name="a")
-        tag_b = Tag.objects.create(name="b")
-        both = Document.objects.create(checksum="1", correspondent=correspondent)
-        both.tags.add(tag_a, tag_b)
-        only_a = Document.objects.create(checksum="2", correspondent=correspondent)
-        only_a.tags.add(tag_a)
-
-        for batch_size in (10_000, 1):
-            with (
-                self.subTest(link_batch_size=batch_size),
-                mock.patch.object(
-                    DocumentViewSet,
-                    "selection_data_link_batch_size",
-                    batch_size,
-                ),
-            ):
-                response = self.client.get(
-                    f"/api/documents/?tags__id__in={tag_a.id},{tag_b.id}"
-                    "&include_selection_data=true",
-                )
-
-                self.assertEqual(response.status_code, status.HTTP_200_OK)
-                counts = {
-                    item["id"]: item["document_count"]
-                    for item in response.data["selection_data"][
-                        "selected_correspondents"
-                    ]
-                }
-                self.assertEqual(counts[correspondent.id], 2)
-
     def test_statistics(self) -> None:
         doc1 = Document.objects.create(
             title="none1",

src/documents/tests/test_api_filter_by_custom_fields.py

@@ -6,6 +6,7 @@ from unittest.mock import patch
 from urllib.parse import quote
 
 import time_machine
+from django.db import connection
 from django.test import override_settings
 from django.utils import timezone
 from rest_framework.test import APITestCase
@@ -20,6 +21,13 @@ from paperless_testing.dirs import DirectoriesMixin
 from paperless_testing.factories import UserFactory
 
 
+def _exact_equal(value: str | None, expected: str) -> bool:
+    """`exact` follows the column collation, which is case insensitive on MariaDB."""
+    if connection.vendor == "mysql" and value is not None:
+        return value.casefold() == expected.casefold()
+    return value == expected
+
+
 class DocumentWrapper:
     """
     Allows Pythonic access to the custom fields associated with the wrapped document.
@@ -287,7 +295,8 @@ class TestCustomFieldsSearch(DirectoriesMixin, APITestCase):
         self._assert_query_match_predicate(
             ["string_field", "exact", "paperless"],
             lambda document: (
-                "string_field" in document and document["string_field"] == "paperless"
+                "string_field" in document
+                and _exact_equal(document["string_field"], "paperless")
             ),
         )
 
@@ -296,7 +305,10 @@ class TestCustomFieldsSearch(DirectoriesMixin, APITestCase):
             ["string_field", "in", ["paperless", "Paperless"]],
             lambda document: (
                 "string_field" in document
-                and document["string_field"] in ("paperless", "Paperless")
+                and any(
+                    _exact_equal(document["string_field"], expected)
+                    for expected in ("paperless", "Paperless")
+                )
             ),
         )
 
@@ -628,22 +640,23 @@ class TestCustomFieldsSearch(DirectoriesMixin, APITestCase):
     # Subset check (document link field only)                   #
     # ==========================================================#
     def test_document_link_contains(self) -> None:
+        linked = {self.documents[1].id, self.documents[2].id}
         # Document link field "contains" performs a subset check.
         self._assert_query_match_predicate(
-            ["documentlink_field", "contains", [1, 2]],
+            ["documentlink_field", "contains", sorted(linked)],
             lambda document: (
                 "documentlink_field" in document
                 and document["documentlink_field"] is not None
-                and set(document["documentlink_field"]) >= {1, 2}
+                and set(document["documentlink_field"]) >= linked
             ),
         )
         # The order of IDs don't matter - this is the same as above.
         self._assert_query_match_predicate(
-            ["documentlink_field", "contains", [2, 1]],
+            ["documentlink_field", "contains", sorted(linked, reverse=True)],
             lambda document: (
                 "documentlink_field" in document
                 and document["documentlink_field"] is not None
-                and set(document["documentlink_field"]) >= {1, 2}
+                and set(document["documentlink_field"]) >= linked
             ),
         )
 
@@ -760,7 +773,7 @@ class TestCustomFieldsSearch(DirectoriesMixin, APITestCase):
             lambda document: (
                 not (
                     "string_field" in document
-                    and document["string_field"] == "paperless"
+                    and _exact_equal(document["string_field"], "paperless")
                 )
             ),
         )

src/documents/tests/test_api_search.py

@@ -944,7 +944,7 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_200_OK)
 
         results = response.data["results"]
-        self.assertEqual({r["id"] for r in results}, {1, 2})
+        self.assertEqual({r["id"] for r in results}, {d1.id, d2.id})
 
     @mock.patch("documents.search._backend.parse_user_query")
     def test_search_parser_bug_surfaces_as_500_not_400(self, m) -> None:
@@ -1801,6 +1801,7 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
     def test_search_sorting(self) -> None:
         u1 = UserFactory(username="user1")
         u2 = UserFactory(username="user2")
+        u3 = UserFactory(username="user3")
         c1 = Correspondent.objects.create(name="corres Ax")
         c2 = Correspondent.objects.create(name="corres Cx")
         c3 = Correspondent.objects.create(name="corres Bx")
@@ -1826,6 +1827,7 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
             content="test",
             archive_serial_number=1,
             title="1",
+            owner=u3,
         )
         Note.objects.create(
             note="This is a note.",
@@ -1878,14 +1880,14 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
             search_query("&ordering=-num_notes"),
             [d1.id, d3.id, d2.id],
         )
-        # owner sort: ORM orders by owner_id (integer); NULLs first in SQLite ASC
+        # owner sort: ORM orders by owner_id (integer)
         self.assertListEqual(
             search_query("&ordering=owner"),
-            [d3.id, d1.id, d2.id],
+            [d1.id, d2.id, d3.id],
         )
         self.assertListEqual(
             search_query("&ordering=-owner"),
-            [d2.id, d1.id, d3.id],
+            [d3.id, d2.id, d1.id],
         )
 
     def test_search_ordering_by_score(self) -> None:

src/documents/tests/test_api_status.py

@@ -5,6 +5,7 @@ from datetime import timedelta
 from pathlib import Path
 from unittest import mock
 
+from django.db import connection
 from django.test import override_settings
 from django.utils import timezone
 from rest_framework import status
@@ -68,7 +69,7 @@ class TestSystemStatus(APITestCase):
         self.assertEqual(response.data["install_type"], "bare-metal")
         self.assertIsNotNone(response.data["storage"]["total"])
         self.assertIsNotNone(response.data["storage"]["available"])
-        self.assertEqual(response.data["database"]["type"], "sqlite")
+        self.assertEqual(response.data["database"]["type"], connection.vendor)
         self.assertIsNotNone(response.data["database"]["url"])
         self.assertEqual(response.data["database"]["status"], "OK")
         self.assertIsNone(response.data["database"]["error"])

src/documents/tests/test_bulk_edit.py

@@ -1031,9 +1031,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
         with self.assertLogs("paperless.bulk_edit", level="ERROR") as cm:
             bulk_edit.merge(doc_ids)
             error_str = cm.output[0]
-            expected_str = (
+            expected_str = f"Error merging document {self.doc2.id}, it will not be included in the merge"
-                "Error merging document 2, it will not be included in the merge"
-            )
             self.assertIn(expected_str, error_str)
 
         mock_consume_file.assert_not_called()
@@ -1193,7 +1191,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
         with self.assertLogs("paperless.bulk_edit", level="ERROR") as cm:
             bulk_edit.split(doc_ids, pages)
             error_str = cm.output[0]
-            expected_str = "Error splitting document 2"
+            expected_str = f"Error splitting document {self.doc2.id}"
             self.assertIn(expected_str, error_str)
 
         mock_consume_file.assert_not_called()

src/documents/tests/test_consumer.py

@@ -248,8 +248,8 @@ class TestConsumer(
         )
         self.assertIsNone(document.correspondent)
         self.assertIsNone(document.document_type)
-        self.assertEqual(document.filename, "0000001.pdf")
+        self.assertEqual(document.filename, f"{document.pk:07}.pdf")
-        self.assertEqual(document.archive_filename, "0000001.pdf")
+        self.assertEqual(document.archive_filename, f"{document.pk:07}.pdf")
 
         self.assertIsFile(document.source_path)
 
@@ -1127,17 +1127,17 @@ class TestConsumer(
         with self.get_consumer(settings.CONSUMPTION_DIR / "simple.png") as consumer:
             consumer.run()
 
-            doc1 = Document.objects.filter(pk=1).first()
+            doc1 = Document.objects.latest("pk")
 
         with self.get_consumer(settings.CONSUMPTION_DIR / "simple.pdf") as consumer:
             consumer.run()
 
-            doc2 = Document.objects.filter(pk=2).first()
+            doc2 = Document.objects.latest("pk")
 
         with self.get_consumer(settings.CONSUMPTION_DIR / "simple.png.pdf") as consumer:
             consumer.run()
 
-            doc3 = Document.objects.filter(pk=3).first()
+            doc3 = Document.objects.latest("pk")
 
         self.assertEqual(doc1.filename, "simple.png")
         self.assertEqual(doc1.archive_filename, "simple.pdf")

src/documents/tests/test_document_list_effective_content_annotation.py

@@ -242,9 +242,7 @@ class TestDocumentListEffectiveContentAnnotation:
         # Only the list query itself reads a content column: no extra query
         # for the skipped prefetch, none for a per-instance fallback
         content_queries = [
-            query
+            query for query in ctx.captured_queries if _content_column() in query["sql"]
-            for query in ctx.captured_queries
-            if '"documents_document"."content"' in query["sql"]
         ]
         assert len(content_queries) == 1
 
@@ -365,6 +363,11 @@ def _get_document_queries(
     return [q for q in ctx.captured_queries if '"django_content_type"' not in q["sql"]]
 
 
+def _content_column() -> str:
+    quote = connection.ops.quote_name
+    return f"{quote('documents_document')}.{quote('content')}"
+
+
 def _get_effective_content_fallback_queries(
     ctx: CaptureQueriesContext,
 ) -> list[dict[str, str]]:
@@ -379,7 +382,7 @@ def _get_effective_content_fallback_queries(
     return [
         q
         for q in ctx.captured_queries
-        if q["sql"].startswith('SELECT "documents_document"."content" FROM')
+        if q["sql"].startswith(f"SELECT {_content_column()} FROM")
     ]
 
 

src/documents/tests/test_file_handling.py

@@ -760,6 +760,9 @@ class TestFileHandlingWithArchive(DirectoriesMixin, FileSystemAssertsMixin, Test
             archive_checksum="B",
         )
 
+        # Without a format the files are renamed to match the primary key
+        original = doc.source_path
+        archive = doc.archive_path
         self.assertIsFile(original)
         self.assertIsFile(archive)
         self.assertIsFile(doc.source_path)
@@ -978,6 +981,9 @@ class TestFileHandlingWithArchive(DirectoriesMixin, FileSystemAssertsMixin, Test
             archive_filename="0000001.pdf",
         )
 
+        # Without a format the files are renamed to match the primary key
+        original = doc.source_path
+        archive = doc.archive_path
         self.assertIsFile(original)
         self.assertIsFile(archive)
         self.assertIsFile(doc.source_path)

src/documents/tests/test_management_exporter.py

@@ -151,6 +151,20 @@ class TestExportImport(
 
         super().setUp()
 
+    def _install_sample_files(self) -> None:
+        documents_dir = Path(self.dirs.media_dir) / "documents"
+        shutil.rmtree(documents_dir)
+        install_document_samples(documents_dir)
+        # Thumbnails are located by primary key, which the database sequence
+        # does not reset between tests on PostgreSQL
+        for number, document in enumerate(
+            (self.d1, self.d2, self.d3, self.d4),
+            start=1,
+        ):
+            source = documents_dir / "thumbnails" / f"{number:07}.webp"
+            if source != document.thumbnail_path:
+                shutil.copy(source, document.thumbnail_path)
+
     def _get_document_from_manifest(self, manifest, id):
         f = list(
             filter(
@@ -204,8 +218,7 @@ class TestExportImport(
         return manifest
 
     def test_exporter(self, *, use_filename_format=False) -> None:
-        shutil.rmtree(Path(self.dirs.media_dir) / "documents")
+        self._install_sample_files()
-        install_document_samples(Path(self.dirs.media_dir) / "documents")
 
         num_permission_objects = Permission.objects.count()
 
@@ -308,8 +321,7 @@ class TestExportImport(
             self.assertEqual(messages.total_issue_count, 0)
 
     def test_exporter_with_filename_format(self) -> None:
-        shutil.rmtree(Path(self.dirs.media_dir) / "documents")
+        self._install_sample_files()
-        install_document_samples(Path(self.dirs.media_dir) / "documents")
 
         with override_settings(
             FILENAME_FORMAT="{created_year}/{correspondent}/{title}",
@@ -317,8 +329,7 @@ class TestExportImport(
             self.test_exporter(use_filename_format=True)
 
     def test_exporter_includes_share_links_and_bundles(self) -> None:
-        shutil.rmtree(Path(self.dirs.media_dir) / "documents")
+        self._install_sample_files()
-        install_document_samples(Path(self.dirs.media_dir) / "documents")
 
         share_link = ShareLink.objects.create(
             slug="share-link-slug",
@@ -416,8 +427,7 @@ class TestExportImport(
             )
 
     def test_update_export_changed_time(self) -> None:
-        shutil.rmtree(Path(self.dirs.media_dir) / "documents")
+        self._install_sample_files()
-        install_document_samples(Path(self.dirs.media_dir) / "documents")
 
         self._do_export()
         self.assertIsFile(self.target / "manifest.json")
@@ -452,8 +462,7 @@ class TestExportImport(
         self.assertEqual(st_mtime_3, st_mtime_4)
 
     def test_update_export_changed_checksum(self) -> None:
-        shutil.rmtree(Path(self.dirs.media_dir) / "documents")
+        self._install_sample_files()
-        install_document_samples(Path(self.dirs.media_dir) / "documents")
 
         self._do_export()
 
@@ -479,8 +488,7 @@ class TestExportImport(
         self.assertIsFile(self.target / "manifest.json")
 
     def test_update_export_deleted_document(self) -> None:
-        shutil.rmtree(Path(self.dirs.media_dir) / "documents")
+        self._install_sample_files()
-        install_document_samples(Path(self.dirs.media_dir) / "documents")
 
         manifest = self._do_export()
 
@@ -511,8 +519,7 @@ class TestExportImport(
 
     @override_settings(FILENAME_FORMAT="{title}/{correspondent}")
     def test_update_export_changed_location(self) -> None:
-        shutil.rmtree(Path(self.dirs.media_dir) / "documents")
+        self._install_sample_files()
-        install_document_samples(Path(self.dirs.media_dir) / "documents")
 
         self._do_export(use_filename_format=True)
         self.assertIsFile(self.target / "wow1" / "c.pdf")
@@ -553,8 +560,7 @@ class TestExportImport(
             - Zipfile is created
             - Zipfile contains exported files
         """
-        shutil.rmtree(Path(self.dirs.media_dir) / "documents")
+        self._install_sample_files()
-        install_document_samples(Path(self.dirs.media_dir) / "documents")
 
         args = ["document_exporter", self.target, "--zip"]
 
@@ -582,8 +588,7 @@ class TestExportImport(
             - Zipfile is created
             - Zipfile contains exported files
         """
-        shutil.rmtree(Path(self.dirs.media_dir) / "documents")
+        self._install_sample_files()
-        install_document_samples(Path(self.dirs.media_dir) / "documents")
 
         args = ["document_exporter", self.target, "--zip", "--use-filename-format"]
 
@@ -618,8 +623,7 @@ class TestExportImport(
             - Zipfile contains exported files
             - The existing file and directory in target are removed
         """
-        shutil.rmtree(Path(self.dirs.media_dir) / "documents")
+        self._install_sample_files()
-        install_document_samples(Path(self.dirs.media_dir) / "documents")
 
         # Create stuff in target directory
         existing_file = self.target / "test.txt"
@@ -714,8 +718,7 @@ class TestExportImport(
             - Manifest.json doesn't contain information about archive files
             - Documents can be imported again
         """
-        shutil.rmtree(Path(self.dirs.media_dir) / "documents")
+        self._install_sample_files()
-        install_document_samples(Path(self.dirs.media_dir) / "documents")
 
         manifest = self._do_export()
         has_archive = False
@@ -757,8 +760,7 @@ class TestExportImport(
             - Manifest.json doesn't contain information about thumbnails
             - Documents can be imported again
         """
-        shutil.rmtree(Path(self.dirs.media_dir) / "documents")
+        self._install_sample_files()
-        install_document_samples(Path(self.dirs.media_dir) / "documents")
 
         manifest = self._do_export()
         has_thumbnail = False
@@ -802,8 +804,7 @@ class TestExportImport(
             - Main manifest.json file doesn't contain information about documents
             - Documents can be imported again
         """
-        shutil.rmtree(Path(self.dirs.media_dir) / "documents")
+        self._install_sample_files()
-        install_document_samples(Path(self.dirs.media_dir) / "documents")
 
         manifest = self._do_export(split_manifest=True)
         has_document = False
@@ -827,8 +828,7 @@ class TestExportImport(
             self.assertEqual(CustomFieldInstance.objects.count(), 1)
 
     def _export_import_barcodes(self, *, split_manifest: bool) -> None:
-        shutil.rmtree(Path(self.dirs.media_dir) / "documents")
+        self._install_sample_files()
-        install_document_samples(Path(self.dirs.media_dir) / "documents")
         DocumentBarcodeFactory(document=self.d1, value="https://example.com")
         DocumentBarcodeFactory(document=self.d2, page=2, value="DE8937")
 
@@ -863,8 +863,7 @@ class TestExportImport(
         THEN:
             - Documents can be imported again
         """
-        shutil.rmtree(Path(self.dirs.media_dir) / "documents")
+        self._install_sample_files()
-        install_document_samples(Path(self.dirs.media_dir) / "documents")
 
         self._do_export(use_folder_prefix=True)
 
@@ -890,8 +889,7 @@ class TestExportImport(
         THEN:
             - Documents can be imported again
         """
-        shutil.rmtree(Path(self.dirs.media_dir) / "documents")
+        self._install_sample_files()
-        install_document_samples(Path(self.dirs.media_dir) / "documents")
 
         self._do_export(use_folder_prefix=True, split_manifest=True)
 
@@ -917,8 +915,7 @@ class TestExportImport(
             - ContentType & Permission objects are not deleted, db transaction rolled back
         """
 
-        shutil.rmtree(Path(self.dirs.media_dir) / "documents")
+        self._install_sample_files()
-        install_document_samples(Path(self.dirs.media_dir) / "documents")
 
         num_content_type_objects = ContentType.objects.count()
         num_permission_objects = Permission.objects.count()
@@ -954,8 +951,7 @@ class TestExportImport(
             self.assertEqual(Permission.objects.count(), num_permission_objects + 1)
 
     def test_exporter_with_auditlog_disabled(self) -> None:
-        shutil.rmtree(Path(self.dirs.media_dir) / "documents")
+        self._install_sample_files()
-        install_document_samples(Path(self.dirs.media_dir) / "documents")
 
         with override_settings(
             AUDIT_LOG_ENABLED=False,
@@ -974,8 +970,7 @@ class TestExportImport(
             - The soft-deleted document, note, and custom field instance
               survive the round-trip with deleted_at preserved
         """
-        shutil.rmtree(Path(self.dirs.media_dir) / "documents")
+        self._install_sample_files()
-        install_document_samples(Path(self.dirs.media_dir) / "documents")
 
         # d1 has self.note and self.cfi1 attached via setUp
         self.d1.delete()

src/documents/tests/test_management_fuzzy.py

@@ -19,6 +19,10 @@ class TestFuzzyMatchCommand(TestCase):
         call_command(
             "document_fuzzy_match",
             "--no-progress-bar",
+            # The default scales with the CPU count and forks, which closes
+            # the test's database connection on PostgreSQL
+            "--processes",
+            "1",
             *args,
             stdout=stdout,
             stderr=stderr,

src/documents/tests/test_management_retagger.py

@@ -425,26 +425,29 @@ class TestRetaggerStoragePath(DirectoriesMixin):
 @pytest.mark.management
 @pytest.mark.django_db
 class TestRetaggerIdRange(DirectoriesMixin):
-    @pytest.mark.usefixtures("documents")
     @pytest.mark.parametrize(
-        ("id_range_args", "expected_count"),
+        ("range_end", "expected_count"),
         [
-            pytest.param(["1", "2"], 1, id="narrow_range_limits_scope"),
+            pytest.param("second", 1, id="narrow_range_limits_scope"),
-            pytest.param(["1", "9999"], 2, id="wide_range_tags_all_matches"),
+            pytest.param("extra", 2, id="wide_range_tags_all_matches"),
         ],
     )
     def test_id_range_limits_scope(
         self,
         tags: TagTuple,
-        id_range_args: list[str],
+        documents: DocumentTuple,
+        range_end: str,
         expected_count: int,
     ) -> None:
-        DocumentFactory(content="NOT the first document")
+        first, second, *_ = documents
+        extra = DocumentFactory(content="NOT the first document")
+        end = {"second": second, "extra": extra}[range_end]
         call_command(
             "document_retagger",
             "--tags",
             "--id-range",
-            *id_range_args,
+            str(first.pk),
+            str(end.pk),
             skip_checks=True,
         )
         tag_first, *_ = tags

src/documents/tests/test_matching_glob_regex.py

@@ -0,0 +1,68 @@
+import re
+from fnmatch import fnmatch
+
+import pytest
+
+from documents.matching import glob_to_database_regex
+from documents.models import Document
+from paperless_testing.factories import DocumentFactory
+
+NAMES = [
+    "invoice.pdf",
+    "Invoice.PDF",
+    "invoice_2024.pdf",
+    "a.b.c",
+    "a-b",
+    "sample [1].pdf",
+    "sample 1.pdf",
+    "x^y",
+    "back\\slash",
+    "dir/file.pdf",
+    "",
+]
+
+PATTERNS = [
+    "*",
+    "invoice.pdf",
+    "*.pdf",
+    "*invoice*pdf",
+    "*a*b*c*",
+    "invoice_????.pdf",
+    "sample [[]1].pdf",
+    "sample [1-2].pdf",
+    "sample [!2].pdf",
+    "sample [^2].pdf",
+    "sample [",
+    "a.b.c",
+    "x^y",
+    "back\\slash",
+    "[]]",
+]
+
+
+@pytest.mark.parametrize("pattern", PATTERNS)
+def test_regex_agrees_with_fnmatch(pattern: str) -> None:
+    regex = glob_to_database_regex(pattern)
+    for name in NAMES:
+        expected = fnmatch(name.lower(), pattern.lower())
+        actual = re.search(regex, name, re.IGNORECASE) is not None
+        assert actual is expected, f"{pattern!r} vs {name!r}"
+
+
+@pytest.mark.parametrize("pattern", ["*", "*a*b*c*", "a*b*c*d*e"])
+def test_regex_avoids_syntax_postgresql_rejects(pattern: str) -> None:
+    regex = glob_to_database_regex(pattern)
+    assert not any(marker in regex for marker in ("(?", "\\Z"))
+
+
+@pytest.mark.django_db
+def test_database_filter_matches_glob() -> None:
+    matching = DocumentFactory(original_filename="Invoice_2024.PDF")
+    DocumentFactory(original_filename="invoice_2024.txt")
+    DocumentFactory(original_filename="receipt.pdf")
+
+    found = Document.objects.filter(
+        original_filename__iregex=glob_to_database_regex("*invoice*.pdf"),
+    )
+
+    assert list(found) == [matching]

src/documents/tests/test_workflows.py

@@ -174,8 +174,8 @@ class TestWorkflows(
         w.save()
 
         self.assertEqual(w.__str__(), "Workflow: Workflow 1")
-        self.assertEqual(trigger.__str__(), "WorkflowTrigger 1")
+        self.assertEqual(trigger.__str__(), f"WorkflowTrigger {trigger.pk}")
-        self.assertEqual(action.__str__(), "WorkflowAction 1")
+        self.assertEqual(action.__str__(), f"WorkflowAction {action.pk}")
 
         test_file = shutil.copy(
             self.SIMPLE_PDF,
@@ -433,7 +433,7 @@ class TestWorkflows(
                 list(document.tags.all()),
                 [self.t1, self.t2, self.t3],
             )
-            self.assertEqual(
+            self.assertCountEqual(
                 list(
                     get_users_with_perms(
                         document,

src/documents/views.py

@@ -5,7 +5,6 @@ import platform
 import re
 import tempfile
 import zipfile
-from collections import Counter
 from collections import deque
 from collections.abc import Callable
 from datetime import datetime
@@ -169,6 +168,7 @@ from documents.permissions import PaperlessNotePermissions
 from documents.permissions import PaperlessObjectPermissions
 from documents.permissions import TrashPermissions
 from documents.permissions import ViewDocumentsPermissions
+from documents.permissions import annotate_document_count_by_ids
 from documents.permissions import annotate_document_count_for_related_queryset
 from documents.permissions import get_document_count_filter_for_user
 from documents.permissions import get_permission_grantees
@@ -1022,83 +1022,71 @@ class DocumentViewSet(
         "custom_field_",
     )
 
-    # Matching document ids are looked up in the link tables in batches of this
-    # size: one literal IN (...) over every match is a multi-MB statement per
-    # relation and, on SQLite, exceeds the bound-parameter limit.
-    selection_data_link_batch_size = 10_000
-
     def _get_selection_data_for_queryset(self, queryset):
-        # The filtered queryset is evaluated once, as the page's own filter is,
+        # Resolve once instead of once per model below. `queryset` can carry an
-        # and never wrapped in a subquery: re-deriving a join or a HAVING
+        # arbitrarily expensive WHERE clause (user filters plus the permission
-        # (shared_by__id) per statement is very slow on some databases, and
+        # filter); re-embedding it as a subquery inside 5 separate Count(...)
-        # regrouping the queryset by another column would miscount.
+        # calls forces the database to re-evaluate that whole thing 5 times, and
-        # order_by() drops the default/user ordering, which only costs a sort.
+        # -- for FK relations especially -- can defeat semi-join planning
-        correspondents: Counter = Counter()
+        # entirely at scale. A concrete id list is cheap to reuse.
-        document_types: Counter = Counter()
+        # order_by() drops the default/user ordering -- irrelevant for a plain
-        storage_paths: Counter = Counter()
+        # id list, but left in place it forces a sort over the full filtered
-        document_ids: list[int] = []
+        # set before the ids can even be collected.
-        seen: set[int] = set()
+        document_ids = list(queryset.order_by().values_list("pk", flat=True))
-        rows = queryset.order_by().values_list(
-            "pk",
-            "correspondent_id",
-            "document_type_id",
-            "storage_path_id",
-        )
-        for pk, correspondent, document_type, storage_path in rows:
-            # A filter can join, repeating a document row.
-            if pk in seen:
-                continue
-            seen.add(pk)
-            document_ids.append(pk)
-            # Documents without a value count under None, which selection()
-            # never looks up.
-            correspondents[correspondent] += 1
-            document_types[document_type] += 1
-            storage_paths[storage_path] += 1
 
-        def link_table_counts(link_model, field_name: str) -> Counter:
+        correspondents = Correspondent.objects.annotate(
-            result: Counter = Counter()
+            document_count=Count(
-            batch_size = self.selection_data_link_batch_size
+                "documents",
-            for start in range(0, len(document_ids), batch_size):
+                filter=Q(documents__id__in=document_ids),
-                result.update(
+                distinct=True,
-                    dict(
-                        link_model.objects.filter(
-                            document_id__in=document_ids[start : start + batch_size],
-                        )
-                        .values_list(field_name)
-                        .annotate(document_count=Count("document_id")),
             ),
         )
-            return result
+        document_types = DocumentType.objects.annotate(
-
+            document_count=Count(
-        def selection(model, counts: Counter) -> list[dict[str, int]]:
+                "documents",
-            # Every object is listed, with 0 for those with no matching document.
+                filter=Q(documents__id__in=document_ids),
-            return [
+                distinct=True,
-                {"id": pk, "document_count": counts.get(pk, 0)}
+            ),
-                for pk in model.objects.values_list("pk", flat=True)
+        )
-            ]
+        storage_paths = StoragePath.objects.annotate(
-
+            document_count=Count(
+                "documents",
+                filter=Q(documents__id__in=document_ids),
+                distinct=True,
+            ),
+        )
+        # Tag and CustomField reach Document through an M2M/through-model table;
+        # a plain Count(filter=...) there is a much more expensive plan than the
+        # FK relations above once the bridge table is large -- see
+        # annotate_document_count_by_ids() for why.
+        tags = annotate_document_count_by_ids(
+            Tag.objects.all(),
+            through_model=Document.tags.through,
+            related_object_field="tag_id",
+            document_ids=document_ids,
+        )
+        custom_fields = annotate_document_count_by_ids(
+            CustomField.objects.all(),
+            through_model=CustomFieldInstance,
+            related_object_field="field_id",
+            document_ids=document_ids,
+        )
         return {
-            "selected_correspondents": selection(
+            "selected_correspondents": [
-                Correspondent,
+                {"id": t.id, "document_count": t.document_count} for t in correspondents
-                correspondents,
+            ],
-            ),
+            "selected_tags": [
-            "selected_tags": selection(
+                {"id": t.id, "document_count": t.document_count} for t in tags
-                Tag,
+            ],
-                link_table_counts(Document.tags.through, "tag_id"),
+            "selected_document_types": [
-            ),
+                {"id": t.id, "document_count": t.document_count} for t in document_types
-            "selected_document_types": selection(
+            ],
-                DocumentType,
+            "selected_storage_paths": [
-                document_types,
+                {"id": t.id, "document_count": t.document_count} for t in storage_paths
-            ),
+            ],
-            "selected_storage_paths": selection(
+            "selected_custom_fields": [
-                StoragePath,
+                {"id": t.id, "document_count": t.document_count} for t in custom_fields
-                storage_paths,
+            ],
-            ),
-            "selected_custom_fields": selection(
-                CustomField,
-                link_table_counts(CustomFieldInstance, "field_id"),
-            ),
         }
 
     @classmethod

src/paperless_ai/tests/test_chat.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import json
+from contextlib import nullcontext
 from typing import TYPE_CHECKING
 from typing import Any
 from unittest.mock import MagicMock
@@ -37,6 +38,13 @@ def patch_embed_model():
     llama_settings.Settings.embed_model = None
 
 
+@pytest.fixture(autouse=True)
+def keep_db_connection():
+    # Closing the connection would roll back the test's transaction on PostgreSQL
+    with patch("paperless_ai.chat.db_connection_released", nullcontext):
+        yield
+
+
 @pytest.fixture(autouse=True)
 def patch_embed_nodes():
     with patch(