Performance: Precompile scoped bytecode cache once per container start

docker/rootfs/etc/s6-overlay/s6-rc.d/init-compile-bytecode/run

@@ -0,0 +1,51 @@
+#!/command/with-contenv /usr/bin/bash
+# shellcheck shell=bash
+declare -r log_prefix="[init-compile-bytecode]"
+
+# PYTHONDONTWRITEBYTECODE=1 is set for the whole container. This unit compiles a
+# scoped set of libraries anyway, to speed up startup without bloating image size.
+
+# Handle the people using a read only file system
+if [[ "${S6_READ_ONLY_ROOT}" == "1" ]]; then
+	echo "${log_prefix} S6_READ_ONLY_ROOT=1, skipping (nothing to write bytecode to)"
+	exit 0
+fi
+
+declare -r site_packages="$(python3 -c 'import site; print(site.getsitepackages()[0])')"
+
+# Deliberately scoped to packages that paperless.settings/paperless/__init__.py import
+# unconditionally on every manage.py invocation (Django itself, the always-loaded
+# INSTALLED_APPS, and celery). This is NOT "compile everything" - the optional AI stack
+# (torch, llama-index, sentence-transformers, ...) is intentionally excluded since it is
+# lazy-imported and large.
+declare -a scope=(
+	"${PAPERLESS_SRC_DIR}"
+	"${site_packages}/django"
+	"${site_packages}/celery"
+	"${site_packages}/kombu"
+	"${site_packages}/rest_framework"
+	"${site_packages}/django_filters"
+	"${site_packages}/whitenoise"
+	"${site_packages}/corsheaders"
+	"${site_packages}/django_extensions"
+	"${site_packages}/guardian"
+	"${site_packages}/allauth"
+	"${site_packages}/drf_spectacular"
+	"${site_packages}/drf_spectacular_sidecar"
+	"${site_packages}/treenode"
+	"${site_packages}/compression_middleware"
+)
+
+declare -a existing_scope=()
+for path in "${scope[@]}"; do
+	[[ -d "${path}" ]] && existing_scope+=("${path}")
+done
+
+echo "${log_prefix} Compiling bytecode for: ${existing_scope[*]}"
+declare -r start_seconds=${SECONDS}
+
+if ! PYTHONDONTWRITEBYTECODE= python3 -m compileall -q "${existing_scope[@]}"; then
+	echo "${log_prefix} WARNING: compileall reported errors (read-only filesystem or unwritable site-packages?); continuing without a bytecode cache"
+fi
+
+echo "${log_prefix} Done in $((SECONDS - start_seconds))s"

docker/rootfs/etc/s6-overlay/s6-rc.d/init-compile-bytecode/type

@@ -0,0 +1 @@
+oneshot

docker/rootfs/etc/s6-overlay/s6-rc.d/init-compile-bytecode/up

@@ -0,0 +1 @@
+/etc/s6-overlay/s6-rc.d/init-compile-bytecode/run

src/documents/bulk_edit.py

@@ -305,74 +305,46 @@ def modify_custom_fields(
         else [(field, None) for field in add_custom_fields]
     )
 
-    custom_fields_by_id: dict[int, CustomField] = {
+    custom_fields = CustomField.objects.filter(
-        cf.id: cf
+        id__in=[int(field) for field, _ in add_custom_fields],
-        for cf in CustomField.objects.filter(
+    ).distinct()
-            id__in=[int(field) for field, _ in add_custom_fields],
-        )
-    }
-    # Deferred, not `.only()`: signal receivers touch other Document fields,
-    # and `.only("pk")` would just turn that into a per-document reload.
-    # `content` is the one field both large and unused here. Skipped
-    # entirely for a remove-only call -- the removal pass below resolves
-    # its own documents.
-    docs_by_id: dict[int, Document] = (
-        {
-            doc.id: doc
-            for doc in Document.objects.filter(id__in=affected_docs).defer("content")
-        }
-        if add_custom_fields
-        else {}
-    )
     for field_id, value in add_custom_fields:
-        custom_field = custom_fields_by_id[field_id]
-        value_field = CustomFieldInstance.TYPE_TO_DATA_STORE_NAME_MAP[
-            custom_field.data_type
-        ]
         for doc_id in affected_docs:
-            defaults = {value_field: value}
+            defaults = {}
-            if (
+            custom_field = custom_fields.get(id=field_id)
-                custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK
+            if custom_field:
-                and value
+                value_field = CustomFieldInstance.TYPE_TO_DATA_STORE_NAME_MAP[
-                and doc_id in value
+                    custom_field.data_type
-            ):
+                ]
-                # Prevent self-linking
+                defaults[value_field] = value
-                continue
+                if (
-            # Not update_or_create(): it fetches an existing row via plain
+                    custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK
-            # `.get()` before calling .save(), so a signal receiver touching
+                    and value
-            # `.field`/`.document` (e.g. auditlog) on that save re-fetches
+                    and doc_id in value
-            # per instance regardless of what's passed in as lookup kwargs.
+                ):
-            # Assigning the cached objects ourselves before .save() avoids
+                    # Prevent self-linking
-            # that for both the create and update case.
+                    continue
-            try:
+            CustomFieldInstance.objects.update_or_create(
-                instance = CustomFieldInstance.objects.get(
+                document_id=doc_id,
-                    document=docs_by_id[doc_id],
+                field_id=field_id,
-                    field=custom_field,
+                defaults=defaults,
-                )
+            )
-            except CustomFieldInstance.DoesNotExist:
-                instance = CustomFieldInstance(
-                    document=docs_by_id[doc_id],
-                    field=custom_field,
-                )
-            instance.document = docs_by_id[doc_id]
-            instance.field = custom_field
-            for attr, val in defaults.items():
-                setattr(instance, attr, val)
-            instance.save()
             if custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK:
-                reflect_doclinks(docs_by_id[doc_id], custom_field, value)
+                doc = Document.objects.get(id=doc_id)
+                reflect_doclinks(doc, custom_field, value)
 
-    # For doc link fields being removed, remove symmetrical links.
+    # For doc link fields that are being removed, remove symmetrical links
-    # select_related here avoids resolving every affected document up front.
     for doclink_being_removed_instance in CustomFieldInstance.objects.filter(
         document_id__in=affected_docs,
         field__id__in=remove_custom_fields,
         field__data_type=CustomField.FieldDataType.DOCUMENTLINK,
         value_document_ids__isnull=False,
-    ).select_related("field", "document"):
+    ):
         for target_doc_id in doclink_being_removed_instance.value:
             remove_doclink(
-                document=doclink_being_removed_instance.document,
+                document=Document.objects.get(
+                    id=doclink_being_removed_instance.document.id,
+                ),
                 field=doclink_being_removed_instance.field,
                 target_doc_id=target_doc_id,
             )
@@ -1205,13 +1177,10 @@ def remove_doclink(
     """
     Removes a 'symmetrical' link to `document` from the target document's existing custom field instance
     """
-    # select_related: a signal receiver (auditlog) touches .document/.field
+    target_doc_field_instance = CustomFieldInstance.objects.filter(
-    # on save() below -- without this, that's a per-call reload query.
+        document_id=target_doc_id,
-    target_doc_field_instance = (
+        field=field,
-        CustomFieldInstance.objects.filter(document_id=target_doc_id, field=field)
+    ).first()
-        .select_related("document", "field")
-        .first()
-    )
     if (
         target_doc_field_instance is not None
         and document.id in target_doc_field_instance.value

src/documents/tests/test_bulk_edit.py

@@ -6,9 +6,7 @@ from unittest import mock
 import pikepdf
 from django.contrib.auth.models import Group
 from django.contrib.auth.models import User
-from django.db import connection
 from django.test import TestCase
-from django.test.utils import CaptureQueriesContext
 from guardian.shortcuts import assign_perm
 from guardian.shortcuts import get_groups_with_perms
 from guardian.shortcuts import get_users_with_perms
@@ -346,202 +344,6 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
         assert _cf_3 is not None
         self.assertNotIn(self.doc3.id, _cf_3.value)
 
-    def test_modify_custom_fields_batches_field_lookup(self) -> None:
-        """
-        GIVEN:
-            - Several documents are being bulk-edited to add several custom
-              fields at once
-        WHEN:
-            - modify_custom_fields runs
-        THEN:
-            - Each CustomField is resolved with one batched query total, not
-              once per (field, document) pair
-        """
-        docs = [
-            Document.objects.create(checksum=f"batch-{i}", title=f"batch-{i}")
-            for i in range(6)
-        ]
-        fields = [
-            CustomField.objects.create(
-                name=f"Batch Field {i}",
-                data_type=CustomField.FieldDataType.STRING,
-            )
-            for i in range(4)
-        ]
-
-        with CaptureQueriesContext(connection) as ctx:
-            bulk_edit.modify_custom_fields(
-                [doc.id for doc in docs],
-                add_custom_fields=[field.id for field in fields],
-                remove_custom_fields=[],
-            )
-
-        field_lookups = [
-            q
-            for q in ctx.captured_queries
-            if 'FROM "documents_customfield"' in q["sql"]
-        ]
-        self.assertEqual(
-            len(field_lookups),
-            1,
-            "Expected a single batched query to resolve the custom fields, "
-            f"got {len(field_lookups)}: {field_lookups}",
-        )
-
-        for doc in docs:
-            self.assertEqual(doc.custom_fields.count(), len(fields))
-
-    def test_modify_custom_fields_batches_document_lookup_for_documentlink(
-        self,
-    ) -> None:
-        """
-        GIVEN:
-            - Several documents are being bulk-edited to add a DOCUMENTLINK
-              custom field at once
-        WHEN:
-            - modify_custom_fields runs
-        THEN:
-            - The Document rows needed to reflect the symmetrical links are
-              resolved with one batched query total, not once per document
-        """
-        docs = [
-            Document.objects.create(checksum=f"link-{i}", title=f"link-{i}")
-            for i in range(6)
-        ]
-        target = Document.objects.create(checksum="link-target", title="link-target")
-        doclink_field = CustomField.objects.create(
-            name="Related",
-            data_type=CustomField.FieldDataType.DOCUMENTLINK,
-        )
-
-        with CaptureQueriesContext(connection) as ctx:
-            bulk_edit.modify_custom_fields(
-                [doc.id for doc in docs],
-                add_custom_fields={doclink_field.id: [target.id]},
-                remove_custom_fields=[],
-            )
-
-        single_document_lookups = [
-            q
-            for q in ctx.captured_queries
-            if 'FROM "documents_document"' in q["sql"]
-            and '"documents_document"."id" = ' in q["sql"]
-        ]
-        self.assertEqual(
-            len(single_document_lookups),
-            0,
-            "Expected document rows to come from a batched query, not "
-            f"per-document lookups, got: {single_document_lookups}",
-        )
-
-        for doc in docs:
-            self.assertEqual(
-                doc.custom_fields.get(field=doclink_field).value,
-                [target.id],
-            )
-
-    def test_modify_custom_fields_update_caches_document_and_field(self) -> None:
-        """
-        GIVEN:
-            - Several documents already have an instance of a custom field
-        WHEN:
-            - modify_custom_fields runs again for the same field, updating
-              the existing instances rather than creating new ones
-        THEN:
-            - No per-instance `.document`/`.field` reload query is issued
-              (e.g. by auditlog's post_save receiver touching them)
-        """
-        docs = [
-            Document.objects.create(checksum=f"update-{i}", title=f"update-{i}")
-            for i in range(6)
-        ]
-        field = CustomField.objects.create(
-            name="Update Field",
-            data_type=CustomField.FieldDataType.STRING,
-        )
-        bulk_edit.modify_custom_fields(
-            [doc.id for doc in docs],
-            add_custom_fields=[field.id],
-            remove_custom_fields=[],
-        )
-
-        with CaptureQueriesContext(connection) as ctx:
-            bulk_edit.modify_custom_fields(
-                [doc.id for doc in docs],
-                add_custom_fields={field.id: "updated value"},
-                remove_custom_fields=[],
-            )
-
-        single_row_reloads = [
-            q
-            for q in ctx.captured_queries
-            if ('FROM "documents_document"' in q["sql"] and '."id" = ' in q["sql"])
-            or ('FROM "documents_customfield"' in q["sql"] and '."id" = ' in q["sql"])
-        ]
-        self.assertEqual(
-            single_row_reloads,
-            [],
-            "Expected no per-instance document/field reload queries when "
-            f"updating existing custom field instances, got: {single_row_reloads}",
-        )
-
-        for doc in docs:
-            self.assertEqual(
-                doc.custom_fields.get(field=field).value,
-                "updated value",
-            )
-
-    def test_modify_custom_fields_removes_symmetrical_doclinks_batched(self) -> None:
-        """
-        GIVEN:
-            - Several source documents link to a shared target via a doc
-              link field
-        WHEN:
-            - The field is removed from all of them in one call
-        THEN:
-            - The symmetrical links are removed from the target
-            - No per-document lookup query is issued, on either side
-        """
-        target = Document.objects.create(checksum="rm-target", title="rm-target")
-        docs = [
-            Document.objects.create(checksum=f"rm-{i}", title=f"rm-{i}")
-            for i in range(6)
-        ]
-        field = CustomField.objects.create(
-            name="Related",
-            data_type=CustomField.FieldDataType.DOCUMENTLINK,
-        )
-        bulk_edit.modify_custom_fields(
-            [doc.id for doc in docs],
-            add_custom_fields={field.id: [target.id]},
-            remove_custom_fields=[],
-        )
-        self.assertEqual(
-            target.custom_fields.get(field=field).value,
-            [d.id for d in docs],
-        )
-
-        with CaptureQueriesContext(connection) as ctx:
-            bulk_edit.modify_custom_fields(
-                [doc.id for doc in docs],
-                add_custom_fields=[],
-                remove_custom_fields=[field.id],
-            )
-
-        single_document_lookups = [
-            q
-            for q in ctx.captured_queries
-            if 'FROM "documents_document"' in q["sql"]
-            and '"documents_document"."id" = ' in q["sql"]
-        ]
-        self.assertEqual(
-            single_document_lookups,
-            [],
-            "Expected batched document resolution, not per-document lookups, "
-            f"got: {single_document_lookups}",
-        )
-        self.assertEqual(target.custom_fields.get(field=field).value, [])
-
     def test_modify_custom_fields_doclink_self_link(self) -> None:
         """
         GIVEN: