prevent overwriting index

Nothing actually passed in something different for the effective_content args, so drop them!

docs/configuration.md

@@ -1215,7 +1215,7 @@ should be a valid crontab(5) expression describing when to run.
 
 : If set to the string "disable", no emails will be fetched automatically.
 
-    Defaults to every ten minutes, with an installation-specific minute offset.
+    Defaults to `*/10 * * * *` or every ten minutes.
 
 #### [`PAPERLESS_TRAIN_TASK_CRON=<cron expression>`](#PAPERLESS_TRAIN_TASK_CRON) {#PAPERLESS_TRAIN_TASK_CRON}
 

src/documents/search/_backend.py

@@ -266,11 +266,7 @@ class WriteBatch:
             if self._lock is not None:
                 self._lock.release()
 
-    def add_or_update(
-        self,
-        document: Document,
-        effective_content: str | None = None,
-    ) -> None:
+    def add_or_update(self, document: Document) -> None:
         """
         Add or update a document in the batch.
 
@@ -280,11 +276,9 @@ class WriteBatch:
 
         Args:
             document: Django Document instance to index
-            effective_content: Override document.content for indexing (used when
-                re-indexing with newer OCR text from document versions)
         """
         self.remove(document.pk)
-        doc = self._backend._build_tantivy_doc(document, effective_content)
+        doc = self._backend._build_tantivy_doc(document)
         self._writer.add_document(doc)
 
     def remove(self, doc_id: int) -> None:
@@ -425,18 +419,17 @@ class TantivyBackend:
     def _build_tantivy_doc(
         self,
         document: Document,
-        effective_content: str | None = None,
         viewer_ids: list[int] | None = None,
         viewer_group_ids: list[int] | None = None,
     ) -> tantivy.Document:
         """Build a tantivy Document from a Django Document instance.
 
-        ``effective_content`` overrides ``document.content`` for indexing —
-        used when re-indexing a root document with a newer version's OCR text.
+        A root document is indexed with its effective content, i.e. the newest
+        version's OCR text, so it is never indexed with its own outdated text.
+        Annotate the queryset with ``annotate_effective_content`` when indexing
+        more than a couple of documents, to resolve that without a query each.
         """
-        content = (
-            effective_content if effective_content is not None else document.content
-        )
+        content = document.get_effective_content() or ""
 
         doc = tantivy.Document()
 
@@ -584,11 +577,7 @@ class TantivyBackend:
 
         return doc
 
-    def add_or_update(
-        self,
-        document: Document,
-        effective_content: str | None = None,
-    ) -> None:
+    def add_or_update(self, document: Document) -> None:
         """
         Add or update a single document with file locking.
 
@@ -601,12 +590,11 @@ class TantivyBackend:
 
         Args:
             document: Django Document instance to index
-            effective_content: Override document.content for indexing
         """
         self._ensure_open()
         try:
             with self.batch_update(lock_timeout=_LOCK_TIMEOUT_SECONDS) as batch:
-                batch.add_or_update(document, effective_content)
+                batch.add_or_update(document)
         except SearchIndexLockError:
             logger.error(
                 "Search index lock exhausted for document %d after %d attempts; "
@@ -1027,7 +1015,6 @@ class TantivyBackend:
             ):
                 doc = self._build_tantivy_doc(
                     document,
-                    document.get_effective_content(),
                     viewer_ids=viewer_ids,
                     viewer_group_ids=viewer_group_ids,
                 )

src/documents/signals/handlers.py

@@ -794,10 +794,12 @@ def cleanup_user_deletion(sender, instance: User | Group, **kwargs) -> None:
 def add_to_index(sender, document, **kwargs) -> None:
     from documents.search import get_backend
 
-    get_backend().add_or_update(
-        document,
-        effective_content=document.get_effective_content(),
-    )
+    # A newly consumed version is not searchable on its own, its content
+    # becomes the effective_content of the root document
+    if document.root_document_id:
+        document = document.root_document
+
+    get_backend().add_or_update(document)
 
 
 def run_workflows_added(

src/documents/tasks.py

@@ -64,6 +64,7 @@ from documents.signals.handlers import send_websocket_document_updated
 from documents.utils import IterWrapper
 from documents.utils import compute_checksum
 from documents.utils import identity
+from documents.versioning import annotate_effective_content
 from documents.workflows.utils import get_workflows_for_trigger
 from paperless.config import AIConfig
 from paperless.logging import consume_task_id
@@ -114,10 +115,7 @@ def index_document(self, document_id: int) -> None:
         )
         return
     with get_backend().batch_update() as batch:
-        batch.add_or_update(
-            document,
-            effective_content=document.get_effective_content(),
-        )
+        batch.add_or_update(document)
 
 
 @shared_task(
@@ -312,7 +310,10 @@ def bulk_update_documents(document_ids) -> None:
     from documents.search import get_backend
 
     document_ids = list(document_ids)
-    documents = Document.objects.filter(id__in=document_ids)
+    # Annotated so indexing below doesn't query the versions of each document
+    documents = annotate_effective_content(
+        Document.objects.filter(id__in=document_ids),
+    )
 
     for doc in documents:
         clear_document_caches(doc.pk)

src/documents/tests/search/test_backend.py

@@ -16,6 +16,7 @@ from documents.search._backend import TantivyBackend
 from documents.search._backend import WriteBatch
 from documents.search._backend import get_backend
 from documents.search._backend import reset_backend
+from documents.signals.handlers import add_to_index
 from documents.tests.factories import CorrespondentFactory
 from documents.tests.factories import DocumentFactory
 from documents.tests.factories import DocumentTypeFactory
@@ -1030,6 +1031,81 @@ class TestHighlightHits:
         assert len(hits) == 0
 
 
+class TestVersionIndexing:
+    """
+    GIVEN:
+        - A root document whose new version has just been consumed, e.g. by
+          the password removal workflow action
+    WHEN:
+        - The consumption finished signal is handled
+    THEN:
+        - The root document is indexed with the new version's content, since
+          versions are not searchable on their own
+    """
+
+    def test_consumed_version_updates_root_entry(
+        self,
+        backend: TantivyBackend,
+        mocker: MockerFixture,
+    ) -> None:
+        root = Document.objects.create(
+            title="Statement",
+            content="",
+            checksum="VER1",
+            pk=90,
+        )
+        backend.add_or_update(root)
+        version = Document.objects.create(
+            title="Statement",
+            content="unprotected statement text",
+            checksum="VER2",
+            pk=91,
+            root_document=root,
+            version_index=1,
+        )
+        mocker.patch("documents.search.get_backend", return_value=backend)
+
+        add_to_index(sender=None, document=version)
+
+        assert backend.search_ids("unprotected", user=None) == [root.pk]
+
+
+class TestEffectiveContentIndexing:
+    """
+    GIVEN:
+        - A root document with a newer version
+    WHEN:
+        - The root document is indexed
+    THEN:
+        - The newest version's content is indexed, never the root's own
+          outdated text
+    """
+
+    def test_root_is_indexed_with_latest_version_content(
+        self,
+        backend: TantivyBackend,
+    ) -> None:
+        root = Document.objects.create(
+            title="Statement",
+            content="stale original text",
+            checksum="EFF1",
+            pk=95,
+        )
+        Document.objects.create(
+            title="Statement",
+            content="latest version text",
+            checksum="EFF2",
+            pk=96,
+            root_document=root,
+            version_index=1,
+        )
+
+        backend.add_or_update(root)
+
+        assert backend.search_ids("latest", user=None) == [root.pk]
+        assert backend.search_ids("stale", user=None) == []
+
+
 class TestIndexDirectoryGarbageCollection:
     """Regression tests for Tantivy segment files leaking on disk when
     multiple long-lived worker processes (Granian/Celery) take turns writing

src/documents/versioning.py

@@ -6,7 +6,10 @@ from typing import TYPE_CHECKING
 from typing import Any
 
 from django.db.models import F
+from django.db.models import OuterRef
 from django.db.models import QuerySet
+from django.db.models import Subquery
+from django.db.models.functions import Coalesce
 
 from documents.models import Document
 
@@ -22,6 +25,24 @@ def versions_newest_first(documents: QuerySet[Document]) -> QuerySet[Document]:
     return documents.order_by(F("version_index").desc(nulls_last=True), "-id")
 
 
+def annotate_effective_content(documents: QuerySet[Document]) -> QuerySet[Document]:
+    """
+    Annotates documents with the content of their newest version, falling back
+    to their own, so get_effective_content() can answer from the row rather
+    than querying for the versions of each document
+    """
+    return documents.annotate(
+        effective_content=Coalesce(
+            Subquery(
+                versions_newest_first(
+                    Document.objects.filter(root_document=OuterRef("pk")),
+                ).values("content")[:1],
+            ),
+            F("content"),
+        ),
+    )
+
+
 def sort_versions_newest_first(documents: list[Document]) -> list[Document]:
     """
     Same sorting as versions_newest_first()

src/paperless/settings/custom.py

@@ -1,7 +1,6 @@
 import datetime
 import logging
 import os
-from hashlib import sha256
 from pathlib import Path
 from typing import Any
 
@@ -173,15 +172,6 @@ def parse_beat_schedule() -> dict:
         # Don't add disabled tasks to the schedule
         if value == "disable":
             continue
-        if (
-            task["env_key"] == "PAPERLESS_EMAIL_TASK_CRON"
-            and task["env_key"] not in os.environ
-        ):
-            # Spread default polling across the ten-minute interval.
-            secret = os.environ["PAPERLESS_SECRET_KEY"].encode()
-            offset = int.from_bytes(sha256(secret).digest()) % 10
-            minutes = ",".join(str(minute) for minute in range(offset, 60, 10))
-            value = f"{minutes} * * * *"
         # I find https://crontab.guru/ super helpful
         # crontab(5) format
         #   - five time-and-date fields

src/paperless/tests/settings/test_custom_parsers.py

@@ -168,7 +168,6 @@ class TestParseHostingSettings:
 def make_expected_schedule(
     overrides: dict[str, dict[str, Any]] | None = None,
     disabled: set[str] | None = None,
-    email_minute: str = "6,16,26,36,46,56",
 ) -> dict[str, Any]:
     """
     Build the expected schedule with optional overrides and disabled tasks.
@@ -186,7 +185,7 @@ def make_expected_schedule(
     schedule: dict[str, Any] = {
         "Check all e-mail accounts": {
             "task": "paperless_mail.tasks.process_mail_accounts",
-            "schedule": crontab(minute=email_minute),
+            "schedule": crontab(minute="*/10"),
             "options": {
                 "expires": mail_expire,
                 "headers": {"trigger_source": "scheduled"},
@@ -267,11 +266,6 @@ class TestParseBeatSchedule:
         ("env", "expected"),
         [
             pytest.param({}, make_expected_schedule(), id="defaults"),
-            pytest.param(
-                {"PAPERLESS_EMAIL_TASK_CRON": "*/10 * * * *"},
-                make_expected_schedule(email_minute="*/10"),
-                id="email-explicit-default",
-            ),
             pytest.param(
                 {"PAPERLESS_EMAIL_TASK_CRON": "*/50 * * * mon"},
                 make_expected_schedule(
@@ -310,11 +304,7 @@ class TestParseBeatSchedule:
         expected: dict[str, Any],
         mocker: MockerFixture,
     ) -> None:
-        mocker.patch.dict(
-            os.environ,
-            {"PAPERLESS_SECRET_KEY": "test-secret", **env},
-            clear=False,
-        )
+        mocker.patch.dict(os.environ, env, clear=False)
         schedule = parse_beat_schedule()
         assert schedule == expected