Breaking: Decouple OCR control from archive file control (#12448)

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com>

src/documents/tests/test_api_app_config.py

@@ -46,7 +46,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
                 "pages": None,
                 "language": None,
                 "mode": None,
-                "skip_archive_file": None,
+                "archive_file_generation": None,
                 "image_dpi": None,
                 "unpaper_clean": None,
                 "deskew": None,

src/documents/tests/test_barcodes.py

@@ -1020,7 +1020,7 @@ class TestTagBarcode(DirectoriesMixin, SampleDirMixin, GetReaderPluginMixin, Tes
         CONSUMER_TAG_BARCODE_SPLIT=True,
         CONSUMER_TAG_BARCODE_MAPPING={"TAG:(.*)": "\\g<1>"},
         CELERY_TASK_ALWAYS_EAGER=True,
-        OCR_MODE="skip",
+        OCR_MODE="auto",
     )
     def test_consume_barcode_file_tag_split_and_assignment(self) -> None:
         """

src/documents/tests/test_consumer.py

@@ -230,7 +230,11 @@ class TestConsumer(
         shutil.copy(src, dst)
         return dst
 
-    @override_settings(FILENAME_FORMAT=None, TIME_ZONE="America/Chicago")
+    @override_settings(
+        FILENAME_FORMAT=None,
+        TIME_ZONE="America/Chicago",
+        ARCHIVE_FILE_GENERATION="always",
+    )
     def testNormalOperation(self) -> None:
         filename = self.get_test_file()
 
@@ -629,7 +633,10 @@ class TestConsumer(
         # Database empty
         self.assertEqual(Document.objects.all().count(), 0)
 
-    @override_settings(FILENAME_FORMAT="{correspondent}/{title}")
+    @override_settings(
+        FILENAME_FORMAT="{correspondent}/{title}",
+        ARCHIVE_FILE_GENERATION="always",
+    )
     def testFilenameHandling(self) -> None:
         with self.get_consumer(
             self.get_test_file(),
@@ -646,7 +653,7 @@ class TestConsumer(
         self._assert_first_last_send_progress()
 
     @mock.patch("documents.consumer.generate_unique_filename")
-    @override_settings(FILENAME_FORMAT="{pk}")
+    @override_settings(FILENAME_FORMAT="{pk}", ARCHIVE_FILE_GENERATION="always")
     def testFilenameHandlingFallsBackWhenGeneratedPathExceedsDbLimit(self, m):
         m.side_effect = lambda doc, archive_filename=False: Path(
             ("a" * 1100 + ".pdf") if not archive_filename else ("b" * 1100 + ".pdf"),
@@ -673,7 +680,10 @@ class TestConsumer(
 
         self._assert_first_last_send_progress()
 
-    @override_settings(FILENAME_FORMAT="{correspondent}/{title}")
+    @override_settings(
+        FILENAME_FORMAT="{correspondent}/{title}",
+        ARCHIVE_FILE_GENERATION="always",
+    )
     @mock.patch("documents.signals.handlers.generate_unique_filename")
     def testFilenameHandlingUnstableFormat(self, m) -> None:
         filenames = ["this", "that", "now this", "i cannot decide"]
@@ -1021,7 +1031,7 @@ class TestConsumer(
         self.assertEqual(Document.objects.count(), 2)
         self._assert_first_last_send_progress()
 
-    @override_settings(FILENAME_FORMAT="{title}")
+    @override_settings(FILENAME_FORMAT="{title}", ARCHIVE_FILE_GENERATION="always")
     @mock.patch("documents.consumer.get_parser_registry")
     def test_similar_filenames(self, m) -> None:
         shutil.copy(
@@ -1132,6 +1142,7 @@ class TestConsumer(
             mock_mail_parser_parse.assert_called_once_with(
                 consumer.working_copy,
                 "message/rfc822",
+                produce_archive=True,
             )
 
 
@@ -1279,7 +1290,14 @@ class PreConsumeTestCase(DirectoriesMixin, GetConsumerMixin, TestCase):
     def test_no_pre_consume_script(self, m) -> None:
         with self.get_consumer(self.test_file) as c:
             c.run()
-            m.assert_not_called()
+            # Verify no pre-consume script subprocess was invoked
+            # (run_subprocess may still be called by _extract_text_for_archive_check)
+            script_calls = [
+                call
+                for call in m.call_args_list
+                if call.args and call.args[0] and call.args[0][0] not in ("pdftotext",)
+            ]
+            self.assertEqual(script_calls, [])
 
     @mock.patch("documents.consumer.run_subprocess")
     @override_settings(PRE_CONSUME_SCRIPT="does-not-exist")
@@ -1295,9 +1313,16 @@ class PreConsumeTestCase(DirectoriesMixin, GetConsumerMixin, TestCase):
                 with self.get_consumer(self.test_file) as c:
                     c.run()
 
-                    m.assert_called_once()
+                    self.assertTrue(m.called)
 
-                    args, _ = m.call_args
+                    # Find the call that invoked the pre-consume script
+                    # (run_subprocess may also be called by _extract_text_for_archive_check)
+                    script_call = next(
+                        call
+                        for call in m.call_args_list
+                        if call.args and call.args[0] and call.args[0][0] == script.name
+                    )
+                    args, _ = script_call
 
                     command = args[0]
                     environment = args[1]

src/documents/tests/test_consumer_archive.py

@@ -0,0 +1,189 @@
+"""Tests for should_produce_archive()."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+from unittest.mock import MagicMock
+
+import pytest
+
+from documents.consumer import should_produce_archive
+
+if TYPE_CHECKING:
+    from pytest_mock import MockerFixture
+
+
+def _parser_instance(
+    *,
+    can_produce: bool = True,
+    requires_rendition: bool = False,
+) -> MagicMock:
+    """Return a mock parser instance with the given capability flags."""
+    instance = MagicMock()
+    instance.can_produce_archive = can_produce
+    instance.requires_pdf_rendition = requires_rendition
+    return instance
+
+
+@pytest.fixture()
+def null_app_config(mocker) -> MagicMock:
+    """Mock ApplicationConfiguration with all fields None → falls back to Django settings."""
+    return mocker.MagicMock(
+        output_type=None,
+        pages=None,
+        language=None,
+        mode=None,
+        archive_file_generation=None,
+        image_dpi=None,
+        unpaper_clean=None,
+        deskew=None,
+        rotate_pages=None,
+        rotate_pages_threshold=None,
+        max_image_pixels=None,
+        color_conversion_strategy=None,
+        user_args=None,
+    )
+
+
+@pytest.fixture(autouse=True)
+def patch_app_config(mocker, null_app_config):
+    """Patch BaseConfig._get_config_instance for all tests in this module."""
+    mocker.patch(
+        "paperless.config.BaseConfig._get_config_instance",
+        return_value=null_app_config,
+    )
+
+
+class TestShouldProduceArchive:
+    @pytest.mark.parametrize(
+        ("generation", "can_produce", "requires_rendition", "mime", "expected"),
+        [
+            pytest.param(
+                "never",
+                True,
+                False,
+                "application/pdf",
+                False,
+                id="never-returns-false",
+            ),
+            pytest.param(
+                "always",
+                True,
+                False,
+                "application/pdf",
+                True,
+                id="always-returns-true",
+            ),
+            pytest.param(
+                "never",
+                True,
+                True,
+                "application/pdf",
+                True,
+                id="requires-rendition-overrides-never",
+            ),
+            pytest.param(
+                "always",
+                False,
+                False,
+                "text/plain",
+                False,
+                id="cannot-produce-overrides-always",
+            ),
+            pytest.param(
+                "always",
+                False,
+                True,
+                "application/pdf",
+                True,
+                id="requires-rendition-wins-even-if-cannot-produce",
+            ),
+            pytest.param(
+                "auto",
+                True,
+                False,
+                "image/tiff",
+                True,
+                id="auto-image-returns-true",
+            ),
+            pytest.param(
+                "auto",
+                True,
+                False,
+                "message/rfc822",
+                False,
+                id="auto-non-pdf-non-image-returns-false",
+            ),
+        ],
+    )
+    def test_generation_setting(
+        self,
+        settings,
+        generation: str,
+        can_produce: bool,  # noqa: FBT001
+        requires_rendition: bool,  # noqa: FBT001
+        mime: str,
+        expected: bool,  # noqa: FBT001
+    ) -> None:
+        settings.ARCHIVE_FILE_GENERATION = generation
+        parser = _parser_instance(
+            can_produce=can_produce,
+            requires_rendition=requires_rendition,
+        )
+        assert should_produce_archive(parser, mime, Path("/tmp/doc")) is expected
+
+    @pytest.mark.parametrize(
+        ("extracted_text", "expected"),
+        [
+            pytest.param(
+                "This is a born-digital PDF with lots of text content. " * 10,
+                False,
+                id="born-digital-long-text-skips-archive",
+            ),
+            pytest.param(None, True, id="no-text-scanned-produces-archive"),
+            pytest.param("tiny", True, id="short-text-treated-as-scanned"),
+        ],
+    )
+    def test_auto_pdf_archive_decision(
+        self,
+        mocker: MockerFixture,
+        settings,
+        extracted_text: str | None,
+        expected: bool,  # noqa: FBT001
+    ) -> None:
+        settings.ARCHIVE_FILE_GENERATION = "auto"
+        mocker.patch("documents.consumer.is_tagged_pdf", return_value=False)
+        mocker.patch("documents.consumer.extract_pdf_text", return_value=extracted_text)
+        parser = _parser_instance(can_produce=True, requires_rendition=False)
+        assert (
+            should_produce_archive(parser, "application/pdf", Path("/tmp/doc.pdf"))
+            is expected
+        )
+
+    def test_tagged_pdf_skips_archive_in_auto_mode(
+        self,
+        mocker: MockerFixture,
+        settings,
+    ) -> None:
+        """Tagged PDFs (e.g. Word exports) are treated as born-digital regardless of text length."""
+        settings.ARCHIVE_FILE_GENERATION = "auto"
+        mocker.patch("documents.consumer.is_tagged_pdf", return_value=True)
+        parser = _parser_instance(can_produce=True, requires_rendition=False)
+        assert (
+            should_produce_archive(parser, "application/pdf", Path("/tmp/doc.pdf"))
+            is False
+        )
+
+    def test_tagged_pdf_does_not_call_pdftotext(
+        self,
+        mocker: MockerFixture,
+        settings,
+    ) -> None:
+        """When a PDF is tagged, pdftotext is not invoked (fast path)."""
+        settings.ARCHIVE_FILE_GENERATION = "auto"
+        mocker.patch("documents.consumer.is_tagged_pdf", return_value=True)
+        mock_extract = mocker.patch("documents.consumer.extract_pdf_text")
+        parser = _parser_instance(can_produce=True, requires_rendition=False)
+        should_produce_archive(parser, "application/pdf", Path("/tmp/doc.pdf"))
+        mock_extract.assert_not_called()

src/documents/tests/test_management.py

@@ -27,7 +27,10 @@ sample_file: Path = Path(__file__).parent / "samples" / "simple.pdf"
 
 
 @pytest.mark.management
-@override_settings(FILENAME_FORMAT="{correspondent}/{title}")
+@override_settings(
+    FILENAME_FORMAT="{correspondent}/{title}",
+    ARCHIVE_FILE_GENERATION="always",
+)
 class TestArchiver(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
     def make_models(self):
         return Document.objects.create(

src/documents/tests/test_tasks.py

@@ -213,6 +213,7 @@ class TestEmptyTrashTask(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
         self.assertEqual(Document.global_objects.count(), 0)
 
 
+@override_settings(ARCHIVE_FILE_GENERATION="always")
 class TestUpdateContent(DirectoriesMixin, TestCase):
     def test_update_content_maybe_archive_file(self) -> None:
         """