Updates the documentation to be correct about the respecting now

docs/configuration.md

@@ -948,10 +948,11 @@ for display in the web interface.
 
     !!! note
 
-        The **remote OCR parser** (Azure AI) always produces a searchable
-        PDF and stores it as the archive copy, regardless of this setting.
-        `ARCHIVE_FILE_GENERATION=never` has no effect when the remote
-        parser handles a document.
+        The **remote OCR parser** (Azure AI) also honors this setting: when
+        no archive is requested (`never`, or `auto` with a born-digital PDF),
+        the remote engine is skipped entirely and locally-extracted text is
+        used instead, avoiding an unnecessary API call and a duplicate text
+        layer.
 
 #### [`PAPERLESS_OCR_CLEAN=<mode>`](#PAPERLESS_OCR_CLEAN) {#PAPERLESS_OCR_CLEAN}
 

docs/migration-v3.md

@@ -187,10 +187,11 @@ PAPERLESS_ARCHIVE_FILE_GENERATION=auto
 
 ### Remote OCR parser
 
-If you use the **remote OCR parser** (Azure AI), note that it always produces a
-searchable PDF and stores it as the archive copy. `ARCHIVE_FILE_GENERATION=never`
-has no effect for documents handled by the remote parser - the archive is produced
-unconditionally by the remote engine.
+If you use the **remote OCR parser** (Azure AI), `ARCHIVE_FILE_GENERATION` is
+honored the same way as for the local engine: when no archive is requested
+(`never`, or `auto` with a born-digital PDF), the remote engine is skipped
+entirely and locally-extracted text is used instead, avoiding an unnecessary
+API call and a duplicate text layer.
 
 ## Search Index (Whoosh -> Tantivy)
 

src/documents/search/_backend.py

@@ -39,7 +39,6 @@ from documents.search._tokenizer import ascii_fold
 from documents.search._tokenizer import autocomplete_tokens
 from documents.search._tokenizer import register_tokenizers
 from documents.utils import IterWrapper
-from documents.utils import QuerySetStream
 from documents.utils import identity
 
 if TYPE_CHECKING:
@@ -1036,15 +1035,14 @@ _EMPTY_VIEWER_GRANT: Final[ViewerGrant] = ViewerGrant(
 )
 
 
-class _DocumentViewerStream(QuerySetStream["Document"]):
+class _DocumentViewerStream:
     """Yield document permission data while batch-loading grants.
 
     Viewer permissions are fetched in batches (see
     ``_bulk_get_viewer_permissions``), but documents are yielded individually so a
     progress bar wrapped around this stream advances per document rather than
-    jumping a whole chunk at a time. ``__len__`` (inherited from
-    ``QuerySetStream``) lets the progress helper still discover the total (it
-    inspects ``QuerySet``/``Sized``).
+    jumping a whole chunk at a time. ``__len__`` lets the progress helper still
+    discover the total (it inspects ``QuerySet``/``Sized``).
 
     The viewer and group ids travel with each document in the yielded pair
     rather than through a separate mutable attribute, so the pairing survives
@@ -1053,11 +1051,18 @@ class _DocumentViewerStream(QuerySetStream["Document"]):
     generator in lock-step.
     """
 
+    def __init__(self, documents: QuerySet[Document], *, chunk_size: int) -> None:
+        self._documents = documents
+        self._chunk_size = chunk_size
+
+    def __len__(self) -> int:
+        return self._documents.count()
+
     def __iter__(self) -> Iterator[tuple[Document, ViewerGrant]]:
         # iterator(chunk_size=…) streams from a server-side cursor instead of
         # materialising the whole queryset in memory; since Django 4.1 it still
         # honours prefetch_related, running the prefetches one batch at a time.
-        documents = self._queryset.iterator(chunk_size=self._chunk_size)
+        documents = self._documents.iterator(chunk_size=self._chunk_size)
         for chunk in chunked(documents, self._chunk_size):
             grants_by_pk = _bulk_get_viewer_permissions([doc.pk for doc in chunk])
             for doc in chunk:

src/documents/tests/test_utils.py

@@ -1,35 +0,0 @@
-import pytest_mock
-
-from documents.utils import QuerySetStream
-
-
-class TestQuerySetStream:
-    def test_len_and_iter_delegate_to_streaming_queryset_methods(
-        self,
-        mocker: pytest_mock.MockerFixture,
-    ) -> None:
-        """
-        GIVEN:
-            - A mock queryset
-        WHEN:
-            - A QuerySetStream wrapping it is measured and iterated
-        THEN:
-            - len() uses count() (not a materializing len()), and iteration
-              uses .iterator(chunk_size=...) (not plain iteration, which
-              would materialize the whole queryset, plus any prefetch
-              caches, into Django's own result cache at once)
-        """
-        mock_queryset = mocker.MagicMock()
-        mock_queryset.count.return_value = 42
-        mock_queryset.iterator.return_value = iter(["row-1", "row-2"])
-        streamed = QuerySetStream(mock_queryset, chunk_size=1000)
-
-        assert len(streamed) == 42
-        assert list(streamed) == ["row-1", "row-2"]
-        # count.call_count isn't asserted exactly: list()'s own size-hint
-        # optimization calls len(streamed) again internally, on top of the
-        # explicit len() call above -- both legitimately delegate to
-        # count(), so only the delegation itself (not the call count) is
-        # the thing being verified here.
-        mock_queryset.count.assert_called_with()
-        mock_queryset.iterator.assert_called_once_with(chunk_size=1000)

src/documents/utils.py

@@ -3,24 +3,16 @@ import logging
 import shutil
 from collections.abc import Callable
 from collections.abc import Iterable
-from collections.abc import Iterator
 from os import utime
 from pathlib import Path
 from subprocess import CompletedProcess
 from subprocess import run
-from typing import TYPE_CHECKING
-from typing import Generic
 from typing import TypeVar
 
 from django.conf import settings
 from PIL import Image
 
-if TYPE_CHECKING:
-    from django.db.models import Model
-    from django.db.models import QuerySet
-
 _T = TypeVar("_T")
-_M = TypeVar("_M", bound="Model")
 
 # A function that wraps an iterable — typically used to inject a progress bar.
 IterWrapper = Callable[[Iterable[_T]], Iterable[_T]]
@@ -31,40 +23,6 @@ def identity(iterable: Iterable[_T]) -> Iterable[_T]:
     return iterable
 
 
-class QuerySetStream(Generic[_M]):
-    """Stream a QuerySet via .iterator(chunk_size=...) instead of
-    materializing it (plus any prefetch caches) all at once, while still
-    supporting len() via count() so a progress bar wrapped around this
-    (e.g. via IterWrapper) shows a real total instead of falling back to
-    indeterminate.
-
-    Plain QuerySet iteration (``for row in queryset:``) is not lazy: Django
-    fetches every matching row in one query and caches the fully-hydrated
-    result in the queryset's own ``_result_cache`` before yielding the
-    first item -- wrapping that in a progress bar or any other iterable
-    adapter doesn't change this, since none of them alter how the
-    underlying queryset produces items. ``.iterator(chunk_size=...)`` is
-    the specific Django API that bypasses ``_result_cache`` and streams
-    from a server-side cursor instead, discarding each chunk once consumed
-    (and, since Django 4.1, still honours ``prefetch_related``, running the
-    prefetches one batch at a time rather than for the whole queryset).
-
-    Subclass to layer additional per-batch work on top (see
-    ``documents.search._backend._DocumentViewerStream``) by overriding
-    ``__iter__`` -- ``__len__`` and the constructor are inherited for free.
-    """
-
-    def __init__(self, queryset: "QuerySet[_M]", *, chunk_size: int) -> None:
-        self._queryset = queryset
-        self._chunk_size = chunk_size
-
-    def __len__(self) -> int:
-        return self._queryset.count()
-
-    def __iter__(self) -> Iterator[_M]:
-        return iter(self._queryset.iterator(chunk_size=self._chunk_size))
-
-
 def _coerce_to_path(
     source: Path | str,
     dest: Path | str,

src/paperless/parsers/remote.py

@@ -3,7 +3,9 @@ Built-in remote-OCR document parser.
 
 Handles documents by sending them to a configured remote OCR engine
 (currently Azure AI Vision / Document Intelligence) and retrieving both
-the extracted text and a searchable PDF with an embedded text layer.
+the extracted text and a searchable PDF with an embedded text layer. For
+born-digital PDFs that need no archive copy, the remote call is skipped
+entirely in favor of locally-extracted text (see ``RemoteDocumentParser.parse``).
 
 When no engine is configured, ``score()`` returns ``None`` so the parser
 is effectively invisible to the registry — the tesseract parser handles
@@ -21,6 +23,8 @@ from typing import Self
 
 from django.conf import settings
 
+from paperless.parsers.utils import extract_pdf_text
+from paperless.parsers.utils import post_process_text
 from paperless.version import __full_version_str__
 
 if TYPE_CHECKING:
@@ -69,8 +73,11 @@ class RemoteDocumentParser:
     """Parse documents via a remote OCR API (currently Azure AI Vision).
 
     This parser sends documents to a remote engine that returns both
-    extracted text and a searchable PDF with an embedded text layer.
-    It does not depend on Tesseract or ocrmypdf.
+    extracted text and a searchable PDF with an embedded text layer,
+    except when ``parse()`` is called with ``produce_archive=False`` for
+    a PDF, in which case the remote call is skipped and only locally
+    extracted text is returned (no archive). It does not depend on
+    Tesseract or ocrmypdf.
 
     Class attributes
     ----------------
@@ -159,8 +166,11 @@ class RemoteDocumentParser:
         Returns
         -------
         bool
-            Always True — the remote engine always returns a PDF with an
-            embedded text layer that serves as the archive copy.
+            Always True — the remote engine is capable of returning a PDF
+            with an embedded text layer to serve as the archive copy.
+            Whether it actually does so for a given document depends on
+            ``produce_archive`` passed to :meth:`parse` (see there for when
+            the remote engine call, and thus archive generation, is skipped).
         """
         return True
 
@@ -217,6 +227,12 @@ class RemoteDocumentParser:
     ) -> None:
         """Send the document to the remote engine and store results.
 
+        When *produce_archive* is False for a PDF, the caller (via
+        ``documents.consumer.should_produce_archive``) has already determined
+        that the document is born-digital and needs no archive — skip the
+        remote engine entirely rather than re-OCRing it and creating a
+        duplicate text layer.
+
         Parameters
         ----------
         document_path:
@@ -224,8 +240,8 @@ class RemoteDocumentParser:
         mime_type:
             Detected MIME type of the document.
         produce_archive:
-            Ignored — the remote engine always returns a searchable PDF,
-            which is stored as the archive copy regardless of this flag.
+            Whether an archive copy is wanted. For PDFs, False skips the
+            remote engine and uses locally-extracted text instead.
         """
         config = RemoteEngineConfig(
             engine=settings.REMOTE_OCR_ENGINE,
@@ -240,6 +256,16 @@ class RemoteDocumentParser:
             self._text = ""
             return
 
+        if not produce_archive and mime_type == "application/pdf":
+            logger.debug(
+                "Remote OCR: skipped — no archive requested, "
+                "using locally-extracted text",
+            )
+            self._text = (
+                post_process_text(extract_pdf_text(document_path, log=logger)) or ""
+            )
+            return
+
         if config.engine == "azureai":
             self._text = self._azure_ai_vision_parse(document_path, config)
 

src/paperless/parsers/tesseract.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 import importlib.resources
 import logging
 import os
-import re
 import shutil
 import tempfile
 from pathlib import Path
@@ -25,9 +24,9 @@ from paperless.config import OcrConfig
 from paperless.models import CleanChoices
 from paperless.models import ModeChoices
 from paperless.models import OutputTypeChoices
-from paperless.parsers.utils import PDF_TEXT_MIN_LENGTH
 from paperless.parsers.utils import extract_pdf_text
-from paperless.parsers.utils import is_tagged_pdf
+from paperless.parsers.utils import pdf_has_digital_text
+from paperless.parsers.utils import post_process_text
 from paperless.parsers.utils import read_file_handle_unicode_errors
 from paperless.version import __full_version_str__
 
@@ -510,10 +509,10 @@ class RasterisedDocumentParser:
 
         if mime_type == "application/pdf":
             text_original = self.extract_text(None, document_path)
-            has_text = text_original is not None and len(text_original) > 0
-            original_has_text = has_text and (
-                is_tagged_pdf(document_path, log=self.log)
-                or len(text_original) > PDF_TEXT_MIN_LENGTH
+            original_has_text = pdf_has_digital_text(
+                document_path,
+                text_original,
+                log=self.log,
             )
         else:
             text_original = None
@@ -658,17 +657,3 @@ class RasterisedDocumentParser:
                     f"No text was found in {document_path}, the content will be empty.",
                 )
                 self.text = ""
-
-
-def post_process_text(text: str | None) -> str | None:
-    if not text:
-        return None
-
-    collapsed_spaces = re.sub(r"([^\S\r\n]+)", " ", text)
-    no_leading_whitespace = re.sub(r"([\n\r]+)([^\S\n\r]+)", "\\1", collapsed_spaces)
-    no_trailing_whitespace = re.sub(r"([^\S\n\r]+)$", "", no_leading_whitespace)
-
-    # TODO: this needs a rework
-    # replace \0 prevents issues with saving to postgres.
-    # text may contain \0 when this character is present in PDF files.
-    return no_trailing_whitespace.strip().replace("\0", " ")

src/paperless/parsers/utils.py

@@ -65,6 +65,63 @@ def is_tagged_pdf(
         return False
 
 
+def pdf_has_digital_text(
+    path: Path,
+    text: str | None,
+    log: logging.Logger | None = None,
+) -> bool:
+    """Return True if a PDF already has a usable, born-digital text layer.
+
+    Combines the tagged-PDF check with an extracted-text length check.
+    Shared by the tesseract and remote OCR parsers to decide whether
+    OCR_MODE=auto/off should skip (re-)OCRing a document.
+
+    Parameters
+    ----------
+    path:
+        Absolute path to the PDF file.
+    text:
+        Text already extracted from the PDF (e.g. via ``extract_pdf_text``),
+        or ``None``.
+    log:
+        Logger for warnings.  Falls back to the module-level logger when omitted.
+
+    Returns
+    -------
+    bool
+        ``True`` when the document already contains a text layer.
+    """
+    return is_tagged_pdf(path, log=log) or (
+        text is not None and len(text) > PDF_TEXT_MIN_LENGTH
+    )
+
+
+def post_process_text(text: str | None) -> str | None:
+    """Normalise whitespace in extracted OCR/PDF text and strip NUL bytes.
+
+    Parameters
+    ----------
+    text:
+        Raw extracted text, or ``None``.
+
+    Returns
+    -------
+    str | None
+        Cleaned text, or ``None`` when *text* is falsy.
+    """
+    if not text:
+        return None
+
+    collapsed_spaces = re.sub(r"([^\S\r\n]+)", " ", text)
+    no_leading_whitespace = re.sub(r"([\n\r]+)([^\S\n\r]+)", "\\1", collapsed_spaces)
+    no_trailing_whitespace = re.sub(r"([^\S\n\r]+)$", "", no_leading_whitespace)
+
+    # TODO: this needs a rework
+    # replace \0 prevents issues with saving to postgres.
+    # text may contain \0 when this character is present in PDF files.
+    return no_trailing_whitespace.strip().replace("\0", " ")
+
+
 def extract_pdf_text(
     path: Path,
     log: logging.Logger | None = None,

src/paperless/tests/parsers/test_remote_parser.py

@@ -336,6 +336,117 @@ class TestRemoteParserParse:
         assert remote_parser.get_date() is None
 
 
+# ---------------------------------------------------------------------------
+# parse() — produce_archive=False skips the remote engine (PDFs only)
+# ---------------------------------------------------------------------------
+
+
+class TestRemoteParserSkipsWhenNoArchiveWanted:
+    """When the caller has already decided no archive is needed for a PDF
+    (documents.consumer.should_produce_archive), the remote engine call is
+    skipped entirely in favor of locally-extracted text.
+    """
+
+    def test_pdf_skips_azure_when_no_archive_requested(
+        self,
+        remote_parser: RemoteDocumentParser,
+        simple_digital_pdf_file: Path,
+        azure_client: Mock,
+    ) -> None:
+        """
+        GIVEN: produce_archive=False for a PDF
+        WHEN:  parse() is called
+        THEN:  Azure is never invoked, no archive is produced, and text
+               comes from local pdftotext extraction
+        """
+        remote_parser.parse(
+            simple_digital_pdf_file,
+            "application/pdf",
+            produce_archive=False,
+        )
+
+        azure_client.begin_analyze_document.assert_not_called()
+        assert remote_parser.get_archive_path() is None
+        assert remote_parser.get_text() != ""
+
+    def test_pdf_no_archive_requested_text_matches_local_extraction(
+        self,
+        remote_parser: RemoteDocumentParser,
+        simple_digital_pdf_file: Path,
+        azure_client: Mock,
+        mocker: MockerFixture,
+    ) -> None:
+        """
+        GIVEN: produce_archive=False for a PDF
+        WHEN:  parse() is called
+        THEN:  the returned text is exactly the locally-extracted text,
+               not anything from the (unused) Azure mock
+        """
+        mocker.patch(
+            "paperless.parsers.remote.extract_pdf_text",
+            return_value="Local digital text.",
+        )
+
+        remote_parser.parse(
+            simple_digital_pdf_file,
+            "application/pdf",
+            produce_archive=False,
+        )
+
+        assert remote_parser.get_text() == "Local digital text."
+
+    def test_pdf_no_archive_requested_closes_no_client(
+        self,
+        remote_parser: RemoteDocumentParser,
+        simple_digital_pdf_file: Path,
+        azure_client: Mock,
+    ) -> None:
+        remote_parser.parse(
+            simple_digital_pdf_file,
+            "application/pdf",
+            produce_archive=False,
+        )
+
+        azure_client.close.assert_not_called()
+
+    def test_non_pdf_still_calls_azure_when_no_archive_requested(
+        self,
+        remote_parser: RemoteDocumentParser,
+        simple_digital_pdf_file: Path,
+        azure_client: Mock,
+    ) -> None:
+        """
+        Images have no local-text fallback, so produce_archive=False does
+        not skip the remote engine for non-PDF MIME types.
+        """
+        remote_parser.parse(
+            simple_digital_pdf_file,
+            "image/png",
+            produce_archive=False,
+        )
+
+        azure_client.begin_analyze_document.assert_called_once()
+        assert remote_parser.get_text() == _DEFAULT_TEXT
+
+    @pytest.mark.usefixtures("no_engine_settings")
+    def test_unconfigured_engine_takes_precedence_over_skip(
+        self,
+        remote_parser: RemoteDocumentParser,
+        simple_digital_pdf_file: Path,
+    ) -> None:
+        """An unconfigured engine still short-circuits before the
+        produce_archive check, returning empty text as before.
+        """
+        remote_parser.parse(
+            simple_digital_pdf_file,
+            "application/pdf",
+            produce_archive=False,
+        )
+
+        assert remote_parser.get_text() == ""
+        assert remote_parser.get_archive_path() is None
+
+
 # ---------------------------------------------------------------------------
 # parse() — Azure failure path
 # ---------------------------------------------------------------------------

src/paperless/tests/parsers/test_tesseract_parser.py

@@ -21,7 +21,7 @@ from documents.parsers import run_convert
 from paperless.models import ModeChoices
 from paperless.parsers import ParserProtocol
 from paperless.parsers.tesseract import RasterisedDocumentParser
-from paperless.parsers.tesseract import post_process_text
+from paperless.parsers.utils import post_process_text
 
 if TYPE_CHECKING:
     from pathlib import Path

src/paperless_ai/indexing.py

@@ -14,7 +14,6 @@ from filelock import Timeout
 from documents.models import Document
 from documents.models import PaperlessTask
 from documents.utils import IterWrapper
-from documents.utils import QuerySetStream
 from documents.utils import identity
 from paperless.config import AIConfig
 from paperless_ai.db import db_connection_released
@@ -33,11 +32,6 @@ logger = logging.getLogger("paperless_ai.indexing")
 RAG_NUM_OUTPUT = 512
 RAG_CHUNK_OVERLAP = 200
 
-# update_llm_index(): row count per .iterator() batch when streaming
-# documents for a rebuild/update via QuerySetStream, matching
-# _DocumentViewerStream's chunk size in documents/search/_backend.py.
-_INDEX_STREAM_CHUNK_SIZE = 1000
-
 
 def queue_llm_index_update_if_needed(*, rebuild: bool, reason: str) -> bool:
     # NOTE: The check-then-enqueue sequence below is non-atomic (TOCTOU): two
@@ -391,9 +385,7 @@ def update_llm_index(
         if rebuild or not store.table_exists():
             logger.info("Rebuilding LLM index.")
             store.drop_table()
-            for document in iter_wrapper(
-                QuerySetStream(documents, chunk_size=_INDEX_STREAM_CHUNK_SIZE),
-            ):
+            for document in iter_wrapper(documents):
                 nodes = build_document_node(document, chunk_size=chunk_size)
                 _embed_nodes(nodes, embed_model)
                 store.add(nodes)
@@ -406,9 +398,7 @@ def update_llm_index(
             )
             existing = store.get_modified_times()
             changed = 0
-            for document in iter_wrapper(
-                QuerySetStream(scoped_documents, chunk_size=_INDEX_STREAM_CHUNK_SIZE),
-            ):
+            for document in iter_wrapper(scoped_documents):
                 doc_id = str(document.id)
                 if existing.get(doc_id) == document.modified.isoformat():
                     continue