diff --git a/src/paperless/parsers/remote.py b/src/paperless/parsers/remote.py index 41989d834..d32caeb88 100644 --- a/src/paperless/parsers/remote.py +++ b/src/paperless/parsers/remote.py @@ -3,7 +3,9 @@ Built-in remote-OCR document parser. Handles documents by sending them to a configured remote OCR engine (currently Azure AI Vision / Document Intelligence) and retrieving both -the extracted text and a searchable PDF with an embedded text layer. +the extracted text and a searchable PDF with an embedded text layer. For +born-digital PDFs that need no archive copy, the remote call is skipped +entirely in favor of locally-extracted text (see ``RemoteDocumentParser.parse``). When no engine is configured, ``score()`` returns ``None`` so the parser is effectively invisible to the registry — the tesseract parser handles @@ -21,6 +23,8 @@ from typing import Self from django.conf import settings +from paperless.parsers.utils import extract_pdf_text +from paperless.parsers.utils import post_process_text from paperless.version import __full_version_str__ if TYPE_CHECKING: @@ -69,8 +73,11 @@ class RemoteDocumentParser: """Parse documents via a remote OCR API (currently Azure AI Vision). This parser sends documents to a remote engine that returns both - extracted text and a searchable PDF with an embedded text layer. - It does not depend on Tesseract or ocrmypdf. + extracted text and a searchable PDF with an embedded text layer, + except when ``parse()`` is called with ``produce_archive=False`` for + a PDF, in which case the remote call is skipped and only locally + extracted text is returned (no archive). It does not depend on + Tesseract or ocrmypdf. Class attributes ---------------- @@ -159,8 +166,11 @@ class RemoteDocumentParser: Returns ------- bool - Always True — the remote engine always returns a PDF with an - embedded text layer that serves as the archive copy. + Always True — the remote engine is capable of returning a PDF + with an embedded text layer to serve as the archive copy. + Whether it actually does so for a given document depends on + ``produce_archive`` passed to :meth:`parse` (see there for when + the remote engine call, and thus archive generation, is skipped). """ return True @@ -217,6 +227,12 @@ class RemoteDocumentParser: ) -> None: """Send the document to the remote engine and store results. + When *produce_archive* is False for a PDF, the caller (via + ``documents.consumer.should_produce_archive``) has already determined + that the document is born-digital and needs no archive — skip the + remote engine entirely rather than re-OCRing it and creating a + duplicate text layer. + Parameters ---------- document_path: @@ -224,8 +240,8 @@ class RemoteDocumentParser: mime_type: Detected MIME type of the document. produce_archive: - Ignored — the remote engine always returns a searchable PDF, - which is stored as the archive copy regardless of this flag. + Whether an archive copy is wanted. For PDFs, False skips the + remote engine and uses locally-extracted text instead. """ config = RemoteEngineConfig( engine=settings.REMOTE_OCR_ENGINE, @@ -240,6 +256,16 @@ class RemoteDocumentParser: self._text = "" return + if not produce_archive and mime_type == "application/pdf": + logger.debug( + "Remote OCR: skipped — no archive requested, " + "using locally-extracted text", + ) + self._text = ( + post_process_text(extract_pdf_text(document_path, log=logger)) or "" + ) + return + if config.engine == "azureai": self._text = self._azure_ai_vision_parse(document_path, config) diff --git a/src/paperless/parsers/tesseract.py b/src/paperless/parsers/tesseract.py index 2e0d791ea..bfb22e682 100644 --- a/src/paperless/parsers/tesseract.py +++ b/src/paperless/parsers/tesseract.py @@ -3,7 +3,6 @@ from __future__ import annotations import importlib.resources import logging import os -import re import shutil import tempfile from pathlib import Path @@ -25,9 +24,9 @@ from paperless.config import OcrConfig from paperless.models import CleanChoices from paperless.models import ModeChoices from paperless.models import OutputTypeChoices -from paperless.parsers.utils import PDF_TEXT_MIN_LENGTH from paperless.parsers.utils import extract_pdf_text -from paperless.parsers.utils import is_tagged_pdf +from paperless.parsers.utils import pdf_has_digital_text +from paperless.parsers.utils import post_process_text from paperless.parsers.utils import read_file_handle_unicode_errors from paperless.version import __full_version_str__ @@ -510,8 +509,10 @@ class RasterisedDocumentParser: if mime_type == "application/pdf": text_original = self.extract_text(None, document_path) - original_has_text = is_tagged_pdf(document_path, log=self.log) or ( - text_original is not None and len(text_original) > PDF_TEXT_MIN_LENGTH + original_has_text = pdf_has_digital_text( + document_path, + text_original, + log=self.log, ) else: text_original = None @@ -656,17 +657,3 @@ class RasterisedDocumentParser: f"No text was found in {document_path}, the content will be empty.", ) self.text = "" - - -def post_process_text(text: str | None) -> str | None: - if not text: - return None - - collapsed_spaces = re.sub(r"([^\S\r\n]+)", " ", text) - no_leading_whitespace = re.sub(r"([\n\r]+)([^\S\n\r]+)", "\\1", collapsed_spaces) - no_trailing_whitespace = re.sub(r"([^\S\n\r]+)$", "", no_leading_whitespace) - - # TODO: this needs a rework - # replace \0 prevents issues with saving to postgres. - # text may contain \0 when this character is present in PDF files. - return no_trailing_whitespace.strip().replace("\0", " ") diff --git a/src/paperless/parsers/utils.py b/src/paperless/parsers/utils.py index 0257ab736..3eb3bafa5 100644 --- a/src/paperless/parsers/utils.py +++ b/src/paperless/parsers/utils.py @@ -65,6 +65,63 @@ def is_tagged_pdf( return False +def pdf_has_digital_text( + path: Path, + text: str | None, + log: logging.Logger | None = None, +) -> bool: + """Return True if a PDF already has a usable, born-digital text layer. + + Combines the tagged-PDF check with an extracted-text length check. + Shared by the tesseract and remote OCR parsers to decide whether + OCR_MODE=auto/off should skip (re-)OCRing a document. + + Parameters + ---------- + path: + Absolute path to the PDF file. + text: + Text already extracted from the PDF (e.g. via ``extract_pdf_text``), + or ``None``. + log: + Logger for warnings. Falls back to the module-level logger when omitted. + + Returns + ------- + bool + ``True`` when the document already contains a text layer. + """ + return is_tagged_pdf(path, log=log) or ( + text is not None and len(text) > PDF_TEXT_MIN_LENGTH + ) + + +def post_process_text(text: str | None) -> str | None: + """Normalise whitespace in extracted OCR/PDF text and strip NUL bytes. + + Parameters + ---------- + text: + Raw extracted text, or ``None``. + + Returns + ------- + str | None + Cleaned text, or ``None`` when *text* is falsy. + """ + if not text: + return None + + collapsed_spaces = re.sub(r"([^\S\r\n]+)", " ", text) + no_leading_whitespace = re.sub(r"([\n\r]+)([^\S\n\r]+)", "\\1", collapsed_spaces) + no_trailing_whitespace = re.sub(r"([^\S\n\r]+)$", "", no_leading_whitespace) + + # TODO: this needs a rework + # replace \0 prevents issues with saving to postgres. + # text may contain \0 when this character is present in PDF files. + return no_trailing_whitespace.strip().replace("\0", " ") + + def extract_pdf_text( path: Path, log: logging.Logger | None = None, diff --git a/src/paperless/tests/parsers/test_remote_parser.py b/src/paperless/tests/parsers/test_remote_parser.py index 99d3342be..7260ff07d 100644 --- a/src/paperless/tests/parsers/test_remote_parser.py +++ b/src/paperless/tests/parsers/test_remote_parser.py @@ -336,6 +336,117 @@ class TestRemoteParserParse: assert remote_parser.get_date() is None +# --------------------------------------------------------------------------- +# parse() — produce_archive=False skips the remote engine (PDFs only) +# --------------------------------------------------------------------------- + + +class TestRemoteParserSkipsWhenNoArchiveWanted: + """When the caller has already decided no archive is needed for a PDF + (documents.consumer.should_produce_archive), the remote engine call is + skipped entirely in favor of locally-extracted text. + """ + + def test_pdf_skips_azure_when_no_archive_requested( + self, + remote_parser: RemoteDocumentParser, + simple_digital_pdf_file: Path, + azure_client: Mock, + ) -> None: + """ + GIVEN: produce_archive=False for a PDF + WHEN: parse() is called + THEN: Azure is never invoked, no archive is produced, and text + comes from local pdftotext extraction + """ + remote_parser.parse( + simple_digital_pdf_file, + "application/pdf", + produce_archive=False, + ) + + azure_client.begin_analyze_document.assert_not_called() + assert remote_parser.get_archive_path() is None + assert remote_parser.get_text() != "" + + def test_pdf_no_archive_requested_text_matches_local_extraction( + self, + remote_parser: RemoteDocumentParser, + simple_digital_pdf_file: Path, + azure_client: Mock, + mocker: MockerFixture, + ) -> None: + """ + GIVEN: produce_archive=False for a PDF + WHEN: parse() is called + THEN: the returned text is exactly the locally-extracted text, + not anything from the (unused) Azure mock + """ + mocker.patch( + "paperless.parsers.remote.extract_pdf_text", + return_value="Local digital text.", + ) + + remote_parser.parse( + simple_digital_pdf_file, + "application/pdf", + produce_archive=False, + ) + + assert remote_parser.get_text() == "Local digital text." + + def test_pdf_no_archive_requested_closes_no_client( + self, + remote_parser: RemoteDocumentParser, + simple_digital_pdf_file: Path, + azure_client: Mock, + ) -> None: + remote_parser.parse( + simple_digital_pdf_file, + "application/pdf", + produce_archive=False, + ) + + azure_client.close.assert_not_called() + + def test_non_pdf_still_calls_azure_when_no_archive_requested( + self, + remote_parser: RemoteDocumentParser, + simple_digital_pdf_file: Path, + azure_client: Mock, + ) -> None: + """ + Images have no local-text fallback, so produce_archive=False does + not skip the remote engine for non-PDF MIME types. + """ + remote_parser.parse( + simple_digital_pdf_file, + "image/png", + produce_archive=False, + ) + + azure_client.begin_analyze_document.assert_called_once() + assert remote_parser.get_text() == _DEFAULT_TEXT + + @pytest.mark.usefixtures("no_engine_settings") + def test_unconfigured_engine_takes_precedence_over_skip( + self, + remote_parser: RemoteDocumentParser, + simple_digital_pdf_file: Path, + ) -> None: + """An unconfigured engine still short-circuits before the + produce_archive check, returning empty text as before. + """ + remote_parser.parse( + simple_digital_pdf_file, + "application/pdf", + produce_archive=False, + ) + + assert remote_parser.get_text() == "" + assert remote_parser.get_archive_path() is None + + # --------------------------------------------------------------------------- # parse() — Azure failure path # --------------------------------------------------------------------------- diff --git a/src/paperless/tests/parsers/test_tesseract_parser.py b/src/paperless/tests/parsers/test_tesseract_parser.py index 0c086745c..f236d5caf 100644 --- a/src/paperless/tests/parsers/test_tesseract_parser.py +++ b/src/paperless/tests/parsers/test_tesseract_parser.py @@ -21,7 +21,7 @@ from documents.parsers import run_convert from paperless.models import ModeChoices from paperless.parsers import ParserProtocol from paperless.parsers.tesseract import RasterisedDocumentParser -from paperless.parsers.tesseract import post_process_text +from paperless.parsers.utils import post_process_text if TYPE_CHECKING: from pathlib import Path