Compare commits

...
7 changed files with 218 additions and 37 deletions
+5 -4
View File
@@ -948,10 +948,11 @@ for display in the web interface.
!!! note !!! note
The **remote OCR parser** (Azure AI) always produces a searchable The **remote OCR parser** (Azure AI) also honors this setting: when
PDF and stores it as the archive copy, regardless of this setting. no archive is requested (`never`, or `auto` with a born-digital PDF),
`ARCHIVE_FILE_GENERATION=never` has no effect when the remote the remote engine is skipped entirely and locally-extracted text is
parser handles a document. used instead, avoiding an unnecessary API call and a duplicate text
layer.
#### [`PAPERLESS_OCR_CLEAN=<mode>`](#PAPERLESS_OCR_CLEAN) {#PAPERLESS_OCR_CLEAN} #### [`PAPERLESS_OCR_CLEAN=<mode>`](#PAPERLESS_OCR_CLEAN) {#PAPERLESS_OCR_CLEAN}
+5 -4
View File
@@ -187,10 +187,11 @@ PAPERLESS_ARCHIVE_FILE_GENERATION=auto
### Remote OCR parser ### Remote OCR parser
If you use the **remote OCR parser** (Azure AI), note that it always produces a If you use the **remote OCR parser** (Azure AI), `ARCHIVE_FILE_GENERATION` is
searchable PDF and stores it as the archive copy. `ARCHIVE_FILE_GENERATION=never` honored the same way as for the local engine: when no archive is requested
has no effect for documents handled by the remote parser - the archive is produced (`never`, or `auto` with a born-digital PDF), the remote engine is skipped
unconditionally by the remote engine. entirely and locally-extracted text is used instead, avoiding an unnecessary
API call and a duplicate text layer.
## Search Index (Whoosh -> Tantivy) ## Search Index (Whoosh -> Tantivy)
+33 -7
View File
@@ -3,7 +3,9 @@ Built-in remote-OCR document parser.
Handles documents by sending them to a configured remote OCR engine Handles documents by sending them to a configured remote OCR engine
(currently Azure AI Vision / Document Intelligence) and retrieving both (currently Azure AI Vision / Document Intelligence) and retrieving both
the extracted text and a searchable PDF with an embedded text layer. the extracted text and a searchable PDF with an embedded text layer. For
born-digital PDFs that need no archive copy, the remote call is skipped
entirely in favor of locally-extracted text (see ``RemoteDocumentParser.parse``).
When no engine is configured, ``score()`` returns ``None`` so the parser When no engine is configured, ``score()`` returns ``None`` so the parser
is effectively invisible to the registry — the tesseract parser handles is effectively invisible to the registry — the tesseract parser handles
@@ -21,6 +23,8 @@ from typing import Self
from django.conf import settings from django.conf import settings
from paperless.parsers.utils import extract_pdf_text
from paperless.parsers.utils import post_process_text
from paperless.version import __full_version_str__ from paperless.version import __full_version_str__
if TYPE_CHECKING: if TYPE_CHECKING:
@@ -69,8 +73,11 @@ class RemoteDocumentParser:
"""Parse documents via a remote OCR API (currently Azure AI Vision). """Parse documents via a remote OCR API (currently Azure AI Vision).
This parser sends documents to a remote engine that returns both This parser sends documents to a remote engine that returns both
extracted text and a searchable PDF with an embedded text layer. extracted text and a searchable PDF with an embedded text layer,
It does not depend on Tesseract or ocrmypdf. except when ``parse()`` is called with ``produce_archive=False`` for
a PDF, in which case the remote call is skipped and only locally
extracted text is returned (no archive). It does not depend on
Tesseract or ocrmypdf.
Class attributes Class attributes
---------------- ----------------
@@ -159,8 +166,11 @@ class RemoteDocumentParser:
Returns Returns
------- -------
bool bool
Always True — the remote engine always returns a PDF with an Always True — the remote engine is capable of returning a PDF
embedded text layer that serves as the archive copy. with an embedded text layer to serve as the archive copy.
Whether it actually does so for a given document depends on
``produce_archive`` passed to :meth:`parse` (see there for when
the remote engine call, and thus archive generation, is skipped).
""" """
return True return True
@@ -217,6 +227,12 @@ class RemoteDocumentParser:
) -> None: ) -> None:
"""Send the document to the remote engine and store results. """Send the document to the remote engine and store results.
When *produce_archive* is False for a PDF, the caller (via
``documents.consumer.should_produce_archive``) has already determined
that the document is born-digital and needs no archive — skip the
remote engine entirely rather than re-OCRing it and creating a
duplicate text layer.
Parameters Parameters
---------- ----------
document_path: document_path:
@@ -224,8 +240,8 @@ class RemoteDocumentParser:
mime_type: mime_type:
Detected MIME type of the document. Detected MIME type of the document.
produce_archive: produce_archive:
Ignored — the remote engine always returns a searchable PDF, Whether an archive copy is wanted. For PDFs, False skips the
which is stored as the archive copy regardless of this flag. remote engine and uses locally-extracted text instead.
""" """
config = RemoteEngineConfig( config = RemoteEngineConfig(
engine=settings.REMOTE_OCR_ENGINE, engine=settings.REMOTE_OCR_ENGINE,
@@ -240,6 +256,16 @@ class RemoteDocumentParser:
self._text = "" self._text = ""
return return
if not produce_archive and mime_type == "application/pdf":
logger.debug(
"Remote OCR: skipped — no archive requested, "
"using locally-extracted text",
)
self._text = (
post_process_text(extract_pdf_text(document_path, log=logger)) or ""
)
return
if config.engine == "azureai": if config.engine == "azureai":
self._text = self._azure_ai_vision_parse(document_path, config) self._text = self._azure_ai_vision_parse(document_path, config)
+6 -21
View File
@@ -3,7 +3,6 @@ from __future__ import annotations
import importlib.resources import importlib.resources
import logging import logging
import os import os
import re
import shutil import shutil
import tempfile import tempfile
from pathlib import Path from pathlib import Path
@@ -25,9 +24,9 @@ from paperless.config import OcrConfig
from paperless.models import CleanChoices from paperless.models import CleanChoices
from paperless.models import ModeChoices from paperless.models import ModeChoices
from paperless.models import OutputTypeChoices from paperless.models import OutputTypeChoices
from paperless.parsers.utils import PDF_TEXT_MIN_LENGTH
from paperless.parsers.utils import extract_pdf_text from paperless.parsers.utils import extract_pdf_text
from paperless.parsers.utils import is_tagged_pdf from paperless.parsers.utils import pdf_has_digital_text
from paperless.parsers.utils import post_process_text
from paperless.parsers.utils import read_file_handle_unicode_errors from paperless.parsers.utils import read_file_handle_unicode_errors
from paperless.version import __full_version_str__ from paperless.version import __full_version_str__
@@ -510,10 +509,10 @@ class RasterisedDocumentParser:
if mime_type == "application/pdf": if mime_type == "application/pdf":
text_original = self.extract_text(None, document_path) text_original = self.extract_text(None, document_path)
has_text = text_original is not None and len(text_original) > 0 original_has_text = pdf_has_digital_text(
original_has_text = has_text and ( document_path,
is_tagged_pdf(document_path, log=self.log) text_original,
or len(text_original) > PDF_TEXT_MIN_LENGTH log=self.log,
) )
else: else:
text_original = None text_original = None
@@ -658,17 +657,3 @@ class RasterisedDocumentParser:
f"No text was found in {document_path}, the content will be empty.", f"No text was found in {document_path}, the content will be empty.",
) )
self.text = "" self.text = ""
def post_process_text(text: str | None) -> str | None:
if not text:
return None
collapsed_spaces = re.sub(r"([^\S\r\n]+)", " ", text)
no_leading_whitespace = re.sub(r"([\n\r]+)([^\S\n\r]+)", "\\1", collapsed_spaces)
no_trailing_whitespace = re.sub(r"([^\S\n\r]+)$", "", no_leading_whitespace)
# TODO: this needs a rework
# replace \0 prevents issues with saving to postgres.
# text may contain \0 when this character is present in PDF files.
return no_trailing_whitespace.strip().replace("\0", " ")
+57
View File
@@ -65,6 +65,63 @@ def is_tagged_pdf(
return False return False
def pdf_has_digital_text(
path: Path,
text: str | None,
log: logging.Logger | None = None,
) -> bool:
"""Return True if a PDF already has a usable, born-digital text layer.
Combines the tagged-PDF check with an extracted-text length check.
Shared by the tesseract and remote OCR parsers to decide whether
OCR_MODE=auto/off should skip (re-)OCRing a document.
Parameters
----------
path:
Absolute path to the PDF file.
text:
Text already extracted from the PDF (e.g. via ``extract_pdf_text``),
or ``None``.
log:
Logger for warnings. Falls back to the module-level logger when omitted.
Returns
-------
bool
``True`` when the document already contains a text layer.
"""
return is_tagged_pdf(path, log=log) or (
text is not None and len(text) > PDF_TEXT_MIN_LENGTH
)
def post_process_text(text: str | None) -> str | None:
"""Normalise whitespace in extracted OCR/PDF text and strip NUL bytes.
Parameters
----------
text:
Raw extracted text, or ``None``.
Returns
-------
str | None
Cleaned text, or ``None`` when *text* is falsy.
"""
if not text:
return None
collapsed_spaces = re.sub(r"([^\S\r\n]+)", " ", text)
no_leading_whitespace = re.sub(r"([\n\r]+)([^\S\n\r]+)", "\\1", collapsed_spaces)
no_trailing_whitespace = re.sub(r"([^\S\n\r]+)$", "", no_leading_whitespace)
# TODO: this needs a rework
# replace \0 prevents issues with saving to postgres.
# text may contain \0 when this character is present in PDF files.
return no_trailing_whitespace.strip().replace("\0", " ")
def extract_pdf_text( def extract_pdf_text(
path: Path, path: Path,
log: logging.Logger | None = None, log: logging.Logger | None = None,
@@ -336,6 +336,117 @@ class TestRemoteParserParse:
assert remote_parser.get_date() is None assert remote_parser.get_date() is None
# ---------------------------------------------------------------------------
# parse() — produce_archive=False skips the remote engine (PDFs only)
# ---------------------------------------------------------------------------
class TestRemoteParserSkipsWhenNoArchiveWanted:
"""When the caller has already decided no archive is needed for a PDF
(documents.consumer.should_produce_archive), the remote engine call is
skipped entirely in favor of locally-extracted text.
"""
def test_pdf_skips_azure_when_no_archive_requested(
self,
remote_parser: RemoteDocumentParser,
simple_digital_pdf_file: Path,
azure_client: Mock,
) -> None:
"""
GIVEN: produce_archive=False for a PDF
WHEN: parse() is called
THEN: Azure is never invoked, no archive is produced, and text
comes from local pdftotext extraction
"""
remote_parser.parse(
simple_digital_pdf_file,
"application/pdf",
produce_archive=False,
)
azure_client.begin_analyze_document.assert_not_called()
assert remote_parser.get_archive_path() is None
assert remote_parser.get_text() != ""
def test_pdf_no_archive_requested_text_matches_local_extraction(
self,
remote_parser: RemoteDocumentParser,
simple_digital_pdf_file: Path,
azure_client: Mock,
mocker: MockerFixture,
) -> None:
"""
GIVEN: produce_archive=False for a PDF
WHEN: parse() is called
THEN: the returned text is exactly the locally-extracted text,
not anything from the (unused) Azure mock
"""
mocker.patch(
"paperless.parsers.remote.extract_pdf_text",
return_value="Local digital text.",
)
remote_parser.parse(
simple_digital_pdf_file,
"application/pdf",
produce_archive=False,
)
assert remote_parser.get_text() == "Local digital text."
def test_pdf_no_archive_requested_closes_no_client(
self,
remote_parser: RemoteDocumentParser,
simple_digital_pdf_file: Path,
azure_client: Mock,
) -> None:
remote_parser.parse(
simple_digital_pdf_file,
"application/pdf",
produce_archive=False,
)
azure_client.close.assert_not_called()
def test_non_pdf_still_calls_azure_when_no_archive_requested(
self,
remote_parser: RemoteDocumentParser,
simple_digital_pdf_file: Path,
azure_client: Mock,
) -> None:
"""
Images have no local-text fallback, so produce_archive=False does
not skip the remote engine for non-PDF MIME types.
"""
remote_parser.parse(
simple_digital_pdf_file,
"image/png",
produce_archive=False,
)
azure_client.begin_analyze_document.assert_called_once()
assert remote_parser.get_text() == _DEFAULT_TEXT
@pytest.mark.usefixtures("no_engine_settings")
def test_unconfigured_engine_takes_precedence_over_skip(
self,
remote_parser: RemoteDocumentParser,
simple_digital_pdf_file: Path,
) -> None:
"""An unconfigured engine still short-circuits before the
produce_archive check, returning empty text as before.
"""
remote_parser.parse(
simple_digital_pdf_file,
"application/pdf",
produce_archive=False,
)
assert remote_parser.get_text() == ""
assert remote_parser.get_archive_path() is None
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# parse() — Azure failure path # parse() — Azure failure path
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -21,7 +21,7 @@ from documents.parsers import run_convert
from paperless.models import ModeChoices from paperless.models import ModeChoices
from paperless.parsers import ParserProtocol from paperless.parsers import ParserProtocol
from paperless.parsers.tesseract import RasterisedDocumentParser from paperless.parsers.tesseract import RasterisedDocumentParser
from paperless.parsers.tesseract import post_process_text from paperless.parsers.utils import post_process_text
if TYPE_CHECKING: if TYPE_CHECKING:
from pathlib import Path from pathlib import Path