mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-07-30 07:44:54 +00:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4aa432148c | ||
|
|
ceebdf1f6a |
@@ -948,10 +948,11 @@ for display in the web interface.
|
|||||||
|
|
||||||
!!! note
|
!!! note
|
||||||
|
|
||||||
The **remote OCR parser** (Azure AI) always produces a searchable
|
The **remote OCR parser** (Azure AI) also honors this setting: when
|
||||||
PDF and stores it as the archive copy, regardless of this setting.
|
no archive is requested (`never`, or `auto` with a born-digital PDF),
|
||||||
`ARCHIVE_FILE_GENERATION=never` has no effect when the remote
|
the remote engine is skipped entirely and locally-extracted text is
|
||||||
parser handles a document.
|
used instead, avoiding an unnecessary API call and a duplicate text
|
||||||
|
layer.
|
||||||
|
|
||||||
#### [`PAPERLESS_OCR_CLEAN=<mode>`](#PAPERLESS_OCR_CLEAN) {#PAPERLESS_OCR_CLEAN}
|
#### [`PAPERLESS_OCR_CLEAN=<mode>`](#PAPERLESS_OCR_CLEAN) {#PAPERLESS_OCR_CLEAN}
|
||||||
|
|
||||||
|
|||||||
@@ -187,10 +187,11 @@ PAPERLESS_ARCHIVE_FILE_GENERATION=auto
|
|||||||
|
|
||||||
### Remote OCR parser
|
### Remote OCR parser
|
||||||
|
|
||||||
If you use the **remote OCR parser** (Azure AI), note that it always produces a
|
If you use the **remote OCR parser** (Azure AI), `ARCHIVE_FILE_GENERATION` is
|
||||||
searchable PDF and stores it as the archive copy. `ARCHIVE_FILE_GENERATION=never`
|
honored the same way as for the local engine: when no archive is requested
|
||||||
has no effect for documents handled by the remote parser - the archive is produced
|
(`never`, or `auto` with a born-digital PDF), the remote engine is skipped
|
||||||
unconditionally by the remote engine.
|
entirely and locally-extracted text is used instead, avoiding an unnecessary
|
||||||
|
API call and a duplicate text layer.
|
||||||
|
|
||||||
## Search Index (Whoosh -> Tantivy)
|
## Search Index (Whoosh -> Tantivy)
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,9 @@ Built-in remote-OCR document parser.
|
|||||||
|
|
||||||
Handles documents by sending them to a configured remote OCR engine
|
Handles documents by sending them to a configured remote OCR engine
|
||||||
(currently Azure AI Vision / Document Intelligence) and retrieving both
|
(currently Azure AI Vision / Document Intelligence) and retrieving both
|
||||||
the extracted text and a searchable PDF with an embedded text layer.
|
the extracted text and a searchable PDF with an embedded text layer. For
|
||||||
|
born-digital PDFs that need no archive copy, the remote call is skipped
|
||||||
|
entirely in favor of locally-extracted text (see ``RemoteDocumentParser.parse``).
|
||||||
|
|
||||||
When no engine is configured, ``score()`` returns ``None`` so the parser
|
When no engine is configured, ``score()`` returns ``None`` so the parser
|
||||||
is effectively invisible to the registry — the tesseract parser handles
|
is effectively invisible to the registry — the tesseract parser handles
|
||||||
@@ -21,6 +23,8 @@ from typing import Self
|
|||||||
|
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
|
|
||||||
|
from paperless.parsers.utils import extract_pdf_text
|
||||||
|
from paperless.parsers.utils import post_process_text
|
||||||
from paperless.version import __full_version_str__
|
from paperless.version import __full_version_str__
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
@@ -69,8 +73,11 @@ class RemoteDocumentParser:
|
|||||||
"""Parse documents via a remote OCR API (currently Azure AI Vision).
|
"""Parse documents via a remote OCR API (currently Azure AI Vision).
|
||||||
|
|
||||||
This parser sends documents to a remote engine that returns both
|
This parser sends documents to a remote engine that returns both
|
||||||
extracted text and a searchable PDF with an embedded text layer.
|
extracted text and a searchable PDF with an embedded text layer,
|
||||||
It does not depend on Tesseract or ocrmypdf.
|
except when ``parse()`` is called with ``produce_archive=False`` for
|
||||||
|
a PDF, in which case the remote call is skipped and only locally
|
||||||
|
extracted text is returned (no archive). It does not depend on
|
||||||
|
Tesseract or ocrmypdf.
|
||||||
|
|
||||||
Class attributes
|
Class attributes
|
||||||
----------------
|
----------------
|
||||||
@@ -159,8 +166,11 @@ class RemoteDocumentParser:
|
|||||||
Returns
|
Returns
|
||||||
-------
|
-------
|
||||||
bool
|
bool
|
||||||
Always True — the remote engine always returns a PDF with an
|
Always True — the remote engine is capable of returning a PDF
|
||||||
embedded text layer that serves as the archive copy.
|
with an embedded text layer to serve as the archive copy.
|
||||||
|
Whether it actually does so for a given document depends on
|
||||||
|
``produce_archive`` passed to :meth:`parse` (see there for when
|
||||||
|
the remote engine call, and thus archive generation, is skipped).
|
||||||
"""
|
"""
|
||||||
return True
|
return True
|
||||||
|
|
||||||
@@ -217,6 +227,12 @@ class RemoteDocumentParser:
|
|||||||
) -> None:
|
) -> None:
|
||||||
"""Send the document to the remote engine and store results.
|
"""Send the document to the remote engine and store results.
|
||||||
|
|
||||||
|
When *produce_archive* is False for a PDF, the caller (via
|
||||||
|
``documents.consumer.should_produce_archive``) has already determined
|
||||||
|
that the document is born-digital and needs no archive — skip the
|
||||||
|
remote engine entirely rather than re-OCRing it and creating a
|
||||||
|
duplicate text layer.
|
||||||
|
|
||||||
Parameters
|
Parameters
|
||||||
----------
|
----------
|
||||||
document_path:
|
document_path:
|
||||||
@@ -224,8 +240,8 @@ class RemoteDocumentParser:
|
|||||||
mime_type:
|
mime_type:
|
||||||
Detected MIME type of the document.
|
Detected MIME type of the document.
|
||||||
produce_archive:
|
produce_archive:
|
||||||
Ignored — the remote engine always returns a searchable PDF,
|
Whether an archive copy is wanted. For PDFs, False skips the
|
||||||
which is stored as the archive copy regardless of this flag.
|
remote engine and uses locally-extracted text instead.
|
||||||
"""
|
"""
|
||||||
config = RemoteEngineConfig(
|
config = RemoteEngineConfig(
|
||||||
engine=settings.REMOTE_OCR_ENGINE,
|
engine=settings.REMOTE_OCR_ENGINE,
|
||||||
@@ -240,6 +256,16 @@ class RemoteDocumentParser:
|
|||||||
self._text = ""
|
self._text = ""
|
||||||
return
|
return
|
||||||
|
|
||||||
|
if not produce_archive and mime_type == "application/pdf":
|
||||||
|
logger.debug(
|
||||||
|
"Remote OCR: skipped — no archive requested, "
|
||||||
|
"using locally-extracted text",
|
||||||
|
)
|
||||||
|
self._text = (
|
||||||
|
post_process_text(extract_pdf_text(document_path, log=logger)) or ""
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
if config.engine == "azureai":
|
if config.engine == "azureai":
|
||||||
self._text = self._azure_ai_vision_parse(document_path, config)
|
self._text = self._azure_ai_vision_parse(document_path, config)
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ from __future__ import annotations
|
|||||||
import importlib.resources
|
import importlib.resources
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import re
|
|
||||||
import shutil
|
import shutil
|
||||||
import tempfile
|
import tempfile
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -25,9 +24,9 @@ from paperless.config import OcrConfig
|
|||||||
from paperless.models import CleanChoices
|
from paperless.models import CleanChoices
|
||||||
from paperless.models import ModeChoices
|
from paperless.models import ModeChoices
|
||||||
from paperless.models import OutputTypeChoices
|
from paperless.models import OutputTypeChoices
|
||||||
from paperless.parsers.utils import PDF_TEXT_MIN_LENGTH
|
|
||||||
from paperless.parsers.utils import extract_pdf_text
|
from paperless.parsers.utils import extract_pdf_text
|
||||||
from paperless.parsers.utils import is_tagged_pdf
|
from paperless.parsers.utils import pdf_has_digital_text
|
||||||
|
from paperless.parsers.utils import post_process_text
|
||||||
from paperless.parsers.utils import read_file_handle_unicode_errors
|
from paperless.parsers.utils import read_file_handle_unicode_errors
|
||||||
from paperless.version import __full_version_str__
|
from paperless.version import __full_version_str__
|
||||||
|
|
||||||
@@ -510,10 +509,10 @@ class RasterisedDocumentParser:
|
|||||||
|
|
||||||
if mime_type == "application/pdf":
|
if mime_type == "application/pdf":
|
||||||
text_original = self.extract_text(None, document_path)
|
text_original = self.extract_text(None, document_path)
|
||||||
has_text = text_original is not None and len(text_original) > 0
|
original_has_text = pdf_has_digital_text(
|
||||||
original_has_text = has_text and (
|
document_path,
|
||||||
is_tagged_pdf(document_path, log=self.log)
|
text_original,
|
||||||
or len(text_original) > PDF_TEXT_MIN_LENGTH
|
log=self.log,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
text_original = None
|
text_original = None
|
||||||
@@ -658,17 +657,3 @@ class RasterisedDocumentParser:
|
|||||||
f"No text was found in {document_path}, the content will be empty.",
|
f"No text was found in {document_path}, the content will be empty.",
|
||||||
)
|
)
|
||||||
self.text = ""
|
self.text = ""
|
||||||
|
|
||||||
|
|
||||||
def post_process_text(text: str | None) -> str | None:
|
|
||||||
if not text:
|
|
||||||
return None
|
|
||||||
|
|
||||||
collapsed_spaces = re.sub(r"([^\S\r\n]+)", " ", text)
|
|
||||||
no_leading_whitespace = re.sub(r"([\n\r]+)([^\S\n\r]+)", "\\1", collapsed_spaces)
|
|
||||||
no_trailing_whitespace = re.sub(r"([^\S\n\r]+)$", "", no_leading_whitespace)
|
|
||||||
|
|
||||||
# TODO: this needs a rework
|
|
||||||
# replace \0 prevents issues with saving to postgres.
|
|
||||||
# text may contain \0 when this character is present in PDF files.
|
|
||||||
return no_trailing_whitespace.strip().replace("\0", " ")
|
|
||||||
|
|||||||
@@ -65,6 +65,63 @@ def is_tagged_pdf(
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def pdf_has_digital_text(
|
||||||
|
path: Path,
|
||||||
|
text: str | None,
|
||||||
|
log: logging.Logger | None = None,
|
||||||
|
) -> bool:
|
||||||
|
"""Return True if a PDF already has a usable, born-digital text layer.
|
||||||
|
|
||||||
|
Combines the tagged-PDF check with an extracted-text length check.
|
||||||
|
Shared by the tesseract and remote OCR parsers to decide whether
|
||||||
|
OCR_MODE=auto/off should skip (re-)OCRing a document.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
path:
|
||||||
|
Absolute path to the PDF file.
|
||||||
|
text:
|
||||||
|
Text already extracted from the PDF (e.g. via ``extract_pdf_text``),
|
||||||
|
or ``None``.
|
||||||
|
log:
|
||||||
|
Logger for warnings. Falls back to the module-level logger when omitted.
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
bool
|
||||||
|
``True`` when the document already contains a text layer.
|
||||||
|
"""
|
||||||
|
return is_tagged_pdf(path, log=log) or (
|
||||||
|
text is not None and len(text) > PDF_TEXT_MIN_LENGTH
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def post_process_text(text: str | None) -> str | None:
|
||||||
|
"""Normalise whitespace in extracted OCR/PDF text and strip NUL bytes.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
text:
|
||||||
|
Raw extracted text, or ``None``.
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
str | None
|
||||||
|
Cleaned text, or ``None`` when *text* is falsy.
|
||||||
|
"""
|
||||||
|
if not text:
|
||||||
|
return None
|
||||||
|
|
||||||
|
collapsed_spaces = re.sub(r"([^\S\r\n]+)", " ", text)
|
||||||
|
no_leading_whitespace = re.sub(r"([\n\r]+)([^\S\n\r]+)", "\\1", collapsed_spaces)
|
||||||
|
no_trailing_whitespace = re.sub(r"([^\S\n\r]+)$", "", no_leading_whitespace)
|
||||||
|
|
||||||
|
# TODO: this needs a rework
|
||||||
|
# replace \0 prevents issues with saving to postgres.
|
||||||
|
# text may contain \0 when this character is present in PDF files.
|
||||||
|
return no_trailing_whitespace.strip().replace("\0", " ")
|
||||||
|
|
||||||
|
|
||||||
def extract_pdf_text(
|
def extract_pdf_text(
|
||||||
path: Path,
|
path: Path,
|
||||||
log: logging.Logger | None = None,
|
log: logging.Logger | None = None,
|
||||||
|
|||||||
@@ -336,6 +336,117 @@ class TestRemoteParserParse:
|
|||||||
assert remote_parser.get_date() is None
|
assert remote_parser.get_date() is None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# parse() — produce_archive=False skips the remote engine (PDFs only)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestRemoteParserSkipsWhenNoArchiveWanted:
|
||||||
|
"""When the caller has already decided no archive is needed for a PDF
|
||||||
|
(documents.consumer.should_produce_archive), the remote engine call is
|
||||||
|
skipped entirely in favor of locally-extracted text.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def test_pdf_skips_azure_when_no_archive_requested(
|
||||||
|
self,
|
||||||
|
remote_parser: RemoteDocumentParser,
|
||||||
|
simple_digital_pdf_file: Path,
|
||||||
|
azure_client: Mock,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN: produce_archive=False for a PDF
|
||||||
|
WHEN: parse() is called
|
||||||
|
THEN: Azure is never invoked, no archive is produced, and text
|
||||||
|
comes from local pdftotext extraction
|
||||||
|
"""
|
||||||
|
remote_parser.parse(
|
||||||
|
simple_digital_pdf_file,
|
||||||
|
"application/pdf",
|
||||||
|
produce_archive=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
azure_client.begin_analyze_document.assert_not_called()
|
||||||
|
assert remote_parser.get_archive_path() is None
|
||||||
|
assert remote_parser.get_text() != ""
|
||||||
|
|
||||||
|
def test_pdf_no_archive_requested_text_matches_local_extraction(
|
||||||
|
self,
|
||||||
|
remote_parser: RemoteDocumentParser,
|
||||||
|
simple_digital_pdf_file: Path,
|
||||||
|
azure_client: Mock,
|
||||||
|
mocker: MockerFixture,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN: produce_archive=False for a PDF
|
||||||
|
WHEN: parse() is called
|
||||||
|
THEN: the returned text is exactly the locally-extracted text,
|
||||||
|
not anything from the (unused) Azure mock
|
||||||
|
"""
|
||||||
|
mocker.patch(
|
||||||
|
"paperless.parsers.remote.extract_pdf_text",
|
||||||
|
return_value="Local digital text.",
|
||||||
|
)
|
||||||
|
|
||||||
|
remote_parser.parse(
|
||||||
|
simple_digital_pdf_file,
|
||||||
|
"application/pdf",
|
||||||
|
produce_archive=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert remote_parser.get_text() == "Local digital text."
|
||||||
|
|
||||||
|
def test_pdf_no_archive_requested_closes_no_client(
|
||||||
|
self,
|
||||||
|
remote_parser: RemoteDocumentParser,
|
||||||
|
simple_digital_pdf_file: Path,
|
||||||
|
azure_client: Mock,
|
||||||
|
) -> None:
|
||||||
|
remote_parser.parse(
|
||||||
|
simple_digital_pdf_file,
|
||||||
|
"application/pdf",
|
||||||
|
produce_archive=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
azure_client.close.assert_not_called()
|
||||||
|
|
||||||
|
def test_non_pdf_still_calls_azure_when_no_archive_requested(
|
||||||
|
self,
|
||||||
|
remote_parser: RemoteDocumentParser,
|
||||||
|
simple_digital_pdf_file: Path,
|
||||||
|
azure_client: Mock,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Images have no local-text fallback, so produce_archive=False does
|
||||||
|
not skip the remote engine for non-PDF MIME types.
|
||||||
|
"""
|
||||||
|
remote_parser.parse(
|
||||||
|
simple_digital_pdf_file,
|
||||||
|
"image/png",
|
||||||
|
produce_archive=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
azure_client.begin_analyze_document.assert_called_once()
|
||||||
|
assert remote_parser.get_text() == _DEFAULT_TEXT
|
||||||
|
|
||||||
|
@pytest.mark.usefixtures("no_engine_settings")
|
||||||
|
def test_unconfigured_engine_takes_precedence_over_skip(
|
||||||
|
self,
|
||||||
|
remote_parser: RemoteDocumentParser,
|
||||||
|
simple_digital_pdf_file: Path,
|
||||||
|
) -> None:
|
||||||
|
"""An unconfigured engine still short-circuits before the
|
||||||
|
produce_archive check, returning empty text as before.
|
||||||
|
"""
|
||||||
|
remote_parser.parse(
|
||||||
|
simple_digital_pdf_file,
|
||||||
|
"application/pdf",
|
||||||
|
produce_archive=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert remote_parser.get_text() == ""
|
||||||
|
assert remote_parser.get_archive_path() is None
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# parse() — Azure failure path
|
# parse() — Azure failure path
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ from documents.parsers import run_convert
|
|||||||
from paperless.models import ModeChoices
|
from paperless.models import ModeChoices
|
||||||
from paperless.parsers import ParserProtocol
|
from paperless.parsers import ParserProtocol
|
||||||
from paperless.parsers.tesseract import RasterisedDocumentParser
|
from paperless.parsers.tesseract import RasterisedDocumentParser
|
||||||
from paperless.parsers.tesseract import post_process_text
|
from paperless.parsers.utils import post_process_text
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|||||||
Reference in New Issue
Block a user