mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-08-10 04:43:19 +00:00
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
50914f784a | ||
|
|
f4b88cae63 | ||
|
|
2516944c18 |
@@ -948,10 +948,11 @@ for display in the web interface.
|
||||
|
||||
!!! note
|
||||
|
||||
The **remote OCR parser** (Azure AI) always produces a searchable
|
||||
PDF and stores it as the archive copy, regardless of this setting.
|
||||
`ARCHIVE_FILE_GENERATION=never` has no effect when the remote
|
||||
parser handles a document.
|
||||
The **remote OCR parser** (Azure AI) also honors this setting: when
|
||||
no archive is requested (`never`, or `auto` with a born-digital PDF),
|
||||
the remote engine is skipped entirely and locally-extracted text is
|
||||
used instead, avoiding an unnecessary API call and a duplicate text
|
||||
layer.
|
||||
|
||||
#### [`PAPERLESS_OCR_CLEAN=<mode>`](#PAPERLESS_OCR_CLEAN) {#PAPERLESS_OCR_CLEAN}
|
||||
|
||||
|
||||
@@ -187,10 +187,11 @@ PAPERLESS_ARCHIVE_FILE_GENERATION=auto
|
||||
|
||||
### Remote OCR parser
|
||||
|
||||
If you use the **remote OCR parser** (Azure AI), note that it always produces a
|
||||
searchable PDF and stores it as the archive copy. `ARCHIVE_FILE_GENERATION=never`
|
||||
has no effect for documents handled by the remote parser - the archive is produced
|
||||
unconditionally by the remote engine.
|
||||
If you use the **remote OCR parser** (Azure AI), `ARCHIVE_FILE_GENERATION` is
|
||||
honored the same way as for the local engine: when no archive is requested
|
||||
(`never`, or `auto` with a born-digital PDF), the remote engine is skipped
|
||||
entirely and locally-extracted text is used instead, avoiding an unnecessary
|
||||
API call and a duplicate text layer.
|
||||
|
||||
## Search Index (Whoosh -> Tantivy)
|
||||
|
||||
|
||||
@@ -3,7 +3,9 @@ Built-in remote-OCR document parser.
|
||||
|
||||
Handles documents by sending them to a configured remote OCR engine
|
||||
(currently Azure AI Vision / Document Intelligence) and retrieving both
|
||||
the extracted text and a searchable PDF with an embedded text layer.
|
||||
the extracted text and a searchable PDF with an embedded text layer. For
|
||||
born-digital PDFs that need no archive copy, the remote call is skipped
|
||||
entirely in favor of locally-extracted text (see ``RemoteDocumentParser.parse``).
|
||||
|
||||
When no engine is configured, ``score()`` returns ``None`` so the parser
|
||||
is effectively invisible to the registry — the tesseract parser handles
|
||||
@@ -22,6 +24,8 @@ from typing import Self
|
||||
from django.conf import settings
|
||||
|
||||
from documents.parsers import ParseError
|
||||
from paperless.parsers.utils import extract_pdf_text
|
||||
from paperless.parsers.utils import post_process_text
|
||||
from paperless.version import __full_version_str__
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -70,8 +74,11 @@ class RemoteDocumentParser:
|
||||
"""Parse documents via a remote OCR API (currently Azure AI Vision).
|
||||
|
||||
This parser sends documents to a remote engine that returns both
|
||||
extracted text and a searchable PDF with an embedded text layer.
|
||||
It does not depend on Tesseract or ocrmypdf.
|
||||
extracted text and a searchable PDF with an embedded text layer,
|
||||
except when ``parse()`` is called with ``produce_archive=False`` for
|
||||
a PDF, in which case the remote call is skipped and only locally
|
||||
extracted text is returned (no archive). It does not depend on
|
||||
Tesseract or ocrmypdf.
|
||||
|
||||
Class attributes
|
||||
----------------
|
||||
@@ -160,8 +167,11 @@ class RemoteDocumentParser:
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
Always True — the remote engine always returns a PDF with an
|
||||
embedded text layer that serves as the archive copy.
|
||||
Always True — the remote engine is capable of returning a PDF
|
||||
with an embedded text layer to serve as the archive copy.
|
||||
Whether it actually does so for a given document depends on
|
||||
``produce_archive`` passed to :meth:`parse` (see there for when
|
||||
the remote engine call, and thus archive generation, is skipped).
|
||||
"""
|
||||
return True
|
||||
|
||||
@@ -218,6 +228,12 @@ class RemoteDocumentParser:
|
||||
) -> None:
|
||||
"""Send the document to the remote engine and store results.
|
||||
|
||||
When *produce_archive* is False for a PDF, the caller (via
|
||||
``documents.consumer.should_produce_archive``) has already determined
|
||||
that the document is born-digital and needs no archive — skip the
|
||||
remote engine entirely rather than re-OCRing it and creating a
|
||||
duplicate text layer.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
document_path:
|
||||
@@ -225,8 +241,8 @@ class RemoteDocumentParser:
|
||||
mime_type:
|
||||
Detected MIME type of the document.
|
||||
produce_archive:
|
||||
Ignored — the remote engine always returns a searchable PDF,
|
||||
which is stored as the archive copy regardless of this flag.
|
||||
Whether an archive copy is wanted. For PDFs, False skips the
|
||||
remote engine and uses locally-extracted text instead.
|
||||
"""
|
||||
config = RemoteEngineConfig(
|
||||
engine=settings.REMOTE_OCR_ENGINE,
|
||||
@@ -241,6 +257,16 @@ class RemoteDocumentParser:
|
||||
self._text = ""
|
||||
return
|
||||
|
||||
if not produce_archive and mime_type == "application/pdf":
|
||||
logger.debug(
|
||||
"Remote OCR: skipped — no archive requested, "
|
||||
"using locally-extracted text",
|
||||
)
|
||||
self._text = (
|
||||
post_process_text(extract_pdf_text(document_path, log=logger)) or ""
|
||||
)
|
||||
return
|
||||
|
||||
if config.engine == "azureai":
|
||||
self._text = self._azure_ai_vision_parse(document_path, config)
|
||||
|
||||
|
||||
@@ -337,6 +337,117 @@ class TestRemoteParserParse:
|
||||
assert remote_parser.get_date() is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# parse() — produce_archive=False skips the remote engine (PDFs only)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRemoteParserSkipsWhenNoArchiveWanted:
|
||||
"""When the caller has already decided no archive is needed for a PDF
|
||||
(documents.consumer.should_produce_archive), the remote engine call is
|
||||
skipped entirely in favor of locally-extracted text.
|
||||
"""
|
||||
|
||||
def test_pdf_skips_azure_when_no_archive_requested(
|
||||
self,
|
||||
remote_parser: RemoteDocumentParser,
|
||||
simple_digital_pdf_file: Path,
|
||||
azure_client: Mock,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN: produce_archive=False for a PDF
|
||||
WHEN: parse() is called
|
||||
THEN: Azure is never invoked, no archive is produced, and text
|
||||
comes from local pdftotext extraction
|
||||
"""
|
||||
remote_parser.parse(
|
||||
simple_digital_pdf_file,
|
||||
"application/pdf",
|
||||
produce_archive=False,
|
||||
)
|
||||
|
||||
azure_client.begin_analyze_document.assert_not_called()
|
||||
assert remote_parser.get_archive_path() is None
|
||||
assert remote_parser.get_text() != ""
|
||||
|
||||
def test_pdf_no_archive_requested_text_matches_local_extraction(
|
||||
self,
|
||||
remote_parser: RemoteDocumentParser,
|
||||
simple_digital_pdf_file: Path,
|
||||
azure_client: Mock,
|
||||
mocker: MockerFixture,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN: produce_archive=False for a PDF
|
||||
WHEN: parse() is called
|
||||
THEN: the returned text is exactly the locally-extracted text,
|
||||
not anything from the (unused) Azure mock
|
||||
"""
|
||||
mocker.patch(
|
||||
"paperless.parsers.remote.extract_pdf_text",
|
||||
return_value="Local digital text.",
|
||||
)
|
||||
|
||||
remote_parser.parse(
|
||||
simple_digital_pdf_file,
|
||||
"application/pdf",
|
||||
produce_archive=False,
|
||||
)
|
||||
|
||||
assert remote_parser.get_text() == "Local digital text."
|
||||
|
||||
def test_pdf_no_archive_requested_closes_no_client(
|
||||
self,
|
||||
remote_parser: RemoteDocumentParser,
|
||||
simple_digital_pdf_file: Path,
|
||||
azure_client: Mock,
|
||||
) -> None:
|
||||
remote_parser.parse(
|
||||
simple_digital_pdf_file,
|
||||
"application/pdf",
|
||||
produce_archive=False,
|
||||
)
|
||||
|
||||
azure_client.close.assert_not_called()
|
||||
|
||||
def test_non_pdf_still_calls_azure_when_no_archive_requested(
|
||||
self,
|
||||
remote_parser: RemoteDocumentParser,
|
||||
simple_digital_pdf_file: Path,
|
||||
azure_client: Mock,
|
||||
) -> None:
|
||||
"""
|
||||
Images have no local-text fallback, so produce_archive=False does
|
||||
not skip the remote engine for non-PDF MIME types.
|
||||
"""
|
||||
remote_parser.parse(
|
||||
simple_digital_pdf_file,
|
||||
"image/png",
|
||||
produce_archive=False,
|
||||
)
|
||||
|
||||
azure_client.begin_analyze_document.assert_called_once()
|
||||
assert remote_parser.get_text() == _DEFAULT_TEXT
|
||||
|
||||
@pytest.mark.usefixtures("no_engine_settings")
|
||||
def test_unconfigured_engine_takes_precedence_over_skip(
|
||||
self,
|
||||
remote_parser: RemoteDocumentParser,
|
||||
simple_digital_pdf_file: Path,
|
||||
) -> None:
|
||||
"""An unconfigured engine still short-circuits before the
|
||||
produce_archive check, returning empty text as before.
|
||||
"""
|
||||
remote_parser.parse(
|
||||
simple_digital_pdf_file,
|
||||
"application/pdf",
|
||||
produce_archive=False,
|
||||
)
|
||||
|
||||
assert remote_parser.get_text() == ""
|
||||
assert remote_parser.get_archive_path() is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# parse() — Azure failure path
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user