mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-08-16 07:43:22 +00:00
Ok, wire up the remote_ocr_mode with allow_remote for consumer
This commit is contained in:
@@ -34,6 +34,7 @@ class DocumentMetadataOverrides:
|
|||||||
skip_asn_if_exists: bool = False
|
skip_asn_if_exists: bool = False
|
||||||
version_label: str | None = None
|
version_label: str | None = None
|
||||||
actor_id: int | None = None
|
actor_id: int | None = None
|
||||||
|
remote_ocr: bool = False
|
||||||
|
|
||||||
def update(self, other: "DocumentMetadataOverrides") -> "DocumentMetadataOverrides":
|
def update(self, other: "DocumentMetadataOverrides") -> "DocumentMetadataOverrides":
|
||||||
"""
|
"""
|
||||||
@@ -57,6 +58,8 @@ class DocumentMetadataOverrides:
|
|||||||
self.actor_id = other.actor_id
|
self.actor_id = other.actor_id
|
||||||
if other.skip_asn_if_exists:
|
if other.skip_asn_if_exists:
|
||||||
self.skip_asn_if_exists = True
|
self.skip_asn_if_exists = True
|
||||||
|
if other.remote_ocr:
|
||||||
|
self.remote_ocr = True
|
||||||
if other.version_label is not None:
|
if other.version_label is not None:
|
||||||
self.version_label = other.version_label
|
self.version_label = other.version_label
|
||||||
|
|
||||||
|
|||||||
+10
-1
@@ -66,6 +66,7 @@ from documents.utils import compute_checksum
|
|||||||
from documents.utils import identity
|
from documents.utils import identity
|
||||||
from documents.workflows.utils import get_workflows_for_trigger
|
from documents.workflows.utils import get_workflows_for_trigger
|
||||||
from paperless.config import AIConfig
|
from paperless.config import AIConfig
|
||||||
|
from paperless.config import RemoteOCRConfig
|
||||||
from paperless.logging import consume_task_id
|
from paperless.logging import consume_task_id
|
||||||
from paperless.parsers import ParserContext
|
from paperless.parsers import ParserContext
|
||||||
from paperless.parsers.registry import get_parser_registry
|
from paperless.parsers.registry import get_parser_registry
|
||||||
@@ -337,10 +338,17 @@ def bulk_update_documents(document_ids) -> None:
|
|||||||
|
|
||||||
|
|
||||||
@shared_task
|
@shared_task
|
||||||
def update_document_content_maybe_archive_file(document_id) -> None:
|
def update_document_content_maybe_archive_file(
|
||||||
|
document_id,
|
||||||
|
*,
|
||||||
|
remote_ocr: bool = False,
|
||||||
|
) -> None:
|
||||||
"""
|
"""
|
||||||
Re-creates OCR content and thumbnail for a document, and archive file if
|
Re-creates OCR content and thumbnail for a document, and archive file if
|
||||||
it exists.
|
it exists.
|
||||||
|
|
||||||
|
Remote OCR is used only when the engine is configured to handle everything
|
||||||
|
or if explicitly asked for via ``remote_ocr``.
|
||||||
"""
|
"""
|
||||||
document = Document.objects.get(id=document_id)
|
document = Document.objects.get(id=document_id)
|
||||||
|
|
||||||
@@ -350,6 +358,7 @@ def update_document_content_maybe_archive_file(document_id) -> None:
|
|||||||
mime_type,
|
mime_type,
|
||||||
document.original_filename or "",
|
document.original_filename or "",
|
||||||
document.source_path,
|
document.source_path,
|
||||||
|
allow_remote=remote_ocr or RemoteOCRConfig().remote_ocr_by_default,
|
||||||
)
|
)
|
||||||
|
|
||||||
if not parser_class:
|
if not parser_class:
|
||||||
|
|||||||
@@ -1559,6 +1559,72 @@ class PostConsumeTestCase(DirectoriesMixin, GetConsumerMixin, TestCase):
|
|||||||
consumer.run_post_consume_script(doc)
|
consumer.run_post_consume_script(doc)
|
||||||
|
|
||||||
|
|
||||||
|
class TestConsumerRemoteOCR(
|
||||||
|
DirectoriesMixin,
|
||||||
|
FileSystemAssertsMixin,
|
||||||
|
GetConsumerMixin,
|
||||||
|
TestCase,
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
The consumer resolves the remote OCR mode and the per-document request from
|
||||||
|
workflows into the allow_remote flag it hands to the parser registry.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def setUp(self) -> None:
|
||||||
|
super().setUp()
|
||||||
|
|
||||||
|
patcher = mock.patch("documents.consumer.get_parser_registry")
|
||||||
|
self.mock_registry = patcher.start()
|
||||||
|
self.mock_registry.return_value.get_parser_for_file.return_value = DummyParser
|
||||||
|
self.addCleanup(patcher.stop)
|
||||||
|
|
||||||
|
def _consume(self, *, overrides: DocumentMetadataOverrides | None = None) -> bool:
|
||||||
|
src = (
|
||||||
|
Path(__file__).parent
|
||||||
|
/ "samples"
|
||||||
|
/ "documents"
|
||||||
|
/ "originals"
|
||||||
|
/ "0000001.pdf"
|
||||||
|
)
|
||||||
|
dst = self.dirs.scratch_dir / "sample.pdf"
|
||||||
|
shutil.copy(src, dst)
|
||||||
|
|
||||||
|
with self.get_consumer(dst, overrides=overrides) as consumer:
|
||||||
|
consumer.run()
|
||||||
|
|
||||||
|
_, kwargs = self.mock_registry.return_value.get_parser_for_file.call_args
|
||||||
|
return kwargs["allow_remote"]
|
||||||
|
|
||||||
|
@override_settings(REMOTE_OCR_MODE="always")
|
||||||
|
def test_always_mode_allows_remote(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN: Remote OCR mode is 'always'.
|
||||||
|
WHEN: A document is consumed without any workflow asking for it.
|
||||||
|
THEN: The registry is allowed to pick the remote parser.
|
||||||
|
"""
|
||||||
|
self.assertTrue(self._consume())
|
||||||
|
|
||||||
|
@override_settings(REMOTE_OCR_MODE="workflow_only")
|
||||||
|
def test_workflow_only_mode_denies_remote_by_default(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN: Remote OCR mode is 'workflow_only'.
|
||||||
|
WHEN: A document is consumed and nothing asked for remote OCR.
|
||||||
|
THEN: The remote parser is excluded.
|
||||||
|
"""
|
||||||
|
self.assertFalse(self._consume())
|
||||||
|
|
||||||
|
@override_settings(REMOTE_OCR_MODE="workflow_only")
|
||||||
|
def test_workflow_only_mode_allows_remote_when_requested(self) -> None:
|
||||||
|
"""
|
||||||
|
GIVEN: Remote OCR mode is 'workflow_only'.
|
||||||
|
WHEN: A workflow set remote_ocr on the metadata overrides.
|
||||||
|
THEN: The registry is allowed to pick the remote parser.
|
||||||
|
"""
|
||||||
|
self.assertTrue(
|
||||||
|
self._consume(overrides=DocumentMetadataOverrides(remote_ocr=True)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class TestMetadataOverrides(TestCase):
|
class TestMetadataOverrides(TestCase):
|
||||||
def test_update_skip_asn_if_exists(self) -> None:
|
def test_update_skip_asn_if_exists(self) -> None:
|
||||||
base = DocumentMetadataOverrides()
|
base = DocumentMetadataOverrides()
|
||||||
@@ -1566,6 +1632,20 @@ class TestMetadataOverrides(TestCase):
|
|||||||
base.update(incoming)
|
base.update(incoming)
|
||||||
self.assertTrue(base.skip_asn_if_exists)
|
self.assertTrue(base.skip_asn_if_exists)
|
||||||
|
|
||||||
|
def test_update_remote_ocr(self) -> None:
|
||||||
|
base = DocumentMetadataOverrides()
|
||||||
|
base.update(DocumentMetadataOverrides(remote_ocr=True))
|
||||||
|
self.assertTrue(base.remote_ocr)
|
||||||
|
|
||||||
|
def test_update_remote_ocr_is_not_unset(self) -> None:
|
||||||
|
"""
|
||||||
|
A later workflow that says nothing must not undo an earlier one that
|
||||||
|
asked for remote OCR.
|
||||||
|
"""
|
||||||
|
base = DocumentMetadataOverrides(remote_ocr=True)
|
||||||
|
base.update(DocumentMetadataOverrides())
|
||||||
|
self.assertTrue(base.remote_ocr)
|
||||||
|
|
||||||
def test_update_actor_and_version_label(self) -> None:
|
def test_update_actor_and_version_label(self) -> None:
|
||||||
base = DocumentMetadataOverrides(
|
base = DocumentMetadataOverrides(
|
||||||
actor_id=1,
|
actor_id=1,
|
||||||
|
|||||||
@@ -287,6 +287,45 @@ class TestUpdateContent(DirectoriesMixin, TestCase):
|
|||||||
self.assertNotEqual(Document.objects.get(pk=doc.pk).content, "test")
|
self.assertNotEqual(Document.objects.get(pk=doc.pk).content, "test")
|
||||||
|
|
||||||
|
|
||||||
|
class TestUpdateContentRemoteOCR(DirectoriesMixin, TestCase):
|
||||||
|
"""
|
||||||
|
Consumption workflows do not run on reprocess, so the remote parser is
|
||||||
|
used only in 'always' mode or when the caller explicitly asks for it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def setUp(self) -> None:
|
||||||
|
super().setUp()
|
||||||
|
|
||||||
|
patcher = mock.patch("documents.tasks.get_parser_registry")
|
||||||
|
self.mock_registry = patcher.start()
|
||||||
|
self.mock_registry.return_value.get_parser_for_file.return_value = None
|
||||||
|
self.addCleanup(patcher.stop)
|
||||||
|
|
||||||
|
self.doc = Document.objects.create(
|
||||||
|
title="test",
|
||||||
|
content="my document",
|
||||||
|
checksum="wow",
|
||||||
|
mime_type="application/pdf",
|
||||||
|
)
|
||||||
|
|
||||||
|
def _allow_remote(self, **kwargs) -> bool:
|
||||||
|
tasks.update_document_content_maybe_archive_file(self.doc.pk, **kwargs)
|
||||||
|
_, call_kwargs = self.mock_registry.return_value.get_parser_for_file.call_args
|
||||||
|
return call_kwargs["allow_remote"]
|
||||||
|
|
||||||
|
@override_settings(REMOTE_OCR_MODE="always")
|
||||||
|
def test_always_mode_allows_remote(self) -> None:
|
||||||
|
self.assertTrue(self._allow_remote())
|
||||||
|
|
||||||
|
@override_settings(REMOTE_OCR_MODE="workflow_only")
|
||||||
|
def test_workflow_only_mode_denies_remote_by_default(self) -> None:
|
||||||
|
self.assertFalse(self._allow_remote())
|
||||||
|
|
||||||
|
@override_settings(REMOTE_OCR_MODE="workflow_only")
|
||||||
|
def test_workflow_only_mode_allows_remote_when_requested(self) -> None:
|
||||||
|
self.assertTrue(self._allow_remote(remote_ocr=True))
|
||||||
|
|
||||||
|
|
||||||
class TestAIIndex(DirectoriesMixin, TestCase):
|
class TestAIIndex(DirectoriesMixin, TestCase):
|
||||||
@override_settings(
|
@override_settings(
|
||||||
AI_ENABLED=True,
|
AI_ENABLED=True,
|
||||||
|
|||||||
@@ -213,6 +213,17 @@ class RemoteOCRConfig(BaseConfig):
|
|||||||
settings.REMOTE_OCR_MODE,
|
settings.REMOTE_OCR_MODE,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def remote_ocr_by_default(self) -> bool:
|
||||||
|
"""
|
||||||
|
Whether every supported document goes to the remote engine.
|
||||||
|
|
||||||
|
When False the remote engine is used only for documents that
|
||||||
|
explicitly asked for it, i.e. a workflow matched during consumption or
|
||||||
|
the user ticked the box when reprocessing.
|
||||||
|
"""
|
||||||
|
return self.remote_ocr_mode == RemoteOCRMode.ALWAYS
|
||||||
|
|
||||||
|
|
||||||
@dataclasses.dataclass
|
@dataclasses.dataclass
|
||||||
class AIConfig(BaseConfig):
|
class AIConfig(BaseConfig):
|
||||||
|
|||||||
@@ -93,3 +93,21 @@ class TestRemoteOCRConfig:
|
|||||||
assert cfg.remote_ocr_engine is None
|
assert cfg.remote_ocr_engine is None
|
||||||
assert cfg.remote_ocr_api_key is None
|
assert cfg.remote_ocr_api_key is None
|
||||||
assert cfg.remote_ocr_endpoint is None
|
assert cfg.remote_ocr_endpoint is None
|
||||||
|
|
||||||
|
|
||||||
|
class TestRemoteOCRByDefault:
|
||||||
|
def test_always_mode(self, make_remote_ocr_config, null_app_config) -> None:
|
||||||
|
cfg = make_remote_ocr_config(
|
||||||
|
null_app_config,
|
||||||
|
REMOTE_OCR_MODE=RemoteOCRMode.ALWAYS,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert cfg.remote_ocr_by_default is True
|
||||||
|
|
||||||
|
def test_workflow_only_mode(self, make_remote_ocr_config, null_app_config) -> None:
|
||||||
|
cfg = make_remote_ocr_config(
|
||||||
|
null_app_config,
|
||||||
|
REMOTE_OCR_MODE=RemoteOCRMode.WORKFLOW_ONLY,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert cfg.remote_ocr_by_default is False
|
||||||
|
|||||||
Reference in New Issue
Block a user