mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-08-11 05:13:18 +00:00
Backend stuff for remote ocr reprocess, add to bulk edit pass in from ui settings
This commit is contained in:
@@ -394,10 +394,16 @@ def delete(doc_ids: list[int]) -> Literal["OK"]:
|
||||
return "OK"
|
||||
|
||||
|
||||
def reprocess(doc_ids: list[int]) -> Literal["OK"]:
|
||||
def reprocess(doc_ids: list[int], *, remote_ocr: bool = False) -> Literal["OK"]:
|
||||
"""
|
||||
Re-run parsing for the given documents.
|
||||
|
||||
Consumption workflows do not run here, so ``remote_ocr`` is how the user
|
||||
asks for the remote engine when it is not configured to handle everything.
|
||||
"""
|
||||
for document_id in doc_ids:
|
||||
update_document_content_maybe_archive_file.apply_async(
|
||||
kwargs={"document_id": document_id},
|
||||
kwargs={"document_id": document_id, "remote_ocr": remote_ocr},
|
||||
headers={"trigger_source": PaperlessTask.TriggerSource.MANUAL},
|
||||
)
|
||||
|
||||
|
||||
@@ -1744,7 +1744,7 @@ class DeleteDocumentsSerializer(DocumentSelectionSerializer):
|
||||
|
||||
|
||||
class ReprocessDocumentsSerializer(DocumentSelectionSerializer):
|
||||
pass
|
||||
remote_ocr = serializers.BooleanField(required=False, default=False)
|
||||
|
||||
|
||||
class BulkEditSerializer(
|
||||
@@ -2086,6 +2086,13 @@ class BulkEditSerializer(
|
||||
f"Page {op['page']} is out of bounds for document with {doc.page_count} pages.",
|
||||
)
|
||||
|
||||
def _validate_parameters_reprocess(self, parameters) -> None:
|
||||
if "remote_ocr" in parameters:
|
||||
if not isinstance(parameters["remote_ocr"], bool):
|
||||
raise serializers.ValidationError("remote_ocr must be a boolean")
|
||||
else:
|
||||
parameters["remote_ocr"] = False
|
||||
|
||||
def validate_parameters_remove_password(self, parameters):
|
||||
if "password" not in parameters:
|
||||
raise serializers.ValidationError("password not specified")
|
||||
@@ -2150,6 +2157,8 @@ class BulkEditSerializer(
|
||||
self._validate_parameters_edit_pdf(parameters, attrs["documents"][0])
|
||||
elif method == bulk_edit.remove_password:
|
||||
self.validate_parameters_remove_password(parameters)
|
||||
elif method == bulk_edit.reprocess:
|
||||
self._validate_parameters_reprocess(parameters)
|
||||
|
||||
return attrs
|
||||
|
||||
|
||||
@@ -532,7 +532,29 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
|
||||
m.assert_called_once()
|
||||
args, kwargs = m.call_args
|
||||
self.assertEqual(args[0], [self.doc1.id])
|
||||
self.assertEqual(len(kwargs), 0)
|
||||
self.assertEqual(kwargs, {"remote_ocr": False})
|
||||
|
||||
@mock.patch("documents.views.bulk_edit.reprocess")
|
||||
def test_reprocess_documents_endpoint_remote_ocr(self, m) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- API data to reprocess a document with remote OCR requested
|
||||
WHEN:
|
||||
- API is called
|
||||
THEN:
|
||||
- reprocess is called with remote_ocr=True
|
||||
"""
|
||||
self.setup_mock(m, "reprocess")
|
||||
response = self.client.post(
|
||||
"/api/documents/reprocess/",
|
||||
json.dumps({"documents": [self.doc1.id], "remote_ocr": True}),
|
||||
content_type="application/json",
|
||||
)
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
m.assert_called_once()
|
||||
args, kwargs = m.call_args
|
||||
self.assertEqual(args[0], [self.doc1.id])
|
||||
self.assertEqual(kwargs, {"remote_ocr": True})
|
||||
|
||||
@mock.patch("documents.serialisers.bulk_edit.set_storage_path")
|
||||
def test_api_set_storage_path(self, m) -> None:
|
||||
@@ -1553,6 +1575,29 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
|
||||
),
|
||||
)
|
||||
|
||||
def test_legacy_bulk_edit_reprocess_invalid_remote_ocr(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- The deprecated bulk_edit endpoint with a non-boolean remote_ocr
|
||||
WHEN:
|
||||
- API is called
|
||||
THEN:
|
||||
- The request is rejected rather than passed through to the task
|
||||
"""
|
||||
response = self.client.post(
|
||||
"/api/documents/bulk_edit/",
|
||||
json.dumps(
|
||||
{
|
||||
"documents": [self.doc1.id],
|
||||
"method": "reprocess",
|
||||
"parameters": {"remote_ocr": "yes please"},
|
||||
},
|
||||
),
|
||||
content_type="application/json",
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
|
||||
@mock.patch("documents.views.bulk_edit.edit_pdf")
|
||||
def test_edit_pdf(self, m) -> None:
|
||||
self.setup_mock(m, "edit_pdf")
|
||||
|
||||
@@ -60,6 +60,10 @@ class TestApiUiSettings(DirectoriesMixin, APITestCase):
|
||||
},
|
||||
"email_enabled": False,
|
||||
"ai_enabled": False,
|
||||
"remote_ocr": {
|
||||
"configured": False,
|
||||
"mode": "always",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
@@ -154,6 +158,50 @@ class TestApiUiSettings(DirectoriesMixin, APITestCase):
|
||||
str(response.data["settings"]),
|
||||
)
|
||||
|
||||
@override_settings(
|
||||
REMOTE_OCR_ENGINE="azureai",
|
||||
REMOTE_OCR_API_KEY="somekey",
|
||||
REMOTE_OCR_ENDPOINT="https://example.cognitiveservices.azure.com",
|
||||
REMOTE_OCR_MODE="workflow_only",
|
||||
)
|
||||
def test_settings_reports_remote_ocr_when_configured(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A fully configured remote OCR engine in workflow_only mode
|
||||
WHEN:
|
||||
- The ui_settings endpoint is called
|
||||
THEN:
|
||||
- The UI is told remote OCR is available and selective, so it can
|
||||
offer it where it would actually change something
|
||||
"""
|
||||
response = self.client.get(self.ENDPOINT, format="json")
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(
|
||||
response.data["settings"]["remote_ocr"],
|
||||
{"configured": True, "mode": "workflow_only"},
|
||||
)
|
||||
|
||||
@override_settings(
|
||||
REMOTE_OCR_ENGINE="azureai",
|
||||
REMOTE_OCR_API_KEY=None,
|
||||
REMOTE_OCR_ENDPOINT=None,
|
||||
)
|
||||
def test_settings_reports_remote_ocr_incompletely_configured(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An engine named but missing its endpoint and API key
|
||||
WHEN:
|
||||
- The ui_settings endpoint is called
|
||||
THEN:
|
||||
- It is reported as not configured, matching what the parser
|
||||
registry will actually do
|
||||
"""
|
||||
response = self.client.get(self.ENDPOINT, format="json")
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertFalse(response.data["settings"]["remote_ocr"]["configured"])
|
||||
|
||||
@override_settings(
|
||||
OAUTH_CALLBACK_BASE_URL="http://localhost:8000",
|
||||
GMAIL_OAUTH_CLIENT_ID="abc123",
|
||||
|
||||
@@ -1782,3 +1782,56 @@ class TestPDFActions(DirectoriesMixin, TestCase):
|
||||
|
||||
self.assertIn("wrong password", str(exc.exception))
|
||||
self.assertIn("Error removing password from document", cm.output[0])
|
||||
|
||||
|
||||
class TestBulkEditReprocess(DirectoriesMixin, TestCase):
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
|
||||
self.doc = Document.objects.create(
|
||||
title="test",
|
||||
checksum="A",
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
|
||||
@mock.patch("documents.bulk_edit.update_document_content_maybe_archive_file")
|
||||
def test_reprocess_defaults_to_local(self, mock_task: mock.Mock) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A reprocess request that says nothing about remote OCR
|
||||
WHEN:
|
||||
- reprocess is called
|
||||
THEN:
|
||||
- The task is queued without asking for the remote engine
|
||||
"""
|
||||
result = bulk_edit.reprocess([self.doc.id])
|
||||
|
||||
self.assertEqual(result, "OK")
|
||||
mock_task.apply_async.assert_called_once()
|
||||
_, kwargs = mock_task.apply_async.call_args
|
||||
self.assertEqual(
|
||||
kwargs["kwargs"],
|
||||
{"document_id": self.doc.id, "remote_ocr": False},
|
||||
)
|
||||
|
||||
@mock.patch("documents.bulk_edit.update_document_content_maybe_archive_file")
|
||||
def test_reprocess_passes_remote_ocr(self, mock_task: mock.Mock) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A reprocess request that explicitly asks for remote OCR
|
||||
WHEN:
|
||||
- reprocess is called
|
||||
THEN:
|
||||
- The request is forwarded to the task for every document
|
||||
"""
|
||||
other = Document.objects.create(
|
||||
title="test2",
|
||||
checksum="B",
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
|
||||
bulk_edit.reprocess([self.doc.id, other.id], remote_ocr=True)
|
||||
|
||||
self.assertEqual(mock_task.apply_async.call_count, 2)
|
||||
for call in mock_task.apply_async.call_args_list:
|
||||
self.assertTrue(call.kwargs["kwargs"]["remote_ocr"])
|
||||
|
||||
@@ -236,8 +236,10 @@ from paperless import version
|
||||
from paperless.celery import app as celery_app
|
||||
from paperless.config import AIConfig
|
||||
from paperless.config import GeneralConfig
|
||||
from paperless.config import RemoteOCRConfig
|
||||
from paperless.models import ApplicationConfiguration
|
||||
from paperless.parsers.registry import get_parser_registry
|
||||
from paperless.parsers.remote import RemoteEngineConfig
|
||||
from paperless.serialisers import GroupSerializer
|
||||
from paperless.serialisers import UserSerializer
|
||||
from paperless.views import StandardPagination
|
||||
@@ -4010,6 +4012,11 @@ class UiSettingsView(GenericAPIView[Any]):
|
||||
|
||||
ui_settings["auditlog_enabled"] = settings.AUDIT_LOG_ENABLED
|
||||
|
||||
ui_settings["remote_ocr"] = {
|
||||
"configured": RemoteEngineConfig.from_app_config().engine_is_valid(),
|
||||
"mode": RemoteOCRConfig().remote_ocr_mode,
|
||||
}
|
||||
|
||||
if settings.GMAIL_OAUTH_ENABLED or settings.OUTLOOK_OAUTH_ENABLED:
|
||||
manager = PaperlessMailOAuth2Manager()
|
||||
if settings.GMAIL_OAUTH_ENABLED:
|
||||
|
||||
Reference in New Issue
Block a user