mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-10-21 15:28:13 +00:00
Remove the handlers, make the matching work
This commit is contained in:
8 files changed
+131
-222
No files matched your search
@@ -13,11 +13,9 @@ class DocumentsConfig(AppConfig):
|
||||
from documents.signals.handlers import add_inbox_tags
|
||||
from documents.signals.handlers import add_or_update_document_in_llm_index
|
||||
from documents.signals.handlers import add_to_index
|
||||
from documents.signals.handlers import capture_old_document_type
|
||||
from documents.signals.handlers import run_workflows_added
|
||||
from documents.signals.handlers import run_workflows_updated
|
||||
from documents.signals.handlers import run_zone_ocr_extraction
|
||||
from documents.signals.handlers import run_zone_ocr_on_type_change
|
||||
from documents.signals.handlers import send_websocket_document_updated
|
||||
from documents.signals.handlers import set_correspondent
|
||||
from documents.signals.handlers import set_document_type
|
||||
@@ -34,14 +32,6 @@ class DocumentsConfig(AppConfig):
|
||||
document_consumption_finished.connect(add_or_update_document_in_llm_index)
|
||||
document_consumption_finished.connect(run_zone_ocr_extraction)
|
||||
|
||||
from django.db.models.signals import post_save
|
||||
from django.db.models.signals import pre_save
|
||||
|
||||
from documents.models import Document
|
||||
|
||||
pre_save.connect(capture_old_document_type, sender=Document)
|
||||
post_save.connect(run_zone_ocr_on_type_change, sender=Document)
|
||||
|
||||
document_updated.connect(run_workflows_updated)
|
||||
document_updated.connect(send_websocket_document_updated)
|
||||
document_updated.connect(add_or_update_document_in_llm_index)
|
||||
|
||||
@@ -1236,19 +1236,10 @@ def remove_doclink(
|
||||
|
||||
def run_zone_ocr(doc_ids: list[int]) -> Literal["OK"]:
|
||||
"""
|
||||
Queue zone OCR for the given documents. Documents whose type has no enabled
|
||||
OCR template are skipped.
|
||||
Queue zone OCR for the given documents. Templates are matched against each
|
||||
document's content when its task runs.
|
||||
"""
|
||||
document_ids = (
|
||||
Document.objects.filter(
|
||||
id__in=doc_ids,
|
||||
document_type__ocr_templates__enabled=True,
|
||||
)
|
||||
.order_by()
|
||||
.values_list("id", flat=True)
|
||||
.distinct()
|
||||
)
|
||||
for document_id in document_ids:
|
||||
for document_id in doc_ids:
|
||||
extract_zones.apply_async(
|
||||
kwargs={"document_id": document_id},
|
||||
headers={"trigger_source": PaperlessTask.TriggerSource.MANUAL},
|
||||
|
||||
@@ -1410,7 +1410,7 @@ def close_connection_pool_on_worker_init(**kwargs) -> None:
|
||||
|
||||
def run_zone_ocr_extraction(sender, document, original_file=None, **kwargs):
|
||||
"""
|
||||
Run zone-based OCR extraction if the document's type has an active template.
|
||||
Run zone-based OCR extraction with the OCR templates that match the document.
|
||||
"""
|
||||
try:
|
||||
from documents.zone_ocr import run_zone_extraction
|
||||
@@ -1423,58 +1423,6 @@ def run_zone_ocr_extraction(sender, document, original_file=None, **kwargs):
|
||||
)
|
||||
|
||||
|
||||
def capture_old_document_type(sender, instance, **kwargs):
|
||||
"""pre_save: remember the document's previous type so the post_save handler
|
||||
can tell whether the type actually changed (vs. every other save)."""
|
||||
if instance.pk:
|
||||
instance._old_document_type_id = (
|
||||
Document.objects.filter(pk=instance.pk)
|
||||
.values_list("document_type_id", flat=True)
|
||||
.first()
|
||||
)
|
||||
else:
|
||||
instance._old_document_type_id = None
|
||||
|
||||
|
||||
def run_zone_ocr_on_type_change(sender, instance, *, created=False, **kwargs):
|
||||
"""
|
||||
Run zone OCR only when a document's TYPE actually changes (and the new type
|
||||
has an enabled template). NOT on every save — zone OCR overwrites fields, so
|
||||
re-running it on each edit would clobber the user's changes. Newly created
|
||||
documents are handled by the consumption signal, and the user can always
|
||||
trigger extraction manually via the run-zone-ocr action.
|
||||
"""
|
||||
if created or not instance.pk or not instance.document_type_id:
|
||||
return
|
||||
|
||||
# Only proceed if the type changed compared to what was in the DB before.
|
||||
old_type = getattr(instance, "_old_document_type_id", None)
|
||||
if old_type == instance.document_type_id:
|
||||
return
|
||||
|
||||
from documents.zone_ocr import get_templates_for_document
|
||||
|
||||
if not get_templates_for_document(instance).exists():
|
||||
return
|
||||
|
||||
try:
|
||||
from documents.zone_ocr import run_zone_extraction
|
||||
|
||||
doc_path = instance.archive_path or instance.source_path
|
||||
if doc_path and Path(doc_path).is_file():
|
||||
logger.info(
|
||||
"Zone OCR: running extraction for document %d (type %d)",
|
||||
instance.pk,
|
||||
instance.document_type_id,
|
||||
)
|
||||
run_zone_extraction(instance, None)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Zone OCR extraction failed for document %s",
|
||||
instance.pk,
|
||||
)
|
||||
|
||||
|
||||
@worker_process_shutdown.connect
|
||||
def close_connection_pool_on_worker_shutdown(**kwargs) -> None: # pragma: no cover
|
||||
"""
|
||||
|
||||
@@ -9,13 +9,15 @@ from rest_framework import status
|
||||
from rest_framework.test import APITestCase
|
||||
|
||||
from documents.models import CustomField
|
||||
from documents.models import DocumentType
|
||||
from documents.models import MatchingModel
|
||||
from documents.models import OcrTemplate
|
||||
from documents.models import OcrTemplateZone
|
||||
from paperless_testing.dirs import DirectoriesMixin
|
||||
from paperless_testing.factories import DocumentFactory
|
||||
from paperless_testing.permissions import grant_object
|
||||
|
||||
CONTENT = "Northwind Power & Light statement"
|
||||
|
||||
|
||||
class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
|
||||
ENDPOINT = "/api/ocr_templates/"
|
||||
@@ -24,7 +26,6 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
|
||||
self.user = User.objects.create_superuser(username="temp_admin")
|
||||
self.client.force_authenticate(user=self.user)
|
||||
|
||||
self.doc_type = DocumentType.objects.create(name="Invoice")
|
||||
self.custom_field_text = CustomField.objects.create(
|
||||
name="Invoice Number",
|
||||
data_type=CustomField.FieldDataType.STRING,
|
||||
@@ -47,7 +48,8 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
|
||||
def _make_template_data(self, **overrides):
|
||||
data = {
|
||||
"name": "Invoice Template",
|
||||
"document_type": self.doc_type.pk,
|
||||
"match": "Northwind",
|
||||
"matching_algorithm": MatchingModel.MATCH_ANY,
|
||||
"default_page": 0,
|
||||
"source_width": 2480,
|
||||
"source_height": 3508,
|
||||
@@ -103,7 +105,8 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
|
||||
|
||||
result = resp.json()
|
||||
self.assertEqual(result["name"], "Invoice Template")
|
||||
self.assertEqual(result["document_type"], self.doc_type.pk)
|
||||
self.assertEqual(result["match"], "Northwind")
|
||||
self.assertEqual(result["matching_algorithm"], MatchingModel.MATCH_ANY)
|
||||
self.assertEqual(len(result["zones"]), 1)
|
||||
self.assertEqual(result["zones"][0]["name"], "Invoice Number")
|
||||
self.assertEqual(OcrTemplate.objects.count(), 1)
|
||||
@@ -144,7 +147,7 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
|
||||
def template(name, owner=None):
|
||||
return OcrTemplate.objects.create(
|
||||
name=name,
|
||||
document_type=self.doc_type,
|
||||
match="Northwind",
|
||||
source_width=1000,
|
||||
source_height=1000,
|
||||
owner=owner,
|
||||
@@ -182,7 +185,7 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
|
||||
)
|
||||
template = OcrTemplate.objects.create(
|
||||
name="Private",
|
||||
document_type=self.doc_type,
|
||||
match="Northwind",
|
||||
source_width=1000,
|
||||
source_height=1000,
|
||||
owner=other,
|
||||
@@ -201,6 +204,27 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
|
||||
self.assertEqual(template.name, "Renamed")
|
||||
self.assertEqual(template.owner, other)
|
||||
|
||||
def test_create_template_rejects_automatic_matching(self):
|
||||
"""
|
||||
GIVEN:
|
||||
- A template payload with automatic matching
|
||||
WHEN:
|
||||
- API request to create it
|
||||
THEN:
|
||||
- The request is rejected, automatic matching needs assigned
|
||||
documents to learn from
|
||||
"""
|
||||
response = self.client.post(
|
||||
self.ENDPOINT,
|
||||
json.dumps(
|
||||
self._make_template_data(matching_algorithm=MatchingModel.MATCH_AUTO),
|
||||
),
|
||||
content_type="application/json",
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
|
||||
self.assertEqual(OcrTemplate.objects.count(), 0)
|
||||
|
||||
def test_create_template_multiple_zones(self):
|
||||
"""
|
||||
GIVEN:
|
||||
@@ -342,7 +366,7 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
|
||||
def test_list_templates(self):
|
||||
template = OcrTemplate.objects.create(
|
||||
name="Test Template",
|
||||
document_type=self.doc_type,
|
||||
match="Northwind",
|
||||
source_width=2480,
|
||||
source_height=3508,
|
||||
)
|
||||
@@ -373,7 +397,7 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
|
||||
"""PUT should replace all zones with the new set."""
|
||||
template = OcrTemplate.objects.create(
|
||||
name="Old Name",
|
||||
document_type=self.doc_type,
|
||||
match="Northwind",
|
||||
source_width=2480,
|
||||
source_height=3508,
|
||||
)
|
||||
@@ -413,7 +437,7 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
|
||||
def test_delete_template_cascades_zones(self):
|
||||
template = OcrTemplate.objects.create(
|
||||
name="To Delete",
|
||||
document_type=self.doc_type,
|
||||
match="Northwind",
|
||||
source_width=2480,
|
||||
source_height=3508,
|
||||
)
|
||||
@@ -441,7 +465,7 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
|
||||
def test_patch_toggle_enabled(self):
|
||||
template = OcrTemplate.objects.create(
|
||||
name="Toggle Test",
|
||||
document_type=self.doc_type,
|
||||
match="Northwind",
|
||||
source_width=2480,
|
||||
source_height=3508,
|
||||
enabled=True,
|
||||
@@ -460,7 +484,7 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
|
||||
"""PATCH without zones field should not delete existing zones."""
|
||||
template = OcrTemplate.objects.create(
|
||||
name="Patch Test",
|
||||
document_type=self.doc_type,
|
||||
match="Northwind",
|
||||
source_width=2480,
|
||||
source_height=3508,
|
||||
)
|
||||
@@ -493,28 +517,34 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
|
||||
)
|
||||
|
||||
@patch("documents.zone_ocr.run_zone_extraction")
|
||||
def test_run_zone_ocr_without_document_type_is_skipped(self, extract):
|
||||
document = DocumentFactory()
|
||||
def test_run_zone_ocr_without_matching_template_is_skipped(self, extract):
|
||||
document = DocumentFactory(content=CONTENT)
|
||||
OcrTemplate.objects.create(
|
||||
name="Other sender",
|
||||
match="Contoso",
|
||||
source_width=1000,
|
||||
source_height=1000,
|
||||
)
|
||||
|
||||
response = self.client.post(f"/api/documents/{document.pk}/run-zone-ocr/")
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(
|
||||
response.json(),
|
||||
{"status": "skipped", "reason": "no_document_type", "results": []},
|
||||
{"status": "skipped", "reason": "no_template", "results": []},
|
||||
)
|
||||
extract.assert_not_called()
|
||||
|
||||
@patch("documents.zone_ocr.run_zone_extraction")
|
||||
def test_run_zone_ocr_without_enabled_template_is_skipped(self, extract):
|
||||
document = DocumentFactory(document_type=self.doc_type)
|
||||
document = DocumentFactory(content=CONTENT)
|
||||
|
||||
for enabled in (None, False):
|
||||
with self.subTest(enabled=enabled):
|
||||
if enabled is False:
|
||||
OcrTemplate.objects.create(
|
||||
name="Disabled template",
|
||||
document_type=self.doc_type,
|
||||
match="Northwind",
|
||||
source_width=1000,
|
||||
source_height=1000,
|
||||
enabled=False,
|
||||
@@ -580,7 +610,7 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
|
||||
codename__in=["view_document", "change_document"],
|
||||
),
|
||||
)
|
||||
document = DocumentFactory(document_type=self.doc_type, owner=owner)
|
||||
document = DocumentFactory(content=CONTENT, owner=owner)
|
||||
self.client.force_authenticate(user=user)
|
||||
|
||||
response = self.client.post(f"/api/documents/{document.pk}/run-zone-ocr/")
|
||||
@@ -594,7 +624,7 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
|
||||
user.user_permissions.add(
|
||||
*Permission.objects.filter(codename__in=["view_document", "add_document"]),
|
||||
)
|
||||
document = DocumentFactory(document_type=self.doc_type, owner=user)
|
||||
document = DocumentFactory(content=CONTENT, owner=user)
|
||||
self.client.force_authenticate(user=user)
|
||||
|
||||
response = self.client.post(f"/api/documents/{document.pk}/run-zone-ocr/")
|
||||
@@ -610,11 +640,11 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
|
||||
@patch("documents.zone_ocr.run_zone_extraction")
|
||||
def test_run_zone_ocr_skips_templates_private_to_the_requester(self, extract):
|
||||
owner = User.objects.create_user(username="document_owner")
|
||||
document = DocumentFactory(document_type=self.doc_type, owner=owner)
|
||||
document = DocumentFactory(content=CONTENT, owner=owner)
|
||||
OcrTemplate.objects.create(
|
||||
name="Requester's private template",
|
||||
owner=self.user,
|
||||
document_type=self.doc_type,
|
||||
match="Northwind",
|
||||
source_width=100,
|
||||
source_height=100,
|
||||
)
|
||||
@@ -638,7 +668,7 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
|
||||
extract,
|
||||
):
|
||||
owner = User.objects.create_user(username="document_owner")
|
||||
document = DocumentFactory(document_type=self.doc_type, owner=owner)
|
||||
document = DocumentFactory(content=CONTENT, owner=owner)
|
||||
document.source_path.write_bytes(b"test PDF")
|
||||
for name, template_owner in [
|
||||
("Owned", owner),
|
||||
@@ -649,7 +679,7 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
|
||||
template = OcrTemplate.objects.create(
|
||||
name=name,
|
||||
owner=template_owner,
|
||||
document_type=self.doc_type,
|
||||
match="Northwind",
|
||||
source_width=100,
|
||||
source_height=100,
|
||||
)
|
||||
@@ -684,7 +714,7 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
|
||||
template = OcrTemplate.objects.create(
|
||||
name="Shared",
|
||||
owner=self.user,
|
||||
document_type=self.doc_type,
|
||||
match="Northwind",
|
||||
source_width=100,
|
||||
source_height=100,
|
||||
)
|
||||
@@ -706,10 +736,10 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
|
||||
|
||||
@patch("documents.zone_ocr.run_zone_extraction")
|
||||
def test_run_zone_ocr_missing_file_is_an_error(self, extract):
|
||||
document = DocumentFactory(document_type=self.doc_type)
|
||||
document = DocumentFactory(content=CONTENT)
|
||||
OcrTemplate.objects.create(
|
||||
name="Template",
|
||||
document_type=self.doc_type,
|
||||
match="Northwind",
|
||||
source_width=1000,
|
||||
source_height=1000,
|
||||
)
|
||||
@@ -723,13 +753,13 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
|
||||
@patch("documents.zone_ocr.run_zone_extraction")
|
||||
def test_run_zone_ocr_reports_completed_results(self, extract):
|
||||
document = DocumentFactory(
|
||||
document_type=self.doc_type,
|
||||
content=CONTENT,
|
||||
mime_type="application/pdf",
|
||||
)
|
||||
document.source_path.write_bytes(b"test PDF")
|
||||
template = OcrTemplate.objects.create(
|
||||
name="Template",
|
||||
document_type=self.doc_type,
|
||||
match="Northwind",
|
||||
source_width=1000,
|
||||
source_height=1000,
|
||||
)
|
||||
|
||||
@@ -21,7 +21,6 @@ from documents.models import CustomField
|
||||
from documents.models import CustomFieldInstance
|
||||
from documents.models import Document
|
||||
from documents.models import DocumentType
|
||||
from documents.models import OcrTemplate
|
||||
from documents.models import StoragePath
|
||||
from documents.models import Tag
|
||||
from documents.permissions import set_permissions_for_objects
|
||||
@@ -1981,45 +1980,28 @@ class TestBulkEditRunZoneOcr:
|
||||
def mock_task(self, mocker: MockerFixture) -> mock.MagicMock:
|
||||
return mocker.patch("documents.bulk_edit.extract_zones")
|
||||
|
||||
def test_queues_documents_with_an_enabled_template(
|
||||
def test_queues_every_selected_document(
|
||||
self,
|
||||
mock_task: mock.MagicMock,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Documents whose type has enabled templates, a disabled template,
|
||||
or no type at all
|
||||
- Selected documents
|
||||
WHEN:
|
||||
- run_zone_ocr is called with all of them
|
||||
- run_zone_ocr is called with them
|
||||
THEN:
|
||||
- Only the document with an enabled template is queued, once
|
||||
- Each document is queued once, templates are matched when its
|
||||
task runs
|
||||
"""
|
||||
invoice = DocumentType.objects.create(name="Invoice")
|
||||
receipt = DocumentType.objects.create(name="Receipt")
|
||||
for name in ("Invoices", "More invoices"):
|
||||
OcrTemplate.objects.create(
|
||||
name=name,
|
||||
document_type=invoice,
|
||||
source_width=1000,
|
||||
source_height=1000,
|
||||
)
|
||||
OcrTemplate.objects.create(
|
||||
name="Receipts",
|
||||
document_type=receipt,
|
||||
source_width=1000,
|
||||
source_height=1000,
|
||||
enabled=False,
|
||||
)
|
||||
templated = DocumentFactory(document_type=invoice)
|
||||
others = [DocumentFactory(document_type=receipt), DocumentFactory()]
|
||||
docs = DocumentFactory.create_batch(2)
|
||||
|
||||
result = bulk_edit.run_zone_ocr([templated.id, *(d.id for d in others)])
|
||||
result = bulk_edit.run_zone_ocr([doc.id for doc in docs])
|
||||
|
||||
assert result == "OK"
|
||||
mock_task.apply_async.assert_called_once()
|
||||
assert mock_task.apply_async.call_args.kwargs["kwargs"] == {
|
||||
"document_id": templated.id,
|
||||
}
|
||||
assert [
|
||||
call.kwargs["kwargs"]["document_id"]
|
||||
for call in mock_task.apply_async.call_args_list
|
||||
] == [doc.id for doc in docs]
|
||||
|
||||
def test_task_runs_extraction(self, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
|
||||
@@ -12,10 +12,9 @@ from django.test import TestCase
|
||||
from documents.models import CustomField
|
||||
from documents.models import CustomFieldInstance
|
||||
from documents.models import Document
|
||||
from documents.models import DocumentType
|
||||
from documents.models import MatchingModel
|
||||
from documents.models import OcrTemplate
|
||||
from documents.models import OcrTemplateZone
|
||||
from documents.signals.handlers import run_zone_ocr_on_type_change
|
||||
from documents.zone_ocr import _apply_transform
|
||||
from documents.zone_ocr import _convert_value
|
||||
from documents.zone_ocr import _detect_mime
|
||||
@@ -32,32 +31,33 @@ class TestTemplateVisibility(DirectoriesMixin, TestCase):
|
||||
super().setUp()
|
||||
self.owner = User.objects.create_user(username="document_owner")
|
||||
self.other = User.objects.create_user(username="other_owner")
|
||||
self.doc_type = DocumentType.objects.create(name="Invoice")
|
||||
self.document = DocumentFactory(owner=self.owner, document_type=self.doc_type)
|
||||
self.document = DocumentFactory(
|
||||
owner=self.owner,
|
||||
content="Northwind Power & Light statement",
|
||||
)
|
||||
self.mine = self.template("Mine", owner=self.owner)
|
||||
self.unowned = self.template("Unowned")
|
||||
self.private = self.template("Private", owner=self.other)
|
||||
|
||||
def template(self, name, **kwargs):
|
||||
kwargs.setdefault("match", "Northwind")
|
||||
kwargs.setdefault("matching_algorithm", MatchingModel.MATCH_ANY)
|
||||
return OcrTemplate.objects.create(
|
||||
name=name,
|
||||
document_type=self.doc_type,
|
||||
source_width=100,
|
||||
source_height=100,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def test_only_owned_unowned_and_shared_templates_apply(self):
|
||||
def test_only_matching_owned_unowned_and_shared_templates_apply(self):
|
||||
shared = self.template("Shared", owner=self.other)
|
||||
grant_object(self.owner, shared, "view_ocrtemplate")
|
||||
self.template("Disabled", owner=self.owner, enabled=False)
|
||||
other_type = DocumentType.objects.create(name="Other")
|
||||
OcrTemplate.objects.create(
|
||||
name="Wrong type",
|
||||
self.template("Other sender", owner=self.owner, match="Contoso")
|
||||
self.template(
|
||||
"No matching",
|
||||
owner=self.owner,
|
||||
document_type=other_type,
|
||||
source_width=100,
|
||||
source_height=100,
|
||||
matching_algorithm=MatchingModel.MATCH_NONE,
|
||||
)
|
||||
self.assertCountEqual(
|
||||
get_templates_for_document(self.document),
|
||||
@@ -72,16 +72,13 @@ class TestTemplateVisibility(DirectoriesMixin, TestCase):
|
||||
|
||||
def test_unowned_documents_only_use_unowned_templates(self):
|
||||
self.document.owner = None
|
||||
self.assertQuerySetEqual(
|
||||
get_templates_for_document(self.document),
|
||||
[self.unowned],
|
||||
)
|
||||
self.assertEqual(get_templates_for_document(self.document), [self.unowned])
|
||||
|
||||
def test_versions_use_the_root_documents_owner(self):
|
||||
version = DocumentFactory(
|
||||
root_document=self.document,
|
||||
owner=self.other,
|
||||
document_type=self.doc_type,
|
||||
content=self.document.content,
|
||||
)
|
||||
self.assertCountEqual(
|
||||
get_templates_for_document(version),
|
||||
@@ -97,7 +94,7 @@ class TestTemplateVisibility(DirectoriesMixin, TestCase):
|
||||
|
||||
def test_inactive_owner_has_no_templates(self):
|
||||
self.owner.is_active = False
|
||||
self.assertFalse(get_templates_for_document(self.document).exists())
|
||||
self.assertEqual(get_templates_for_document(self.document), [])
|
||||
|
||||
@patch("documents.zone_ocr._process_template")
|
||||
def test_extraction_never_processes_private_templates(self, process):
|
||||
@@ -120,26 +117,6 @@ class TestTemplateVisibility(DirectoriesMixin, TestCase):
|
||||
[self.mine, self.unowned],
|
||||
)
|
||||
|
||||
@patch("documents.zone_ocr.run_zone_extraction")
|
||||
def test_type_change_skips_documents_with_only_private_templates(self, extract):
|
||||
self.mine.delete()
|
||||
self.unowned.delete()
|
||||
self.document.source_path.write_bytes(b"test PDF")
|
||||
self.document._old_document_type_id = None
|
||||
|
||||
run_zone_ocr_on_type_change(Document, self.document)
|
||||
|
||||
extract.assert_not_called()
|
||||
|
||||
@patch("documents.zone_ocr.run_zone_extraction")
|
||||
def test_type_change_runs_when_a_template_is_visible(self, extract):
|
||||
self.document.source_path.write_bytes(b"test PDF")
|
||||
self.document._old_document_type_id = None
|
||||
|
||||
run_zone_ocr_on_type_change(Document, self.document)
|
||||
|
||||
extract.assert_called_once_with(self.document, None)
|
||||
|
||||
|
||||
class TestApplyTransform(TestCase):
|
||||
"""Tests for the _apply_transform function."""
|
||||
@@ -393,36 +370,45 @@ class TestRunZoneExtraction(TestCase):
|
||||
"""Tests for the full extraction pipeline."""
|
||||
|
||||
def setUp(self):
|
||||
self.doc_type = DocumentType.objects.create(name="Invoice")
|
||||
self.custom_field = CustomField.objects.create(
|
||||
name="Invoice Number",
|
||||
data_type=CustomField.FieldDataType.STRING,
|
||||
)
|
||||
|
||||
def test_skips_document_without_type(self):
|
||||
doc = Document.objects.create(
|
||||
title="No Type",
|
||||
content="test",
|
||||
mime_type="application/pdf",
|
||||
@patch("documents.zone_ocr._process_template")
|
||||
def test_skips_template_that_does_not_match(self, mock_process):
|
||||
template = OcrTemplate.objects.create(
|
||||
name="Other sender",
|
||||
match="Contoso",
|
||||
source_width=2480,
|
||||
source_height=3508,
|
||||
)
|
||||
OcrTemplateZone.objects.create(
|
||||
template=template,
|
||||
name="Zone",
|
||||
custom_field=self.custom_field,
|
||||
x=0,
|
||||
y=0,
|
||||
width=100,
|
||||
height=50,
|
||||
)
|
||||
run_zone_extraction(doc, Path("/nonexistent"))
|
||||
self.assertEqual(CustomFieldInstance.objects.count(), 0)
|
||||
|
||||
def test_skips_document_without_matching_template(self):
|
||||
other_type = DocumentType.objects.create(name="Other")
|
||||
doc = Document.objects.create(
|
||||
title="No Template",
|
||||
content="test",
|
||||
mime_type="application/pdf",
|
||||
document_type=other_type,
|
||||
)
|
||||
run_zone_extraction(doc, Path("/nonexistent"))
|
||||
self.assertEqual(CustomFieldInstance.objects.count(), 0)
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=".pdf") as f:
|
||||
f.write(b"%PDF-1.4 fake")
|
||||
f.flush()
|
||||
run_zone_extraction(doc, Path(f.name))
|
||||
|
||||
mock_process.assert_not_called()
|
||||
|
||||
def test_skips_disabled_template(self):
|
||||
template = OcrTemplate.objects.create(
|
||||
name="Disabled",
|
||||
document_type=self.doc_type,
|
||||
match="test",
|
||||
source_width=2480,
|
||||
source_height=3508,
|
||||
enabled=False,
|
||||
@@ -441,7 +427,6 @@ class TestRunZoneExtraction(TestCase):
|
||||
title="Test",
|
||||
content="test",
|
||||
mime_type="application/pdf",
|
||||
document_type=self.doc_type,
|
||||
)
|
||||
run_zone_extraction(doc, Path("/nonexistent"))
|
||||
self.assertEqual(CustomFieldInstance.objects.count(), 0)
|
||||
@@ -449,7 +434,7 @@ class TestRunZoneExtraction(TestCase):
|
||||
def test_skips_template_with_no_zones(self):
|
||||
OcrTemplate.objects.create(
|
||||
name="Empty",
|
||||
document_type=self.doc_type,
|
||||
match="test",
|
||||
source_width=2480,
|
||||
source_height=3508,
|
||||
enabled=True,
|
||||
@@ -459,7 +444,6 @@ class TestRunZoneExtraction(TestCase):
|
||||
title="Test",
|
||||
content="test",
|
||||
mime_type="application/pdf",
|
||||
document_type=self.doc_type,
|
||||
)
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=".pdf") as f:
|
||||
@@ -472,7 +456,7 @@ class TestRunZoneExtraction(TestCase):
|
||||
def test_calls_process_for_enabled_template(self, mock_process):
|
||||
template = OcrTemplate.objects.create(
|
||||
name="Active",
|
||||
document_type=self.doc_type,
|
||||
match="test",
|
||||
source_width=2480,
|
||||
source_height=3508,
|
||||
enabled=True,
|
||||
@@ -491,7 +475,6 @@ class TestRunZoneExtraction(TestCase):
|
||||
title="Test",
|
||||
content="test",
|
||||
mime_type="application/pdf",
|
||||
document_type=self.doc_type,
|
||||
)
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=".pdf") as f:
|
||||
@@ -508,7 +491,7 @@ class TestRunZoneExtraction(TestCase):
|
||||
|
||||
template = OcrTemplate.objects.create(
|
||||
name="Failing",
|
||||
document_type=self.doc_type,
|
||||
match="test",
|
||||
source_width=2480,
|
||||
source_height=3508,
|
||||
enabled=True,
|
||||
@@ -527,7 +510,6 @@ class TestRunZoneExtraction(TestCase):
|
||||
title="Test",
|
||||
content="test",
|
||||
mime_type="application/pdf",
|
||||
document_type=self.doc_type,
|
||||
)
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=".pdf") as f:
|
||||
@@ -542,18 +524,17 @@ class TestRunZoneExtraction(TestCase):
|
||||
title="Test",
|
||||
content="test",
|
||||
mime_type="application/pdf",
|
||||
document_type=self.doc_type,
|
||||
)
|
||||
# No template, so it exits early — but shouldn't crash on None
|
||||
run_zone_extraction(doc, None)
|
||||
|
||||
@patch("documents.zone_ocr._process_template")
|
||||
def test_multiple_templates_all_process(self, mock_process):
|
||||
"""Multiple enabled templates for the same type should all run."""
|
||||
"""Multiple matching templates should all run."""
|
||||
for i in range(3):
|
||||
template = OcrTemplate.objects.create(
|
||||
name=f"Template {i}",
|
||||
document_type=self.doc_type,
|
||||
match="test",
|
||||
source_width=2480,
|
||||
source_height=3508,
|
||||
enabled=True,
|
||||
@@ -572,7 +553,6 @@ class TestRunZoneExtraction(TestCase):
|
||||
title="Test",
|
||||
content="test",
|
||||
mime_type="application/pdf",
|
||||
document_type=self.doc_type,
|
||||
)
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=".pdf") as f:
|
||||
|
||||
@@ -2286,7 +2286,7 @@ class DocumentViewSet(
|
||||
fields={
|
||||
"status": serializers.ChoiceField(choices=["completed", "skipped"]),
|
||||
"reason": serializers.ChoiceField(
|
||||
choices=["no_document_type", "no_template"],
|
||||
choices=["no_template"],
|
||||
required=False,
|
||||
),
|
||||
"results": serializers.ListField(child=serializers.DictField()),
|
||||
@@ -2307,15 +2307,10 @@ class DocumentViewSet(
|
||||
) or not has_perms_owner_aware(request.user, "change_document", document):
|
||||
return HttpResponseForbidden("Insufficient permissions")
|
||||
|
||||
if not document.document_type_id:
|
||||
return Response(
|
||||
{"status": "skipped", "reason": "no_document_type", "results": []},
|
||||
)
|
||||
|
||||
from documents.zone_ocr import get_templates_for_document
|
||||
|
||||
templates = get_templates_for_document(document)
|
||||
if not templates.exists():
|
||||
if not templates:
|
||||
return Response(
|
||||
{"status": "skipped", "reason": "no_template", "results": []},
|
||||
)
|
||||
@@ -2334,7 +2329,7 @@ class DocumentViewSet(
|
||||
# Collect results
|
||||
results = []
|
||||
builtin_labels = {"title": "Title", "asn": "ASN", "created": "Created"}
|
||||
for template in templates.prefetch_related("zones", "zones__custom_field"):
|
||||
for template in templates:
|
||||
for zone in template.zones.all():
|
||||
target = getattr(zone, "target", None) or "custom_field"
|
||||
if target == "custom_field" and zone.custom_field_id:
|
||||
|
||||
+12
-19
@@ -21,6 +21,7 @@ from pathlib import Path
|
||||
from django.conf import settings
|
||||
from PIL import Image
|
||||
|
||||
from documents.matching import matches
|
||||
from documents.models import CustomField
|
||||
from documents.models import CustomFieldInstance
|
||||
from documents.models import Document
|
||||
@@ -33,19 +34,18 @@ from documents.versioning import get_root_document
|
||||
logger = logging.getLogger("paperless.zone_ocr")
|
||||
|
||||
|
||||
def get_templates_for_document(document: Document):
|
||||
def get_templates_for_document(document: Document) -> list[OcrTemplate]:
|
||||
"""
|
||||
The enabled templates for the document's type that its owner can view, the
|
||||
same rule as matching. A version is owned through its root document.
|
||||
The enabled templates that match the document's content and that its owner
|
||||
can view, the same rule as matching. A version is owned through its root
|
||||
document.
|
||||
"""
|
||||
templates = OcrTemplate.objects.filter(
|
||||
document_type_id=document.document_type_id,
|
||||
enabled=True,
|
||||
)
|
||||
owner = get_root_document(document).owner
|
||||
return templates.filter(
|
||||
templates = OcrTemplate.objects.filter(
|
||||
enabled=True,
|
||||
id__in=permitted_object_ids(owner, OcrTemplate, "view_ocrtemplate"),
|
||||
)
|
||||
).prefetch_related("zones", "zones__custom_field")
|
||||
return [template for template in templates if matches(template, document)]
|
||||
|
||||
|
||||
def run_zone_extraction(
|
||||
@@ -53,18 +53,11 @@ def run_zone_extraction(
|
||||
original_file: Path | None,
|
||||
) -> None:
|
||||
"""
|
||||
Run zone-based OCR extraction for a document if its type has an active template.
|
||||
Run zone-based OCR extraction for every template that matches a document.
|
||||
Called from the document_consumption_finished signal handler.
|
||||
"""
|
||||
if not document.document_type_id:
|
||||
return
|
||||
|
||||
templates = get_templates_for_document(document).prefetch_related(
|
||||
"zones",
|
||||
"zones__custom_field",
|
||||
)
|
||||
|
||||
if not templates.exists():
|
||||
templates = get_templates_for_document(document)
|
||||
if not templates:
|
||||
return
|
||||
|
||||
# Resolve the document file: prefer archive (PDF/A), then source, then signal arg
|
||||
|
||||
Reference in new issue
Block a user