Remove the handlers, make the matching work

This commit is contained in:
shamoon committed 2026-10-20 13:39:09 -07:00
1 parent bd73fe1e80
commit bedefd85f2
8 files changed
+131 -222

No files matched your search

+57 -27
View File
@@ -9,13 +9,15 @@ from rest_framework import status
from rest_framework.test import APITestCase
from documents.models import CustomField
from documents.models import DocumentType
from documents.models import MatchingModel
from documents.models import OcrTemplate
from documents.models import OcrTemplateZone
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory
from paperless_testing.permissions import grant_object
CONTENT = "Northwind Power & Light statement"
class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
ENDPOINT = "/api/ocr_templates/"
@@ -24,7 +26,6 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
self.user = User.objects.create_superuser(username="temp_admin")
self.client.force_authenticate(user=self.user)
self.doc_type = DocumentType.objects.create(name="Invoice")
self.custom_field_text = CustomField.objects.create(
name="Invoice Number",
data_type=CustomField.FieldDataType.STRING,
@@ -47,7 +48,8 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
def _make_template_data(self, **overrides):
data = {
"name": "Invoice Template",
"document_type": self.doc_type.pk,
"match": "Northwind",
"matching_algorithm": MatchingModel.MATCH_ANY,
"default_page": 0,
"source_width": 2480,
"source_height": 3508,
@@ -103,7 +105,8 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
result = resp.json()
self.assertEqual(result["name"], "Invoice Template")
self.assertEqual(result["document_type"], self.doc_type.pk)
self.assertEqual(result["match"], "Northwind")
self.assertEqual(result["matching_algorithm"], MatchingModel.MATCH_ANY)
self.assertEqual(len(result["zones"]), 1)
self.assertEqual(result["zones"][0]["name"], "Invoice Number")
self.assertEqual(OcrTemplate.objects.count(), 1)
@@ -144,7 +147,7 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
def template(name, owner=None):
return OcrTemplate.objects.create(
name=name,
document_type=self.doc_type,
match="Northwind",
source_width=1000,
source_height=1000,
owner=owner,
@@ -182,7 +185,7 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
)
template = OcrTemplate.objects.create(
name="Private",
document_type=self.doc_type,
match="Northwind",
source_width=1000,
source_height=1000,
owner=other,
@@ -201,6 +204,27 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
self.assertEqual(template.name, "Renamed")
self.assertEqual(template.owner, other)
def test_create_template_rejects_automatic_matching(self):
"""
GIVEN:
- A template payload with automatic matching
WHEN:
- API request to create it
THEN:
- The request is rejected, automatic matching needs assigned
documents to learn from
"""
response = self.client.post(
self.ENDPOINT,
json.dumps(
self._make_template_data(matching_algorithm=MatchingModel.MATCH_AUTO),
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertEqual(OcrTemplate.objects.count(), 0)
def test_create_template_multiple_zones(self):
"""
GIVEN:
@@ -342,7 +366,7 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
def test_list_templates(self):
template = OcrTemplate.objects.create(
name="Test Template",
document_type=self.doc_type,
match="Northwind",
source_width=2480,
source_height=3508,
)
@@ -373,7 +397,7 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
"""PUT should replace all zones with the new set."""
template = OcrTemplate.objects.create(
name="Old Name",
document_type=self.doc_type,
match="Northwind",
source_width=2480,
source_height=3508,
)
@@ -413,7 +437,7 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
def test_delete_template_cascades_zones(self):
template = OcrTemplate.objects.create(
name="To Delete",
document_type=self.doc_type,
match="Northwind",
source_width=2480,
source_height=3508,
)
@@ -441,7 +465,7 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
def test_patch_toggle_enabled(self):
template = OcrTemplate.objects.create(
name="Toggle Test",
document_type=self.doc_type,
match="Northwind",
source_width=2480,
source_height=3508,
enabled=True,
@@ -460,7 +484,7 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
"""PATCH without zones field should not delete existing zones."""
template = OcrTemplate.objects.create(
name="Patch Test",
document_type=self.doc_type,
match="Northwind",
source_width=2480,
source_height=3508,
)
@@ -493,28 +517,34 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
)
@patch("documents.zone_ocr.run_zone_extraction")
def test_run_zone_ocr_without_document_type_is_skipped(self, extract):
document = DocumentFactory()
def test_run_zone_ocr_without_matching_template_is_skipped(self, extract):
document = DocumentFactory(content=CONTENT)
OcrTemplate.objects.create(
name="Other sender",
match="Contoso",
source_width=1000,
source_height=1000,
)
response = self.client.post(f"/api/documents/{document.pk}/run-zone-ocr/")
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(
response.json(),
{"status": "skipped", "reason": "no_document_type", "results": []},
{"status": "skipped", "reason": "no_template", "results": []},
)
extract.assert_not_called()
@patch("documents.zone_ocr.run_zone_extraction")
def test_run_zone_ocr_without_enabled_template_is_skipped(self, extract):
document = DocumentFactory(document_type=self.doc_type)
document = DocumentFactory(content=CONTENT)
for enabled in (None, False):
with self.subTest(enabled=enabled):
if enabled is False:
OcrTemplate.objects.create(
name="Disabled template",
document_type=self.doc_type,
match="Northwind",
source_width=1000,
source_height=1000,
enabled=False,
@@ -580,7 +610,7 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
codename__in=["view_document", "change_document"],
),
)
document = DocumentFactory(document_type=self.doc_type, owner=owner)
document = DocumentFactory(content=CONTENT, owner=owner)
self.client.force_authenticate(user=user)
response = self.client.post(f"/api/documents/{document.pk}/run-zone-ocr/")
@@ -594,7 +624,7 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
user.user_permissions.add(
*Permission.objects.filter(codename__in=["view_document", "add_document"]),
)
document = DocumentFactory(document_type=self.doc_type, owner=user)
document = DocumentFactory(content=CONTENT, owner=user)
self.client.force_authenticate(user=user)
response = self.client.post(f"/api/documents/{document.pk}/run-zone-ocr/")
@@ -610,11 +640,11 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
@patch("documents.zone_ocr.run_zone_extraction")
def test_run_zone_ocr_skips_templates_private_to_the_requester(self, extract):
owner = User.objects.create_user(username="document_owner")
document = DocumentFactory(document_type=self.doc_type, owner=owner)
document = DocumentFactory(content=CONTENT, owner=owner)
OcrTemplate.objects.create(
name="Requester's private template",
owner=self.user,
document_type=self.doc_type,
match="Northwind",
source_width=100,
source_height=100,
)
@@ -638,7 +668,7 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
extract,
):
owner = User.objects.create_user(username="document_owner")
document = DocumentFactory(document_type=self.doc_type, owner=owner)
document = DocumentFactory(content=CONTENT, owner=owner)
document.source_path.write_bytes(b"test PDF")
for name, template_owner in [
("Owned", owner),
@@ -649,7 +679,7 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
template = OcrTemplate.objects.create(
name=name,
owner=template_owner,
document_type=self.doc_type,
match="Northwind",
source_width=100,
source_height=100,
)
@@ -684,7 +714,7 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
template = OcrTemplate.objects.create(
name="Shared",
owner=self.user,
document_type=self.doc_type,
match="Northwind",
source_width=100,
source_height=100,
)
@@ -706,10 +736,10 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
@patch("documents.zone_ocr.run_zone_extraction")
def test_run_zone_ocr_missing_file_is_an_error(self, extract):
document = DocumentFactory(document_type=self.doc_type)
document = DocumentFactory(content=CONTENT)
OcrTemplate.objects.create(
name="Template",
document_type=self.doc_type,
match="Northwind",
source_width=1000,
source_height=1000,
)
@@ -723,13 +753,13 @@ class TestOcrTemplatesAPI(DirectoriesMixin, APITestCase):
@patch("documents.zone_ocr.run_zone_extraction")
def test_run_zone_ocr_reports_completed_results(self, extract):
document = DocumentFactory(
document_type=self.doc_type,
content=CONTENT,
mime_type="application/pdf",
)
document.source_path.write_bytes(b"test PDF")
template = OcrTemplate.objects.create(
name="Template",
document_type=self.doc_type,
match="Northwind",
source_width=1000,
source_height=1000,
)
+11 -29
View File
@@ -21,7 +21,6 @@ from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import OcrTemplate
from documents.models import StoragePath
from documents.models import Tag
from documents.permissions import set_permissions_for_objects
@@ -1981,45 +1980,28 @@ class TestBulkEditRunZoneOcr:
def mock_task(self, mocker: MockerFixture) -> mock.MagicMock:
return mocker.patch("documents.bulk_edit.extract_zones")
def test_queues_documents_with_an_enabled_template(
def test_queues_every_selected_document(
self,
mock_task: mock.MagicMock,
) -> None:
"""
GIVEN:
- Documents whose type has enabled templates, a disabled template,
or no type at all
- Selected documents
WHEN:
- run_zone_ocr is called with all of them
- run_zone_ocr is called with them
THEN:
- Only the document with an enabled template is queued, once
- Each document is queued once, templates are matched when its
task runs
"""
invoice = DocumentType.objects.create(name="Invoice")
receipt = DocumentType.objects.create(name="Receipt")
for name in ("Invoices", "More invoices"):
OcrTemplate.objects.create(
name=name,
document_type=invoice,
source_width=1000,
source_height=1000,
)
OcrTemplate.objects.create(
name="Receipts",
document_type=receipt,
source_width=1000,
source_height=1000,
enabled=False,
)
templated = DocumentFactory(document_type=invoice)
others = [DocumentFactory(document_type=receipt), DocumentFactory()]
docs = DocumentFactory.create_batch(2)
result = bulk_edit.run_zone_ocr([templated.id, *(d.id for d in others)])
result = bulk_edit.run_zone_ocr([doc.id for doc in docs])
assert result == "OK"
mock_task.apply_async.assert_called_once()
assert mock_task.apply_async.call_args.kwargs["kwargs"] == {
"document_id": templated.id,
}
assert [
call.kwargs["kwargs"]["document_id"]
for call in mock_task.apply_async.call_args_list
] == [doc.id for doc in docs]
def test_task_runs_extraction(self, mocker: MockerFixture) -> None:
"""
+44 -64
View File
@@ -12,10 +12,9 @@ from django.test import TestCase
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import MatchingModel
from documents.models import OcrTemplate
from documents.models import OcrTemplateZone
from documents.signals.handlers import run_zone_ocr_on_type_change
from documents.zone_ocr import _apply_transform
from documents.zone_ocr import _convert_value
from documents.zone_ocr import _detect_mime
@@ -32,32 +31,33 @@ class TestTemplateVisibility(DirectoriesMixin, TestCase):
super().setUp()
self.owner = User.objects.create_user(username="document_owner")
self.other = User.objects.create_user(username="other_owner")
self.doc_type = DocumentType.objects.create(name="Invoice")
self.document = DocumentFactory(owner=self.owner, document_type=self.doc_type)
self.document = DocumentFactory(
owner=self.owner,
content="Northwind Power & Light statement",
)
self.mine = self.template("Mine", owner=self.owner)
self.unowned = self.template("Unowned")
self.private = self.template("Private", owner=self.other)
def template(self, name, **kwargs):
kwargs.setdefault("match", "Northwind")
kwargs.setdefault("matching_algorithm", MatchingModel.MATCH_ANY)
return OcrTemplate.objects.create(
name=name,
document_type=self.doc_type,
source_width=100,
source_height=100,
**kwargs,
)
def test_only_owned_unowned_and_shared_templates_apply(self):
def test_only_matching_owned_unowned_and_shared_templates_apply(self):
shared = self.template("Shared", owner=self.other)
grant_object(self.owner, shared, "view_ocrtemplate")
self.template("Disabled", owner=self.owner, enabled=False)
other_type = DocumentType.objects.create(name="Other")
OcrTemplate.objects.create(
name="Wrong type",
self.template("Other sender", owner=self.owner, match="Contoso")
self.template(
"No matching",
owner=self.owner,
document_type=other_type,
source_width=100,
source_height=100,
matching_algorithm=MatchingModel.MATCH_NONE,
)
self.assertCountEqual(
get_templates_for_document(self.document),
@@ -72,16 +72,13 @@ class TestTemplateVisibility(DirectoriesMixin, TestCase):
def test_unowned_documents_only_use_unowned_templates(self):
self.document.owner = None
self.assertQuerySetEqual(
get_templates_for_document(self.document),
[self.unowned],
)
self.assertEqual(get_templates_for_document(self.document), [self.unowned])
def test_versions_use_the_root_documents_owner(self):
version = DocumentFactory(
root_document=self.document,
owner=self.other,
document_type=self.doc_type,
content=self.document.content,
)
self.assertCountEqual(
get_templates_for_document(version),
@@ -97,7 +94,7 @@ class TestTemplateVisibility(DirectoriesMixin, TestCase):
def test_inactive_owner_has_no_templates(self):
self.owner.is_active = False
self.assertFalse(get_templates_for_document(self.document).exists())
self.assertEqual(get_templates_for_document(self.document), [])
@patch("documents.zone_ocr._process_template")
def test_extraction_never_processes_private_templates(self, process):
@@ -120,26 +117,6 @@ class TestTemplateVisibility(DirectoriesMixin, TestCase):
[self.mine, self.unowned],
)
@patch("documents.zone_ocr.run_zone_extraction")
def test_type_change_skips_documents_with_only_private_templates(self, extract):
self.mine.delete()
self.unowned.delete()
self.document.source_path.write_bytes(b"test PDF")
self.document._old_document_type_id = None
run_zone_ocr_on_type_change(Document, self.document)
extract.assert_not_called()
@patch("documents.zone_ocr.run_zone_extraction")
def test_type_change_runs_when_a_template_is_visible(self, extract):
self.document.source_path.write_bytes(b"test PDF")
self.document._old_document_type_id = None
run_zone_ocr_on_type_change(Document, self.document)
extract.assert_called_once_with(self.document, None)
class TestApplyTransform(TestCase):
"""Tests for the _apply_transform function."""
@@ -393,36 +370,45 @@ class TestRunZoneExtraction(TestCase):
"""Tests for the full extraction pipeline."""
def setUp(self):
self.doc_type = DocumentType.objects.create(name="Invoice")
self.custom_field = CustomField.objects.create(
name="Invoice Number",
data_type=CustomField.FieldDataType.STRING,
)
def test_skips_document_without_type(self):
doc = Document.objects.create(
title="No Type",
content="test",
mime_type="application/pdf",
@patch("documents.zone_ocr._process_template")
def test_skips_template_that_does_not_match(self, mock_process):
template = OcrTemplate.objects.create(
name="Other sender",
match="Contoso",
source_width=2480,
source_height=3508,
)
OcrTemplateZone.objects.create(
template=template,
name="Zone",
custom_field=self.custom_field,
x=0,
y=0,
width=100,
height=50,
)
run_zone_extraction(doc, Path("/nonexistent"))
self.assertEqual(CustomFieldInstance.objects.count(), 0)
def test_skips_document_without_matching_template(self):
other_type = DocumentType.objects.create(name="Other")
doc = Document.objects.create(
title="No Template",
content="test",
mime_type="application/pdf",
document_type=other_type,
)
run_zone_extraction(doc, Path("/nonexistent"))
self.assertEqual(CustomFieldInstance.objects.count(), 0)
with tempfile.NamedTemporaryFile(suffix=".pdf") as f:
f.write(b"%PDF-1.4 fake")
f.flush()
run_zone_extraction(doc, Path(f.name))
mock_process.assert_not_called()
def test_skips_disabled_template(self):
template = OcrTemplate.objects.create(
name="Disabled",
document_type=self.doc_type,
match="test",
source_width=2480,
source_height=3508,
enabled=False,
@@ -441,7 +427,6 @@ class TestRunZoneExtraction(TestCase):
title="Test",
content="test",
mime_type="application/pdf",
document_type=self.doc_type,
)
run_zone_extraction(doc, Path("/nonexistent"))
self.assertEqual(CustomFieldInstance.objects.count(), 0)
@@ -449,7 +434,7 @@ class TestRunZoneExtraction(TestCase):
def test_skips_template_with_no_zones(self):
OcrTemplate.objects.create(
name="Empty",
document_type=self.doc_type,
match="test",
source_width=2480,
source_height=3508,
enabled=True,
@@ -459,7 +444,6 @@ class TestRunZoneExtraction(TestCase):
title="Test",
content="test",
mime_type="application/pdf",
document_type=self.doc_type,
)
with tempfile.NamedTemporaryFile(suffix=".pdf") as f:
@@ -472,7 +456,7 @@ class TestRunZoneExtraction(TestCase):
def test_calls_process_for_enabled_template(self, mock_process):
template = OcrTemplate.objects.create(
name="Active",
document_type=self.doc_type,
match="test",
source_width=2480,
source_height=3508,
enabled=True,
@@ -491,7 +475,6 @@ class TestRunZoneExtraction(TestCase):
title="Test",
content="test",
mime_type="application/pdf",
document_type=self.doc_type,
)
with tempfile.NamedTemporaryFile(suffix=".pdf") as f:
@@ -508,7 +491,7 @@ class TestRunZoneExtraction(TestCase):
template = OcrTemplate.objects.create(
name="Failing",
document_type=self.doc_type,
match="test",
source_width=2480,
source_height=3508,
enabled=True,
@@ -527,7 +510,6 @@ class TestRunZoneExtraction(TestCase):
title="Test",
content="test",
mime_type="application/pdf",
document_type=self.doc_type,
)
with tempfile.NamedTemporaryFile(suffix=".pdf") as f:
@@ -542,18 +524,17 @@ class TestRunZoneExtraction(TestCase):
title="Test",
content="test",
mime_type="application/pdf",
document_type=self.doc_type,
)
# No template, so it exits early — but shouldn't crash on None
run_zone_extraction(doc, None)
@patch("documents.zone_ocr._process_template")
def test_multiple_templates_all_process(self, mock_process):
"""Multiple enabled templates for the same type should all run."""
"""Multiple matching templates should all run."""
for i in range(3):
template = OcrTemplate.objects.create(
name=f"Template {i}",
document_type=self.doc_type,
match="test",
source_width=2480,
source_height=3508,
enabled=True,
@@ -572,7 +553,6 @@ class TestRunZoneExtraction(TestCase):
title="Test",
content="test",
mime_type="application/pdf",
document_type=self.doc_type,
)
with tempfile.NamedTemporaryFile(suffix=".pdf") as f: