mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-11-21 04:52:40 +00:00
Fix: report a last-page zone as missing when the page count is unknown
An image counts as one page and a PDF without a stored count is counted from the file.
This commit is contained in:
1 parent
6b8b946567
commit
e81bdff6c2
3 files changed
+121
-10
No files matched your search
@@ -44,6 +44,7 @@ from documents.zone_ocr import _parse_number
|
||||
from documents.zone_ocr import _prepare_field_update
|
||||
from documents.zone_ocr import _process_template
|
||||
from documents.zone_ocr import _render_pages
|
||||
from documents.zone_ocr import _resolve_page_idx
|
||||
from documents.zone_ocr import _zone_language
|
||||
from documents.zone_ocr import extract_zone_preview
|
||||
from documents.zone_ocr import get_templates_for_content
|
||||
@@ -3704,6 +3705,38 @@ class TestResolveZoneSource:
|
||||
)
|
||||
|
||||
|
||||
class TestResolvePageIdx:
|
||||
"""Tests for _resolve_page_idx."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("page_value", "page_count", "expected"),
|
||||
[
|
||||
pytest.param(None, 4, 0, id="blank-first-page"),
|
||||
pytest.param(1, 4, 0, id="page-1"),
|
||||
pytest.param(3, 4, 2, id="page-3"),
|
||||
pytest.param(-1, 4, 3, id="last-page-with-count"),
|
||||
pytest.param(-1, None, None, id="last-page-count-unknown"),
|
||||
pytest.param(-1, 0, None, id="last-page-count-zero"),
|
||||
],
|
||||
)
|
||||
def test_resolves_page_to_index(
|
||||
self,
|
||||
page_value: int | None,
|
||||
page_count: int | None,
|
||||
expected: int | None,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A zone page value and a document page count
|
||||
WHEN:
|
||||
- The page is resolved to a 0-based index
|
||||
THEN:
|
||||
- Pages are 1-based, blank is the first page, and the last page
|
||||
without a known page count cannot be located
|
||||
"""
|
||||
assert _resolve_page_idx(page_value, page_count) == expected
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@pytest.mark.usefixtures("paperless_dirs")
|
||||
class TestRunZoneExtraction:
|
||||
@@ -3884,6 +3917,63 @@ class TestRunZoneExtraction:
|
||||
)
|
||||
assert {r.template for r in results} == {"First", "Second"}
|
||||
|
||||
def test_last_page_zone_without_page_count_is_missing_page(
|
||||
self,
|
||||
mock_zone_ocr_run: MagicMock,
|
||||
document: Document,
|
||||
source: Path,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A matching template with a last-page zone and a PDF document whose
|
||||
page count is unknown and cannot be read from the file
|
||||
WHEN:
|
||||
- Zone extraction runs
|
||||
THEN:
|
||||
- The zone is reported as missing_page, no text is read and the
|
||||
document is not changed
|
||||
"""
|
||||
document.page_count = None
|
||||
document.mime_type = "application/pdf"
|
||||
document.save()
|
||||
document.source_path.write_bytes(b"not a pdf")
|
||||
template = OcrTemplateFactory(name="Matching", match="test")
|
||||
zone = make_zone(template, "Last", page=-1)
|
||||
|
||||
results = run_zone_extraction(document, templates=[template])
|
||||
|
||||
assert [(r.template, r.zone, r.reason) for r in results] == [
|
||||
("Matching", zone.name, ZoneOcrReason.MISSING_PAGE),
|
||||
]
|
||||
mock_zone_ocr_run.assert_not_called()
|
||||
document.refresh_from_db()
|
||||
assert document.title == "Test"
|
||||
|
||||
def test_last_page_zone_on_an_image_without_page_count_reads_the_image(
|
||||
self,
|
||||
document: Document,
|
||||
source: Path,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A matching template with a last-page title zone and an image
|
||||
document whose page count is unknown
|
||||
WHEN:
|
||||
- Zone extraction runs
|
||||
THEN:
|
||||
- The image is read as its only page and the title is written
|
||||
"""
|
||||
document.page_count = None
|
||||
document.save()
|
||||
template = OcrTemplateFactory(name="Matching", match="test")
|
||||
make_zone(template, "Last", page=-1)
|
||||
|
||||
results = run_zone_extraction(document, templates=[template])
|
||||
|
||||
assert [r.reason for r in results] == [None]
|
||||
document.refresh_from_db()
|
||||
assert document.title == "extracted"
|
||||
|
||||
def test_reports_missing_file_when_document_has_no_file(
|
||||
self,
|
||||
mock_zone_ocr_run: MagicMock,
|
||||
|
||||
@@ -47,6 +47,7 @@ from documents.permissions import permitted_object_ids
|
||||
from documents.regex import safe_regex_fullmatch
|
||||
from documents.validators import monetary_validator
|
||||
from documents.versioning import get_root_document
|
||||
from paperless.parsers.utils import get_page_count_for_pdf
|
||||
from paperless.utils import OCR_TO_DATEPARSER_LANGUAGES
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -329,13 +330,15 @@ def plan_zone_extraction(
|
||||
for template, zones in template_zones
|
||||
]
|
||||
|
||||
if not any(zones for _, zones in template_zones):
|
||||
return []
|
||||
page_count = _effective_page_count(doc_path, page_count, is_image=is_image)
|
||||
pages = {
|
||||
_resolve_page_idx(zone.page, page_count)
|
||||
page_idx
|
||||
for _, zones in template_zones
|
||||
for zone in zones
|
||||
if (page_idx := _resolve_page_idx(zone.page, page_count)) is not None
|
||||
}
|
||||
if not pages:
|
||||
return []
|
||||
plans = []
|
||||
settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(dir=settings.SCRATCH_DIR) as tmp_dir:
|
||||
@@ -438,13 +441,30 @@ def zone_results_written(results: list[ZoneOcrResult]) -> bool:
|
||||
return any(result.reason is None for result in results)
|
||||
|
||||
|
||||
def _resolve_page_idx(page_value: int | None, page_count: int | None) -> int:
|
||||
def _effective_page_count(
|
||||
doc_path: Path,
|
||||
page_count: int | None,
|
||||
*,
|
||||
is_image: bool,
|
||||
) -> int | None:
|
||||
"""The stored page count, else 1 for an image, else the pages in the PDF."""
|
||||
if page_count:
|
||||
return page_count
|
||||
if is_image:
|
||||
return 1
|
||||
return get_page_count_for_pdf(doc_path)
|
||||
|
||||
|
||||
def _resolve_page_idx(page_value: int | None, page_count: int | None) -> int | None:
|
||||
"""
|
||||
Resolve a zone's 1-based page or -1 (last page) to a 0-based index, defaulting
|
||||
to the first page. Without a page count, the last page resolves to the first.
|
||||
to the first page. The last page without a known page count resolves to None,
|
||||
meaning the zone cannot be located.
|
||||
"""
|
||||
if page_value == -1:
|
||||
return max((page_count or 1) - 1, 0)
|
||||
if not page_count:
|
||||
return None
|
||||
return page_count - 1
|
||||
return max((page_value or 1) - 1, 0)
|
||||
|
||||
|
||||
@@ -480,7 +500,7 @@ def _plan_template(
|
||||
page_idx = _resolve_page_idx(zone.page, page_count)
|
||||
extracted = None
|
||||
reason: ZoneOcrReason | None = ZoneOcrReason.MISSING_PAGE
|
||||
if page_idx in page_images:
|
||||
if page_idx is not None and page_idx in page_images:
|
||||
extraction = _extract_zone(
|
||||
page_images[page_idx],
|
||||
zone,
|
||||
@@ -852,18 +872,19 @@ def extract_zone_preview(
|
||||
is_image: bool,
|
||||
) -> ZoneExtraction:
|
||||
"""Extract a zone without writing its value to the document."""
|
||||
page_count = _effective_page_count(doc_path, page_count, is_image=is_image)
|
||||
page_idx = _resolve_page_idx(zone.page, page_count)
|
||||
settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(dir=settings.SCRATCH_DIR) as tmp_dir:
|
||||
tmp_path = Path(tmp_dir)
|
||||
page_images = _render_pages(
|
||||
doc_path,
|
||||
{page_idx},
|
||||
set() if page_idx is None else {page_idx},
|
||||
tmp_path,
|
||||
page_count,
|
||||
is_image=is_image,
|
||||
)
|
||||
if page_idx not in page_images:
|
||||
if page_idx is None or page_idx not in page_images:
|
||||
return ZoneExtraction()
|
||||
|
||||
return _extract_zone(
|
||||
|
||||
Reference in new issue
Block a user