amorfo77/paperless-ngx
1
0 Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2026-10-20 23:08:14 +00:00
Code Issues Projects Releases Wiki Activity
Files
a05afd654ca7643dcd18e59bb491ebef6dbb435a
paperless-ngx/src/paperless/tests/parsers/test_tesseract_parser.py
T
Trenton HandClaude Sonnet 5.5 a05afd654c Feature: Generate PDF thumbnails via poppler utils (#14420) 
* Chore: Generate PDF thumbnails with pdftoppm and Pillow instead of ImageMagick

Replace the ImageMagick convert based PDF thumbnail path with pdftoppm and
Pillow. Keeps thumbnails at 500px wide for small and odd sized pages, covers
the qpdf repair and double failure paths, removes the unused run_convert
helper, hardens thumbnail encoding, deprecates PAPERLESS_CONVERT_MEMORY_LIMIT
and PAPERLESS_CONVERT_TMPDIR, and corrects the related docs and CI.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

* Chore: Supersample the PDF thumbnail render 2x

Render the PDF page at twice the target size and downsample, with edge case
handling for small and odd sized pages, simplified helpers and tests, and
trimmed comments.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

* Fix: use a fractional thumbnail DPI so oversized pages render a bounded raster

* Deprecate the unused PAPERLESS_GS_BINARY setting too

* Warn at startup when pdftoppm is missing

* Resample PDF thumbnails once instead of twice with LANCZOS, still never enlarging

* Read PDF page size from pdfinfo so it matches pdftoppm

* Mark the thumbnail constants Final

* Comment each pdftoppm option
2026-10-20 11:22:42 -07:00

1359 lines
45 KiB
Python
Raw Blame History

"""
Tests for paperless.parsers.tesseract.RasterisedDocumentParser.
All tests use fixtures defined in conftest.py for parser lifecycle and
sample-file access. Settings-dependent tests mutate parser.settings
directly rather than going through the database.
"""
from __future__ import annotations
import re
import shutil
import unicodedata
from typing import TYPE_CHECKING
import pytest
from ocrmypdf import SubprocessOutputError
from PIL import Image
from documents.parsers import ParseError
from documents.parsers import rasterize_pdf_page_to_png
from paperless.models import ModeChoices
from paperless.parsers import ParserProtocol
from paperless.parsers.tesseract import RasterisedDocumentParser
from paperless.parsers.utils import is_tagged_pdf
if TYPE_CHECKING:
from pathlib import Path
from unittest.mock import MagicMock
from pytest_mock import MockerFixture
from paperless.tests.parsers.conftest import MakeTesseractParser
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def assert_ordered_substrings(content: str, strings: list[str]) -> None:
"""Assert all *strings* appear in *content* in the given order."""
indices: list[int] = []
for s in strings:
assert s in content, f"{s!r} not found in content"
indices.append(content.index(s))
assert indices == sorted(indices), f"Strings out of order in content: {strings}"
# ---------------------------------------------------------------------------
# Protocol compliance
# ---------------------------------------------------------------------------
class TestRasterisedDocumentParserProtocol:
"""Verify class-level protocol attributes and classmethods — no DB, no parser."""
def test_class_attributes_present(self) -> None:
assert RasterisedDocumentParser.name
assert RasterisedDocumentParser.version
assert RasterisedDocumentParser.author
assert RasterisedDocumentParser.url
def test_supported_mime_types_returns_dict(self) -> None:
mime_types = RasterisedDocumentParser.supported_mime_types()
assert isinstance(mime_types, dict)
for mime in (
"application/pdf",
"image/jpeg",
"image/png",
"image/tiff",
"image/gif",
"image/bmp",
"image/webp",
"image/heic",
):
assert mime in mime_types
@pytest.mark.parametrize(
("mime_type", "expected"),
[
pytest.param("application/pdf", 10, id="pdf"),
pytest.param("image/jpeg", 10, id="jpeg"),
pytest.param("image/png", 10, id="png"),
pytest.param("image/tiff", 10, id="tiff"),
pytest.param("image/gif", 10, id="gif"),
pytest.param("image/bmp", 10, id="bmp"),
pytest.param("image/webp", 10, id="webp"),
pytest.param("image/heic", 10, id="heic"),
pytest.param("text/plain", None, id="text-unsupported"),
pytest.param("application/msword", None, id="word-unsupported"),
],
)
def test_score(self, mime_type: str, expected: int | None) -> None:
assert RasterisedDocumentParser.score(mime_type, "file.pdf") == expected
def test_isinstance_satisfies_protocol(
self,
tesseract_parser: RasterisedDocumentParser,
) -> None:
assert isinstance(tesseract_parser, ParserProtocol)
def test_can_produce_archive_is_true(
self,
tesseract_parser: RasterisedDocumentParser,
) -> None:
assert tesseract_parser.can_produce_archive is True
def test_requires_pdf_rendition_is_false(
self,
tesseract_parser: RasterisedDocumentParser,
) -> None:
assert tesseract_parser.requires_pdf_rendition is False
# ---------------------------------------------------------------------------
# Lifecycle
# ---------------------------------------------------------------------------
class TestRasterisedDocumentParserLifecycle:
"""Context-manager cleanup — no DB."""
def test_tempdir_cleaned_up_on_exit(
self,
mocker: MockerFixture,
null_app_config: MagicMock,
) -> None:
mocker.patch(
"paperless.config.BaseConfig._get_config_instance",
return_value=null_app_config,
)
with RasterisedDocumentParser() as parser:
tempdir = parser.tempdir
assert tempdir.exists()
assert not tempdir.exists()
def test_tempdir_cleaned_up_after_exception(
self,
mocker: MockerFixture,
null_app_config: MagicMock,
) -> None:
mocker.patch(
"paperless.config.BaseConfig._get_config_instance",
return_value=null_app_config,
)
tempdir: Path | None = None
with pytest.raises(RuntimeError):
with RasterisedDocumentParser() as parser:
tempdir = parser.tempdir
raise RuntimeError("boom")
assert tempdir is not None and not tempdir.exists()
# ---------------------------------------------------------------------------
# Page count
# ---------------------------------------------------------------------------
class TestGetPageCount:
def test_single_page_pdf(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
assert (
tesseract_parser.get_page_count(
tesseract_samples_dir / "simple-digital.pdf",
"application/pdf",
)
== 1
)
def test_multi_page_pdf(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
assert (
tesseract_parser.get_page_count(
tesseract_samples_dir / "multi-page-mixed.pdf",
"application/pdf",
)
== 6
)
def test_password_protected_returns_none(
self,
mocker: MockerFixture,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
caplog,
) -> None:
"""
GIVEN:
- pikepdf raises when opening a protected PDF
WHEN:
- Page count is requested
THEN:
- Returns None and logs a warning
"""
mocker.patch("pikepdf.Pdf.open", side_effect=Exception("password required"))
import logging
with caplog.at_level(logging.WARNING):
page_count = tesseract_parser.get_page_count(
tesseract_samples_dir / "simple-digital.pdf",
"application/pdf",
)
assert page_count is None
assert any(
"Unable to determine PDF page count" in r.message for r in caplog.records
)
def test_non_pdf_returns_none(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
assert (
tesseract_parser.get_page_count(
tesseract_samples_dir / "simple.png",
"image/png",
)
is None
)
# ---------------------------------------------------------------------------
# DPI helpers
# ---------------------------------------------------------------------------
class TestDpiHelpers:
@pytest.mark.parametrize(
("filename", "expected_dpi"),
[
pytest.param("simple-no-dpi.png", None, id="no-dpi"),
pytest.param("simple.png", 72, id="with-dpi"),
],
)
def test_get_dpi(
self,
filename: str,
expected_dpi: int | None,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
assert (
tesseract_parser.get_dpi(str(tesseract_samples_dir / filename))
== expected_dpi
)
def test_calculate_a4_dpi(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
assert (
tesseract_parser.calculate_a4_dpi(
str(tesseract_samples_dir / "simple-no-dpi.png"),
)
== 62
)
# ---------------------------------------------------------------------------
# Thumbnail
# ---------------------------------------------------------------------------
class TestGetThumbnail:
def test_thumbnail_is_file(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
thumb = tesseract_parser.get_thumbnail(
tesseract_samples_dir / "simple-digital.pdf",
"application/pdf",
)
assert thumb.is_file()
@pytest.mark.parametrize(
("filename", "expected_height"),
[
pytest.param("simple-digital.pdf", 647, id="portrait-letter"),
pytest.param("rotated.pdf", 386, id="landscape"),
],
)
def test_thumbnail_is_correct_format_and_size(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
filename: str,
expected_height: int,
) -> None:
"""
GIVEN:
- A portrait or landscape PDF
WHEN:
- A thumbnail is generated
THEN:
- A 500px wide WebP keeping the page's aspect ratio
"""
thumb = tesseract_parser.get_thumbnail(
tesseract_samples_dir / filename,
"application/pdf",
)
with Image.open(thumb) as im:
assert im.format == "WEBP"
assert im.width == 500
assert im.height == pytest.approx(expected_height, abs=2)
def test_thumbnail_fallback_on_pdftoppm_error(
self,
mocker: MockerFixture,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
"""
GIVEN:
- Rasterizing the original PDF fails
WHEN:
- A thumbnail is generated
THEN:
- The PDF is repaired with qpdf and a real thumbnail is rendered
"""
original = tesseract_samples_dir / "simple-digital.pdf"
def _fail_on_original(in_path: Path, out_path: Path, **kwargs) -> None:
if in_path == original:
raise ParseError("Does not compute.")
rasterize_pdf_page_to_png(in_path, out_path, **kwargs)
rasterize = mocker.patch(
"documents.parsers.rasterize_pdf_page_to_png",
side_effect=_fail_on_original,
)
thumb = tesseract_parser.get_thumbnail(original, "application/pdf")
assert rasterize.call_count == 2
assert thumb.is_file()
assert thumb.name == "convert_qpdf.webp"
with Image.open(thumb) as im:
assert im.format == "WEBP"
assert im.width == 500
def test_thumbnail_encrypted_pdf(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
thumb = tesseract_parser.get_thumbnail(
tesseract_samples_dir / "encrypted.pdf",
"application/pdf",
)
assert thumb.is_file()
# ---------------------------------------------------------------------------
# extract_text
# ---------------------------------------------------------------------------
class TestExtractText:
def test_extract_text_from_digital_pdf(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
text = tesseract_parser.extract_text(
None,
tesseract_samples_dir / "simple-digital.pdf",
)
assert text is not None
assert "This is a test document." in text.strip()
# ---------------------------------------------------------------------------
# Parse — PDF modes
# ---------------------------------------------------------------------------
class TestParsePdf:
def test_simple_digital_creates_archive(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
"""
GIVEN:
- Multi-page digital PDF with sufficient text layer
- Default settings (mode=auto, produce_archive=True)
WHEN:
- Document is parsed
THEN:
- Archive is created (AUTO mode + text present + produce_archive=True
→ PDF/A conversion via skip_text)
- Text is extracted
"""
tesseract_parser.parse(
tesseract_samples_dir / "multi-page-digital.pdf",
"application/pdf",
)
assert tesseract_parser.archive_path is not None
assert tesseract_parser.archive_path.is_file()
_text = tesseract_parser.get_text()
assert _text is not None
assert_ordered_substrings(
_text.lower(),
["page 1", "page 2", "page 3"],
)
def test_with_form_default(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
tesseract_parser.parse(
tesseract_samples_dir / "with-form.pdf",
"application/pdf",
)
assert tesseract_parser.archive_path is not None
assert tesseract_parser.archive_path.is_file()
assert_ordered_substrings(
tesseract_parser.get_text(),
["Please enter your name in here:", "This is a PDF document with a form."],
)
def test_with_form_redo_no_archive_when_not_requested(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
tesseract_parser.settings.mode = ModeChoices.REDO
tesseract_parser.parse(
tesseract_samples_dir / "with-form.pdf",
"application/pdf",
produce_archive=False,
)
assert tesseract_parser.archive_path is None
assert_ordered_substrings(
tesseract_parser.get_text(),
["Please enter your name in here:", "This is a PDF document with a form."],
)
def test_with_form_force(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
tesseract_parser.settings.mode = ModeChoices.FORCE
tesseract_parser.parse(
tesseract_samples_dir / "with-form.pdf",
"application/pdf",
)
assert_ordered_substrings(
tesseract_parser.get_text(),
["Please enter your name in here:", "This is a PDF document with a form."],
)
def test_signed_skip_mode_no_archive(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
tesseract_parser.settings.mode = ModeChoices.AUTO
tesseract_parser.parse(tesseract_samples_dir / "signed.pdf", "application/pdf")
assert tesseract_parser.archive_path is None
assert_ordered_substrings(
tesseract_parser.get_text(),
[
"This is a digitally signed PDF, created with Acrobat Pro for the Paperless project to enable",
"automated testing of signed/encrypted PDFs",
],
)
def test_encrypted_skip_mode_empty_text(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
tesseract_parser.settings.mode = ModeChoices.AUTO
tesseract_parser.parse(
tesseract_samples_dir / "encrypted.pdf",
"application/pdf",
)
assert tesseract_parser.archive_path is None
assert tesseract_parser.get_text() == ""
def test_gs_rendering_error_raises_parse_error(
self,
mocker: MockerFixture,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
mocker.patch(
"ocrmypdf.ocr",
side_effect=SubprocessOutputError("Ghostscript PDF/A rendering failed"),
)
with pytest.raises(ParseError):
tesseract_parser.parse(
tesseract_samples_dir / "simple-digital.pdf",
"application/pdf",
)
# ---------------------------------------------------------------------------
# Parse — images
# ---------------------------------------------------------------------------
class TestParseImages:
def test_simple_png(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
tesseract_parser.parse(tesseract_samples_dir / "simple.png", "image/png")
assert tesseract_parser.archive_path is not None
assert tesseract_parser.archive_path.is_file()
assert_ordered_substrings(
tesseract_parser.get_text(),
["This is a test document."],
)
def test_simple_alpha_png(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
tmp_path: Path,
) -> None:
dest = tmp_path / "simple-alpha.png"
shutil.copy(tesseract_samples_dir / "simple-alpha.png", dest)
tesseract_parser.parse(dest, "image/png")
assert tesseract_parser.archive_path is not None
assert tesseract_parser.archive_path.is_file()
assert_ordered_substrings(
tesseract_parser.get_text(),
["This is a test document."],
)
def test_no_dpi_with_default_dpi(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
tesseract_parser.settings.image_dpi = 72
tesseract_parser.parse(tesseract_samples_dir / "simple-no-dpi.png", "image/png")
assert tesseract_parser.archive_path is not None
assert tesseract_parser.archive_path.is_file()
_text = tesseract_parser.get_text()
assert _text is not None
assert "this is a test document." in _text.lower()
def test_no_dpi_no_fallback_raises(
self,
mocker: MockerFixture,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
mocker.patch.object(tesseract_parser, "calculate_a4_dpi", return_value=None)
with pytest.raises(ParseError):
tesseract_parser.parse(
tesseract_samples_dir / "simple-no-dpi.png",
"image/png",
)
# ---------------------------------------------------------------------------
# Parse — multi-page PDF
# ---------------------------------------------------------------------------
class TestParseMultiPage:
def test_multi_page_digital(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
tesseract_parser.parse(
tesseract_samples_dir / "multi-page-digital.pdf",
"application/pdf",
)
assert tesseract_parser.archive_path is not None
assert tesseract_parser.archive_path.is_file()
_text = tesseract_parser.get_text()
assert _text is not None
assert_ordered_substrings(
_text.lower(),
["page 1", "page 2", "page 3"],
)
@pytest.mark.parametrize(
"mode",
[
pytest.param("auto", id="auto"),
pytest.param("redo", id="redo"),
pytest.param("force", id="force"),
],
)
def test_multi_page_digital_pages_2(
self,
mode: str,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
tesseract_parser.settings.pages = 2
tesseract_parser.settings.mode = mode
tesseract_parser.parse(
tesseract_samples_dir / "multi-page-digital.pdf",
"application/pdf",
)
assert tesseract_parser.archive_path is not None
_text = tesseract_parser.get_text()
assert _text is not None
assert_ordered_substrings(
_text.lower(),
["page 1", "page 2", "page 3"],
)
def test_multi_page_images_skip(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
tesseract_parser.settings.mode = ModeChoices.AUTO
tesseract_parser.parse(
tesseract_samples_dir / "multi-page-images.pdf",
"application/pdf",
)
assert tesseract_parser.archive_path is not None
_text = tesseract_parser.get_text()
assert _text is not None
assert_ordered_substrings(
_text.lower(),
["page 1", "page 2", "page 3"],
)
def test_multi_page_images_redo_pages_2(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
"""
GIVEN:
- File with image-only pages
- OCR of only pages 1 and 2 requested
- Mode: redo
WHEN:
- Document is parsed
THEN:
- Pages 1 and 2 extracted; page 3 absent
"""
tesseract_parser.settings.pages = 2
tesseract_parser.settings.mode = ModeChoices.REDO
tesseract_parser.parse(
tesseract_samples_dir / "multi-page-images.pdf",
"application/pdf",
)
assert tesseract_parser.archive_path is not None
text = tesseract_parser.get_text()
assert text is not None
text = text.lower()
assert_ordered_substrings(text, ["page 1", "page 2"])
assert "page 3" not in text
def test_multi_page_images_force_page_1(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
"""
GIVEN:
- File with image-only pages
- Only page 1 requested
- Mode: force
WHEN:
- Document is parsed
THEN:
- Only page 1 extracted
"""
tesseract_parser.settings.pages = 1
tesseract_parser.settings.mode = ModeChoices.FORCE
tesseract_parser.parse(
tesseract_samples_dir / "multi-page-images.pdf",
"application/pdf",
)
assert tesseract_parser.archive_path is not None
text = tesseract_parser.get_text()
assert text is not None
text = text.lower()
assert "page 1" in text
assert "page 2" not in text
assert "page 3" not in text
def test_multi_page_tiff(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
"""
GIVEN:
- Multi-page TIFF image
WHEN:
- Image is parsed
THEN:
- Text from all pages extracted
"""
tesseract_parser.parse(
tesseract_samples_dir / "multi-page-images.tiff",
"image/tiff",
)
assert tesseract_parser.archive_path is not None
_text = tesseract_parser.get_text()
assert _text is not None
assert_ordered_substrings(
_text.lower(),
["page 1", "page 2", "page 3"],
)
def test_multi_page_tiff_alpha(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
tmp_path: Path,
) -> None:
"""
GIVEN:
- Multi-page TIFF with alpha channel
WHEN:
- Image is parsed
THEN:
- Text from all pages extracted
"""
dest = tmp_path / "alpha.tiff"
shutil.copy(tesseract_samples_dir / "multi-page-images-alpha.tiff", dest)
tesseract_parser.parse(dest, "image/tiff")
assert tesseract_parser.archive_path is not None
_text = tesseract_parser.get_text()
assert _text is not None
assert_ordered_substrings(
_text.lower(),
["page 1", "page 2", "page 3"],
)
def test_multi_page_tiff_alpha_srgb(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
tmp_path: Path,
) -> None:
"""
GIVEN:
- Multi-page TIFF with alpha channel and sRGB colorspace
WHEN:
- Image is parsed
THEN:
- Text from all pages extracted
"""
dest = tmp_path / "alpha-rgb.tiff"
shutil.copy(tesseract_samples_dir / "multi-page-images-alpha-rgb.tiff", dest)
tesseract_parser.parse(dest, "image/tiff")
assert tesseract_parser.archive_path is not None
_text = tesseract_parser.get_text()
assert _text is not None
assert_ordered_substrings(
_text.lower(),
["page 1", "page 2", "page 3"],
)
# ---------------------------------------------------------------------------
# Parse — skip_noarchive / skip_archive_file
# ---------------------------------------------------------------------------
class TestSkipArchive:
def test_skip_noarchive_with_text_layer(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
"""
GIVEN:
- File with existing text layer
- Mode: auto, produce_archive=False
WHEN:
- Document is parsed
THEN:
- Text extracted from original; no archive created (text exists +
produce_archive=False skips OCRmyPDF entirely)
"""
tesseract_parser.settings.mode = ModeChoices.AUTO
tesseract_parser.parse(
tesseract_samples_dir / "multi-page-digital.pdf",
"application/pdf",
produce_archive=False,
)
assert tesseract_parser.archive_path is None
_text = tesseract_parser.get_text()
assert _text is not None
assert_ordered_substrings(
_text.lower(),
["page 1", "page 2", "page 3"],
)
def test_skip_noarchive_image_only_creates_archive(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
"""
GIVEN:
- File with image-only pages (no text layer)
- Mode: auto, skip_archive_file: auto
WHEN:
- Document is parsed
THEN:
- Text extracted; archive created (OCR needed, no existing text)
"""
tesseract_parser.settings.mode = ModeChoices.AUTO
tesseract_parser.parse(
tesseract_samples_dir / "multi-page-images.pdf",
"application/pdf",
)
assert tesseract_parser.archive_path is not None
_text = tesseract_parser.get_text()
assert _text is not None
assert_ordered_substrings(
_text.lower(),
["page 1", "page 2", "page 3"],
)
@pytest.mark.parametrize(
("produce_archive", "filename", "expect_archive"),
[
pytest.param(
True,
"multi-page-digital.pdf",
True,
id="produce-archive-with-text",
),
pytest.param(
True,
"multi-page-images.pdf",
True,
id="produce-archive-no-text",
),
pytest.param(
False,
"multi-page-digital.pdf",
False,
id="no-archive-with-text-layer",
),
pytest.param(
False,
"multi-page-images.pdf",
False,
id="no-archive-no-text-layer",
),
],
)
def test_produce_archive_flag(
self,
produce_archive: bool, # noqa: FBT001
filename: str,
expect_archive: bool, # noqa: FBT001
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
"""
GIVEN:
- Various PDFs (with and without text layers)
- produce_archive flag set to True or False
WHEN:
- Document is parsed
THEN:
- archive_path is set if and only if produce_archive=True
- Text is always extracted
"""
tesseract_parser.settings.mode = ModeChoices.AUTO
tesseract_parser.parse(
tesseract_samples_dir / filename,
"application/pdf",
produce_archive=produce_archive,
)
text = tesseract_parser.get_text()
assert text is not None
text = text.lower()
assert_ordered_substrings(text, ["page 1", "page 2", "page 3"])
if expect_archive:
assert tesseract_parser.archive_path is not None
else:
assert tesseract_parser.archive_path is None
def test_tagged_pdf_skips_ocr_in_auto_mode(
self,
mocker: MockerFixture,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
"""
GIVEN:
- A tagged PDF (e.g. exported from Word, /MarkInfo /Marked true)
- Mode: auto, produce_archive=False
WHEN:
- Document is parsed
THEN:
- OCRmyPDF is not invoked (tagged ⇒ original_has_text=True)
- Text is extracted from the original via pdftotext
- No archive is produced
"""
tesseract_parser.settings.mode = ModeChoices.AUTO
mock_ocr = mocker.patch("ocrmypdf.ocr")
tesseract_parser.parse(
tesseract_samples_dir / "simple-digital.pdf",
"application/pdf",
produce_archive=False,
)
mock_ocr.assert_not_called()
assert tesseract_parser.archive_path is None
assert tesseract_parser.get_text()
def test_tagged_pdf_without_text_does_not_skip_ocr(
self,
mocker: MockerFixture,
tesseract_parser: RasterisedDocumentParser,
tagged_no_text_pdf_file: Path,
) -> None:
"""
GIVEN:
- A real PDF that reports itself as tagged (/MarkInfo /Marked
true) but whose only pdftotext output is layout padding (a
lone form-feed byte), not real text (see GitHub issue #13387,
originally reported against #13349's tagged-PDF handling)
- Mode: auto, produce_archive=False
WHEN:
- Document is parsed
THEN:
- The tag alone is not trusted as "has text"; OCRmyPDF still runs
"""
assert is_tagged_pdf(tagged_no_text_pdf_file) is True
tesseract_parser.settings.mode = ModeChoices.AUTO
mock_ocr = mocker.patch("ocrmypdf.ocr")
tesseract_parser.parse(
tagged_no_text_pdf_file,
"application/pdf",
produce_archive=False,
)
mock_ocr.assert_called()
def test_tagged_pdf_produces_pdfa_archive_without_ocr(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
"""
GIVEN:
- A tagged PDF (e.g. exported from Word, /MarkInfo /Marked true)
- Mode: auto, produce_archive=True
WHEN:
- Document is parsed
THEN:
- OCRmyPDF runs with skip_text (PDF/A conversion only, no OCR)
- Archive is produced
- Text is preserved from the original
"""
tesseract_parser.settings.mode = ModeChoices.AUTO
tesseract_parser.parse(
tesseract_samples_dir / "simple-digital.pdf",
"application/pdf",
produce_archive=True,
)
assert tesseract_parser.archive_path is not None
assert tesseract_parser.get_text()
# ---------------------------------------------------------------------------
# Parse — mixed pages / sidecar
# ---------------------------------------------------------------------------
class TestParseMixed:
def test_multi_page_mixed_skip_mode(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
"""
GIVEN:
- File with text in some pages (image) and some pages (digital)
- Mode: auto (skip_text), skip_archive_file: always
WHEN:
- Document is parsed
THEN:
- All pages extracted; archive created; sidecar notes skipped pages
"""
tesseract_parser.settings.mode = ModeChoices.AUTO
tesseract_parser.parse(
tesseract_samples_dir / "multi-page-mixed.pdf",
"application/pdf",
)
assert tesseract_parser.archive_path is not None
assert tesseract_parser.archive_path.is_file()
_text = tesseract_parser.get_text()
assert _text is not None
assert_ordered_substrings(
_text.lower(),
["page 1", "page 2", "page 3", "page 4", "page 5", "page 6"],
)
sidecar = (tesseract_parser.tempdir / "sidecar.txt").read_text()
assert "[OCR skipped on page(s) 4-6]" in sidecar
def test_single_page_mixed_redo_mode(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
"""
GIVEN:
- Single page with both text and image content
- Mode: redo
WHEN:
- Document is parsed
THEN:
- Both text layer and image text extracted; archive created
"""
tesseract_parser.settings.mode = ModeChoices.REDO
tesseract_parser.parse(
tesseract_samples_dir / "single-page-mixed.pdf",
"application/pdf",
)
assert tesseract_parser.archive_path is not None
assert tesseract_parser.archive_path.is_file()
_text = tesseract_parser.get_text()
assert _text is not None
assert_ordered_substrings(
_text.lower(),
[
"this is some normal text, present on page 1 of the document.",
"this is some text, but in an image, also on page 1.",
"this is further text on page 1.",
],
)
sidecar = (tesseract_parser.tempdir / "sidecar.txt").read_text().lower()
assert "this is some text, but in an image, also on page 1." in sidecar
assert (
"this is some normal text, present on page 1 of the document."
not in sidecar
)
def test_multi_page_mixed_skip_noarchive(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
"""
GIVEN:
- File with mixed pages (some with text, some image-only)
- Mode: auto, produce_archive=False
WHEN:
- Document is parsed
THEN:
- No archive created (produce_archive=False); text from text layer present
"""
tesseract_parser.settings.mode = ModeChoices.AUTO
tesseract_parser.parse(
tesseract_samples_dir / "multi-page-mixed.pdf",
"application/pdf",
produce_archive=False,
)
assert tesseract_parser.archive_path is None
_text = tesseract_parser.get_text()
assert _text is not None
assert_ordered_substrings(
_text.lower(),
["page 4", "page 5", "page 6"],
)
# ---------------------------------------------------------------------------
# Parse — rotation
# ---------------------------------------------------------------------------
class TestParseRotate:
def test_rotate_auto_mode(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
tesseract_parser.settings.mode = ModeChoices.AUTO
tesseract_parser.settings.rotate = True
tesseract_parser.parse(tesseract_samples_dir / "rotated.pdf", "application/pdf")
assert_ordered_substrings(
tesseract_parser.get_text(),
[
"This is the text that appears on the first page. It\u2019s a lot of text.",
"Even if the pages are rotated, OCRmyPDF still gets the job done.",
"This is a really weird file with lots of nonsense text.",
"If you read this, it\u2019s your own fault. Also check your screen orientation.",
],
)
# ---------------------------------------------------------------------------
# Parse — RTL
# ---------------------------------------------------------------------------
class TestParseRtl:
def test_rtl_language_detected(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
"""
GIVEN:
- PDF with RTL Arabic text in its text layer (short: 18 chars)
- mode=off, produce_archive=True: PDF/A conversion via skip_text, no OCR engine
WHEN:
- Document is parsed
THEN:
- Arabic content is extracted from the PDF text layer (normalised for bidi)
Note: The RTL PDF has a short text layer (< VALID_TEXT_LENGTH=50) so AUTO mode
would attempt full OCR, which fails due to PriorOcrFoundError and falls back to
force-ocr with English Tesseract (producing garbage). Using mode="off" forces
skip_text=True so the Arabic text layer is preserved through PDF/A conversion.
"""
tesseract_parser.settings.mode = ModeChoices.OFF
tesseract_parser.parse(
tesseract_samples_dir / "rtl-test.pdf",
"application/pdf",
)
_text = tesseract_parser.get_text()
assert _text is not None
normalised = "".join(
ch
for ch in unicodedata.normalize("NFKC", _text)
if unicodedata.category(ch) != "Cf" and not ch.isspace()
)
assert "ةرازو" in normalised
# pdftotext uses Arabic Yeh (U+064A) where ocrmypdf used Farsi Yeh (U+06CC)
assert any(token in normalised for token in ("ةیلخادلا", "الاخليد", "ةيلخادال"))
# ---------------------------------------------------------------------------
# Parse — OCRmyPDF parameters
# ---------------------------------------------------------------------------
@pytest.mark.django_db
class TestOcrmypdfParameters:
"""Tests that inspect the dict passed to ocrmypdf.
These create parsers inline via make_tesseract_parser with specific
Django settings overrides so OcrConfig picks them up at construction time.
"""
def test_basic_parameter_mapping(
self,
make_tesseract_parser: MakeTesseractParser,
) -> None:
with make_tesseract_parser() as parser:
params = parser.construct_ocrmypdf_parameters(
input_file="input.pdf",
output_file="output.pdf",
sidecar_file="sidecar.txt",
mime_type="application/pdf",
safe_fallback=False,
)
assert params["input_file_or_options"] == "input.pdf"
assert params["output_file"] == "output.pdf"
assert params["sidecar"] == "sidecar.txt"
@pytest.mark.parametrize(
("ocr_clean", "expected_clean", "expected_clean_final"),
[
pytest.param("none", False, False, id="clean-none"),
pytest.param("clean", True, False, id="clean-clean"),
],
)
def test_clean_option(
self,
ocr_clean: str,
*,
expected_clean: bool,
expected_clean_final: bool,
make_tesseract_parser: MakeTesseractParser,
) -> None:
with make_tesseract_parser(OCR_CLEAN=ocr_clean) as parser:
params = parser.construct_ocrmypdf_parameters("", "", "", "")
assert ("clean" in params) == expected_clean
assert ("clean_final" in params) == expected_clean_final
def test_clean_final_auto_mode(
self,
make_tesseract_parser: MakeTesseractParser,
) -> None:
with make_tesseract_parser(OCR_CLEAN="clean-final", OCR_MODE="auto") as parser:
params = parser.construct_ocrmypdf_parameters("", "", "", "")
assert params["clean_final"] is True
assert "clean" not in params
def test_clean_final_redo_mode_falls_back_to_clean(
self,
make_tesseract_parser: MakeTesseractParser,
) -> None:
with make_tesseract_parser(OCR_CLEAN="clean-final", OCR_MODE="redo") as parser:
params = parser.construct_ocrmypdf_parameters("", "", "", "")
assert params["clean"] is True
assert "clean_final" not in params
@pytest.mark.parametrize(
("ocr_mode", "ocr_deskew", "expect_deskew"),
[
pytest.param("auto", True, True, id="auto-deskew-on"),
pytest.param("redo", True, False, id="redo-deskew-off"),
pytest.param("auto", False, False, id="auto-no-deskew"),
],
)
def test_deskew_option(
self,
ocr_mode: str,
*,
ocr_deskew: bool,
expect_deskew: bool,
make_tesseract_parser: MakeTesseractParser,
) -> None:
with make_tesseract_parser(OCR_MODE=ocr_mode, OCR_DESKEW=ocr_deskew) as parser:
params = parser.construct_ocrmypdf_parameters("", "", "", "")
assert ("deskew" in params) == expect_deskew
def test_max_image_pixels_positive(
self,
make_tesseract_parser: MakeTesseractParser,
) -> None:
with make_tesseract_parser(OCR_MAX_IMAGE_PIXELS=1_000_001.0) as parser:
params = parser.construct_ocrmypdf_parameters("", "", "", "")
assert "max_image_mpixels" in params
assert abs(params["max_image_mpixels"] - 1.0) < 1e-4
def test_max_image_pixels_negative_omitted(
self,
make_tesseract_parser: MakeTesseractParser,
) -> None:
with make_tesseract_parser(OCR_MAX_IMAGE_PIXELS=-1_000_001.0) as parser:
params = parser.construct_ocrmypdf_parameters("", "", "", "")
assert "max_image_mpixels" not in params
# ---------------------------------------------------------------------------
# Parse — file type matrix
# ---------------------------------------------------------------------------
class TestParserFileTypes:
@pytest.mark.parametrize(
("filename", "mime_type"),
[
pytest.param("simple.bmp", "image/bmp", id="bmp"),
pytest.param("simple.jpg", "image/jpeg", id="jpeg"),
pytest.param("simple.tif", "image/tiff", id="tiff"),
],
)
def test_simple_image_contains_test_text(
self,
filename: str,
mime_type: str,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
tesseract_parser.parse(tesseract_samples_dir / filename, mime_type)
assert tesseract_parser.archive_path is not None
assert tesseract_parser.archive_path.is_file()
_text = tesseract_parser.get_text()
assert _text is not None
assert "this is a test document" in _text.lower()
def test_heic(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
tesseract_parser.parse(tesseract_samples_dir / "simple.heic", "image/heic")
assert tesseract_parser.archive_path is not None
_text = tesseract_parser.get_text()
assert _text is not None
assert "pizza" in _text.lower()
def test_gif_with_explicit_dpi(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
tesseract_parser.settings.image_dpi = 200
tesseract_parser.parse(tesseract_samples_dir / "simple.gif", "image/gif")
assert tesseract_parser.archive_path is not None
_text = tesseract_parser.get_text()
assert _text is not None
assert "this is a test document" in _text.lower()
def test_webp_with_explicit_dpi(
self,
tesseract_parser: RasterisedDocumentParser,
tesseract_samples_dir: Path,
) -> None:
tesseract_parser.settings.image_dpi = 72
tesseract_parser.parse(tesseract_samples_dir / "document.webp", "image/webp")
assert tesseract_parser.archive_path is not None
_text = tesseract_parser.get_text()
assert _text is not None
assert re.search(
r"this is a ?webp document, created 11/14/2022\.",
_text.lower(),
)
# ---------------------------------------------------------------------------
# Registry
# ---------------------------------------------------------------------------
# Remote ocr config from ApplicationConfiguration needs DB access
@pytest.mark.django_db
class TestRasterisedDocumentParserRegistry:
def test_registered_in_defaults(self) -> None:
from paperless.parsers.registry import ParserRegistry
registry = ParserRegistry()
registry.register_defaults()
assert RasterisedDocumentParser in registry._builtins
@pytest.mark.parametrize(
("mime_type", "filename"),
[
pytest.param("application/pdf", "doc.pdf", id="pdf"),
pytest.param("image/png", "image.png", id="png"),
pytest.param("image/jpeg", "photo.jpg", id="jpeg"),
pytest.param("image/tiff", "scan.tif", id="tiff"),
],
)
def test_get_parser_for_supported_mime(
self,
mime_type: str,
filename: str,
) -> None:
from paperless.parsers.registry import get_parser_registry
registry = get_parser_registry()
assert (
registry.get_parser_for_file(mime_type, filename)
is RasterisedDocumentParser
)
Reference in new issue View Git Blame Copy Permalink