diff --git a/pyproject.toml b/pyproject.toml index d41a918c0..6bbdf0f48 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -248,9 +248,7 @@ lint.per-file-ignores."docker/wait-for-redis.py" = [ lint.per-file-ignores."src/documents/models.py" = [ "SIM115", ] -lint.per-file-ignores."src/paperless_tesseract/tests/test_parser.py" = [ - "RUF001", -] + lint.isort.force-single-line = true [tool.codespell] diff --git a/src/documents/consumer.py b/src/documents/consumer.py index ba2bba473..81d9eb456 100644 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -54,6 +54,7 @@ from documents.utils import run_subprocess from paperless.parsers import ParserContext from paperless.parsers.mail import MailDocumentParser from paperless.parsers.remote import RemoteDocumentParser +from paperless.parsers.tesseract import RasterisedDocumentParser from paperless.parsers.text import TextDocumentParser from paperless.parsers.tika import TikaDocumentParser @@ -74,6 +75,7 @@ def _parser_cleanup(parser: DocumentParser) -> None: parser, ( MailDocumentParser, + RasterisedDocumentParser, RemoteDocumentParser, TextDocumentParser, TikaDocumentParser, @@ -463,6 +465,7 @@ class ConsumerPlugin( document_parser, ( MailDocumentParser, + RasterisedDocumentParser, RemoteDocumentParser, TextDocumentParser, TikaDocumentParser, diff --git a/src/documents/management/commands/document_thumbnails.py b/src/documents/management/commands/document_thumbnails.py index e4ae88766..1756f8754 100644 --- a/src/documents/management/commands/document_thumbnails.py +++ b/src/documents/management/commands/document_thumbnails.py @@ -4,6 +4,11 @@ import shutil from documents.management.commands.base import PaperlessCommand from documents.models import Document from documents.parsers import get_parser_class_for_mime_type +from paperless.parsers.mail import MailDocumentParser +from paperless.parsers.remote import RemoteDocumentParser +from paperless.parsers.tesseract import RasterisedDocumentParser +from paperless.parsers.text import TextDocumentParser +from paperless.parsers.tika import TikaDocumentParser logger = logging.getLogger("paperless.management.thumbnails") @@ -22,16 +27,38 @@ def _process_document(doc_id: int) -> None: parser = parser_class(logging_group=None) + parser_is_new_style = isinstance( + parser, + ( + MailDocumentParser, + RasterisedDocumentParser, + RemoteDocumentParser, + TextDocumentParser, + TikaDocumentParser, + ), + ) + + # TODO(stumpylog): Remove branch in the future when all parsers use new protocol + if parser_is_new_style: + parser.__enter__() + try: - thumb = parser.get_thumbnail( - document.source_path, - document.mime_type, - document.get_public_filename(), - ) + # TODO(stumpylog): Remove branch in the future when all parsers use new protocol + if parser_is_new_style: + thumb = parser.get_thumbnail(document.source_path, document.mime_type) + else: + thumb = parser.get_thumbnail( + document.source_path, + document.mime_type, + document.get_public_filename(), + ) shutil.move(thumb, document.thumbnail_path) finally: # TODO(stumpylog): Cleanup once all parsers are handled - parser.cleanup() + if parser_is_new_style: + parser.__exit__(None, None, None) + else: + parser.cleanup() class Command(PaperlessCommand): diff --git a/src/documents/tasks.py b/src/documents/tasks.py index 947da878f..a8ca0cc5f 100644 --- a/src/documents/tasks.py +++ b/src/documents/tasks.py @@ -68,6 +68,7 @@ from paperless.config import AIConfig from paperless.parsers import ParserContext from paperless.parsers.mail import MailDocumentParser from paperless.parsers.remote import RemoteDocumentParser +from paperless.parsers.tesseract import RasterisedDocumentParser from paperless.parsers.text import TextDocumentParser from paperless.parsers.tika import TikaDocumentParser from paperless_ai.indexing import llm_index_add_or_update_document @@ -326,6 +327,7 @@ def update_document_content_maybe_archive_file(document_id) -> None: parser, ( MailDocumentParser, + RasterisedDocumentParser, RemoteDocumentParser, TextDocumentParser, TikaDocumentParser, @@ -440,7 +442,13 @@ def update_document_content_maybe_archive_file(document_id) -> None: # TODO(stumpylog): Remove branch in the future when all parsers use new protocol if isinstance( parser, - (MailDocumentParser, TextDocumentParser, TikaDocumentParser), + ( + MailDocumentParser, + RasterisedDocumentParser, + RemoteDocumentParser, + TextDocumentParser, + TikaDocumentParser, + ), ): parser.__exit__(None, None, None) else: diff --git a/src/documents/tests/test_parsers.py b/src/documents/tests/test_parsers.py index 7be6ad20d..5ea1b361e 100644 --- a/src/documents/tests/test_parsers.py +++ b/src/documents/tests/test_parsers.py @@ -9,9 +9,9 @@ from documents.parsers import get_default_file_extension from documents.parsers import get_parser_class_for_mime_type from documents.parsers import get_supported_file_extensions from documents.parsers import is_file_ext_supported +from paperless.parsers.tesseract import RasterisedDocumentParser from paperless.parsers.text import TextDocumentParser from paperless.parsers.tika import TikaDocumentParser -from paperless_tesseract.parsers import RasterisedDocumentParser class TestParserDiscovery(TestCase): diff --git a/src/paperless/parsers/registry.py b/src/paperless/parsers/registry.py index dc227ce7a..7effe554f 100644 --- a/src/paperless/parsers/registry.py +++ b/src/paperless/parsers/registry.py @@ -195,6 +195,7 @@ class ParserRegistry: """ from paperless.parsers.mail import MailDocumentParser from paperless.parsers.remote import RemoteDocumentParser + from paperless.parsers.tesseract import RasterisedDocumentParser from paperless.parsers.text import TextDocumentParser from paperless.parsers.tika import TikaDocumentParser @@ -202,6 +203,7 @@ class ParserRegistry: self.register_builtin(RemoteDocumentParser) self.register_builtin(TikaDocumentParser) self.register_builtin(MailDocumentParser) + self.register_builtin(RasterisedDocumentParser) # ------------------------------------------------------------------ # Discovery diff --git a/src/paperless_tesseract/parsers.py b/src/paperless/parsers/tesseract.py similarity index 72% rename from src/paperless_tesseract/parsers.py rename to src/paperless/parsers/tesseract.py index 73532caa0..99cff36aa 100644 --- a/src/paperless_tesseract/parsers.py +++ b/src/paperless/parsers/tesseract.py @@ -1,13 +1,18 @@ +from __future__ import annotations + +import logging import os import re +import shutil import tempfile from pathlib import Path from typing import TYPE_CHECKING +from typing import Any +from typing import Self from django.conf import settings from PIL import Image -from documents.parsers import DocumentParser from documents.parsers import ParseError from documents.parsers import make_thumbnail_from_pdf from documents.utils import maybe_override_pixel_limit @@ -16,6 +21,28 @@ from paperless.config import OcrConfig from paperless.models import ArchiveFileChoices from paperless.models import CleanChoices from paperless.models import ModeChoices +from paperless.parsers.utils import read_file_handle_unicode_errors +from paperless.version import __full_version_str__ + +if TYPE_CHECKING: + import datetime + from types import TracebackType + + from paperless.parsers import MetadataEntry + from paperless.parsers import ParserContext + +logger = logging.getLogger("paperless.parsing.tesseract") + +_SUPPORTED_MIME_TYPES: dict[str, str] = { + "application/pdf": ".pdf", + "image/jpeg": ".jpg", + "image/png": ".png", + "image/tiff": ".tif", + "image/gif": ".gif", + "image/bmp": ".bmp", + "image/webp": ".webp", + "image/heic": ".heic", +} class NoTextFoundException(Exception): @@ -26,81 +53,125 @@ class RtlLanguageException(Exception): pass -class RasterisedDocumentParser(DocumentParser): +class RasterisedDocumentParser: """ This parser uses Tesseract to try and get some text out of a rasterised image, whether it's a PDF, or other graphical format (JPEG, TIFF, etc.) """ - logging_name = "paperless.parsing.tesseract" + name: str = "Paperless-ngx Tesseract OCR Parser" + version: str = __full_version_str__ + author: str = "Paperless-ngx Contributors" + url: str = "https://github.com/paperless-ngx/paperless-ngx" - def get_settings(self) -> OcrConfig: - """ - This parser uses the OCR configuration settings to parse documents - """ - return OcrConfig() + # ------------------------------------------------------------------ + # Class methods + # ------------------------------------------------------------------ - def get_page_count(self, document_path, mime_type): - page_count = None - if mime_type == "application/pdf": - try: - import pikepdf + @classmethod + def supported_mime_types(cls) -> dict[str, str]: + return _SUPPORTED_MIME_TYPES - with pikepdf.Pdf.open(document_path) as pdf: - page_count = len(pdf.pages) - except Exception as e: - self.log.warning( - f"Unable to determine PDF page count {document_path}: {e}", - ) - return page_count + @classmethod + def score( + cls, + mime_type: str, + filename: str, + path: Path | None = None, + ) -> int | None: + if mime_type in _SUPPORTED_MIME_TYPES: + return 10 + return None - def extract_metadata(self, document_path, mime_type): - result = [] - if mime_type == "application/pdf": - import pikepdf + # ------------------------------------------------------------------ + # Properties + # ------------------------------------------------------------------ - namespace_pattern = re.compile(r"\{(.*)\}(.*)") + @property + def can_produce_archive(self) -> bool: + return True - pdf = pikepdf.open(document_path) - meta = pdf.open_metadata() - for key, value in meta.items(): - if isinstance(value, list): - value = " ".join([str(e) for e in value]) - value = str(value) - try: - m = namespace_pattern.match(key) - if m is None: # pragma: no cover - continue - namespace = m.group(1) - key_value = m.group(2) - try: - namespace.encode("utf-8") - key_value.encode("utf-8") - except UnicodeEncodeError as e: # pragma: no cover - self.log.debug(f"Skipping metadata key {key}: {e}") - continue - result.append( - { - "namespace": namespace, - "prefix": meta.REVERSE_NS[namespace], - "key": key_value, - "value": value, - }, - ) - except Exception as e: - self.log.warning( - f"Error while reading metadata {key}: {value}. Error: {e}", - ) - return result + @property + def requires_pdf_rendition(self) -> bool: + return False - def get_thumbnail(self, document_path, mime_type, file_name=None): + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + def __init__(self, logging_group: object = None) -> None: + settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True) + self.tempdir = Path( + tempfile.mkdtemp(prefix="paperless-", dir=settings.SCRATCH_DIR), + ) + self.settings = OcrConfig() + self.archive_path: Path | None = None + self.text: str | None = None + self.date: datetime.datetime | None = None + self.log = logger + + def __enter__(self) -> Self: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + logger.debug("Cleaning up temporary directory %s", self.tempdir) + shutil.rmtree(self.tempdir, ignore_errors=True) + + # ------------------------------------------------------------------ + # Core parsing interface + # ------------------------------------------------------------------ + + def configure(self, context: ParserContext) -> None: + pass + + # ------------------------------------------------------------------ + # Result accessors + # ------------------------------------------------------------------ + + def get_text(self) -> str | None: + return self.text + + def get_date(self) -> datetime.datetime | None: + return self.date + + def get_archive_path(self) -> Path | None: + return self.archive_path + + # ------------------------------------------------------------------ + # Thumbnail, page count, and metadata + # ------------------------------------------------------------------ + + def get_thumbnail(self, document_path: Path, mime_type: str) -> Path: return make_thumbnail_from_pdf( - self.archive_path or document_path, + self.archive_path or Path(document_path), self.tempdir, - self.logging_group, ) - def is_image(self, mime_type) -> bool: + def get_page_count(self, document_path: Path, mime_type: str) -> int | None: + if mime_type == "application/pdf": + from paperless.parsers.utils import get_page_count_for_pdf + + return get_page_count_for_pdf(Path(document_path), log=self.log) + return None + + def extract_metadata( + self, + document_path: Path, + mime_type: str, + ) -> list[MetadataEntry]: + if mime_type != "application/pdf": + return [] + + from paperless.parsers.utils import extract_pdf_metadata + + return extract_pdf_metadata(Path(document_path), log=self.log) + + def is_image(self, mime_type: str) -> bool: return mime_type in [ "image/png", "image/jpeg", @@ -111,25 +182,25 @@ class RasterisedDocumentParser(DocumentParser): "image/heic", ] - def has_alpha(self, image) -> bool: + def has_alpha(self, image: Path) -> bool: with Image.open(image) as im: return im.mode in ("RGBA", "LA") - def remove_alpha(self, image_path: str) -> Path: + def remove_alpha(self, image_path: Path) -> Path: no_alpha_image = Path(self.tempdir) / "image-no-alpha" run_subprocess( [ settings.CONVERT_BINARY, "-alpha", "off", - image_path, - no_alpha_image, + str(image_path), + str(no_alpha_image), ], logger=self.log, ) return no_alpha_image - def get_dpi(self, image) -> int | None: + def get_dpi(self, image: Path) -> int | None: try: with Image.open(image) as im: x, _ = im.info["dpi"] @@ -138,7 +209,7 @@ class RasterisedDocumentParser(DocumentParser): self.log.warning(f"Error while getting DPI from image {image}: {e}") return None - def calculate_a4_dpi(self, image) -> int | None: + def calculate_a4_dpi(self, image: Path) -> int | None: try: with Image.open(image) as im: width, _ = im.size @@ -156,6 +227,7 @@ class RasterisedDocumentParser(DocumentParser): sidecar_file: Path | None, pdf_file: Path, ) -> str | None: + text: str | None = None # When re-doing OCR, the sidecar contains ONLY the new text, not # the whole text, so do not utilize it in that case if ( @@ -163,7 +235,7 @@ class RasterisedDocumentParser(DocumentParser): and sidecar_file.is_file() and self.settings.mode != "redo" ): - text = self.read_file_handle_unicode_errors(sidecar_file) + text = read_file_handle_unicode_errors(sidecar_file) if "[OCR skipped on page" not in text: # This happens when there's already text in the input file. @@ -191,12 +263,12 @@ class RasterisedDocumentParser(DocumentParser): "-layout", "-enc", "UTF-8", - pdf_file, + str(pdf_file), tmp.name, ], logger=self.log, ) - text = self.read_file_handle_unicode_errors(Path(tmp.name)) + text = read_file_handle_unicode_errors(Path(tmp.name)) return post_process_text(text) @@ -211,16 +283,14 @@ class RasterisedDocumentParser(DocumentParser): def construct_ocrmypdf_parameters( self, - input_file, - mime_type, - output_file, - sidecar_file, + input_file: Path, + mime_type: str, + output_file: Path, + sidecar_file: Path, *, - safe_fallback=False, - ): - if TYPE_CHECKING: - assert isinstance(self.settings, OcrConfig) - ocrmypdf_args = { + safe_fallback: bool = False, + ) -> dict[str, Any]: + ocrmypdf_args: dict[str, Any] = { "input_file_or_options": input_file, "output_file": output_file, # need to use threads, since this will be run in daemonized @@ -330,7 +400,13 @@ class RasterisedDocumentParser(DocumentParser): return ocrmypdf_args - def parse(self, document_path: Path, mime_type, file_name=None) -> None: + def parse( + self, + document_path: Path, + mime_type: str, + *, + produce_archive: bool = True, + ) -> None: # This forces tesseract to use one core per page. os.environ["OMP_THREAD_LIMIT"] = "1" VALID_TEXT_LENGTH = 50 @@ -458,7 +534,7 @@ class RasterisedDocumentParser(DocumentParser): self.text = "" -def post_process_text(text): +def post_process_text(text: str | None) -> str | None: if not text: return None diff --git a/src/paperless/parsers/utils.py b/src/paperless/parsers/utils.py index b72f31a28..2e6ce7061 100644 --- a/src/paperless/parsers/utils.py +++ b/src/paperless/parsers/utils.py @@ -20,6 +20,34 @@ if TYPE_CHECKING: logger = logging.getLogger("paperless.parsers.utils") +def read_file_handle_unicode_errors( + filepath: Path, + log: logging.Logger | None = None, +) -> str: + """Read a file as UTF-8 text, replacing invalid bytes rather than raising. + + Parameters + ---------- + filepath: + Absolute path to the file to read. + log: + Logger to use for warnings. Falls back to the module-level logger + when omitted. + + Returns + ------- + str + File content as a string, with any invalid UTF-8 sequences replaced + by the Unicode replacement character. + """ + _log = log or logger + try: + return filepath.read_text(encoding="utf-8") + except UnicodeDecodeError as e: + _log.warning("Unicode error during text reading, continuing: %s", e) + return filepath.read_bytes().decode("utf-8", errors="replace") + + def get_page_count_for_pdf( document_path: Path, log: logging.Logger | None = None, @@ -107,7 +135,7 @@ def extract_pdf_metadata( try: namespace.encode("utf-8") key_value.encode("utf-8") - except UnicodeEncodeError as enc_err: + except UnicodeEncodeError as enc_err: # pragma: no cover _log.debug("Skipping metadata key %s: %s", key, enc_err) continue diff --git a/src/paperless/tests/parsers/conftest.py b/src/paperless/tests/parsers/conftest.py index 5a22f24ab..a484f02c8 100644 --- a/src/paperless/tests/parsers/conftest.py +++ b/src/paperless/tests/parsers/conftest.py @@ -6,20 +6,29 @@ so it is easy to see which files belong to which test module. from __future__ import annotations +from contextlib import contextmanager from typing import TYPE_CHECKING import pytest +from django.test import override_settings from paperless.parsers.mail import MailDocumentParser from paperless.parsers.remote import RemoteDocumentParser +from paperless.parsers.tesseract import RasterisedDocumentParser from paperless.parsers.text import TextDocumentParser from paperless.parsers.tika import TikaDocumentParser if TYPE_CHECKING: + from collections.abc import Callable from collections.abc import Generator from pathlib import Path + from unittest.mock import MagicMock from pytest_django.fixtures import SettingsWrapper + from pytest_mock import MockerFixture + + #: Type for the ``make_tesseract_parser`` fixture factory. + MakeTesseractParser = Callable[..., Generator[RasterisedDocumentParser, None, None]] # ------------------------------------------------------------------ @@ -411,3 +420,381 @@ def nginx_base_url() -> Generator[str, None, None]: The base URL for the nginx HTTP server we expect to be alive """ yield "http://localhost:8080" + + +# ------------------------------------------------------------------ +# Tesseract parser sample files +# ------------------------------------------------------------------ + + +@pytest.fixture(scope="session") +def tesseract_samples_dir(samples_dir: Path) -> Path: + """Absolute path to the tesseract parser sample files directory. + + Returns + ------- + Path + ``/tesseract/`` + """ + return samples_dir / "tesseract" + + +@pytest.fixture(scope="session") +def document_webp_file(tesseract_samples_dir: Path) -> Path: + """Path to a WebP document sample file. + + Returns + ------- + Path + Absolute path to ``tesseract/document.webp``. + """ + return tesseract_samples_dir / "document.webp" + + +@pytest.fixture(scope="session") +def encrypted_pdf_file(tesseract_samples_dir: Path) -> Path: + """Path to an encrypted PDF sample file. + + Returns + ------- + Path + Absolute path to ``tesseract/encrypted.pdf``. + """ + return tesseract_samples_dir / "encrypted.pdf" + + +@pytest.fixture(scope="session") +def multi_page_digital_pdf_file(tesseract_samples_dir: Path) -> Path: + """Path to a multi-page digital PDF sample file. + + Returns + ------- + Path + Absolute path to ``tesseract/multi-page-digital.pdf``. + """ + return tesseract_samples_dir / "multi-page-digital.pdf" + + +@pytest.fixture(scope="session") +def multi_page_images_alpha_rgb_tiff_file(tesseract_samples_dir: Path) -> Path: + """Path to a multi-page TIFF with alpha channel in RGB. + + Returns + ------- + Path + Absolute path to ``tesseract/multi-page-images-alpha-rgb.tiff``. + """ + return tesseract_samples_dir / "multi-page-images-alpha-rgb.tiff" + + +@pytest.fixture(scope="session") +def multi_page_images_alpha_tiff_file(tesseract_samples_dir: Path) -> Path: + """Path to a multi-page TIFF with alpha channel. + + Returns + ------- + Path + Absolute path to ``tesseract/multi-page-images-alpha.tiff``. + """ + return tesseract_samples_dir / "multi-page-images-alpha.tiff" + + +@pytest.fixture(scope="session") +def multi_page_images_pdf_file(tesseract_samples_dir: Path) -> Path: + """Path to a multi-page PDF with images. + + Returns + ------- + Path + Absolute path to ``tesseract/multi-page-images.pdf``. + """ + return tesseract_samples_dir / "multi-page-images.pdf" + + +@pytest.fixture(scope="session") +def multi_page_images_tiff_file(tesseract_samples_dir: Path) -> Path: + """Path to a multi-page TIFF sample file. + + Returns + ------- + Path + Absolute path to ``tesseract/multi-page-images.tiff``. + """ + return tesseract_samples_dir / "multi-page-images.tiff" + + +@pytest.fixture(scope="session") +def multi_page_mixed_pdf_file(tesseract_samples_dir: Path) -> Path: + """Path to a multi-page mixed PDF sample file. + + Returns + ------- + Path + Absolute path to ``tesseract/multi-page-mixed.pdf``. + """ + return tesseract_samples_dir / "multi-page-mixed.pdf" + + +@pytest.fixture(scope="session") +def no_text_alpha_png_file(tesseract_samples_dir: Path) -> Path: + """Path to a PNG with alpha channel and no text. + + Returns + ------- + Path + Absolute path to ``tesseract/no-text-alpha.png``. + """ + return tesseract_samples_dir / "no-text-alpha.png" + + +@pytest.fixture(scope="session") +def rotated_pdf_file(tesseract_samples_dir: Path) -> Path: + """Path to a rotated PDF sample file. + + Returns + ------- + Path + Absolute path to ``tesseract/rotated.pdf``. + """ + return tesseract_samples_dir / "rotated.pdf" + + +@pytest.fixture(scope="session") +def rtl_test_pdf_file(tesseract_samples_dir: Path) -> Path: + """Path to an RTL test PDF sample file. + + Returns + ------- + Path + Absolute path to ``tesseract/rtl-test.pdf``. + """ + return tesseract_samples_dir / "rtl-test.pdf" + + +@pytest.fixture(scope="session") +def signed_pdf_file(tesseract_samples_dir: Path) -> Path: + """Path to a signed PDF sample file. + + Returns + ------- + Path + Absolute path to ``tesseract/signed.pdf``. + """ + return tesseract_samples_dir / "signed.pdf" + + +@pytest.fixture(scope="session") +def simple_alpha_png_file(tesseract_samples_dir: Path) -> Path: + """Path to a simple PNG with alpha channel. + + Returns + ------- + Path + Absolute path to ``tesseract/simple-alpha.png``. + """ + return tesseract_samples_dir / "simple-alpha.png" + + +@pytest.fixture(scope="session") +def simple_digital_pdf_file(tesseract_samples_dir: Path) -> Path: + """Path to a simple digital PDF sample file. + + Returns + ------- + Path + Absolute path to ``tesseract/simple-digital.pdf``. + """ + return tesseract_samples_dir / "simple-digital.pdf" + + +@pytest.fixture(scope="session") +def simple_no_dpi_png_file(tesseract_samples_dir: Path) -> Path: + """Path to a simple PNG without DPI information. + + Returns + ------- + Path + Absolute path to ``tesseract/simple-no-dpi.png``. + """ + return tesseract_samples_dir / "simple-no-dpi.png" + + +@pytest.fixture(scope="session") +def simple_bmp_file(tesseract_samples_dir: Path) -> Path: + """Path to a simple BMP sample file. + + Returns + ------- + Path + Absolute path to ``tesseract/simple.bmp``. + """ + return tesseract_samples_dir / "simple.bmp" + + +@pytest.fixture(scope="session") +def simple_gif_file(tesseract_samples_dir: Path) -> Path: + """Path to a simple GIF sample file. + + Returns + ------- + Path + Absolute path to ``tesseract/simple.gif``. + """ + return tesseract_samples_dir / "simple.gif" + + +@pytest.fixture(scope="session") +def simple_heic_file(tesseract_samples_dir: Path) -> Path: + """Path to a simple HEIC sample file. + + Returns + ------- + Path + Absolute path to ``tesseract/simple.heic``. + """ + return tesseract_samples_dir / "simple.heic" + + +@pytest.fixture(scope="session") +def simple_jpg_file(tesseract_samples_dir: Path) -> Path: + """Path to a simple JPG sample file. + + Returns + ------- + Path + Absolute path to ``tesseract/simple.jpg``. + """ + return tesseract_samples_dir / "simple.jpg" + + +@pytest.fixture(scope="session") +def simple_png_file(tesseract_samples_dir: Path) -> Path: + """Path to a simple PNG sample file. + + Returns + ------- + Path + Absolute path to ``tesseract/simple.png``. + """ + return tesseract_samples_dir / "simple.png" + + +@pytest.fixture(scope="session") +def simple_tif_file(tesseract_samples_dir: Path) -> Path: + """Path to a simple TIF sample file. + + Returns + ------- + Path + Absolute path to ``tesseract/simple.tif``. + """ + return tesseract_samples_dir / "simple.tif" + + +@pytest.fixture(scope="session") +def single_page_mixed_pdf_file(tesseract_samples_dir: Path) -> Path: + """Path to a single-page mixed PDF sample file. + + Returns + ------- + Path + Absolute path to ``tesseract/single-page-mixed.pdf``. + """ + return tesseract_samples_dir / "single-page-mixed.pdf" + + +@pytest.fixture(scope="session") +def with_form_pdf_file(tesseract_samples_dir: Path) -> Path: + """Path to a PDF with form sample file. + + Returns + ------- + Path + Absolute path to ``tesseract/with-form.pdf``. + """ + return tesseract_samples_dir / "with-form.pdf" + + +# ------------------------------------------------------------------ +# Tesseract parser instance and settings helpers +# ------------------------------------------------------------------ + + +@pytest.fixture() +def null_app_config(mocker: MockerFixture) -> MagicMock: + """Return a MagicMock with all OcrConfig fields set to None. + + This allows the parser to fall back to Django settings instead of + hitting the database. + + Returns + ------- + MagicMock + Mock config with all fields as None + """ + return mocker.MagicMock( + output_type=None, + pages=None, + language=None, + mode=None, + skip_archive_file=None, + image_dpi=None, + unpaper_clean=None, + deskew=None, + rotate_pages=None, + rotate_pages_threshold=None, + max_image_pixels=None, + color_conversion_strategy=None, + user_args=None, + ) + + +@pytest.fixture() +def tesseract_parser( + mocker: MockerFixture, + null_app_config: MagicMock, +) -> Generator[RasterisedDocumentParser, None, None]: + """Yield a RasterisedDocumentParser and clean up its temporary directory afterwards. + + Patches the config system to avoid database access. + + Yields + ------ + RasterisedDocumentParser + A ready-to-use parser instance. + """ + mocker.patch( + "paperless.config.BaseConfig._get_config_instance", + return_value=null_app_config, + ) + with RasterisedDocumentParser() as parser: + yield parser + + +@pytest.fixture() +def make_tesseract_parser( + mocker: MockerFixture, + null_app_config: MagicMock, +) -> MakeTesseractParser: + """Return a factory for creating RasterisedDocumentParser with Django settings overrides. + + This fixture is useful for tests that need to create parsers with different + settings configurations. + + Returns + ------- + Callable[..., contextmanager[RasterisedDocumentParser]] + A context manager factory that accepts Django settings overrides + """ + mocker.patch( + "paperless.config.BaseConfig._get_config_instance", + return_value=null_app_config, + ) + + @contextmanager + def _make_parser(**django_settings_overrides): + with override_settings(**django_settings_overrides): + with RasterisedDocumentParser() as parser: + yield parser + + return _make_parser diff --git a/src/paperless/tests/parsers/test_remote_parser.py b/src/paperless/tests/parsers/test_remote_parser.py index d0b9effba..69199a6e8 100644 --- a/src/paperless/tests/parsers/test_remote_parser.py +++ b/src/paperless/tests/parsers/test_remote_parser.py @@ -481,12 +481,17 @@ class TestRemoteParserRegistry: assert parser_cls is RemoteDocumentParser @pytest.mark.usefixtures("no_engine_settings") - def test_get_parser_returns_none_for_pdf_when_not_configured(self) -> None: - """With no tesseract parser registered yet, PDF has no handler if remote is off.""" + def test_get_parser_returns_none_for_unsupported_type_when_not_configured( + self, + ) -> None: + """With remote off and a truly unsupported MIME type, registry returns None.""" from paperless.parsers.registry import ParserRegistry registry = ParserRegistry() registry.register_defaults() - parser_cls = registry.get_parser_for_file("application/pdf", "doc.pdf") + parser_cls = registry.get_parser_for_file( + "application/x-unknown-format", + "doc.xyz", + ) assert parser_cls is None diff --git a/src/paperless_tesseract/tests/test_parser_custom_settings.py b/src/paperless/tests/parsers/test_tesseract_custom_settings.py similarity index 99% rename from src/paperless_tesseract/tests/test_parser_custom_settings.py rename to src/paperless/tests/parsers/test_tesseract_custom_settings.py index a4db8a2aa..60d1486f4 100644 --- a/src/paperless_tesseract/tests/test_parser_custom_settings.py +++ b/src/paperless/tests/parsers/test_tesseract_custom_settings.py @@ -10,7 +10,7 @@ from paperless.models import CleanChoices from paperless.models import ColorConvertChoices from paperless.models import ModeChoices from paperless.models import OutputTypeChoices -from paperless_tesseract.parsers import RasterisedDocumentParser +from paperless.parsers.tesseract import RasterisedDocumentParser class TestParserSettingsFromDb(DirectoriesMixin, FileSystemAssertsMixin, TestCase): diff --git a/src/paperless/tests/parsers/test_tesseract_parser.py b/src/paperless/tests/parsers/test_tesseract_parser.py new file mode 100644 index 000000000..daa7020c7 --- /dev/null +++ b/src/paperless/tests/parsers/test_tesseract_parser.py @@ -0,0 +1,1174 @@ +""" +Tests for paperless.parsers.tesseract.RasterisedDocumentParser. + +All tests use fixtures defined in conftest.py for parser lifecycle and +sample-file access. Settings-dependent tests mutate parser.settings +directly rather than going through the database. +""" + +from __future__ import annotations + +import re +import shutil +import unicodedata +from typing import TYPE_CHECKING + +import pytest +from ocrmypdf import SubprocessOutputError + +from documents.parsers import ParseError +from documents.parsers import run_convert +from paperless.parsers import ParserProtocol +from paperless.parsers.tesseract import RasterisedDocumentParser +from paperless.parsers.tesseract import post_process_text + +if TYPE_CHECKING: + from pathlib import Path + from unittest.mock import MagicMock + + from pytest_mock import MockerFixture + + from paperless.tests.parsers.conftest import MakeTesseractParser + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def assert_ordered_substrings(content: str, strings: list[str]) -> None: + """Assert all *strings* appear in *content* in the given order.""" + indices: list[int] = [] + for s in strings: + assert s in content, f"{s!r} not found in content" + indices.append(content.index(s)) + assert indices == sorted(indices), f"Strings out of order in content: {strings}" + + +# --------------------------------------------------------------------------- +# Protocol compliance +# --------------------------------------------------------------------------- + + +class TestRasterisedDocumentParserProtocol: + """Verify class-level protocol attributes and classmethods — no DB, no parser.""" + + def test_class_attributes_present(self) -> None: + assert RasterisedDocumentParser.name + assert RasterisedDocumentParser.version + assert RasterisedDocumentParser.author + assert RasterisedDocumentParser.url + + def test_supported_mime_types_returns_dict(self) -> None: + mime_types = RasterisedDocumentParser.supported_mime_types() + assert isinstance(mime_types, dict) + for mime in ( + "application/pdf", + "image/jpeg", + "image/png", + "image/tiff", + "image/gif", + "image/bmp", + "image/webp", + "image/heic", + ): + assert mime in mime_types + + @pytest.mark.parametrize( + ("mime_type", "expected"), + [ + pytest.param("application/pdf", 10, id="pdf"), + pytest.param("image/jpeg", 10, id="jpeg"), + pytest.param("image/png", 10, id="png"), + pytest.param("image/tiff", 10, id="tiff"), + pytest.param("image/gif", 10, id="gif"), + pytest.param("image/bmp", 10, id="bmp"), + pytest.param("image/webp", 10, id="webp"), + pytest.param("image/heic", 10, id="heic"), + pytest.param("text/plain", None, id="text-unsupported"), + pytest.param("application/msword", None, id="word-unsupported"), + ], + ) + def test_score(self, mime_type: str, expected: int | None) -> None: + assert RasterisedDocumentParser.score(mime_type, "file.pdf") == expected + + def test_isinstance_satisfies_protocol( + self, + tesseract_parser: RasterisedDocumentParser, + ) -> None: + assert isinstance(tesseract_parser, ParserProtocol) + + def test_can_produce_archive_is_true( + self, + tesseract_parser: RasterisedDocumentParser, + ) -> None: + assert tesseract_parser.can_produce_archive is True + + def test_requires_pdf_rendition_is_false( + self, + tesseract_parser: RasterisedDocumentParser, + ) -> None: + assert tesseract_parser.requires_pdf_rendition is False + + +# --------------------------------------------------------------------------- +# Lifecycle +# --------------------------------------------------------------------------- + + +class TestRasterisedDocumentParserLifecycle: + """Context-manager cleanup — no DB.""" + + def test_tempdir_cleaned_up_on_exit( + self, + mocker: MockerFixture, + null_app_config: MagicMock, + ) -> None: + mocker.patch( + "paperless.config.BaseConfig._get_config_instance", + return_value=null_app_config, + ) + with RasterisedDocumentParser() as parser: + tempdir = parser.tempdir + assert tempdir.exists() + assert not tempdir.exists() + + def test_tempdir_cleaned_up_after_exception( + self, + mocker: MockerFixture, + null_app_config: MagicMock, + ) -> None: + mocker.patch( + "paperless.config.BaseConfig._get_config_instance", + return_value=null_app_config, + ) + tempdir: Path | None = None + with pytest.raises(RuntimeError): + with RasterisedDocumentParser() as parser: + tempdir = parser.tempdir + raise RuntimeError("boom") + assert tempdir is not None and not tempdir.exists() + + +# --------------------------------------------------------------------------- +# post_process_text +# --------------------------------------------------------------------------- + + +class TestPostProcessText: + @pytest.mark.parametrize( + ("source", "expected"), + [ + pytest.param( + "simple string", + "simple string", + id="collapse-spaces", + ), + pytest.param( + "simple newline\n testing string", + "simple newline\ntesting string", + id="preserve-newline", + ), + pytest.param( + "utf-8 строка с пробелами в конце ", # noqa: RUF001 + "utf-8 строка с пробелами в конце", # noqa: RUF001 + id="utf8-trailing-spaces", + ), + ], + ) + def test_post_process_text(self, source: str, expected: str) -> None: + assert post_process_text(source) == expected + + +# --------------------------------------------------------------------------- +# Page count +# --------------------------------------------------------------------------- + + +class TestGetPageCount: + def test_single_page_pdf( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + assert ( + tesseract_parser.get_page_count( + tesseract_samples_dir / "simple-digital.pdf", + "application/pdf", + ) + == 1 + ) + + def test_multi_page_pdf( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + assert ( + tesseract_parser.get_page_count( + tesseract_samples_dir / "multi-page-mixed.pdf", + "application/pdf", + ) + == 6 + ) + + def test_password_protected_returns_none( + self, + mocker: MockerFixture, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + caplog, + ) -> None: + """ + GIVEN: + - pikepdf raises when opening a protected PDF + WHEN: + - Page count is requested + THEN: + - Returns None and logs a warning + """ + mocker.patch("pikepdf.Pdf.open", side_effect=Exception("password required")) + import logging + + with caplog.at_level(logging.WARNING): + page_count = tesseract_parser.get_page_count( + tesseract_samples_dir / "simple-digital.pdf", + "application/pdf", + ) + assert page_count is None + assert any( + "Unable to determine PDF page count" in r.message for r in caplog.records + ) + + def test_non_pdf_returns_none( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + assert ( + tesseract_parser.get_page_count( + tesseract_samples_dir / "simple.png", + "image/png", + ) + is None + ) + + +# --------------------------------------------------------------------------- +# DPI helpers +# --------------------------------------------------------------------------- + + +class TestDpiHelpers: + @pytest.mark.parametrize( + ("filename", "expected_dpi"), + [ + pytest.param("simple-no-dpi.png", None, id="no-dpi"), + pytest.param("simple.png", 72, id="with-dpi"), + ], + ) + def test_get_dpi( + self, + filename: str, + expected_dpi: int | None, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + assert ( + tesseract_parser.get_dpi(str(tesseract_samples_dir / filename)) + == expected_dpi + ) + + def test_calculate_a4_dpi( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + assert ( + tesseract_parser.calculate_a4_dpi( + str(tesseract_samples_dir / "simple-no-dpi.png"), + ) + == 62 + ) + + +# --------------------------------------------------------------------------- +# Thumbnail +# --------------------------------------------------------------------------- + + +class TestGetThumbnail: + def test_thumbnail_is_file( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + thumb = tesseract_parser.get_thumbnail( + tesseract_samples_dir / "simple-digital.pdf", + "application/pdf", + ) + assert thumb.is_file() + + def test_thumbnail_fallback_on_convert_error( + self, + mocker: MockerFixture, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + def _raise_on_pdf(input_file, output_file, **kwargs) -> None: + if ".pdf" in str(input_file): + raise ParseError("Does not compute.") + run_convert(input_file=input_file, output_file=output_file, **kwargs) + + mocker.patch("documents.parsers.run_convert", side_effect=_raise_on_pdf) + + thumb = tesseract_parser.get_thumbnail( + tesseract_samples_dir / "simple-digital.pdf", + "application/pdf", + ) + assert thumb.is_file() + + def test_thumbnail_encrypted_pdf( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + thumb = tesseract_parser.get_thumbnail( + tesseract_samples_dir / "encrypted.pdf", + "application/pdf", + ) + assert thumb.is_file() + + +# --------------------------------------------------------------------------- +# extract_text +# --------------------------------------------------------------------------- + + +class TestExtractText: + def test_extract_text_from_digital_pdf( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + text = tesseract_parser.extract_text( + None, + tesseract_samples_dir / "simple-digital.pdf", + ) + assert text is not None + assert "This is a test document." in text.strip() + + +# --------------------------------------------------------------------------- +# Parse — PDF modes +# --------------------------------------------------------------------------- + + +class TestParsePdf: + def test_simple_digital_creates_archive( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + tesseract_parser.parse( + tesseract_samples_dir / "simple-digital.pdf", + "application/pdf", + ) + assert tesseract_parser.archive_path is not None + assert tesseract_parser.archive_path.is_file() + assert_ordered_substrings( + tesseract_parser.get_text(), + ["This is a test document."], + ) + + def test_with_form_default( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + tesseract_parser.parse( + tesseract_samples_dir / "with-form.pdf", + "application/pdf", + ) + assert tesseract_parser.archive_path is not None + assert tesseract_parser.archive_path.is_file() + assert_ordered_substrings( + tesseract_parser.get_text(), + ["Please enter your name in here:", "This is a PDF document with a form."], + ) + + def test_with_form_redo_produces_no_archive( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + tesseract_parser.settings.mode = "redo" + tesseract_parser.parse( + tesseract_samples_dir / "with-form.pdf", + "application/pdf", + ) + assert tesseract_parser.archive_path is None + assert_ordered_substrings( + tesseract_parser.get_text(), + ["Please enter your name in here:", "This is a PDF document with a form."], + ) + + def test_with_form_force( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + tesseract_parser.settings.mode = "force" + tesseract_parser.parse( + tesseract_samples_dir / "with-form.pdf", + "application/pdf", + ) + assert_ordered_substrings( + tesseract_parser.get_text(), + ["Please enter your name in here:", "This is a PDF document with a form."], + ) + + def test_signed_skip_mode_no_archive( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + tesseract_parser.settings.mode = "skip" + tesseract_parser.parse(tesseract_samples_dir / "signed.pdf", "application/pdf") + assert tesseract_parser.archive_path is None + assert_ordered_substrings( + tesseract_parser.get_text(), + [ + "This is a digitally signed PDF, created with Acrobat Pro for the Paperless project to enable", + "automated testing of signed/encrypted PDFs", + ], + ) + + def test_encrypted_skip_mode_empty_text( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + tesseract_parser.settings.mode = "skip" + tesseract_parser.parse( + tesseract_samples_dir / "encrypted.pdf", + "application/pdf", + ) + assert tesseract_parser.archive_path is None + assert tesseract_parser.get_text() == "" + + def test_gs_rendering_error_raises_parse_error( + self, + mocker: MockerFixture, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + mocker.patch( + "ocrmypdf.ocr", + side_effect=SubprocessOutputError("Ghostscript PDF/A rendering failed"), + ) + with pytest.raises(ParseError): + tesseract_parser.parse( + tesseract_samples_dir / "simple-digital.pdf", + "application/pdf", + ) + + +# --------------------------------------------------------------------------- +# Parse — images +# --------------------------------------------------------------------------- + + +class TestParseImages: + def test_simple_png( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + tesseract_parser.parse(tesseract_samples_dir / "simple.png", "image/png") + assert tesseract_parser.archive_path is not None + assert tesseract_parser.archive_path.is_file() + assert_ordered_substrings( + tesseract_parser.get_text(), + ["This is a test document."], + ) + + def test_simple_alpha_png( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + tmp_path: Path, + ) -> None: + dest = tmp_path / "simple-alpha.png" + shutil.copy(tesseract_samples_dir / "simple-alpha.png", dest) + tesseract_parser.parse(dest, "image/png") + assert tesseract_parser.archive_path is not None + assert tesseract_parser.archive_path.is_file() + assert_ordered_substrings( + tesseract_parser.get_text(), + ["This is a test document."], + ) + + def test_no_dpi_with_default_dpi( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + tesseract_parser.settings.image_dpi = 72 + tesseract_parser.parse(tesseract_samples_dir / "simple-no-dpi.png", "image/png") + assert tesseract_parser.archive_path is not None + assert tesseract_parser.archive_path.is_file() + assert "this is a test document." in tesseract_parser.get_text().lower() + + def test_no_dpi_no_fallback_raises( + self, + mocker: MockerFixture, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + mocker.patch.object(tesseract_parser, "calculate_a4_dpi", return_value=None) + with pytest.raises(ParseError): + tesseract_parser.parse( + tesseract_samples_dir / "simple-no-dpi.png", + "image/png", + ) + + +# --------------------------------------------------------------------------- +# Parse — multi-page PDF +# --------------------------------------------------------------------------- + + +class TestParseMultiPage: + def test_multi_page_digital( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + tesseract_parser.parse( + tesseract_samples_dir / "multi-page-digital.pdf", + "application/pdf", + ) + assert tesseract_parser.archive_path is not None + assert tesseract_parser.archive_path.is_file() + assert_ordered_substrings( + tesseract_parser.get_text().lower(), + ["page 1", "page 2", "page 3"], + ) + + @pytest.mark.parametrize( + "mode", + [ + pytest.param("skip", id="skip"), + pytest.param("redo", id="redo"), + pytest.param("force", id="force"), + ], + ) + def test_multi_page_digital_pages_2( + self, + mode: str, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + tesseract_parser.settings.pages = 2 + tesseract_parser.settings.mode = mode + tesseract_parser.parse( + tesseract_samples_dir / "multi-page-digital.pdf", + "application/pdf", + ) + assert tesseract_parser.archive_path is not None + assert_ordered_substrings( + tesseract_parser.get_text().lower(), + ["page 1", "page 2", "page 3"], + ) + + def test_multi_page_images_skip( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + tesseract_parser.settings.mode = "skip" + tesseract_parser.parse( + tesseract_samples_dir / "multi-page-images.pdf", + "application/pdf", + ) + assert tesseract_parser.archive_path is not None + assert_ordered_substrings( + tesseract_parser.get_text().lower(), + ["page 1", "page 2", "page 3"], + ) + + def test_multi_page_images_redo_pages_2( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + """ + GIVEN: + - File with image-only pages + - OCR of only pages 1 and 2 requested + - Mode: redo + WHEN: + - Document is parsed + THEN: + - Pages 1 and 2 extracted; page 3 absent + """ + tesseract_parser.settings.pages = 2 + tesseract_parser.settings.mode = "redo" + tesseract_parser.parse( + tesseract_samples_dir / "multi-page-images.pdf", + "application/pdf", + ) + assert tesseract_parser.archive_path is not None + text = tesseract_parser.get_text().lower() + assert_ordered_substrings(text, ["page 1", "page 2"]) + assert "page 3" not in text + + def test_multi_page_images_force_page_1( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + """ + GIVEN: + - File with image-only pages + - Only page 1 requested + - Mode: force + WHEN: + - Document is parsed + THEN: + - Only page 1 extracted + """ + tesseract_parser.settings.pages = 1 + tesseract_parser.settings.mode = "force" + tesseract_parser.parse( + tesseract_samples_dir / "multi-page-images.pdf", + "application/pdf", + ) + assert tesseract_parser.archive_path is not None + text = tesseract_parser.get_text().lower() + assert "page 1" in text + assert "page 2" not in text + assert "page 3" not in text + + def test_multi_page_tiff( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + """ + GIVEN: + - Multi-page TIFF image + WHEN: + - Image is parsed + THEN: + - Text from all pages extracted + """ + tesseract_parser.parse( + tesseract_samples_dir / "multi-page-images.tiff", + "image/tiff", + ) + assert tesseract_parser.archive_path is not None + assert_ordered_substrings( + tesseract_parser.get_text().lower(), + ["page 1", "page 2", "page 3"], + ) + + def test_multi_page_tiff_alpha( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + tmp_path: Path, + ) -> None: + """ + GIVEN: + - Multi-page TIFF with alpha channel + WHEN: + - Image is parsed + THEN: + - Text from all pages extracted + """ + dest = tmp_path / "alpha.tiff" + shutil.copy(tesseract_samples_dir / "multi-page-images-alpha.tiff", dest) + tesseract_parser.parse(dest, "image/tiff") + assert tesseract_parser.archive_path is not None + assert_ordered_substrings( + tesseract_parser.get_text().lower(), + ["page 1", "page 2", "page 3"], + ) + + def test_multi_page_tiff_alpha_srgb( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + tmp_path: Path, + ) -> None: + """ + GIVEN: + - Multi-page TIFF with alpha channel and sRGB colorspace + WHEN: + - Image is parsed + THEN: + - Text from all pages extracted + """ + dest = tmp_path / "alpha-rgb.tiff" + shutil.copy(tesseract_samples_dir / "multi-page-images-alpha-rgb.tiff", dest) + tesseract_parser.parse(dest, "image/tiff") + assert tesseract_parser.archive_path is not None + assert_ordered_substrings( + tesseract_parser.get_text().lower(), + ["page 1", "page 2", "page 3"], + ) + + +# --------------------------------------------------------------------------- +# Parse — skip_noarchive / skip_archive_file +# --------------------------------------------------------------------------- + + +class TestSkipArchive: + def test_skip_noarchive_with_text_layer( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + """ + GIVEN: + - File with existing text layer + - Mode: skip_noarchive + WHEN: + - Document is parsed + THEN: + - Text extracted; no archive created + """ + tesseract_parser.settings.mode = "skip_noarchive" + tesseract_parser.parse( + tesseract_samples_dir / "multi-page-digital.pdf", + "application/pdf", + ) + assert tesseract_parser.archive_path is None + assert_ordered_substrings( + tesseract_parser.get_text().lower(), + ["page 1", "page 2", "page 3"], + ) + + def test_skip_noarchive_image_only_creates_archive( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + """ + GIVEN: + - File with image-only pages (no text layer) + - Mode: skip_noarchive + WHEN: + - Document is parsed + THEN: + - Text extracted; archive created (OCR needed) + """ + tesseract_parser.settings.mode = "skip_noarchive" + tesseract_parser.parse( + tesseract_samples_dir / "multi-page-images.pdf", + "application/pdf", + ) + assert tesseract_parser.archive_path is not None + assert_ordered_substrings( + tesseract_parser.get_text().lower(), + ["page 1", "page 2", "page 3"], + ) + + @pytest.mark.parametrize( + ("skip_archive_file", "filename", "expect_archive"), + [ + pytest.param("never", "multi-page-digital.pdf", True, id="never-with-text"), + pytest.param("never", "multi-page-images.pdf", True, id="never-no-text"), + pytest.param( + "with_text", + "multi-page-digital.pdf", + False, + id="with-text-layer", + ), + pytest.param( + "with_text", + "multi-page-images.pdf", + True, + id="with-text-no-layer", + ), + pytest.param( + "always", + "multi-page-digital.pdf", + False, + id="always-with-text", + ), + pytest.param("always", "multi-page-images.pdf", False, id="always-no-text"), + ], + ) + def test_skip_archive_file_setting( + self, + skip_archive_file: str, + filename: str, + expect_archive: str, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + tesseract_parser.settings.skip_archive_file = skip_archive_file + tesseract_parser.parse(tesseract_samples_dir / filename, "application/pdf") + text = tesseract_parser.get_text().lower() + assert_ordered_substrings(text, ["page 1", "page 2", "page 3"]) + if expect_archive: + assert tesseract_parser.archive_path is not None + else: + assert tesseract_parser.archive_path is None + + +# --------------------------------------------------------------------------- +# Parse — mixed pages / sidecar +# --------------------------------------------------------------------------- + + +class TestParseMixed: + def test_multi_page_mixed_skip_mode( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + """ + GIVEN: + - File with text in some pages (image) and some pages (digital) + - Mode: skip + WHEN: + - Document is parsed + THEN: + - All pages extracted; archive created; sidecar notes skipped pages + """ + tesseract_parser.settings.mode = "skip" + tesseract_parser.parse( + tesseract_samples_dir / "multi-page-mixed.pdf", + "application/pdf", + ) + assert tesseract_parser.archive_path is not None + assert tesseract_parser.archive_path.is_file() + assert_ordered_substrings( + tesseract_parser.get_text().lower(), + ["page 1", "page 2", "page 3", "page 4", "page 5", "page 6"], + ) + sidecar = (tesseract_parser.tempdir / "sidecar.txt").read_text() + assert "[OCR skipped on page(s) 4-6]" in sidecar + + def test_single_page_mixed_redo_mode( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + """ + GIVEN: + - Single page with both text and image content + - Mode: redo + WHEN: + - Document is parsed + THEN: + - Both text layer and image text extracted; archive created + """ + tesseract_parser.settings.mode = "redo" + tesseract_parser.parse( + tesseract_samples_dir / "single-page-mixed.pdf", + "application/pdf", + ) + assert tesseract_parser.archive_path is not None + assert tesseract_parser.archive_path.is_file() + assert_ordered_substrings( + tesseract_parser.get_text().lower(), + [ + "this is some normal text, present on page 1 of the document.", + "this is some text, but in an image, also on page 1.", + "this is further text on page 1.", + ], + ) + sidecar = (tesseract_parser.tempdir / "sidecar.txt").read_text().lower() + assert "this is some text, but in an image, also on page 1." in sidecar + assert ( + "this is some normal text, present on page 1 of the document." + not in sidecar + ) + + def test_multi_page_mixed_skip_noarchive( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + """ + GIVEN: + - File with mixed pages + - Mode: skip_noarchive + WHEN: + - Document is parsed + THEN: + - No archive created (file has text layer); later-page text present + """ + tesseract_parser.settings.mode = "skip_noarchive" + tesseract_parser.parse( + tesseract_samples_dir / "multi-page-mixed.pdf", + "application/pdf", + ) + assert tesseract_parser.archive_path is None + assert_ordered_substrings( + tesseract_parser.get_text().lower(), + ["page 4", "page 5", "page 6"], + ) + + +# --------------------------------------------------------------------------- +# Parse — rotation +# --------------------------------------------------------------------------- + + +class TestParseRotate: + def test_rotate_skip_mode( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + tesseract_parser.settings.mode = "skip" + tesseract_parser.settings.rotate = True + tesseract_parser.parse(tesseract_samples_dir / "rotated.pdf", "application/pdf") + assert_ordered_substrings( + tesseract_parser.get_text(), + [ + "This is the text that appears on the first page. It\u2019s a lot of text.", + "Even if the pages are rotated, OCRmyPDF still gets the job done.", + "This is a really weird file with lots of nonsense text.", + "If you read this, it\u2019s your own fault. Also check your screen orientation.", + ], + ) + + +# --------------------------------------------------------------------------- +# Parse — RTL +# --------------------------------------------------------------------------- + + +class TestParseRtl: + def test_rtl_language_detected( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + """ + GIVEN: + - PDF with RTL Arabic text + WHEN: + - Document is parsed + THEN: + - Arabic content is extracted (normalised for bidi) + """ + tesseract_parser.parse( + tesseract_samples_dir / "rtl-test.pdf", + "application/pdf", + ) + normalised = "".join( + ch + for ch in unicodedata.normalize("NFKC", tesseract_parser.get_text()) + if unicodedata.category(ch) != "Cf" and not ch.isspace() + ) + assert "ةرازو" in normalised + assert any(token in normalised for token in ("ةیلخادلا", "الاخليد")) + + +# --------------------------------------------------------------------------- +# Parse — OCRmyPDF parameters +# --------------------------------------------------------------------------- + + +@pytest.mark.django_db +class TestOcrmypdfParameters: + """Tests that inspect the dict passed to ocrmypdf. + + These create parsers inline via make_tesseract_parser with specific + Django settings overrides so OcrConfig picks them up at construction time. + """ + + def test_basic_parameter_mapping( + self, + make_tesseract_parser: MakeTesseractParser, + ) -> None: + with make_tesseract_parser() as parser: + params = parser.construct_ocrmypdf_parameters( + input_file="input.pdf", + output_file="output.pdf", + sidecar_file="sidecar.txt", + mime_type="application/pdf", + safe_fallback=False, + ) + assert params["input_file_or_options"] == "input.pdf" + assert params["output_file"] == "output.pdf" + assert params["sidecar"] == "sidecar.txt" + + @pytest.mark.parametrize( + ("ocr_clean", "expected_clean", "expected_clean_final"), + [ + pytest.param("none", False, False, id="clean-none"), + pytest.param("clean", True, False, id="clean-clean"), + ], + ) + def test_clean_option( + self, + ocr_clean: str, + *, + expected_clean: bool, + expected_clean_final: bool, + make_tesseract_parser: MakeTesseractParser, + ) -> None: + with make_tesseract_parser(OCR_CLEAN=ocr_clean) as parser: + params = parser.construct_ocrmypdf_parameters("", "", "", "") + assert ("clean" in params) == expected_clean + assert ("clean_final" in params) == expected_clean_final + + def test_clean_final_skip_mode( + self, + make_tesseract_parser: MakeTesseractParser, + ) -> None: + with make_tesseract_parser(OCR_CLEAN="clean-final", OCR_MODE="skip") as parser: + params = parser.construct_ocrmypdf_parameters("", "", "", "") + assert params["clean_final"] is True + assert "clean" not in params + + def test_clean_final_redo_mode_falls_back_to_clean( + self, + make_tesseract_parser: MakeTesseractParser, + ) -> None: + with make_tesseract_parser(OCR_CLEAN="clean-final", OCR_MODE="redo") as parser: + params = parser.construct_ocrmypdf_parameters("", "", "", "") + assert params["clean"] is True + assert "clean_final" not in params + + @pytest.mark.parametrize( + ("ocr_mode", "ocr_deskew", "expect_deskew"), + [ + pytest.param("skip", True, True, id="skip-deskew-on"), + pytest.param("redo", True, False, id="redo-deskew-off"), + pytest.param("skip", False, False, id="skip-no-deskew"), + ], + ) + def test_deskew_option( + self, + ocr_mode: str, + *, + ocr_deskew: bool, + expect_deskew: bool, + make_tesseract_parser: MakeTesseractParser, + ) -> None: + with make_tesseract_parser(OCR_MODE=ocr_mode, OCR_DESKEW=ocr_deskew) as parser: + params = parser.construct_ocrmypdf_parameters("", "", "", "") + assert ("deskew" in params) == expect_deskew + + def test_max_image_pixels_positive( + self, + make_tesseract_parser: MakeTesseractParser, + ) -> None: + with make_tesseract_parser(OCR_MAX_IMAGE_PIXELS=1_000_001.0) as parser: + params = parser.construct_ocrmypdf_parameters("", "", "", "") + assert "max_image_mpixels" in params + assert abs(params["max_image_mpixels"] - 1.0) < 1e-4 + + def test_max_image_pixels_negative_omitted( + self, + make_tesseract_parser: MakeTesseractParser, + ) -> None: + with make_tesseract_parser(OCR_MAX_IMAGE_PIXELS=-1_000_001.0) as parser: + params = parser.construct_ocrmypdf_parameters("", "", "", "") + assert "max_image_mpixels" not in params + + +# --------------------------------------------------------------------------- +# Parse — file type matrix +# --------------------------------------------------------------------------- + + +class TestParserFileTypes: + @pytest.mark.parametrize( + ("filename", "mime_type"), + [ + pytest.param("simple.bmp", "image/bmp", id="bmp"), + pytest.param("simple.jpg", "image/jpeg", id="jpeg"), + pytest.param("simple.tif", "image/tiff", id="tiff"), + ], + ) + def test_simple_image_contains_test_text( + self, + filename: str, + mime_type: str, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + tesseract_parser.parse(tesseract_samples_dir / filename, mime_type) + assert tesseract_parser.archive_path is not None + assert tesseract_parser.archive_path.is_file() + assert "this is a test document" in tesseract_parser.get_text().lower() + + def test_heic( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + tesseract_parser.parse(tesseract_samples_dir / "simple.heic", "image/heic") + assert tesseract_parser.archive_path is not None + assert "pizza" in tesseract_parser.get_text().lower() + + def test_gif_with_explicit_dpi( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + tesseract_parser.settings.image_dpi = 200 + tesseract_parser.parse(tesseract_samples_dir / "simple.gif", "image/gif") + assert tesseract_parser.archive_path is not None + assert "this is a test document" in tesseract_parser.get_text().lower() + + def test_webp_with_explicit_dpi( + self, + tesseract_parser: RasterisedDocumentParser, + tesseract_samples_dir: Path, + ) -> None: + tesseract_parser.settings.image_dpi = 72 + tesseract_parser.parse(tesseract_samples_dir / "document.webp", "image/webp") + assert tesseract_parser.archive_path is not None + assert re.search( + r"this is a ?webp document, created 11/14/2022\.", + tesseract_parser.get_text().lower(), + ) + + +# --------------------------------------------------------------------------- +# Registry +# --------------------------------------------------------------------------- + + +class TestRasterisedDocumentParserRegistry: + def test_registered_in_defaults(self) -> None: + from paperless.parsers.registry import ParserRegistry + + registry = ParserRegistry() + registry.register_defaults() + assert RasterisedDocumentParser in registry._builtins + + @pytest.mark.parametrize( + ("mime_type", "filename"), + [ + pytest.param("application/pdf", "doc.pdf", id="pdf"), + pytest.param("image/png", "image.png", id="png"), + pytest.param("image/jpeg", "photo.jpg", id="jpeg"), + pytest.param("image/tiff", "scan.tif", id="tiff"), + ], + ) + def test_get_parser_for_supported_mime( + self, + mime_type: str, + filename: str, + ) -> None: + from paperless.parsers.registry import get_parser_registry + + registry = get_parser_registry() + assert ( + registry.get_parser_for_file(mime_type, filename) + is RasterisedDocumentParser + ) diff --git a/src/paperless/tests/parsers/test_text_parser.py b/src/paperless/tests/parsers/test_text_parser.py index 091d8287e..fd2a57857 100644 --- a/src/paperless/tests/parsers/test_text_parser.py +++ b/src/paperless/tests/parsers/test_text_parser.py @@ -256,6 +256,9 @@ class TestTextParserRegistry: from paperless.parsers.registry import get_parser_registry registry = get_parser_registry() - parser_cls = registry.get_parser_for_file("application/pdf", "doc.pdf") + parser_cls = registry.get_parser_for_file( + "application/x-unknown-format", + "doc.xyz", + ) assert parser_cls is None diff --git a/src/paperless_tesseract/tests/samples/document.webp b/src/paperless/tests/samples/tesseract/document.webp similarity index 100% rename from src/paperless_tesseract/tests/samples/document.webp rename to src/paperless/tests/samples/tesseract/document.webp diff --git a/src/paperless_tesseract/tests/samples/encrypted.pdf b/src/paperless/tests/samples/tesseract/encrypted.pdf similarity index 100% rename from src/paperless_tesseract/tests/samples/encrypted.pdf rename to src/paperless/tests/samples/tesseract/encrypted.pdf diff --git a/src/paperless_tesseract/tests/samples/multi-page-digital.pdf b/src/paperless/tests/samples/tesseract/multi-page-digital.pdf similarity index 100% rename from src/paperless_tesseract/tests/samples/multi-page-digital.pdf rename to src/paperless/tests/samples/tesseract/multi-page-digital.pdf diff --git a/src/paperless_tesseract/tests/samples/multi-page-images-alpha-rgb.tiff b/src/paperless/tests/samples/tesseract/multi-page-images-alpha-rgb.tiff similarity index 100% rename from src/paperless_tesseract/tests/samples/multi-page-images-alpha-rgb.tiff rename to src/paperless/tests/samples/tesseract/multi-page-images-alpha-rgb.tiff diff --git a/src/paperless_tesseract/tests/samples/multi-page-images-alpha.tiff b/src/paperless/tests/samples/tesseract/multi-page-images-alpha.tiff similarity index 100% rename from src/paperless_tesseract/tests/samples/multi-page-images-alpha.tiff rename to src/paperless/tests/samples/tesseract/multi-page-images-alpha.tiff diff --git a/src/paperless_tesseract/tests/samples/multi-page-images.pdf b/src/paperless/tests/samples/tesseract/multi-page-images.pdf similarity index 100% rename from src/paperless_tesseract/tests/samples/multi-page-images.pdf rename to src/paperless/tests/samples/tesseract/multi-page-images.pdf diff --git a/src/paperless_tesseract/tests/samples/multi-page-images.tiff b/src/paperless/tests/samples/tesseract/multi-page-images.tiff similarity index 100% rename from src/paperless_tesseract/tests/samples/multi-page-images.tiff rename to src/paperless/tests/samples/tesseract/multi-page-images.tiff diff --git a/src/paperless_tesseract/tests/samples/multi-page-mixed.pdf b/src/paperless/tests/samples/tesseract/multi-page-mixed.pdf similarity index 100% rename from src/paperless_tesseract/tests/samples/multi-page-mixed.pdf rename to src/paperless/tests/samples/tesseract/multi-page-mixed.pdf diff --git a/src/paperless_tesseract/tests/samples/no-text-alpha.png b/src/paperless/tests/samples/tesseract/no-text-alpha.png similarity index 100% rename from src/paperless_tesseract/tests/samples/no-text-alpha.png rename to src/paperless/tests/samples/tesseract/no-text-alpha.png diff --git a/src/paperless_tesseract/tests/samples/rotated.pdf b/src/paperless/tests/samples/tesseract/rotated.pdf similarity index 100% rename from src/paperless_tesseract/tests/samples/rotated.pdf rename to src/paperless/tests/samples/tesseract/rotated.pdf diff --git a/src/paperless_tesseract/tests/samples/rtl-test.pdf b/src/paperless/tests/samples/tesseract/rtl-test.pdf similarity index 100% rename from src/paperless_tesseract/tests/samples/rtl-test.pdf rename to src/paperless/tests/samples/tesseract/rtl-test.pdf diff --git a/src/paperless_tesseract/tests/samples/signed.pdf b/src/paperless/tests/samples/tesseract/signed.pdf similarity index 100% rename from src/paperless_tesseract/tests/samples/signed.pdf rename to src/paperless/tests/samples/tesseract/signed.pdf diff --git a/src/paperless_tesseract/tests/samples/simple-alpha.png b/src/paperless/tests/samples/tesseract/simple-alpha.png similarity index 100% rename from src/paperless_tesseract/tests/samples/simple-alpha.png rename to src/paperless/tests/samples/tesseract/simple-alpha.png diff --git a/src/paperless_tesseract/tests/samples/simple-digital.pdf b/src/paperless/tests/samples/tesseract/simple-digital.pdf similarity index 100% rename from src/paperless_tesseract/tests/samples/simple-digital.pdf rename to src/paperless/tests/samples/tesseract/simple-digital.pdf diff --git a/src/paperless_tesseract/tests/samples/simple-no-dpi.png b/src/paperless/tests/samples/tesseract/simple-no-dpi.png similarity index 100% rename from src/paperless_tesseract/tests/samples/simple-no-dpi.png rename to src/paperless/tests/samples/tesseract/simple-no-dpi.png diff --git a/src/paperless_tesseract/tests/samples/simple.bmp b/src/paperless/tests/samples/tesseract/simple.bmp similarity index 100% rename from src/paperless_tesseract/tests/samples/simple.bmp rename to src/paperless/tests/samples/tesseract/simple.bmp diff --git a/src/paperless_tesseract/tests/samples/simple.gif b/src/paperless/tests/samples/tesseract/simple.gif similarity index 100% rename from src/paperless_tesseract/tests/samples/simple.gif rename to src/paperless/tests/samples/tesseract/simple.gif diff --git a/src/paperless_tesseract/tests/samples/simple.heic b/src/paperless/tests/samples/tesseract/simple.heic similarity index 100% rename from src/paperless_tesseract/tests/samples/simple.heic rename to src/paperless/tests/samples/tesseract/simple.heic diff --git a/src/paperless_tesseract/tests/samples/simple.jpg b/src/paperless/tests/samples/tesseract/simple.jpg similarity index 100% rename from src/paperless_tesseract/tests/samples/simple.jpg rename to src/paperless/tests/samples/tesseract/simple.jpg diff --git a/src/paperless_tesseract/tests/samples/simple.png b/src/paperless/tests/samples/tesseract/simple.png similarity index 100% rename from src/paperless_tesseract/tests/samples/simple.png rename to src/paperless/tests/samples/tesseract/simple.png diff --git a/src/paperless_tesseract/tests/samples/simple.tif b/src/paperless/tests/samples/tesseract/simple.tif similarity index 100% rename from src/paperless_tesseract/tests/samples/simple.tif rename to src/paperless/tests/samples/tesseract/simple.tif diff --git a/src/paperless_tesseract/tests/samples/single-page-mixed.pdf b/src/paperless/tests/samples/tesseract/single-page-mixed.pdf similarity index 100% rename from src/paperless_tesseract/tests/samples/single-page-mixed.pdf rename to src/paperless/tests/samples/tesseract/single-page-mixed.pdf diff --git a/src/paperless_tesseract/tests/samples/with-form.pdf b/src/paperless/tests/samples/tesseract/with-form.pdf similarity index 100% rename from src/paperless_tesseract/tests/samples/with-form.pdf rename to src/paperless/tests/samples/tesseract/with-form.pdf diff --git a/src/paperless_tesseract/signals.py b/src/paperless_tesseract/signals.py index e4d8449ed..d80d13614 100644 --- a/src/paperless_tesseract/signals.py +++ b/src/paperless_tesseract/signals.py @@ -1,10 +1,23 @@ -def get_parser(*args, **kwargs): - from paperless_tesseract.parsers import RasterisedDocumentParser +from __future__ import annotations +from typing import Any + + +def get_parser(*args: Any, **kwargs: Any) -> Any: + from paperless.parsers.tesseract import RasterisedDocumentParser + + # RasterisedDocumentParser accepts logging_group for constructor compatibility but + # does not store or use it (no legacy DocumentParser base class). + # progress_callback is also not used. Both may arrive as a positional arg + # (consumer) or a keyword arg (views); *args absorbs the positional form, + # kwargs.pop handles the keyword form. Phase 4 will replace this signal + # path with the new ParserRegistry so the shim can be removed at that point. + kwargs.pop("logging_group", None) + kwargs.pop("progress_callback", None) return RasterisedDocumentParser(*args, **kwargs) -def tesseract_consumer_declaration(sender, **kwargs): +def tesseract_consumer_declaration(sender: Any, **kwargs: Any) -> dict[str, Any]: return { "parser": get_parser, "weight": 0, diff --git a/src/paperless_tesseract/tests/test_parser.py b/src/paperless_tesseract/tests/test_parser.py deleted file mode 100644 index 2703f30dd..000000000 --- a/src/paperless_tesseract/tests/test_parser.py +++ /dev/null @@ -1,924 +0,0 @@ -import shutil -import tempfile -import unicodedata -import uuid -from pathlib import Path -from unittest import mock - -from django.test import TestCase -from django.test import override_settings -from ocrmypdf import SubprocessOutputError - -from documents.parsers import ParseError -from documents.parsers import run_convert -from documents.tests.utils import DirectoriesMixin -from documents.tests.utils import FileSystemAssertsMixin -from paperless_tesseract.parsers import RasterisedDocumentParser -from paperless_tesseract.parsers import post_process_text - - -class TestParser(DirectoriesMixin, FileSystemAssertsMixin, TestCase): - SAMPLE_FILES = Path(__file__).resolve().parent / "samples" - - def assertContainsStrings(self, content, strings) -> None: - # Asserts that all strings appear in content, in the given order. - indices = [] - for s in strings: - if s in content: - indices.append(content.index(s)) - else: - self.fail(f"'{s}' is not in '{content}'") - self.assertListEqual(indices, sorted(indices)) - - def test_post_process_text(self) -> None: - text_cases = [ - ("simple string", "simple string"), - ("simple newline\n testing string", "simple newline\ntesting string"), - ( - "utf-8 строка с пробелами в конце ", - "utf-8 строка с пробелами в конце", - ), - ] - - for source, result in text_cases: - actual_result = post_process_text(source) - self.assertEqual( - result, - actual_result, - f"strip_exceess_whitespace({source}) != '{result}', but '{actual_result}'", - ) - - def test_get_text_from_pdf(self) -> None: - parser = RasterisedDocumentParser(uuid.uuid4()) - text = parser.extract_text( - None, - self.SAMPLE_FILES / "simple-digital.pdf", - ) - - self.assertContainsStrings(text.strip(), ["This is a test document."]) - - def test_get_page_count(self) -> None: - """ - GIVEN: - - PDF file with a single page - - PDF file with multiple pages - WHEN: - - The number of pages is requested - THEN: - - The method returns 1 as the expected number of pages - - The method returns the correct number of pages (6) - """ - parser = RasterisedDocumentParser(uuid.uuid4()) - page_count = parser.get_page_count( - str(self.SAMPLE_FILES / "simple-digital.pdf"), - "application/pdf", - ) - self.assertEqual(page_count, 1) - - page_count = parser.get_page_count( - str(self.SAMPLE_FILES / "multi-page-mixed.pdf"), - "application/pdf", - ) - self.assertEqual(page_count, 6) - - def test_get_page_count_password_protected(self) -> None: - """ - GIVEN: - - Password protected PDF file - WHEN: - - The number of pages is requested - THEN: - - The method returns None - """ - parser = RasterisedDocumentParser(uuid.uuid4()) - with self.assertLogs("paperless.parsing.tesseract", level="WARNING") as cm: - page_count = parser.get_page_count( - str(self.SAMPLE_FILES / "password-protected.pdf"), - "application/pdf", - ) - self.assertEqual(page_count, None) - self.assertIn("Unable to determine PDF page count", cm.output[0]) - - def test_thumbnail(self) -> None: - parser = RasterisedDocumentParser(uuid.uuid4()) - thumb = parser.get_thumbnail( - str(self.SAMPLE_FILES / "simple-digital.pdf"), - "application/pdf", - ) - self.assertIsFile(thumb) - - @mock.patch("documents.parsers.run_convert") - def test_thumbnail_fallback(self, m) -> None: - def call_convert(input_file, output_file, **kwargs) -> None: - if ".pdf" in str(input_file): - raise ParseError("Does not compute.") - else: - run_convert(input_file=input_file, output_file=output_file, **kwargs) - - m.side_effect = call_convert - - parser = RasterisedDocumentParser(uuid.uuid4()) - thumb = parser.get_thumbnail( - str(self.SAMPLE_FILES / "simple-digital.pdf"), - "application/pdf", - ) - self.assertIsFile(thumb) - - def test_thumbnail_encrypted(self) -> None: - parser = RasterisedDocumentParser(uuid.uuid4()) - thumb = parser.get_thumbnail( - str(self.SAMPLE_FILES / "encrypted.pdf"), - "application/pdf", - ) - self.assertIsFile(thumb) - - def test_get_dpi(self) -> None: - parser = RasterisedDocumentParser(None) - - dpi = parser.get_dpi(str(self.SAMPLE_FILES / "simple-no-dpi.png")) - self.assertEqual(dpi, None) - - dpi = parser.get_dpi(str(self.SAMPLE_FILES / "simple.png")) - self.assertEqual(dpi, 72) - - def test_simple_digital(self) -> None: - parser = RasterisedDocumentParser(None) - - parser.parse( - str(self.SAMPLE_FILES / "simple-digital.pdf"), - "application/pdf", - ) - - self.assertIsFile(parser.archive_path) - - self.assertContainsStrings(parser.get_text(), ["This is a test document."]) - - def test_with_form(self) -> None: - parser = RasterisedDocumentParser(None) - - parser.parse( - str(self.SAMPLE_FILES / "with-form.pdf"), - "application/pdf", - ) - - self.assertIsFile(parser.archive_path) - - self.assertContainsStrings( - parser.get_text(), - ["Please enter your name in here:", "This is a PDF document with a form."], - ) - - @override_settings(OCR_MODE="redo") - def test_with_form_error(self) -> None: - parser = RasterisedDocumentParser(None) - - parser.parse( - str(self.SAMPLE_FILES / "with-form.pdf"), - "application/pdf", - ) - - self.assertIsNone(parser.archive_path) - self.assertContainsStrings( - parser.get_text(), - ["Please enter your name in here:", "This is a PDF document with a form."], - ) - - @override_settings(OCR_MODE="skip") - def test_signed(self) -> None: - parser = RasterisedDocumentParser(None) - - parser.parse(str(self.SAMPLE_FILES / "signed.pdf"), "application/pdf") - - self.assertIsNone(parser.archive_path) - self.assertContainsStrings( - parser.get_text(), - [ - "This is a digitally signed PDF, created with Acrobat Pro for the Paperless project to enable", - "automated testing of signed/encrypted PDFs", - ], - ) - - @override_settings(OCR_MODE="skip") - def test_encrypted(self) -> None: - parser = RasterisedDocumentParser(None) - - parser.parse( - str(self.SAMPLE_FILES / "encrypted.pdf"), - "application/pdf", - ) - - self.assertIsNone(parser.archive_path) - self.assertEqual(parser.get_text(), "") - - @override_settings(OCR_MODE="redo") - def test_with_form_error_notext(self) -> None: - parser = RasterisedDocumentParser(None) - parser.parse( - str(self.SAMPLE_FILES / "with-form.pdf"), - "application/pdf", - ) - - self.assertContainsStrings( - parser.get_text(), - ["Please enter your name in here:", "This is a PDF document with a form."], - ) - - @override_settings(OCR_MODE="force") - def test_with_form_force(self) -> None: - parser = RasterisedDocumentParser(None) - - parser.parse( - str(self.SAMPLE_FILES / "with-form.pdf"), - "application/pdf", - ) - - self.assertContainsStrings( - parser.get_text(), - ["Please enter your name in here:", "This is a PDF document with a form."], - ) - - def test_image_simple(self) -> None: - parser = RasterisedDocumentParser(None) - - parser.parse(str(self.SAMPLE_FILES / "simple.png"), "image/png") - - self.assertIsFile(parser.archive_path) - - self.assertContainsStrings(parser.get_text(), ["This is a test document."]) - - def test_image_simple_alpha(self) -> None: - parser = RasterisedDocumentParser(None) - - with tempfile.TemporaryDirectory() as tempdir: - # Copy sample file to temp directory, as the parsing changes the file - # and this makes it modified to Git - sample_file = self.SAMPLE_FILES / "simple-alpha.png" - dest_file = Path(tempdir) / "simple-alpha.png" - shutil.copy(sample_file, dest_file) - - parser.parse(str(dest_file), "image/png") - - self.assertIsFile(parser.archive_path) - - self.assertContainsStrings(parser.get_text(), ["This is a test document."]) - - def test_image_calc_a4_dpi(self) -> None: - parser = RasterisedDocumentParser(None) - - dpi = parser.calculate_a4_dpi( - str(self.SAMPLE_FILES / "simple-no-dpi.png"), - ) - - self.assertEqual(dpi, 62) - - @mock.patch("paperless_tesseract.parsers.RasterisedDocumentParser.calculate_a4_dpi") - def test_image_dpi_fail(self, m) -> None: - m.return_value = None - parser = RasterisedDocumentParser(None) - - def f() -> None: - parser.parse( - str(self.SAMPLE_FILES / "simple-no-dpi.png"), - "image/png", - ) - - self.assertRaises(ParseError, f) - - @override_settings(OCR_IMAGE_DPI=72, MAX_IMAGE_PIXELS=0) - def test_image_no_dpi_default(self) -> None: - parser = RasterisedDocumentParser(None) - - parser.parse(str(self.SAMPLE_FILES / "simple-no-dpi.png"), "image/png") - - self.assertIsFile(parser.archive_path) - - self.assertContainsStrings( - parser.get_text().lower(), - ["this is a test document."], - ) - - def test_multi_page(self) -> None: - parser = RasterisedDocumentParser(None) - parser.parse( - str(self.SAMPLE_FILES / "multi-page-digital.pdf"), - "application/pdf", - ) - self.assertIsFile(parser.archive_path) - self.assertContainsStrings( - parser.get_text().lower(), - ["page 1", "page 2", "page 3"], - ) - - @override_settings(OCR_PAGES=2, OCR_MODE="skip") - def test_multi_page_pages_skip(self) -> None: - parser = RasterisedDocumentParser(None) - parser.parse( - str(self.SAMPLE_FILES / "multi-page-digital.pdf"), - "application/pdf", - ) - self.assertIsFile(parser.archive_path) - self.assertContainsStrings( - parser.get_text().lower(), - ["page 1", "page 2", "page 3"], - ) - - @override_settings(OCR_PAGES=2, OCR_MODE="redo") - def test_multi_page_pages_redo(self) -> None: - parser = RasterisedDocumentParser(None) - parser.parse( - str(self.SAMPLE_FILES / "multi-page-digital.pdf"), - "application/pdf", - ) - self.assertIsFile(parser.archive_path) - self.assertContainsStrings( - parser.get_text().lower(), - ["page 1", "page 2", "page 3"], - ) - - @override_settings(OCR_PAGES=2, OCR_MODE="force") - def test_multi_page_pages_force(self) -> None: - parser = RasterisedDocumentParser(None) - parser.parse( - str(self.SAMPLE_FILES / "multi-page-digital.pdf"), - "application/pdf", - ) - self.assertIsFile(parser.archive_path) - self.assertContainsStrings( - parser.get_text().lower(), - ["page 1", "page 2", "page 3"], - ) - - @override_settings(OCR_MODE="skip") - def test_multi_page_analog_pages_skip(self) -> None: - parser = RasterisedDocumentParser(None) - parser.parse( - str(self.SAMPLE_FILES / "multi-page-images.pdf"), - "application/pdf", - ) - self.assertIsFile(parser.archive_path) - self.assertContainsStrings( - parser.get_text().lower(), - ["page 1", "page 2", "page 3"], - ) - - @override_settings(OCR_PAGES=2, OCR_MODE="redo") - def test_multi_page_analog_pages_redo(self) -> None: - """ - GIVEN: - - File with text contained in images but no text layer - - OCR of only pages 1 and 2 requested - - OCR mode set to redo - WHEN: - - Document is parsed - THEN: - - Text of page 1 and 2 extracted - - An archive file is created - """ - parser = RasterisedDocumentParser(None) - parser.parse( - str(self.SAMPLE_FILES / "multi-page-images.pdf"), - "application/pdf", - ) - self.assertIsFile(parser.archive_path) - self.assertContainsStrings(parser.get_text().lower(), ["page 1", "page 2"]) - self.assertNotIn("page 3", parser.get_text().lower()) - - @override_settings(OCR_PAGES=1, OCR_MODE="force") - def test_multi_page_analog_pages_force(self) -> None: - """ - GIVEN: - - File with text contained in images but no text layer - - OCR of only page 1 requested - - OCR mode set to force - WHEN: - - Document is parsed - THEN: - - Only text of page 1 is extracted - - An archive file is created - """ - parser = RasterisedDocumentParser(None) - parser.parse( - str(self.SAMPLE_FILES / "multi-page-images.pdf"), - "application/pdf", - ) - self.assertIsFile(parser.archive_path) - self.assertContainsStrings(parser.get_text().lower(), ["page 1"]) - self.assertNotIn("page 2", parser.get_text().lower()) - self.assertNotIn("page 3", parser.get_text().lower()) - - @override_settings(OCR_MODE="skip_noarchive") - def test_skip_noarchive_withtext(self) -> None: - """ - GIVEN: - - File with existing text layer - - OCR mode set to skip_noarchive - WHEN: - - Document is parsed - THEN: - - Text from images is extracted - - No archive file is created - """ - parser = RasterisedDocumentParser(None) - parser.parse( - str(self.SAMPLE_FILES / "multi-page-digital.pdf"), - "application/pdf", - ) - self.assertIsNone(parser.archive_path) - self.assertContainsStrings( - parser.get_text().lower(), - ["page 1", "page 2", "page 3"], - ) - - @override_settings(OCR_MODE="skip_noarchive") - def test_skip_noarchive_notext(self) -> None: - """ - GIVEN: - - File with text contained in images but no text layer - - OCR mode set to skip_noarchive - WHEN: - - Document is parsed - THEN: - - Text from images is extracted - - An archive file is created with the OCRd text - """ - parser = RasterisedDocumentParser(None) - parser.parse( - str(self.SAMPLE_FILES / "multi-page-images.pdf"), - "application/pdf", - ) - - self.assertContainsStrings( - parser.get_text().lower(), - ["page 1", "page 2", "page 3"], - ) - - self.assertIsNotNone(parser.archive_path) - - @override_settings(OCR_SKIP_ARCHIVE_FILE="never") - def test_skip_archive_never_withtext(self) -> None: - """ - GIVEN: - - File with existing text layer - - OCR_SKIP_ARCHIVE_FILE set to never - WHEN: - - Document is parsed - THEN: - - Text from text layer is extracted - - Archive file is created - """ - parser = RasterisedDocumentParser(None) - parser.parse( - str(self.SAMPLE_FILES / "multi-page-digital.pdf"), - "application/pdf", - ) - self.assertIsNotNone(parser.archive_path) - self.assertContainsStrings( - parser.get_text().lower(), - ["page 1", "page 2", "page 3"], - ) - - @override_settings(OCR_SKIP_ARCHIVE_FILE="never") - def test_skip_archive_never_withimages(self) -> None: - """ - GIVEN: - - File with text contained in images but no text layer - - OCR_SKIP_ARCHIVE_FILE set to never - WHEN: - - Document is parsed - THEN: - - Text from images is extracted - - Archive file is created - """ - parser = RasterisedDocumentParser(None) - parser.parse( - str(self.SAMPLE_FILES / "multi-page-images.pdf"), - "application/pdf", - ) - self.assertIsNotNone(parser.archive_path) - self.assertContainsStrings( - parser.get_text().lower(), - ["page 1", "page 2", "page 3"], - ) - - @override_settings(OCR_SKIP_ARCHIVE_FILE="with_text") - def test_skip_archive_withtext_withtext(self) -> None: - """ - GIVEN: - - File with existing text layer - - OCR_SKIP_ARCHIVE_FILE set to with_text - WHEN: - - Document is parsed - THEN: - - Text from text layer is extracted - - No archive file is created - """ - parser = RasterisedDocumentParser(None) - parser.parse( - str(self.SAMPLE_FILES / "multi-page-digital.pdf"), - "application/pdf", - ) - self.assertIsNone(parser.archive_path) - self.assertContainsStrings( - parser.get_text().lower(), - ["page 1", "page 2", "page 3"], - ) - - @override_settings(OCR_SKIP_ARCHIVE_FILE="with_text") - def test_skip_archive_withtext_withimages(self) -> None: - """ - GIVEN: - - File with text contained in images but no text layer - - OCR_SKIP_ARCHIVE_FILE set to with_text - WHEN: - - Document is parsed - THEN: - - Text from images is extracted - - Archive file is created - """ - parser = RasterisedDocumentParser(None) - parser.parse( - str(self.SAMPLE_FILES / "multi-page-images.pdf"), - "application/pdf", - ) - self.assertIsNotNone(parser.archive_path) - self.assertContainsStrings( - parser.get_text().lower(), - ["page 1", "page 2", "page 3"], - ) - - @override_settings(OCR_SKIP_ARCHIVE_FILE="always") - def test_skip_archive_always_withtext(self) -> None: - """ - GIVEN: - - File with existing text layer - - OCR_SKIP_ARCHIVE_FILE set to always - WHEN: - - Document is parsed - THEN: - - Text from text layer is extracted - - No archive file is created - """ - parser = RasterisedDocumentParser(None) - parser.parse( - str(self.SAMPLE_FILES / "multi-page-digital.pdf"), - "application/pdf", - ) - self.assertIsNone(parser.archive_path) - self.assertContainsStrings( - parser.get_text().lower(), - ["page 1", "page 2", "page 3"], - ) - - @override_settings(OCR_SKIP_ARCHIVE_FILE="always") - def test_skip_archive_always_withimages(self) -> None: - """ - GIVEN: - - File with text contained in images but no text layer - - OCR_SKIP_ARCHIVE_FILE set to always - WHEN: - - Document is parsed - THEN: - - Text from images is extracted - - No archive file is created - """ - parser = RasterisedDocumentParser(None) - parser.parse( - str(self.SAMPLE_FILES / "multi-page-images.pdf"), - "application/pdf", - ) - self.assertIsNone(parser.archive_path) - self.assertContainsStrings( - parser.get_text().lower(), - ["page 1", "page 2", "page 3"], - ) - - @override_settings(OCR_MODE="skip") - def test_multi_page_mixed(self) -> None: - """ - GIVEN: - - File with some text contained in images and some in text layer - - OCR mode set to skip - WHEN: - - Document is parsed - THEN: - - Text from images is extracted - - An archive file is created with the OCRd text and the original text - """ - parser = RasterisedDocumentParser(None) - parser.parse( - str(self.SAMPLE_FILES / "multi-page-mixed.pdf"), - "application/pdf", - ) - self.assertIsNotNone(parser.archive_path) - self.assertIsFile(parser.archive_path) - self.assertContainsStrings( - parser.get_text().lower(), - ["page 1", "page 2", "page 3", "page 4", "page 5", "page 6"], - ) - - with (parser.tempdir / "sidecar.txt").open() as f: - sidecar = f.read() - - self.assertIn("[OCR skipped on page(s) 4-6]", sidecar) - - @override_settings(OCR_MODE="redo") - def test_single_page_mixed(self) -> None: - """ - GIVEN: - - File with some text contained in images and some in text layer - - Text and images are mixed on the same page - - OCR mode set to redo - WHEN: - - Document is parsed - THEN: - - Text from images is extracted - - Full content of the file is parsed (not just the image text) - - An archive file is created with the OCRd text and the original text - """ - parser = RasterisedDocumentParser(None) - parser.parse( - str(self.SAMPLE_FILES / "single-page-mixed.pdf"), - "application/pdf", - ) - self.assertIsNotNone(parser.archive_path) - self.assertIsFile(parser.archive_path) - self.assertContainsStrings( - parser.get_text().lower(), - [ - "this is some normal text, present on page 1 of the document.", - "this is some text, but in an image, also on page 1.", - "this is further text on page 1.", - ], - ) - - with (parser.tempdir / "sidecar.txt").open() as f: - sidecar = f.read().lower() - - self.assertIn("this is some text, but in an image, also on page 1.", sidecar) - self.assertNotIn( - "this is some normal text, present on page 1 of the document.", - sidecar, - ) - - @override_settings(OCR_MODE="skip_noarchive") - def test_multi_page_mixed_no_archive(self) -> None: - """ - GIVEN: - - File with some text contained in images and some in text layer - - OCR mode set to skip_noarchive - WHEN: - - Document is parsed - THEN: - - Text from images is extracted - - No archive file is created as original file contains text - """ - parser = RasterisedDocumentParser(None) - parser.parse( - str(self.SAMPLE_FILES / "multi-page-mixed.pdf"), - "application/pdf", - ) - self.assertIsNone(parser.archive_path) - self.assertContainsStrings( - parser.get_text().lower(), - ["page 4", "page 5", "page 6"], - ) - - @override_settings(OCR_MODE="skip", OCR_ROTATE_PAGES=True) - def test_rotate(self) -> None: - parser = RasterisedDocumentParser(None) - parser.parse(str(self.SAMPLE_FILES / "rotated.pdf"), "application/pdf") - self.assertContainsStrings( - parser.get_text(), - [ - "This is the text that appears on the first page. It’s a lot of text.", - "Even if the pages are rotated, OCRmyPDF still gets the job done.", - "This is a really weird file with lots of nonsense text.", - "If you read this, it’s your own fault. Also check your screen orientation.", - ], - ) - - def test_multi_page_tiff(self) -> None: - """ - GIVEN: - - Multi-page TIFF image - WHEN: - - Image is parsed - THEN: - - Text from all pages extracted - """ - parser = RasterisedDocumentParser(None) - parser.parse( - str(self.SAMPLE_FILES / "multi-page-images.tiff"), - "image/tiff", - ) - self.assertIsFile(parser.archive_path) - self.assertContainsStrings( - parser.get_text().lower(), - ["page 1", "page 2", "page 3"], - ) - - def test_multi_page_tiff_alpha(self) -> None: - """ - GIVEN: - - Multi-page TIFF image - - Image include an alpha channel - WHEN: - - Image is parsed - THEN: - - Text from all pages extracted - """ - parser = RasterisedDocumentParser(None) - sample_file = self.SAMPLE_FILES / "multi-page-images-alpha.tiff" - with tempfile.NamedTemporaryFile() as tmp_file: - shutil.copy(sample_file, tmp_file.name) - parser.parse( - tmp_file.name, - "image/tiff", - ) - self.assertIsFile(parser.archive_path) - self.assertContainsStrings( - parser.get_text().lower(), - ["page 1", "page 2", "page 3"], - ) - - def test_multi_page_tiff_alpha_srgb(self) -> None: - """ - GIVEN: - - Multi-page TIFF image - - Image include an alpha channel - - Image is srgb colorspace - WHEN: - - Image is parsed - THEN: - - Text from all pages extracted - """ - parser = RasterisedDocumentParser(None) - sample_file = str( - self.SAMPLE_FILES / "multi-page-images-alpha-rgb.tiff", - ) - with tempfile.NamedTemporaryFile() as tmp_file: - shutil.copy(sample_file, tmp_file.name) - parser.parse( - tmp_file.name, - "image/tiff", - ) - self.assertIsFile(parser.archive_path) - self.assertContainsStrings( - parser.get_text().lower(), - ["page 1", "page 2", "page 3"], - ) - - def test_ocrmypdf_parameters(self) -> None: - parser = RasterisedDocumentParser(None) - params = parser.construct_ocrmypdf_parameters( - input_file="input.pdf", - output_file="output.pdf", - sidecar_file="sidecar.txt", - mime_type="application/pdf", - safe_fallback=False, - ) - - self.assertEqual(params["input_file_or_options"], "input.pdf") - self.assertEqual(params["output_file"], "output.pdf") - self.assertEqual(params["sidecar"], "sidecar.txt") - - with override_settings(OCR_CLEAN="none"): - parser = RasterisedDocumentParser(None) - params = parser.construct_ocrmypdf_parameters("", "", "", "") - self.assertNotIn("clean", params) - self.assertNotIn("clean_final", params) - - with override_settings(OCR_CLEAN="clean"): - parser = RasterisedDocumentParser(None) - params = parser.construct_ocrmypdf_parameters("", "", "", "") - self.assertTrue(params["clean"]) - self.assertNotIn("clean_final", params) - - with override_settings(OCR_CLEAN="clean-final", OCR_MODE="skip"): - parser = RasterisedDocumentParser(None) - params = parser.construct_ocrmypdf_parameters("", "", "", "") - self.assertTrue(params["clean_final"]) - self.assertNotIn("clean", params) - - with override_settings(OCR_CLEAN="clean-final", OCR_MODE="redo"): - parser = RasterisedDocumentParser(None) - params = parser.construct_ocrmypdf_parameters("", "", "", "") - self.assertTrue(params["clean"]) - self.assertNotIn("clean_final", params) - - with override_settings(OCR_DESKEW=True, OCR_MODE="skip"): - parser = RasterisedDocumentParser(None) - params = parser.construct_ocrmypdf_parameters("", "", "", "") - self.assertTrue(params["deskew"]) - - with override_settings(OCR_DESKEW=True, OCR_MODE="redo"): - parser = RasterisedDocumentParser(None) - params = parser.construct_ocrmypdf_parameters("", "", "", "") - self.assertNotIn("deskew", params) - - with override_settings(OCR_DESKEW=False, OCR_MODE="skip"): - parser = RasterisedDocumentParser(None) - params = parser.construct_ocrmypdf_parameters("", "", "", "") - self.assertNotIn("deskew", params) - - with override_settings(OCR_MAX_IMAGE_PIXELS=1_000_001.0): - parser = RasterisedDocumentParser(None) - params = parser.construct_ocrmypdf_parameters("", "", "", "") - self.assertIn("max_image_mpixels", params) - self.assertAlmostEqual(params["max_image_mpixels"], 1, places=4) - - with override_settings(OCR_MAX_IMAGE_PIXELS=-1_000_001.0): - parser = RasterisedDocumentParser(None) - params = parser.construct_ocrmypdf_parameters("", "", "", "") - self.assertNotIn("max_image_mpixels", params) - - def test_rtl_language_detection(self) -> None: - """ - GIVEN: - - File with text in an RTL language - WHEN: - - Document is parsed - THEN: - - Text from the document is extracted - """ - parser = RasterisedDocumentParser(None) - - parser.parse( - str(self.SAMPLE_FILES / "rtl-test.pdf"), - "application/pdf", - ) - - # OCR output for RTL text varies across platforms/versions due to - # bidi controls and presentation forms; normalize before assertion. - normalized_text = "".join( - char - for char in unicodedata.normalize("NFKC", parser.get_text()) - if unicodedata.category(char) != "Cf" and not char.isspace() - ) - - self.assertIn("ةرازو", normalized_text) - self.assertTrue( - any(token in normalized_text for token in ("ةیلخادلا", "الاخليد")), - ) - - @mock.patch("ocrmypdf.ocr") - def test_gs_rendering_error(self, m) -> None: - m.side_effect = SubprocessOutputError("Ghostscript PDF/A rendering failed") - parser = RasterisedDocumentParser(None) - - self.assertRaises( - ParseError, - parser.parse, - str(self.SAMPLE_FILES / "simple-digital.pdf"), - "application/pdf", - ) - - -class TestParserFileTypes(DirectoriesMixin, FileSystemAssertsMixin, TestCase): - SAMPLE_FILES = Path(__file__).parent / "samples" - - def test_bmp(self) -> None: - parser = RasterisedDocumentParser(None) - parser.parse(str(self.SAMPLE_FILES / "simple.bmp"), "image/bmp") - self.assertIsFile(parser.archive_path) - self.assertIn("this is a test document", parser.get_text().lower()) - - def test_jpg(self) -> None: - parser = RasterisedDocumentParser(None) - parser.parse(str(self.SAMPLE_FILES / "simple.jpg"), "image/jpeg") - self.assertIsFile(parser.archive_path) - self.assertIn("this is a test document", parser.get_text().lower()) - - def test_heic(self) -> None: - parser = RasterisedDocumentParser(None) - parser.parse(str(self.SAMPLE_FILES / "simple.heic"), "image/heic") - self.assertIsFile(parser.archive_path) - self.assertIn("pizza", parser.get_text().lower()) - - @override_settings(OCR_IMAGE_DPI=200) - def test_gif(self) -> None: - parser = RasterisedDocumentParser(None) - parser.parse(str(self.SAMPLE_FILES / "simple.gif"), "image/gif") - self.assertIsFile(parser.archive_path) - self.assertIn("this is a test document", parser.get_text().lower()) - - def test_tiff(self) -> None: - parser = RasterisedDocumentParser(None) - parser.parse(str(self.SAMPLE_FILES / "simple.tif"), "image/tiff") - self.assertIsFile(parser.archive_path) - self.assertIn("this is a test document", parser.get_text().lower()) - - @override_settings(OCR_IMAGE_DPI=72) - def test_webp(self) -> None: - parser = RasterisedDocumentParser(None) - parser.parse( - str(self.SAMPLE_FILES / "document.webp"), - "image/webp", - ) - self.assertIsFile(parser.archive_path) - # Older tesseracts consistently mangle the space between "a webp", - # tesseract 5.3.0 seems to do a better job, so we're accepting both - self.assertRegex( - parser.get_text().lower(), - r"this is a ?webp document, created 11/14/2022.", - )