mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-10-23 08:08:55 +00:00
366 lines
13 KiB
Python
366 lines
13 KiB
Python
"""Tests for paperless.parsers.utils helpers."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import codecs
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING
|
|
|
|
import pikepdf
|
|
import pytest
|
|
|
|
from paperless.parsers.utils import get_pdf_first_page_size_points
|
|
from paperless.parsers.utils import is_tagged_pdf
|
|
from paperless.parsers.utils import pdf_born_digital_text
|
|
from paperless.parsers.utils import post_process_text
|
|
from paperless.parsers.utils import read_file_handle_unicode_errors
|
|
|
|
if TYPE_CHECKING:
|
|
from pytest_mock import MockerFixture
|
|
|
|
|
|
class TestReadFileHandleUnicodeErrors:
|
|
def test_plain_utf8(self, tmp_path: Path) -> None:
|
|
f = tmp_path / "plain.txt"
|
|
f.write_bytes(b"hello world")
|
|
assert read_file_handle_unicode_errors(f) == "hello world"
|
|
|
|
def test_utf8_bom(self, tmp_path: Path) -> None:
|
|
f = tmp_path / "bom.txt"
|
|
f.write_bytes(codecs.BOM_UTF8 + b"hello")
|
|
assert read_file_handle_unicode_errors(f) == "hello"
|
|
|
|
def test_utf16_le(self, tmp_path: Path) -> None:
|
|
f = tmp_path / "utf16le.txt"
|
|
f.write_bytes(codecs.BOM_UTF16_LE + "hello".encode("utf-16-le"))
|
|
assert read_file_handle_unicode_errors(f) == "hello"
|
|
|
|
def test_utf16_be(self, tmp_path: Path) -> None:
|
|
f = tmp_path / "utf16be.txt"
|
|
f.write_bytes(codecs.BOM_UTF16_BE + "hello".encode("utf-16-be"))
|
|
assert read_file_handle_unicode_errors(f) == "hello"
|
|
|
|
def test_nul_bytes_stripped(self, tmp_path: Path) -> None:
|
|
f = tmp_path / "null-bytes.txt"
|
|
f.write_bytes(b"foo\x00bar")
|
|
assert read_file_handle_unicode_errors(f) == "foobar"
|
|
|
|
def test_invalid_utf8_replaced(self, tmp_path: Path) -> None:
|
|
f = tmp_path / "bad.txt"
|
|
f.write_bytes(b"ok\x80\x81bad")
|
|
result = read_file_handle_unicode_errors(f)
|
|
assert "ok" in result
|
|
assert "bad" in result
|
|
assert "\x00" not in result
|
|
|
|
|
|
class TestIsTaggedPdf:
|
|
def test_tagged_pdf_returns_true(self, simple_digital_pdf_file: Path) -> None:
|
|
assert is_tagged_pdf(simple_digital_pdf_file) is True
|
|
|
|
def test_untagged_pdf_returns_false(self, multi_page_images_pdf_file: Path) -> None:
|
|
assert is_tagged_pdf(multi_page_images_pdf_file) is False
|
|
|
|
def test_nonexistent_path_returns_false(self) -> None:
|
|
assert is_tagged_pdf(Path("/nonexistent/file.pdf")) is False
|
|
|
|
def test_corrupt_pdf_returns_false(self, tmp_path: Path) -> None:
|
|
bad = tmp_path / "bad.pdf"
|
|
bad.write_bytes(b"not a pdf")
|
|
assert is_tagged_pdf(bad) is False
|
|
|
|
|
|
class TestGetPdfFirstPageSizePoints:
|
|
@staticmethod
|
|
def _write_pdf(
|
|
path: Path,
|
|
*,
|
|
media_box: tuple[float, float, float, float] = (0, 0, 600, 800),
|
|
crop_box: tuple[float, float, float, float] | None = None,
|
|
page_rotate: int | None = None,
|
|
inherited_rotate: int | None = None,
|
|
) -> Path:
|
|
pdf = pikepdf.new()
|
|
pdf.add_blank_page(page_size=(media_box[2], media_box[3]))
|
|
page = pdf.pages[0]
|
|
page.obj.MediaBox = pikepdf.Array(media_box)
|
|
if crop_box is not None:
|
|
page.obj.CropBox = pikepdf.Array(crop_box)
|
|
if page_rotate is not None:
|
|
page.obj.Rotate = page_rotate
|
|
if inherited_rotate is not None:
|
|
pdf.Root.Pages.Rotate = inherited_rotate
|
|
pdf.save(path)
|
|
return path
|
|
|
|
def test_letter_sample(self, simple_digital_pdf_file: Path) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A US Letter sample PDF with no CropBox and no rotation
|
|
WHEN:
|
|
- The first page size is requested
|
|
THEN:
|
|
- The MediaBox size in points is returned
|
|
"""
|
|
assert get_pdf_first_page_size_points(simple_digital_pdf_file) == (
|
|
612.0,
|
|
792.0,
|
|
)
|
|
|
|
@pytest.mark.parametrize(
|
|
("rotate", "expected"),
|
|
[
|
|
pytest.param(0, (600.0, 800.0), id="rotate-0"),
|
|
pytest.param(90, (800.0, 600.0), id="rotate-90"),
|
|
pytest.param(180, (600.0, 800.0), id="rotate-180"),
|
|
pytest.param(270, (800.0, 600.0), id="rotate-270"),
|
|
pytest.param(-90, (800.0, 600.0), id="rotate-negative-90"),
|
|
],
|
|
)
|
|
def test_page_rotation_swaps_dimensions(
|
|
self,
|
|
tmp_path: Path,
|
|
rotate: int,
|
|
expected: tuple[float, float],
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A portrait PDF page with /Rotate set directly on the page
|
|
WHEN:
|
|
- The first page size is requested
|
|
THEN:
|
|
- Width and height are swapped for quarter-turn rotations only
|
|
"""
|
|
pdf_path = self._write_pdf(tmp_path / "rotated.pdf", page_rotate=rotate)
|
|
assert get_pdf_first_page_size_points(pdf_path) == expected
|
|
|
|
def test_inherited_rotation_swaps_dimensions(self, tmp_path: Path) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A page inheriting /Rotate 90 from the /Pages node
|
|
WHEN:
|
|
- The first page size is requested
|
|
THEN:
|
|
- Width and height are swapped
|
|
"""
|
|
pdf_path = self._write_pdf(tmp_path / "inherited.pdf", inherited_rotate=90)
|
|
assert get_pdf_first_page_size_points(pdf_path) == (800.0, 600.0)
|
|
|
|
def test_crop_box_preferred_over_media_box(self, tmp_path: Path) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A PDF page with a CropBox smaller than its MediaBox
|
|
WHEN:
|
|
- The first page size is requested
|
|
THEN:
|
|
- The CropBox size is returned
|
|
"""
|
|
pdf_path = self._write_pdf(
|
|
tmp_path / "cropped.pdf",
|
|
crop_box=(50, 100, 350, 500),
|
|
)
|
|
assert get_pdf_first_page_size_points(pdf_path) == (300.0, 400.0)
|
|
|
|
def test_crop_box_clipped_to_media_box(self, tmp_path: Path) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A PDF page whose CropBox extends well past its MediaBox
|
|
WHEN:
|
|
- The first page size is requested
|
|
THEN:
|
|
- The size is the CropBox clipped to the MediaBox, as pdftoppm renders it
|
|
"""
|
|
pdf_path = self._write_pdf(
|
|
tmp_path / "oversized-crop.pdf",
|
|
media_box=(0, 0, 200, 200),
|
|
crop_box=(-200, -200, 600, 600),
|
|
)
|
|
assert get_pdf_first_page_size_points(pdf_path) == (200.0, 200.0)
|
|
|
|
def test_crop_box_with_rotation(self, tmp_path: Path) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A rotated PDF page with a CropBox smaller than its MediaBox
|
|
WHEN:
|
|
- The first page size is requested
|
|
THEN:
|
|
- The CropBox size is returned with width and height swapped
|
|
"""
|
|
pdf_path = self._write_pdf(
|
|
tmp_path / "cropped-rotated.pdf",
|
|
media_box=(0, 0, 600, 300),
|
|
crop_box=(100, 50, 400, 250),
|
|
page_rotate=270,
|
|
)
|
|
assert get_pdf_first_page_size_points(pdf_path) == (200.0, 300.0)
|
|
|
|
def test_pdfinfo_not_installed_returns_none(
|
|
self,
|
|
mocker: MockerFixture,
|
|
simple_digital_pdf_file: Path,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- The pdfinfo binary cannot be executed
|
|
WHEN:
|
|
- The first page size is requested
|
|
THEN:
|
|
- None is returned and nothing is raised
|
|
"""
|
|
mocker.patch("subprocess.run", side_effect=FileNotFoundError("pdfinfo"))
|
|
assert get_pdf_first_page_size_points(simple_digital_pdf_file) is None
|
|
|
|
def test_degenerate_box_returns_none(self, tmp_path: Path) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A PDF page whose box has zero width
|
|
WHEN:
|
|
- The first page size is requested
|
|
THEN:
|
|
- None is returned
|
|
"""
|
|
pdf_path = self._write_pdf(
|
|
tmp_path / "degenerate.pdf",
|
|
media_box=(0, 0, 600, 800),
|
|
crop_box=(100, 0, 100, 800),
|
|
)
|
|
assert get_pdf_first_page_size_points(pdf_path) is None
|
|
|
|
def test_nonexistent_path_returns_none(self) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A path that does not exist
|
|
WHEN:
|
|
- The first page size is requested
|
|
THEN:
|
|
- None is returned and nothing is raised
|
|
"""
|
|
assert get_pdf_first_page_size_points(Path("/nonexistent/file.pdf")) is None
|
|
|
|
def test_corrupt_pdf_returns_none(self, tmp_path: Path) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A file that is not a PDF
|
|
WHEN:
|
|
- The first page size is requested
|
|
THEN:
|
|
- None is returned and nothing is raised
|
|
"""
|
|
bad = tmp_path / "bad.pdf"
|
|
bad.write_bytes(b"not a pdf")
|
|
assert get_pdf_first_page_size_points(bad) is None
|
|
|
|
def test_encrypted_pdf_returns_none(self, parser_samples_dir: Path) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A password protected PDF
|
|
WHEN:
|
|
- The first page size is requested
|
|
THEN:
|
|
- None is returned and nothing is raised
|
|
"""
|
|
encrypted = parser_samples_dir / "tesseract" / "encrypted.pdf"
|
|
assert get_pdf_first_page_size_points(encrypted) is None
|
|
|
|
|
|
class TestPostProcessText:
|
|
@pytest.mark.parametrize(
|
|
("source", "expected"),
|
|
[
|
|
pytest.param(
|
|
"simple string",
|
|
"simple string",
|
|
id="collapse-spaces",
|
|
),
|
|
pytest.param(
|
|
"simple newline\n testing string",
|
|
"simple newline\ntesting string",
|
|
id="preserve-newline",
|
|
),
|
|
pytest.param(
|
|
"utf-8 строка с пробелами в конце ", # noqa: RUF001
|
|
"utf-8 строка с пробелами в конце", # noqa: RUF001
|
|
id="utf8-trailing-spaces",
|
|
),
|
|
pytest.param(None, None, id="none-input"),
|
|
pytest.param("", None, id="empty-string"),
|
|
pytest.param(" \n\x0c \n ", None, id="whitespace-and-formfeed-only"),
|
|
],
|
|
)
|
|
def test_post_process_text(
|
|
self,
|
|
source: str | None,
|
|
expected: str | None,
|
|
) -> None:
|
|
assert post_process_text(source) == expected
|
|
|
|
|
|
class TestPdfBornDigitalText:
|
|
"""Regression coverage for GH #13387.
|
|
|
|
should_produce_archive() and RasterisedDocumentParser.parse() must agree
|
|
on whether a PDF has real text, so both go through this one function.
|
|
"""
|
|
|
|
@pytest.mark.parametrize(
|
|
("extracted", "tagged", "expected_text", "expected_born_digital"),
|
|
[
|
|
pytest.param("tiny", True, "tiny", True, id="tagged-with-real-text"),
|
|
pytest.param("tiny", False, "tiny", False, id="untagged-below-min-length"),
|
|
pytest.param(
|
|
"x" * 51,
|
|
False,
|
|
"x" * 51,
|
|
True,
|
|
id="untagged-above-min-length",
|
|
),
|
|
pytest.param(None, True, None, False, id="tagged-but-no-text"),
|
|
],
|
|
)
|
|
def test_born_digital_decision(
|
|
self,
|
|
mocker: MockerFixture,
|
|
tmp_path: Path,
|
|
extracted: str | None,
|
|
tagged: bool, # noqa: FBT001
|
|
expected_text: str | None,
|
|
expected_born_digital: bool, # noqa: FBT001
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A PDF whose pdftotext output and /MarkInfo tag status vary
|
|
WHEN:
|
|
- pdf_born_digital_text() is called
|
|
THEN:
|
|
- The normalized text and born-digital verdict match; the tag
|
|
alone never counts as "has text"
|
|
"""
|
|
mocker.patch(
|
|
"paperless.parsers.utils.extract_pdf_text",
|
|
return_value=extracted,
|
|
)
|
|
mocker.patch("paperless.parsers.utils.is_tagged_pdf", return_value=tagged)
|
|
text, born_digital = pdf_born_digital_text(tmp_path / "doc.pdf")
|
|
assert text == expected_text
|
|
assert born_digital is expected_born_digital
|
|
|
|
def test_tagged_but_textless_pdf_is_not_born_digital(
|
|
self,
|
|
tagged_no_text_pdf_file: Path,
|
|
) -> None:
|
|
"""
|
|
GIVEN:
|
|
- A real PDF that is tagged (/MarkInfo /Marked true) but whose
|
|
only "text" is layout padding (a stray form-feed byte)
|
|
WHEN:
|
|
- pdf_born_digital_text() is called with no mocking
|
|
THEN:
|
|
- The normalized text is None and the PDF is not treated as
|
|
born-digital. The raw, unnormalized pdftotext output is
|
|
non-empty for this file, which is exactly what caused the
|
|
archive decision to disagree with the OCR decision in #13387.
|
|
"""
|
|
text, born_digital = pdf_born_digital_text(tagged_no_text_pdf_file)
|
|
assert text is None
|
|
assert born_digital is False
|