diff --git a/src/conftest.py b/src/conftest.py index 9bfcbd199..ef4e61ca2 100644 --- a/src/conftest.py +++ b/src/conftest.py @@ -5,8 +5,20 @@ this file is imported for every session, so anything heavy belongs inside the fixture body that needs it. """ +from __future__ import annotations + +from typing import TYPE_CHECKING + import pytest +if TYPE_CHECKING: + from collections.abc import Generator + from pathlib import Path + + from pytest_django.fixtures import Settings + + from paperless_testing.dirs import PaperlessDirs + @pytest.fixture(scope="session", autouse=True) def faker_session_locale() -> str: @@ -33,3 +45,27 @@ def _clear_content_type_caches() -> None: ContentType.objects.clear_cache() clear_ct_cache() + + +@pytest.fixture +def paperless_dirs( + tmp_path: Path, + settings: Settings, +) -> Generator[PaperlessDirs, None, None]: + """The standard temp directory layout, applied to Django settings.""" + from documents.search import reset_backend + from paperless_testing.dirs import build_paperless_dirs + from paperless_testing.dirs import dirs_settings + + dirs = build_paperless_dirs(tmp_path) + for name, value in dirs_settings(dirs).items(): + setattr(settings, name, value) + + # Not directory settings, but they are needed alongside the layout by the + # sanity checker tests. + settings.IGNORABLE_FILES = {".DS_Store", "Thumbs.db", "desktop.ini"} + settings.APP_LOGO = "" + + reset_backend() + yield dirs + reset_backend() diff --git a/src/documents/tests/conftest.py b/src/documents/tests/conftest.py index 4788c04ba..aae70c882 100644 --- a/src/documents/tests/conftest.py +++ b/src/documents/tests/conftest.py @@ -1,6 +1,5 @@ import shutil from collections.abc import Generator -from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING @@ -16,16 +15,7 @@ UserModelT = get_user_model() if TYPE_CHECKING: from documents.models import Document - - -@dataclass(frozen=True, slots=True) -class PaperlessDirs: - """Standard Paperless-ngx directory layout for tests.""" - - media: Path - originals: Path - archive: Path - thumbnails: Path + from paperless_testing.dirs import PaperlessDirs @pytest.fixture(scope="session") @@ -34,52 +24,24 @@ def samples_dir() -> Path: return Path(__file__).parent / "samples" / "documents" -@pytest.fixture() -def paperless_dirs(tmp_path: Path) -> PaperlessDirs: - """Create and return the directory structure for testing.""" - media = tmp_path / "media" - dirs = PaperlessDirs( - media=media, - originals=media / "documents" / "originals", - archive=media / "documents" / "archive", - thumbnails=media / "documents" / "thumbnails", - ) - for d in (dirs.originals, dirs.archive, dirs.thumbnails): - d.mkdir(parents=True) - return dirs - - -@pytest.fixture() -def _media_settings(paperless_dirs: PaperlessDirs, settings) -> None: - """Configure Django settings to point at temp directories.""" - settings.MEDIA_ROOT = paperless_dirs.media - settings.ORIGINALS_DIR = paperless_dirs.originals - settings.ARCHIVE_DIR = paperless_dirs.archive - settings.THUMBNAIL_DIR = paperless_dirs.thumbnails - settings.MEDIA_LOCK = paperless_dirs.media / "media.lock" - settings.IGNORABLE_FILES = {".DS_Store", "Thumbs.db", "desktop.ini"} - settings.APP_LOGO = "" - - @pytest.fixture() def sample_doc( - paperless_dirs: PaperlessDirs, - _media_settings: None, + paperless_dirs: "PaperlessDirs", samples_dir: Path, ) -> "Document": """Create a document with valid files and matching checksums.""" - with filelock.FileLock(paperless_dirs.media / "media.lock"): + with filelock.FileLock(paperless_dirs.media_lock): shutil.copy( samples_dir / "originals" / "0000001.pdf", - paperless_dirs.originals / "0000001.pdf", + paperless_dirs.originals_dir / "0000001.pdf", ) shutil.copy( samples_dir / "archive" / "0000001.pdf", - paperless_dirs.archive / "0000001.pdf", + paperless_dirs.archive_dir / "0000001.pdf", ) shutil.copy( samples_dir / "thumbnails" / "0000001.webp", - paperless_dirs.thumbnails / "0000001.webp", + paperless_dirs.thumbnail_dir / "0000001.webp", ) return DocumentFactory( diff --git a/src/documents/tests/management/test_management_sanity_checker.py b/src/documents/tests/management/test_management_sanity_checker.py index 1d3a84ad9..99513c640 100644 --- a/src/documents/tests/management/test_management_sanity_checker.py +++ b/src/documents/tests/management/test_management_sanity_checker.py @@ -19,7 +19,7 @@ from paperless_testing.factories import DocumentFactory if TYPE_CHECKING: from documents.models import Document - from documents.tests.conftest import PaperlessDirs + from paperless_testing.dirs import PaperlessDirs def _render_to_string(messages: SanityCheckMessages) -> str: @@ -71,7 +71,7 @@ class TestRenderResultsWithIssues: assert "INFO" in output assert "No OCR data" in output - @pytest.mark.usefixtures("_media_settings") + @pytest.mark.usefixtures("paperless_dirs") def test_global_message(self) -> None: msgs = SanityCheckMessages() msgs.warning(None, "Orphaned file: /tmp/stray.pdf") @@ -87,7 +87,7 @@ class TestRenderResultsWithIssues: assert "Thumbnail missing" in output assert "Checksum mismatch" in output - @pytest.mark.usefixtures("_media_settings") + @pytest.mark.usefixtures("paperless_dirs") def test_unknown_doc_pk(self) -> None: msgs = SanityCheckMessages() msgs.error(99999, "Ghost document") @@ -184,7 +184,6 @@ class TestDocumentSanityCheckerCommand: assert "ERROR" in output assert "Original of document does not exist" in output - @pytest.mark.usefixtures("_media_settings") def test_checksum_mismatch(self, paperless_dirs: PaperlessDirs) -> None: """Lightweight document with zero-byte files triggers checksum mismatch.""" doc = DocumentFactory( diff --git a/src/documents/tests/test_api_post_document_nfc.py b/src/documents/tests/test_api_post_document_nfc.py index 5b3088b01..2eff44e27 100644 --- a/src/documents/tests/test_api_post_document_nfc.py +++ b/src/documents/tests/test_api_post_document_nfc.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import unicodedata from typing import TYPE_CHECKING from unittest import mock @@ -7,8 +9,11 @@ import pytest from django.core.files.uploadedfile import SimpleUploadedFile if TYPE_CHECKING: + from rest_framework.test import APIClient + from documents.data_models import ConsumableDocument from documents.data_models import DocumentMetadataOverrides + from paperless_testing.dirs import PaperlessDirs @pytest.fixture() @@ -18,22 +23,14 @@ def consume_file_mock(): yield m -@pytest.fixture() -def directories(tmp_path, settings, _media_settings): - scratch = tmp_path / "scratch" - scratch.mkdir() - settings.SCRATCH_DIR = scratch - return scratch - - @pytest.mark.django_db class TestPostDocumentNFCNormalization: def test_nfd_filename_normalized_to_nfc( self, - admin_client, + admin_client: APIClient, consume_file_mock: mock.MagicMock, - directories, - ): + paperless_dirs: PaperlessDirs, + ) -> None: """Uploaded file with NFD filename must have its name stored as NFC.""" nfd = unicodedata.normalize("NFD", "Rechnung März.pdf") nfc = unicodedata.normalize("NFC", "Rechnung März.pdf") @@ -69,10 +66,10 @@ class TestPostDocumentNFCNormalization: def test_already_nfc_filename_unchanged( self, - admin_client, + admin_client: APIClient, consume_file_mock: mock.MagicMock, - directories, - ): + paperless_dirs: PaperlessDirs, + ) -> None: """Uploaded file with already-NFC filename must pass through unchanged.""" nfc = unicodedata.normalize("NFC", "Invoice_2024.pdf") diff --git a/src/documents/tests/test_management_exporter.py b/src/documents/tests/test_management_exporter.py index 2ffded5e6..f85a8c4bf 100644 --- a/src/documents/tests/test_management_exporter.py +++ b/src/documents/tests/test_management_exporter.py @@ -49,8 +49,8 @@ from documents.settings import EXPORTER_SHARE_LINK_BUNDLE_NAME from documents.tests.utils import DirectoriesMixin from documents.tests.utils import FileSystemAssertsMixin from documents.tests.utils import SampleDirMixin -from documents.tests.utils import paperless_environment from paperless_mail.models import MailAccount +from paperless_testing.dirs import paperless_environment @pytest.mark.management diff --git a/src/documents/tests/test_permission_filtering_security.py b/src/documents/tests/test_permission_filtering_security.py index 638be2d41..8d30c7c6c 100644 --- a/src/documents/tests/test_permission_filtering_security.py +++ b/src/documents/tests/test_permission_filtering_security.py @@ -1,6 +1,7 @@ from __future__ import annotations from http import HTTPStatus +from typing import TYPE_CHECKING from unittest.mock import patch import pytest @@ -30,6 +31,9 @@ from paperless_testing.factories import DocumentTypeFactory from paperless_testing.factories import StoragePathFactory from paperless_testing.factories import TagFactory +if TYPE_CHECKING: + from paperless_testing.dirs import PaperlessDirs + def assert_visible_document_ids(actual_ids, *, expected_visible, expected_hidden): actual_ids = set(actual_ids) @@ -369,10 +373,9 @@ class TestBulkEditChangePermissionBoundary: class TestBulkDownloadPermissionChecksRootDocument: def test_download_requires_global_view_permission( self, - rest_api_client, - paperless_dirs, - _media_settings, - ): + rest_api_client: APIClient, + paperless_dirs: PaperlessDirs, + ) -> None: owner = User.objects.create_user(username="owner") requester = User.objects.create_user(username="requester") root = DocumentFactory(owner=owner) @@ -390,10 +393,9 @@ class TestBulkDownloadPermissionChecksRootDocument: def test_permission_checked_on_root_not_on_version( self, - rest_api_client, - paperless_dirs, - _media_settings, - ): + rest_api_client: APIClient, + paperless_dirs: PaperlessDirs, + ) -> None: owner = User.objects.create_user(username="owner") requester = User.objects.create_user(username="requester") requester.user_permissions.add( diff --git a/src/documents/tests/test_sanity_check.py b/src/documents/tests/test_sanity_check.py index f282f3ea5..e04367f7b 100644 --- a/src/documents/tests/test_sanity_check.py +++ b/src/documents/tests/test_sanity_check.py @@ -19,7 +19,7 @@ if TYPE_CHECKING: from collections.abc import Iterable from documents.models import Document - from documents.tests.conftest import PaperlessDirs + from paperless_testing.dirs import PaperlessDirs class TestSanityCheckMessages: @@ -46,14 +46,14 @@ class TestSanityCheckMessages: class TestCheckSanityNoDocuments: """Sanity checks against an empty archive.""" - @pytest.mark.usefixtures("_media_settings") + @pytest.mark.usefixtures("paperless_dirs") def test_no_documents(self) -> None: messages = check_sanity() assert not messages.has_error assert not messages.has_warning assert messages.total_issue_count == 0 - @pytest.mark.usefixtures("_media_settings") + @pytest.mark.usefixtures("paperless_dirs") def test_no_issues_logs_clean(self, caplog: pytest.LogCaptureFixture) -> None: messages = check_sanity() with caplog.at_level(logging.INFO, logger="paperless.sanity_checker"): @@ -214,18 +214,17 @@ class TestCheckSanityOrphans: sample_doc: Document, paperless_dirs: PaperlessDirs, ) -> None: - (paperless_dirs.originals / "orphan.pdf").touch() + (paperless_dirs.originals_dir / "orphan.pdf").touch() messages = check_sanity() assert messages.has_warning assert any("Orphaned file" in m["message"] for m in messages[None]) - @pytest.mark.usefixtures("_media_settings") def test_ignorable_files_not_flagged( self, paperless_dirs: PaperlessDirs, ) -> None: - (paperless_dirs.media / ".DS_Store").touch() - (paperless_dirs.media / "desktop.ini").touch() + (paperless_dirs.media_dir / ".DS_Store").touch() + (paperless_dirs.media_dir / "desktop.ini").touch() messages = check_sanity() assert not messages.has_warning @@ -269,13 +268,13 @@ class TestCheckSanityLogMessages: paperless_dirs: PaperlessDirs, caplog: pytest.LogCaptureFixture, ) -> None: - (paperless_dirs.originals / "orphan.pdf").touch() + (paperless_dirs.originals_dir / "orphan.pdf").touch() messages = check_sanity() with caplog.at_level(logging.WARNING, logger="paperless.sanity_checker"): messages.log_messages() assert "Orphaned file" in caplog.text - @pytest.mark.usefixtures("_media_settings") + @pytest.mark.usefixtures("paperless_dirs") def test_logs_unknown_doc_pk(self, caplog: pytest.LogCaptureFixture) -> None: """A doc PK not in the DB logs 'Unknown' as the title.""" messages = check_sanity() diff --git a/src/documents/tests/utils.py b/src/documents/tests/utils.py index d699ccf5e..16bc16f3d 100644 --- a/src/documents/tests/utils.py +++ b/src/documents/tests/utils.py @@ -79,17 +79,6 @@ def remove_dirs(dirs) -> None: dirs.settings_override.disable() -@contextmanager -def paperless_environment(): - dirs = None - try: - dirs = setup_directories() - yield dirs - finally: - if dirs: - remove_dirs(dirs) - - def util_call_with_backoff( method_or_callable: Callable, args: list | tuple, diff --git a/src/paperless/tests/test_checks.py b/src/paperless/tests/test_checks.py index 94325ae95..20e13b66d 100644 --- a/src/paperless/tests/test_checks.py +++ b/src/paperless/tests/test_checks.py @@ -1,6 +1,5 @@ import os from collections.abc import Callable -from dataclasses import dataclass from pathlib import Path from unittest import mock @@ -19,35 +18,7 @@ from paperless.checks import check_v3_minimum_upgrade_version from paperless.checks import debug_mode_check from paperless.checks import paths_check from paperless.checks import settings_values_check - - -@dataclass(frozen=True, slots=True) -class PaperlessTestDirs: - data_dir: Path - media_dir: Path - consumption_dir: Path - - -# TODO: consolidate with documents/tests/conftest.py PaperlessDirs/paperless_dirs -# once the paperless and documents test suites are ready to share fixtures. -@pytest.fixture() -def directories(tmp_path: Path, settings: Settings) -> PaperlessTestDirs: - data_dir = tmp_path / "data" - media_dir = tmp_path / "media" - consumption_dir = tmp_path / "consumption" - - for d in (data_dir, media_dir, consumption_dir): - d.mkdir() - - settings.DATA_DIR = data_dir - settings.MEDIA_ROOT = media_dir - settings.CONSUMPTION_DIR = consumption_dir - - return PaperlessTestDirs( - data_dir=data_dir, - media_dir=media_dir, - consumption_dir=consumption_dir, - ) +from paperless_testing.dirs import PaperlessDirs class TestChecks: @@ -58,7 +29,7 @@ class TestChecks: settings.CONVERT_BINARY = "uuuhh" assert len(binaries_check(None)) == 1 - @pytest.mark.usefixtures("directories") + @pytest.mark.usefixtures("paperless_dirs") def test_paths_check(self) -> None: assert paths_check(None) == [] @@ -73,17 +44,17 @@ class TestChecks: for msg in msgs: assert msg.msg.endswith("is set but doesn't exist.") - def test_paths_check_no_access(self, directories: PaperlessTestDirs) -> None: - directories.data_dir.chmod(0o000) - directories.media_dir.chmod(0o000) - directories.consumption_dir.chmod(0o000) + def test_paths_check_no_access(self, paperless_dirs: PaperlessDirs) -> None: + paperless_dirs.data_dir.chmod(0o000) + paperless_dirs.media_dir.chmod(0o000) + paperless_dirs.consumption_dir.chmod(0o000) try: msgs = paths_check(None) finally: - directories.data_dir.chmod(0o777) - directories.media_dir.chmod(0o777) - directories.consumption_dir.chmod(0o777) + paperless_dirs.data_dir.chmod(0o777) + paperless_dirs.media_dir.chmod(0o777) + paperless_dirs.consumption_dir.chmod(0o777) assert len(msgs) == 3 for msg in msgs: diff --git a/src/paperless_testing/dirs.py b/src/paperless_testing/dirs.py new file mode 100644 index 000000000..266bad307 --- /dev/null +++ b/src/paperless_testing/dirs.py @@ -0,0 +1,131 @@ +"""The Paperless-ngx temp directory layout, stated once. + +``build_paperless_dirs`` owns where things go and creates them. +``dirs_settings`` owns the mapping onto Django setting names and is pure. +Everything else in the test suite is a caller of these two. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import TYPE_CHECKING +from typing import TypedDict + +from django.test import override_settings + +if TYPE_CHECKING: + from collections.abc import Iterator + + +@dataclass(frozen=True, slots=True) +class PaperlessDirs: + """Standard Paperless-ngx directory layout for tests.""" + + data_dir: Path + scratch_dir: Path + media_dir: Path + consumption_dir: Path + static_dir: Path + index_dir: Path + originals_dir: Path + thumbnail_dir: Path + archive_dir: Path + logging_dir: Path + model_file: Path + media_lock: Path + + +class DirSettings(TypedDict): + """The Django settings the layout above maps onto.""" + + DATA_DIR: Path + SCRATCH_DIR: Path + MEDIA_ROOT: Path + ORIGINALS_DIR: Path + THUMBNAIL_DIR: Path + ARCHIVE_DIR: Path + CONSUMPTION_DIR: Path + LOGGING_DIR: Path + INDEX_DIR: Path + STATIC_ROOT: Path + MODEL_FILE: Path + MEDIA_LOCK: Path + + +def build_paperless_dirs(root: Path) -> PaperlessDirs: + """Compute the layout under root and create the directories.""" + data_dir = root / "data" + media_dir = root / "media" + documents_dir = media_dir / "documents" + + dirs = PaperlessDirs( + data_dir=data_dir, + scratch_dir=root / "scratch", + media_dir=media_dir, + consumption_dir=root / "consume", + static_dir=root / "static", + index_dir=data_dir / "index", + originals_dir=documents_dir / "originals", + thumbnail_dir=documents_dir / "thumbnails", + archive_dir=documents_dir / "archive", + logging_dir=data_dir / "log", + model_file=data_dir / "classification_model.pickle", + media_lock=media_dir / "media.lock", + ) + + for directory in ( + dirs.data_dir, + dirs.scratch_dir, + dirs.media_dir, + dirs.consumption_dir, + dirs.static_dir, + dirs.index_dir, + dirs.originals_dir, + dirs.thumbnail_dir, + dirs.archive_dir, + dirs.logging_dir, + ): + directory.mkdir(parents=True, exist_ok=True) + + return dirs + + +def dirs_settings(dirs: PaperlessDirs) -> DirSettings: + """Map the layout onto Django setting names.""" + return DirSettings( + DATA_DIR=dirs.data_dir, + SCRATCH_DIR=dirs.scratch_dir, + MEDIA_ROOT=dirs.media_dir, + ORIGINALS_DIR=dirs.originals_dir, + THUMBNAIL_DIR=dirs.thumbnail_dir, + ARCHIVE_DIR=dirs.archive_dir, + CONSUMPTION_DIR=dirs.consumption_dir, + LOGGING_DIR=dirs.logging_dir, + INDEX_DIR=dirs.index_dir, + STATIC_ROOT=dirs.static_dir, + MODEL_FILE=dirs.model_file, + MEDIA_LOCK=dirs.media_lock, + ) + + +@contextmanager +def paperless_environment() -> Iterator[PaperlessDirs]: + """A second, isolated environment for the duration of the block. + + Only for tests needing a fresh environment part way through a test body, + which a fixture cannot provide. Everything else uses the paperless_dirs + fixture. + """ + from documents.search import reset_backend + + with TemporaryDirectory(ignore_cleanup_errors=True) as tmp: + dirs = build_paperless_dirs(Path(tmp)) + with override_settings(**dirs_settings(dirs)): + reset_backend() + try: + yield dirs + finally: + reset_backend()