diff --git a/src/conftest.py b/src/conftest.py index 187ca5982..781cdff61 100644 --- a/src/conftest.py +++ b/src/conftest.py @@ -20,6 +20,7 @@ if TYPE_CHECKING: from rest_framework.test import APIClient from paperless_testing.dirs import PaperlessDirs + from paperless_testing.fakes.progress import FakeProgressManager @pytest.fixture(scope="session", autouse=True) @@ -136,3 +137,15 @@ def user_client(rest_api_client: APIClient, regular_user: User) -> APIClient: rest_api_client.force_authenticate(user=regular_user) rest_api_client.credentials(HTTP_ACCEPT="application/json; version=10") return rest_api_client + + +@pytest.fixture +def fake_progress_manager( + monkeypatch: pytest.MonkeyPatch, +) -> type[FakeProgressManager]: + """Replace documents.tasks.ProgressManager with the fake, so consuming a file + in a test never tries to reach a broker.""" + from paperless_testing.fakes.progress import FakeProgressManager + + monkeypatch.setattr("documents.tasks.ProgressManager", FakeProgressManager) + return FakeProgressManager diff --git a/src/documents/tests/search/test_migration_fulltext_query_field_prefixes.py b/src/documents/tests/search/test_migration_fulltext_query_field_prefixes.py index df02e5efb..e2be0915f 100644 --- a/src/documents/tests/search/test_migration_fulltext_query_field_prefixes.py +++ b/src/documents/tests/search/test_migration_fulltext_query_field_prefixes.py @@ -1,6 +1,6 @@ import pytest -from documents.tests.utils import TestMigrations +from paperless_testing.migrations import TestMigrations pytestmark = pytest.mark.search diff --git a/src/documents/tests/test_api_app_config.py b/src/documents/tests/test_api_app_config.py index a670c2375..ab05135aa 100644 --- a/src/documents/tests/test_api_app_config.py +++ b/src/documents/tests/test_api_app_config.py @@ -10,11 +10,11 @@ from PIL.PngImagePlugin import PngInfo from rest_framework import status from rest_framework.test import APITestCase -from documents.tests.utils import read_streaming_response from paperless.models import ApplicationConfiguration from paperless.models import ColorConvertChoices from paperless_testing.dirs import DirectoriesMixin from paperless_testing.factories import UserFactory +from paperless_testing.http import read_streaming_response class TestApiAppConfig(DirectoriesMixin, APITestCase): diff --git a/src/documents/tests/test_api_bulk_download.py b/src/documents/tests/test_api_bulk_download.py index 1c5558bd6..e0c29d3cd 100644 --- a/src/documents/tests/test_api_bulk_download.py +++ b/src/documents/tests/test_api_bulk_download.py @@ -13,9 +13,9 @@ from documents.models import Correspondent from documents.models import Document from documents.models import DocumentType from documents.tests.utils import SampleDirMixin -from documents.tests.utils import read_streaming_response from paperless_testing.dirs import DirectoriesMixin from paperless_testing.factories import UserFactory +from paperless_testing.http import read_streaming_response from paperless_testing.permissions import grant_global diff --git a/src/documents/tests/test_api_document_versions.py b/src/documents/tests/test_api_document_versions.py index 5bc02f9c6..92c8ede99 100644 --- a/src/documents/tests/test_api_document_versions.py +++ b/src/documents/tests/test_api_document_versions.py @@ -16,11 +16,11 @@ from documents.data_models import DocumentSource from documents.filters import EffectiveContentFilter from documents.filters import TitleContentFilter from documents.models import Document -from documents.tests.utils import read_streaming_response from documents.versioning import annotate_effective_content from documents.views import DocumentSelectionMixin from paperless_testing.dirs import DirectoriesMixin from paperless_testing.factories import UserFactory +from paperless_testing.http import read_streaming_response from paperless_testing.permissions import grant_global if TYPE_CHECKING: diff --git a/src/documents/tests/test_api_documents.py b/src/documents/tests/test_api_documents.py index b5640c646..defba88d3 100644 --- a/src/documents/tests/test_api_documents.py +++ b/src/documents/tests/test_api_documents.py @@ -48,11 +48,11 @@ from documents.models import WorkflowAction from documents.models import WorkflowTrigger from documents.signals.handlers import run_workflows from documents.tests.utils import ConsumeTaskMixin -from documents.tests.utils import read_streaming_response from paperless_testing.dirs import DirectoriesMixin from paperless_testing.factories import DocumentFactory from paperless_testing.factories import TagFactory from paperless_testing.factories import UserFactory +from paperless_testing.http import read_streaming_response from paperless_testing.permissions import grant_all_global from paperless_testing.permissions import grant_global from paperless_testing.permissions import grant_object diff --git a/src/documents/tests/test_barcodes.py b/src/documents/tests/test_barcodes.py index 2ad4868d8..c21f75fbc 100644 --- a/src/documents/tests/test_barcodes.py +++ b/src/documents/tests/test_barcodes.py @@ -2,8 +2,8 @@ import shutil from collections.abc import Generator from contextlib import contextmanager from pathlib import Path -from unittest import mock +import pytest from django.conf import settings from django.test import TestCase from django.test import override_settings @@ -18,11 +18,11 @@ from documents.models import Document from documents.models import Tag from documents.plugins.base import StopConsumeTaskError from documents.tests.utils import ConsumeTaskMixin -from documents.tests.utils import DummyProgressManager -from documents.tests.utils import FileSystemAssertsMixin from documents.tests.utils import SampleDirMixin from paperless.models import ApplicationConfiguration +from paperless_testing.assertions import FileSystemAssertsMixin from paperless_testing.dirs import DirectoriesMixin +from paperless_testing.fakes.progress import FakeProgressManager class GetReaderPluginMixin: @@ -31,7 +31,7 @@ class GetReaderPluginMixin: reader = BarcodePlugin( ConsumableDocument(DocumentSource.ConsumeFolder, original_file=filepath), DocumentMetadataOverrides(), - DummyProgressManager(filepath.name, None), + FakeProgressManager(filepath.name, None), self.dirs.scratch_dir, "task-id", ) @@ -86,6 +86,7 @@ class TestBarcode( self.assertDictEqual(separator_page_numbers, {1: False}) @override_settings(CONSUMER_ENABLE_ASN_BARCODE=True) + @pytest.mark.usefixtures("fake_progress_manager") def test_asn_barcode_duplicate_in_trash_fails(self) -> None: """ GIVEN: @@ -110,15 +111,14 @@ class TestBarcode( dupe_asn = settings.SCRATCH_DIR / "barcode-39-asn-123-second.pdf" shutil.copy(test_file, dupe_asn) - with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): - with self.assertRaisesRegex(ConsumerError, r"ASN 123.*trash"): - tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ConsumeFolder, - original_file=dupe_asn, - ), - None, - ) + with self.assertRaisesRegex(ConsumerError, r"ASN 123.*trash"): + tasks.consume_file( + ConsumableDocument( + source=DocumentSource.ConsumeFolder, + original_file=dupe_asn, + ), + None, + ) @override_settings( CONSUMER_BARCODE_TIFF_SUPPORT=True, @@ -606,6 +606,7 @@ class TestBarcodeNewConsume( TestCase, ): @override_settings(CONSUMER_ENABLE_BARCODES=True) + @pytest.mark.usefixtures("fake_progress_manager") def test_consume_barcode_file(self) -> None: """ GIVEN: @@ -624,34 +625,33 @@ class TestBarcodeNewConsume( overrides = DocumentMetadataOverrides(tag_ids=[1, 2, 9]) - with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): - self.assertEqual( - tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ConsumeFolder, - original_file=temp_copy, - ), - overrides, + self.assertEqual( + tasks.consume_file( + ConsumableDocument( + source=DocumentSource.ConsumeFolder, + original_file=temp_copy, ), - {"reason": "Barcode splitting complete!"}, - ) - # 2 new document consume tasks created - self.assertEqual(self.consume_file_mock.call_count, 2) + overrides, + ), + {"reason": "Barcode splitting complete!"}, + ) + # 2 new document consume tasks created + self.assertEqual(self.consume_file_mock.call_count, 2) - self.assertIsNotFile(temp_copy) + self.assertIsNotFile(temp_copy) - # Check the split files exist - # Check the original_path is set - # Check the source is unchanged - # Check the overrides are unchanged - for ( - new_input_doc, - new_doc_overrides, - ) in self.get_all_consume_task_call_args(): - self.assertIsFile(new_input_doc.original_file) - self.assertEqual(new_input_doc.original_path, temp_copy) - self.assertEqual(new_input_doc.source, DocumentSource.ConsumeFolder) - self.assertEqual(overrides, new_doc_overrides) + # Check the split files exist + # Check the original_path is set + # Check the source is unchanged + # Check the overrides are unchanged + for ( + new_input_doc, + new_doc_overrides, + ) in self.get_all_consume_task_call_args(): + self.assertIsFile(new_input_doc.original_file) + self.assertEqual(new_input_doc.original_path, temp_copy) + self.assertEqual(new_input_doc.source, DocumentSource.ConsumeFolder) + self.assertEqual(overrides, new_doc_overrides) class TestAsnBarcode(DirectoriesMixin, SampleDirMixin, GetReaderPluginMixin, TestCase): @@ -660,7 +660,7 @@ class TestAsnBarcode(DirectoriesMixin, SampleDirMixin, GetReaderPluginMixin, Tes reader = BarcodePlugin( ConsumableDocument(DocumentSource.ConsumeFolder, original_file=filepath), DocumentMetadataOverrides(), - DummyProgressManager(filepath.name, None), + FakeProgressManager(filepath.name, None), self.dirs.scratch_dir, "task-id", ) @@ -745,6 +745,7 @@ class TestAsnBarcode(DirectoriesMixin, SampleDirMixin, GetReaderPluginMixin, Tes self.assertEqual(asn, None) @override_settings(CONSUMER_ENABLE_ASN_BARCODE=True) + @pytest.mark.usefixtures("fake_progress_manager") def test_consume_barcode_file_asn_assignment(self) -> None: """ GIVEN: @@ -762,19 +763,18 @@ class TestAsnBarcode(DirectoriesMixin, SampleDirMixin, GetReaderPluginMixin, Tes dst = settings.SCRATCH_DIR / "barcode-39-asn-123.pdf" shutil.copy(test_file, dst) - with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): - tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ConsumeFolder, - original_file=dst, - ), - None, - ) + tasks.consume_file( + ConsumableDocument( + source=DocumentSource.ConsumeFolder, + original_file=dst, + ), + None, + ) - document = Document.objects.first() - assert document is not None + document = Document.objects.first() + assert document is not None - self.assertEqual(document.archive_serial_number, 123) + self.assertEqual(document.archive_serial_number, 123) def test_scan_file_for_qrcode_without_upscale(self) -> None: """ @@ -819,7 +819,7 @@ class TestTagBarcode(DirectoriesMixin, SampleDirMixin, GetReaderPluginMixin, Tes reader = BarcodePlugin( ConsumableDocument(DocumentSource.ConsumeFolder, original_file=filepath), DocumentMetadataOverrides(), - DummyProgressManager(filepath.name, None), + FakeProgressManager(filepath.name, None), self.dirs.scratch_dir, "task-id", ) @@ -1024,6 +1024,7 @@ class TestTagBarcode(DirectoriesMixin, SampleDirMixin, GetReaderPluginMixin, Tes CELERY_TASK_ALWAYS_EAGER=True, OCR_MODE="auto", ) + @pytest.mark.usefixtures("fake_progress_manager") def test_consume_barcode_file_tag_split_and_assignment(self) -> None: """ GIVEN: @@ -1042,34 +1043,33 @@ class TestTagBarcode(DirectoriesMixin, SampleDirMixin, GetReaderPluginMixin, Tes dst = settings.SCRATCH_DIR / "split-by-tag-basic.pdf" shutil.copy(test_file, dst) - with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): - result = tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ConsumeFolder, - original_file=dst, - ), - None, - ) + result = tasks.consume_file( + ConsumableDocument( + source=DocumentSource.ConsumeFolder, + original_file=dst, + ), + None, + ) - self.assertEqual(result, {"reason": "Barcode splitting complete!"}) + self.assertEqual(result, {"reason": "Barcode splitting complete!"}) - documents = Document.objects.all().order_by("id") - self.assertEqual(documents.count(), 3) + documents = Document.objects.all().order_by("id") + self.assertEqual(documents.count(), 3) - doc1 = documents[0] - self.assertEqual(doc1.tags.count(), 0) + doc1 = documents[0] + self.assertEqual(doc1.tags.count(), 0) - doc2 = documents[1] - self.assertEqual(doc2.tags.count(), 1) - _tag_1 = doc2.tags.first() - assert _tag_1 is not None - self.assertEqual(_tag_1.name, "invoice") + doc2 = documents[1] + self.assertEqual(doc2.tags.count(), 1) + _tag_1 = doc2.tags.first() + assert _tag_1 is not None + self.assertEqual(_tag_1.name, "invoice") - doc3 = documents[2] - self.assertEqual(doc3.tags.count(), 1) - _tag_2 = doc3.tags.first() - assert _tag_2 is not None - self.assertEqual(_tag_2.name, "receipt") + doc3 = documents[2] + self.assertEqual(doc3.tags.count(), 1) + _tag_2 = doc3.tags.first() + assert _tag_2 is not None + self.assertEqual(_tag_2.name, "receipt") @override_settings( CONSUMER_ENABLE_TAG_BARCODE=True, diff --git a/src/documents/tests/test_consumer.py b/src/documents/tests/test_consumer.py index e18c7e8a0..a4e48379f 100644 --- a/src/documents/tests/test_consumer.py +++ b/src/documents/tests/test_consumer.py @@ -30,12 +30,12 @@ from documents.models import Tag from documents.parsers import ParseError from documents.plugins.helpers import ProgressStatusOptions from documents.tasks import sanity_check -from documents.tests.utils import DummyProgressManager -from documents.tests.utils import FileSystemAssertsMixin from documents.tests.utils import GetConsumerMixin from paperless_mail.models import MailRule +from paperless_testing.assertions import FileSystemAssertsMixin from paperless_testing.dirs import DirectoriesMixin from paperless_testing.factories import UserFactory +from paperless_testing.fakes.progress import FakeProgressManager class _BaseNewStyleParser: @@ -777,7 +777,7 @@ class TestConsumer( ) version_file = self.get_test_file2() - status = DummyProgressManager(version_file.name, None) + status = FakeProgressManager(version_file.name, None) overrides = DocumentMetadataOverrides( version_label="v2", actor_id=actor.pk, @@ -840,7 +840,7 @@ class TestConsumer( assert root_doc is not None version_file = self.get_test_file2() - status = DummyProgressManager(version_file.name, None) + status = FakeProgressManager(version_file.name, None) overrides = DocumentMetadataOverrides( filename="valid_pdf_version-upload", actor_id=999999, @@ -897,7 +897,7 @@ class TestConsumer( assert root_doc is not None def consume_version(version_file: Path) -> Document: - status = DummyProgressManager(version_file.name, None) + status = FakeProgressManager(version_file.name, None) overrides = DocumentMetadataOverrides() doc = ConsumableDocument( DocumentSource.ApiUpload, diff --git a/src/documents/tests/test_double_sided.py b/src/documents/tests/test_double_sided.py index 1e9b046c4..7ea806e93 100644 --- a/src/documents/tests/test_double_sided.py +++ b/src/documents/tests/test_double_sided.py @@ -2,8 +2,8 @@ import datetime as dt import os import shutil from pathlib import Path -from unittest import mock +import pytest from django.test import TestCase from django.test import override_settings from pdfminer.high_level import extract_text @@ -15,12 +15,12 @@ from documents.data_models import ConsumableDocument from documents.data_models import DocumentSource from documents.double_sided import STAGING_FILE_NAME from documents.double_sided import TIMEOUT_MINUTES -from documents.tests.utils import DummyProgressManager -from documents.tests.utils import FileSystemAssertsMixin from documents.tests.utils import SampleDirMixin +from paperless_testing.assertions import FileSystemAssertsMixin from paperless_testing.dirs import DirectoriesMixin +@pytest.mark.usefixtures("fake_progress_manager") @override_settings( CONSUMER_RECURSIVE=True, CONSUMER_ENABLE_COLLATE_DOUBLE_SIDED=True, @@ -46,17 +46,13 @@ class TestDoubleSided( dst = self.double_sided_dir / dstname dst.parent.mkdir(parents=True, exist_ok=True) shutil.copy(src, dst) - with mock.patch( - "documents.tasks.ProgressManager", - DummyProgressManager, - ): - msg = tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ConsumeFolder, - original_file=dst, - ), - None, - ) + msg = tasks.consume_file( + ConsumableDocument( + source=DocumentSource.ConsumeFolder, + original_file=dst, + ), + None, + ) self.assertIsNotFile(dst) return msg diff --git a/src/documents/tests/test_file_handling.py b/src/documents/tests/test_file_handling.py index e7f8354b5..46c54f98f 100644 --- a/src/documents/tests/test_file_handling.py +++ b/src/documents/tests/test_file_handling.py @@ -29,7 +29,7 @@ from documents.models import DocumentType from documents.models import StoragePath from documents.serialisers import DocumentSerializer from documents.tasks import empty_trash -from documents.tests.utils import FileSystemAssertsMixin +from paperless_testing.assertions import FileSystemAssertsMixin from paperless_testing.dirs import DirectoriesMixin from paperless_testing.factories import DocumentFactory from paperless_testing.factories import UserFactory diff --git a/src/documents/tests/test_management.py b/src/documents/tests/test_management.py index 27375ecee..a5a2e6233 100644 --- a/src/documents/tests/test_management.py +++ b/src/documents/tests/test_management.py @@ -20,7 +20,7 @@ if TYPE_CHECKING: from documents.file_handling import generate_filename from documents.models import Document from documents.tasks import update_document_content_maybe_archive_file -from documents.tests.utils import FileSystemAssertsMixin +from paperless_testing.assertions import FileSystemAssertsMixin from paperless_testing.dirs import DirectoriesMixin sample_file: Path = Path(__file__).parent / "samples" / "simple.pdf" diff --git a/src/documents/tests/test_management_exporter.py b/src/documents/tests/test_management_exporter.py index 4083244bb..428960025 100644 --- a/src/documents/tests/test_management_exporter.py +++ b/src/documents/tests/test_management_exporter.py @@ -45,9 +45,9 @@ from documents.models import WorkflowTrigger from documents.sanity_checker import check_sanity from documents.settings import EXPORTER_FILE_NAME from documents.settings import EXPORTER_SHARE_LINK_BUNDLE_NAME -from documents.tests.utils import FileSystemAssertsMixin from documents.tests.utils import SampleDirMixin from paperless_mail.models import MailAccount +from paperless_testing.assertions import FileSystemAssertsMixin from paperless_testing.dirs import DirectoriesMixin from paperless_testing.dirs import paperless_environment from paperless_testing.permissions import grant_object diff --git a/src/documents/tests/test_management_importer.py b/src/documents/tests/test_management_importer.py index a965a1037..4f64fe137 100644 --- a/src/documents/tests/test_management_importer.py +++ b/src/documents/tests/test_management_importer.py @@ -15,8 +15,8 @@ from documents.management.commands.document_importer import _deserialize_record from documents.models import Document from documents.settings import EXPORTER_ARCHIVE_NAME from documents.settings import EXPORTER_FILE_NAME -from documents.tests.utils import FileSystemAssertsMixin from documents.tests.utils import SampleDirMixin +from paperless_testing.assertions import FileSystemAssertsMixin from paperless_testing.dirs import DirectoriesMixin diff --git a/src/documents/tests/test_management_thumbnails.py b/src/documents/tests/test_management_thumbnails.py index c72dd1b5e..4da0caf5a 100644 --- a/src/documents/tests/test_management_thumbnails.py +++ b/src/documents/tests/test_management_thumbnails.py @@ -9,7 +9,7 @@ from django.test import TestCase from documents.management.commands.document_thumbnails import _process_document from documents.models import Document from documents.parsers import get_default_thumbnail -from documents.tests.utils import FileSystemAssertsMixin +from paperless_testing.assertions import FileSystemAssertsMixin from paperless_testing.dirs import DirectoriesMixin diff --git a/src/documents/tests/test_migration_saved_view_visibility.py b/src/documents/tests/test_migration_saved_view_visibility.py index c4996761a..c46a8c8e0 100644 --- a/src/documents/tests/test_migration_saved_view_visibility.py +++ b/src/documents/tests/test_migration_saved_view_visibility.py @@ -1,4 +1,4 @@ -from documents.tests.utils import TestMigrations +from paperless_testing.migrations import TestMigrations SAVED_VIEWS_KEY = "saved_views" DASHBOARD_VIEWS_VISIBLE_IDS_KEY = "dashboard_views_visible_ids" diff --git a/src/documents/tests/test_migration_sha256_checksums.py b/src/documents/tests/test_migration_sha256_checksums.py index 4a53b724c..7063d96bd 100644 --- a/src/documents/tests/test_migration_sha256_checksums.py +++ b/src/documents/tests/test_migration_sha256_checksums.py @@ -7,7 +7,7 @@ from django.conf import settings from django.db import connection from django.test import override_settings -from documents.tests.utils import TestMigrations +from paperless_testing.migrations import TestMigrations def _sha256(data: bytes) -> str: diff --git a/src/documents/tests/test_migration_share_link_bundle.py b/src/documents/tests/test_migration_share_link_bundle.py index 1d56469ca..e5eaefda3 100644 --- a/src/documents/tests/test_migration_share_link_bundle.py +++ b/src/documents/tests/test_migration_share_link_bundle.py @@ -1,4 +1,4 @@ -from documents.tests.utils import TestMigrations +from paperless_testing.migrations import TestMigrations class TestMigrateShareLinkBundlePermissions(TestMigrations): diff --git a/src/documents/tests/test_tasks.py b/src/documents/tests/test_tasks.py index 778ab1855..6829beb6b 100644 --- a/src/documents/tests/test_tasks.py +++ b/src/documents/tests/test_tasks.py @@ -18,7 +18,7 @@ from documents.models import WorkflowAction from documents.sanity_checker import SanityCheckFailedException from documents.sanity_checker import SanityCheckMessages from documents.tests.test_classifier import dummy_preprocess -from documents.tests.utils import FileSystemAssertsMixin +from paperless_testing.assertions import FileSystemAssertsMixin from paperless_testing.dirs import DirectoriesMixin diff --git a/src/documents/tests/test_views.py b/src/documents/tests/test_views.py index c20d247d1..df2601120 100644 --- a/src/documents/tests/test_views.py +++ b/src/documents/tests/test_views.py @@ -28,12 +28,12 @@ from documents.models import StoragePath from documents.models import Tag from documents.models import UiSettings from documents.signals.handlers import update_llm_suggestions_cache -from documents.tests.utils import read_streaming_response from paperless.models import ApplicationConfiguration from paperless_ai.exceptions import LLMProviderError from paperless_ai.exceptions import LLMTimeoutError from paperless_testing.dirs import DirectoriesMixin from paperless_testing.factories import UserFactory +from paperless_testing.http import read_streaming_response from paperless_testing.permissions import grant_global from paperless_testing.permissions import grant_object diff --git a/src/documents/tests/test_workflows.py b/src/documents/tests/test_workflows.py index 7364ec8c2..ee59f8de0 100644 --- a/src/documents/tests/test_workflows.py +++ b/src/documents/tests/test_workflows.py @@ -63,12 +63,11 @@ from documents.models import WorkflowTrigger from documents.plugins.base import StopConsumeTaskError from documents.serialisers import WorkflowTriggerSerializer from documents.signals import document_consumption_finished -from documents.tests.utils import DummyProgressManager -from documents.tests.utils import FileSystemAssertsMixin from documents.tests.utils import SampleDirMixin from documents.workflows.actions import execute_password_removal_action from paperless_mail.models import MailAccount from paperless_mail.models import MailRule +from paperless_testing.assertions import FileSystemAssertsMixin from paperless_testing.dirs import DirectoriesMixin from paperless_testing.factories import UserFactory from paperless_testing.permissions import grant_object @@ -128,6 +127,7 @@ class TestWorkflows( return super().setUp() + @pytest.mark.usefixtures("fake_progress_manager") def test_workflow_match(self) -> None: """ GIVEN: @@ -180,74 +180,74 @@ class TestWorkflows( self.dirs.scratch_dir / "simple.pdf", ) - with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): - with self.assertLogs("paperless.matching", level="INFO") as cm: - tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ConsumeFolder, - original_file=test_file, - ), - None, - ) + with self.assertLogs("paperless.matching", level="INFO") as cm: + tasks.consume_file( + ConsumableDocument( + source=DocumentSource.ConsumeFolder, + original_file=test_file, + ), + None, + ) - document = Document.objects.first() - assert document is not None - self.assertEqual(document.correspondent, self.c) - self.assertEqual(document.document_type, self.dt) - self.assertEqual(list(document.tags.all()), [self.t1, self.t2, self.t3]) - self.assertEqual(document.storage_path, self.sp) - self.assertEqual(document.owner, self.user2) - self.assertEqual( - list( - get_users_with_perms( - document, - only_with_perms_in=["view_document"], - ), + document = Document.objects.first() + assert document is not None + self.assertEqual(document.correspondent, self.c) + self.assertEqual(document.document_type, self.dt) + self.assertEqual(list(document.tags.all()), [self.t1, self.t2, self.t3]) + self.assertEqual(document.storage_path, self.sp) + self.assertEqual(document.owner, self.user2) + self.assertEqual( + list( + get_users_with_perms( + document, + only_with_perms_in=["view_document"], ), - [self.user3], - ) - self.assertEqual( - list( - get_groups_with_perms( - document, - ), + ), + [self.user3], + ) + self.assertEqual( + list( + get_groups_with_perms( + document, ), - [self.group1], - ) - self.assertEqual( - list( - get_users_with_perms( - document, - only_with_perms_in=["change_document"], - ), + ), + [self.group1], + ) + self.assertEqual( + list( + get_users_with_perms( + document, + only_with_perms_in=["change_document"], ), - [self.user3], - ) - self.assertEqual( - list( - get_groups_with_perms( - document, - ), + ), + [self.user3], + ) + self.assertEqual( + list( + get_groups_with_perms( + document, ), - [self.group1], - ) - self.assertEqual( - document.title, - f"Doc from {self.c.name}", - ) - self.assertEqual( - list(document.custom_fields.all().values_list("field", flat=True)), - [self.cf1.pk, self.cf2.pk], - ) - self.assertEqual( - document.custom_fields.get(field=self.cf2.pk).value, - 42, - ) + ), + [self.group1], + ) + self.assertEqual( + document.title, + f"Doc from {self.c.name}", + ) + self.assertEqual( + list(document.custom_fields.all().values_list("field", flat=True)), + [self.cf1.pk, self.cf2.pk], + ) + self.assertEqual( + document.custom_fields.get(field=self.cf2.pk).value, + 42, + ) info = cm.output[0] expected_str = f"Document matched {trigger} from {w}" self.assertIn(expected_str, info) + @pytest.mark.usefixtures("fake_progress_manager") def test_workflow_match_mailrule(self) -> None: """ GIVEN: @@ -292,65 +292,65 @@ class TestWorkflows( self.dirs.scratch_dir / "simple.pdf", ) - with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): - with self.assertLogs("paperless.matching", level="INFO") as cm: - tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ConsumeFolder, - original_file=test_file, - mailrule_id=self.rule1.pk, + with self.assertLogs("paperless.matching", level="INFO") as cm: + tasks.consume_file( + ConsumableDocument( + source=DocumentSource.ConsumeFolder, + original_file=test_file, + mailrule_id=self.rule1.pk, + ), + None, + ) + document = Document.objects.first() + assert document is not None + self.assertEqual(document.correspondent, self.c) + self.assertEqual(document.document_type, self.dt) + self.assertEqual(list(document.tags.all()), [self.t1, self.t2, self.t3]) + self.assertEqual(document.storage_path, self.sp) + self.assertEqual(document.owner, self.user2) + self.assertEqual( + list( + get_users_with_perms( + document, + only_with_perms_in=["view_document"], ), - None, - ) - document = Document.objects.first() - assert document is not None - self.assertEqual(document.correspondent, self.c) - self.assertEqual(document.document_type, self.dt) - self.assertEqual(list(document.tags.all()), [self.t1, self.t2, self.t3]) - self.assertEqual(document.storage_path, self.sp) - self.assertEqual(document.owner, self.user2) - self.assertEqual( - list( - get_users_with_perms( - document, - only_with_perms_in=["view_document"], - ), + ), + [self.user3], + ) + self.assertEqual( + list( + get_groups_with_perms( + document, ), - [self.user3], - ) - self.assertEqual( - list( - get_groups_with_perms( - document, - ), + ), + [self.group1], + ) + self.assertEqual( + list( + get_users_with_perms( + document, + only_with_perms_in=["change_document"], ), - [self.group1], - ) - self.assertEqual( - list( - get_users_with_perms( - document, - only_with_perms_in=["change_document"], - ), + ), + [self.user3], + ) + self.assertEqual( + list( + get_groups_with_perms( + document, ), - [self.user3], - ) - self.assertEqual( - list( - get_groups_with_perms( - document, - ), - ), - [self.group1], - ) - self.assertEqual( - document.title, - f"Doc from {self.c.name}", - ) + ), + [self.group1], + ) + self.assertEqual( + document.title, + f"Doc from {self.c.name}", + ) info = cm.output[0] expected_str = f"Document matched {trigger} from {w}" self.assertIn(expected_str, info) + @pytest.mark.usefixtures("fake_progress_manager") def test_workflow_match_multiple(self) -> None: """ GIVEN: @@ -411,42 +411,42 @@ class TestWorkflows( self.dirs.scratch_dir / "simple.pdf", ) - with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): - with self.assertLogs("paperless.matching", level="INFO") as cm: - tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ConsumeFolder, - original_file=test_file, + with self.assertLogs("paperless.matching", level="INFO") as cm: + tasks.consume_file( + ConsumableDocument( + source=DocumentSource.ConsumeFolder, + original_file=test_file, + ), + None, + ) + document = Document.objects.first() + assert document is not None + # workflow 1 + self.assertEqual(document.document_type, self.dt) + # workflow 2 + self.assertEqual(document.correspondent, self.c2) + self.assertEqual(document.storage_path, self.sp) + # workflow 1 & 2 + self.assertEqual( + list(document.tags.all()), + [self.t1, self.t2, self.t3], + ) + self.assertEqual( + list( + get_users_with_perms( + document, + only_with_perms_in=["view_document"], ), - None, - ) - document = Document.objects.first() - assert document is not None - # workflow 1 - self.assertEqual(document.document_type, self.dt) - # workflow 2 - self.assertEqual(document.correspondent, self.c2) - self.assertEqual(document.storage_path, self.sp) - # workflow 1 & 2 - self.assertEqual( - list(document.tags.all()), - [self.t1, self.t2, self.t3], - ) - self.assertEqual( - list( - get_users_with_perms( - document, - only_with_perms_in=["view_document"], - ), - ), - [self.user2, self.user3], - ) + ), + [self.user2, self.user3], + ) expected_str = f"Document matched {trigger1} from {w1}" self.assertIn(expected_str, cm.output[0]) expected_str = f"Document matched {trigger2} from {w2}" self.assertIn(expected_str, cm.output[1]) + @pytest.mark.usefixtures("fake_progress_manager") def test_workflow_fnmatch_path(self) -> None: """ GIVEN: @@ -480,22 +480,22 @@ class TestWorkflows( self.dirs.scratch_dir / "simple.pdf", ) - with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): - with self.assertLogs("paperless.matching", level="DEBUG") as cm: - tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ConsumeFolder, - original_file=test_file, - ), - None, - ) - document = Document.objects.first() - assert document is not None - self.assertEqual(document.title, "Doc fnmatch title") + with self.assertLogs("paperless.matching", level="DEBUG") as cm: + tasks.consume_file( + ConsumableDocument( + source=DocumentSource.ConsumeFolder, + original_file=test_file, + ), + None, + ) + document = Document.objects.first() + assert document is not None + self.assertEqual(document.title, "Doc fnmatch title") expected_str = f"Document matched {trigger} from {w}" self.assertIn(expected_str, cm.output[0]) + @pytest.mark.usefixtures("fake_progress_manager") def test_workflow_no_match_filename(self) -> None: """ GIVEN: @@ -533,47 +533,47 @@ class TestWorkflows( self.dirs.scratch_dir / "simple.pdf", ) - with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): - with self.assertLogs("paperless.matching", level="DEBUG") as cm: - tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ConsumeFolder, - original_file=test_file, - ), - None, - ) - document = Document.objects.first() - assert document is not None - self.assertIsNone(document.correspondent) - self.assertIsNone(document.document_type) - self.assertEqual(document.tags.all().count(), 0) - self.assertIsNone(document.storage_path) - self.assertIsNone(document.owner) - self.assertEqual( - get_users_with_perms( - document, - only_with_perms_in=["view_document"], - ).count(), - 0, - ) - group_perms: QuerySet[Any] = get_groups_with_perms(document) - self.assertEqual(group_perms.count(), 0) - self.assertEqual( - get_users_with_perms( - document, - only_with_perms_in=["change_document"], - ).count(), - 0, - ) - group_perms: QuerySet[Any] = get_groups_with_perms(document) - self.assertEqual(group_perms.count(), 0) - self.assertEqual(document.title, "simple") + with self.assertLogs("paperless.matching", level="DEBUG") as cm: + tasks.consume_file( + ConsumableDocument( + source=DocumentSource.ConsumeFolder, + original_file=test_file, + ), + None, + ) + document = Document.objects.first() + assert document is not None + self.assertIsNone(document.correspondent) + self.assertIsNone(document.document_type) + self.assertEqual(document.tags.all().count(), 0) + self.assertIsNone(document.storage_path) + self.assertIsNone(document.owner) + self.assertEqual( + get_users_with_perms( + document, + only_with_perms_in=["view_document"], + ).count(), + 0, + ) + group_perms: QuerySet[Any] = get_groups_with_perms(document) + self.assertEqual(group_perms.count(), 0) + self.assertEqual( + get_users_with_perms( + document, + only_with_perms_in=["change_document"], + ).count(), + 0, + ) + group_perms: QuerySet[Any] = get_groups_with_perms(document) + self.assertEqual(group_perms.count(), 0) + self.assertEqual(document.title, "simple") expected_str = f"Document did not match {w}" self.assertIn(expected_str, cm.output[0]) expected_str = f"Document filename {test_file.name} does not match" self.assertIn(expected_str, cm.output[1]) + @pytest.mark.usefixtures("fake_progress_manager") def test_workflow_no_match_path(self) -> None: """ GIVEN: @@ -610,41 +610,40 @@ class TestWorkflows( self.dirs.scratch_dir / "simple.pdf", ) - with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): - with self.assertLogs("paperless.matching", level="DEBUG") as cm: - tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ConsumeFolder, - original_file=test_file, - ), - None, - ) - document = Document.objects.first() - assert document is not None - self.assertIsNone(document.correspondent) - self.assertIsNone(document.document_type) - self.assertEqual(document.tags.all().count(), 0) - self.assertIsNone(document.storage_path) - self.assertIsNone(document.owner) - self.assertEqual( - get_users_with_perms( - document, - only_with_perms_in=["view_document"], - ).count(), - 0, - ) - group_perms: QuerySet[Any] = get_groups_with_perms(document) - self.assertEqual(group_perms.count(), 0) - self.assertEqual( - get_users_with_perms( - document, - only_with_perms_in=["change_document"], - ).count(), - 0, - ) - group_perms: QuerySet[Any] = get_groups_with_perms(document) - self.assertEqual(group_perms.count(), 0) - self.assertEqual(document.title, "simple") + with self.assertLogs("paperless.matching", level="DEBUG") as cm: + tasks.consume_file( + ConsumableDocument( + source=DocumentSource.ConsumeFolder, + original_file=test_file, + ), + None, + ) + document = Document.objects.first() + assert document is not None + self.assertIsNone(document.correspondent) + self.assertIsNone(document.document_type) + self.assertEqual(document.tags.all().count(), 0) + self.assertIsNone(document.storage_path) + self.assertIsNone(document.owner) + self.assertEqual( + get_users_with_perms( + document, + only_with_perms_in=["view_document"], + ).count(), + 0, + ) + group_perms: QuerySet[Any] = get_groups_with_perms(document) + self.assertEqual(group_perms.count(), 0) + self.assertEqual( + get_users_with_perms( + document, + only_with_perms_in=["change_document"], + ).count(), + 0, + ) + group_perms: QuerySet[Any] = get_groups_with_perms(document) + self.assertEqual(group_perms.count(), 0) + self.assertEqual(document.title, "simple") expected_str = f"Document did not match {w}" self.assertIn(expected_str, cm.output[0]) @@ -653,6 +652,7 @@ class TestWorkflows( ) self.assertIn(expected_str, cm.output[1]) + @pytest.mark.usefixtures("fake_progress_manager") def test_workflow_no_match_mail_rule(self) -> None: """ GIVEN: @@ -689,48 +689,48 @@ class TestWorkflows( self.dirs.scratch_dir / "simple.pdf", ) - with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): - with self.assertLogs("paperless.matching", level="DEBUG") as cm: - tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ConsumeFolder, - original_file=test_file, - mailrule_id=99, - ), - None, - ) - document = Document.objects.first() - assert document is not None - self.assertIsNone(document.correspondent) - self.assertIsNone(document.document_type) - self.assertEqual(document.tags.all().count(), 0) - self.assertIsNone(document.storage_path) - self.assertIsNone(document.owner) - self.assertEqual( - get_users_with_perms( - document, - only_with_perms_in=["view_document"], - ).count(), - 0, - ) - group_perms: QuerySet[Any] = get_groups_with_perms(document) - self.assertEqual(group_perms.count(), 0) - self.assertEqual( - get_users_with_perms( - document, - only_with_perms_in=["change_document"], - ).count(), - 0, - ) - group_perms: QuerySet[Any] = get_groups_with_perms(document) - self.assertEqual(group_perms.count(), 0) - self.assertEqual(document.title, "simple") + with self.assertLogs("paperless.matching", level="DEBUG") as cm: + tasks.consume_file( + ConsumableDocument( + source=DocumentSource.ConsumeFolder, + original_file=test_file, + mailrule_id=99, + ), + None, + ) + document = Document.objects.first() + assert document is not None + self.assertIsNone(document.correspondent) + self.assertIsNone(document.document_type) + self.assertEqual(document.tags.all().count(), 0) + self.assertIsNone(document.storage_path) + self.assertIsNone(document.owner) + self.assertEqual( + get_users_with_perms( + document, + only_with_perms_in=["view_document"], + ).count(), + 0, + ) + group_perms: QuerySet[Any] = get_groups_with_perms(document) + self.assertEqual(group_perms.count(), 0) + self.assertEqual( + get_users_with_perms( + document, + only_with_perms_in=["change_document"], + ).count(), + 0, + ) + group_perms: QuerySet[Any] = get_groups_with_perms(document) + self.assertEqual(group_perms.count(), 0) + self.assertEqual(document.title, "simple") expected_str = f"Document did not match {w}" self.assertIn(expected_str, cm.output[0]) expected_str = "Document mail rule 99 !=" self.assertIn(expected_str, cm.output[1]) + @pytest.mark.usefixtures("fake_progress_manager") def test_workflow_no_match_source(self) -> None: """ GIVEN: @@ -767,41 +767,40 @@ class TestWorkflows( self.dirs.scratch_dir / "simple.pdf", ) - with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): - with self.assertLogs("paperless.matching", level="DEBUG") as cm: - tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ApiUpload, - original_file=test_file, - ), - None, - ) - document = Document.objects.first() - assert document is not None - self.assertIsNone(document.correspondent) - self.assertIsNone(document.document_type) - self.assertEqual(document.tags.all().count(), 0) - self.assertIsNone(document.storage_path) - self.assertIsNone(document.owner) - self.assertEqual( - get_users_with_perms( - document, - only_with_perms_in=["view_document"], - ).count(), - 0, - ) - group_perms: QuerySet[Any] = get_groups_with_perms(document) - self.assertEqual(group_perms.count(), 0) - self.assertEqual( - get_users_with_perms( - document, - only_with_perms_in=["change_document"], - ).count(), - 0, - ) - group_perms: QuerySet[Any] = get_groups_with_perms(document) - self.assertEqual(group_perms.count(), 0) - self.assertEqual(document.title, "simple") + with self.assertLogs("paperless.matching", level="DEBUG") as cm: + tasks.consume_file( + ConsumableDocument( + source=DocumentSource.ApiUpload, + original_file=test_file, + ), + None, + ) + document = Document.objects.first() + assert document is not None + self.assertIsNone(document.correspondent) + self.assertIsNone(document.document_type) + self.assertEqual(document.tags.all().count(), 0) + self.assertIsNone(document.storage_path) + self.assertIsNone(document.owner) + self.assertEqual( + get_users_with_perms( + document, + only_with_perms_in=["view_document"], + ).count(), + 0, + ) + group_perms: QuerySet[Any] = get_groups_with_perms(document) + self.assertEqual(group_perms.count(), 0) + self.assertEqual( + get_users_with_perms( + document, + only_with_perms_in=["change_document"], + ).count(), + 0, + ) + group_perms: QuerySet[Any] = get_groups_with_perms(document) + self.assertEqual(group_perms.count(), 0) + self.assertEqual(document.title, "simple") expected_str = f"Document did not match {w}" self.assertIn(expected_str, cm.output[0]) @@ -843,6 +842,7 @@ class TestWorkflows( expected_str = f"No matching triggers with type {WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED} found" self.assertIn(expected_str, cm.output[1]) + @pytest.mark.usefixtures("fake_progress_manager") def test_workflow_repeat_custom_fields(self) -> None: """ GIVEN: @@ -878,21 +878,20 @@ class TestWorkflows( self.dirs.scratch_dir / "simple.pdf", ) - with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): - with self.assertLogs("paperless.matching", level="INFO") as cm: - tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ConsumeFolder, - original_file=test_file, - ), - None, - ) - document = Document.objects.first() - assert document is not None - self.assertEqual( - list(document.custom_fields.all().values_list("field", flat=True)), - [self.cf1.pk], - ) + with self.assertLogs("paperless.matching", level="INFO") as cm: + tasks.consume_file( + ConsumableDocument( + source=DocumentSource.ConsumeFolder, + original_file=test_file, + ), + None, + ) + document = Document.objects.first() + assert document is not None + self.assertEqual( + list(document.custom_fields.all().values_list("field", flat=True)), + [self.cf1.pk], + ) expected_str = f"Document matched {trigger} from {w}" self.assertIn(expected_str, cm.output[0]) @@ -1964,6 +1963,7 @@ class TestWorkflows( self.assertEqual(doc.custom_fields.all().count(), 1) + @pytest.mark.usefixtures("fake_progress_manager") def test_document_consumption_workflow_month_placeholder_addded(self) -> None: trigger = WorkflowTrigger.objects.create( type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION, @@ -1989,20 +1989,19 @@ class TestWorkflows( self.SAMPLE_DIR / "simple.pdf", self.dirs.scratch_dir / "simple.pdf", ) - with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): - tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ApiUpload, - original_file=test_file, - ), - None, - ) - document = Document.objects.first() - assert document is not None - self.assertRegex( - document.title, - r"Doc added in \w{3,}", - ) # Match any 3-letter month name + tasks.consume_file( + ConsumableDocument( + source=DocumentSource.ApiUpload, + original_file=test_file, + ), + None, + ) + document = Document.objects.first() + assert document is not None + self.assertRegex( + document.title, + r"Doc added in \w{3,}", + ) # Match any 3-letter month name def test_document_updated_workflow_existing_custom_field_empty_value(self) -> None: """ @@ -3125,6 +3124,7 @@ class TestWorkflows( group_perms: QuerySet[Any] = get_groups_with_perms(doc) self.assertNotIn(self.group1, group_perms) + @pytest.mark.usefixtures("fake_progress_manager") def test_removal_action_document_consumed(self) -> None: """ GIVEN: @@ -3189,74 +3189,74 @@ class TestWorkflows( self.dirs.scratch_dir / "simple.pdf", ) - with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): - with self.assertLogs("paperless.matching", level="INFO") as cm: - tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ConsumeFolder, - original_file=test_file, - ), - None, - ) + with self.assertLogs("paperless.matching", level="INFO") as cm: + tasks.consume_file( + ConsumableDocument( + source=DocumentSource.ConsumeFolder, + original_file=test_file, + ), + None, + ) - document = Document.objects.first() - assert document is not None + document = Document.objects.first() + assert document is not None - self.assertIsNone(document.correspondent) - self.assertIsNone(document.document_type) - self.assertEqual( - list(document.tags.all()), - [self.t2, self.t3], - ) - self.assertIsNone(document.storage_path) - self.assertIsNone(document.owner) - self.assertEqual( - list( - get_users_with_perms( - document, - only_with_perms_in=["view_document"], - ), + self.assertIsNone(document.correspondent) + self.assertIsNone(document.document_type) + self.assertEqual( + list(document.tags.all()), + [self.t2, self.t3], + ) + self.assertIsNone(document.storage_path) + self.assertIsNone(document.owner) + self.assertEqual( + list( + get_users_with_perms( + document, + only_with_perms_in=["view_document"], ), - [self.user2], - ) - self.assertEqual( - list( - get_groups_with_perms( - document, - ), + ), + [self.user2], + ) + self.assertEqual( + list( + get_groups_with_perms( + document, ), - [self.group2], - ) - self.assertEqual( - list( - get_users_with_perms( - document, - only_with_perms_in=["change_document"], - ), + ), + [self.group2], + ) + self.assertEqual( + list( + get_users_with_perms( + document, + only_with_perms_in=["change_document"], ), - [self.user2], - ) - self.assertEqual( - list( - get_groups_with_perms( - document, - ), + ), + [self.user2], + ) + self.assertEqual( + list( + get_groups_with_perms( + document, ), - [self.group2], - ) - self.assertEqual( - document.title, - "Doc from None", - ) - self.assertEqual( - list(document.custom_fields.all().values_list("field", flat=True)), - [self.cf2.pk], - ) + ), + [self.group2], + ) + self.assertEqual( + document.title, + "Doc from None", + ) + self.assertEqual( + list(document.custom_fields.all().values_list("field", flat=True)), + [self.cf2.pk], + ) info = cm.output[0] expected_str = f"Document matched {trigger} from {w}" self.assertIn(expected_str, info) + @pytest.mark.usefixtures("fake_progress_manager") def test_removal_action_document_consumed_remove_all(self) -> None: """ GIVEN: @@ -3313,49 +3313,48 @@ class TestWorkflows( self.dirs.scratch_dir / "simple.pdf", ) - with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): - with self.assertLogs("paperless.matching", level="INFO") as cm: - tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ConsumeFolder, - original_file=test_file, - ), - None, - ) - document = Document.objects.first() - assert document is not None - self.assertIsNone(document.correspondent) - self.assertIsNone(document.document_type) - self.assertEqual(document.tags.all().count(), 0) + with self.assertLogs("paperless.matching", level="INFO") as cm: + tasks.consume_file( + ConsumableDocument( + source=DocumentSource.ConsumeFolder, + original_file=test_file, + ), + None, + ) + document = Document.objects.first() + assert document is not None + self.assertIsNone(document.correspondent) + self.assertIsNone(document.document_type) + self.assertEqual(document.tags.all().count(), 0) - self.assertIsNone(document.storage_path) - self.assertIsNone(document.owner) - self.assertEqual( - get_users_with_perms( - document, - only_with_perms_in=["view_document"], - ).count(), - 0, - ) - group_perms: QuerySet[Any] = get_groups_with_perms(document) - self.assertEqual(group_perms.count(), 0) - self.assertEqual( - get_users_with_perms( - document, - only_with_perms_in=["change_document"], - ).count(), - 0, - ) - group_perms: QuerySet[Any] = get_groups_with_perms(document) - self.assertEqual(group_perms.count(), 0) - self.assertEqual( - document.custom_fields.all() - .values_list( - "field", - ) - .count(), - 0, + self.assertIsNone(document.storage_path) + self.assertIsNone(document.owner) + self.assertEqual( + get_users_with_perms( + document, + only_with_perms_in=["view_document"], + ).count(), + 0, + ) + group_perms: QuerySet[Any] = get_groups_with_perms(document) + self.assertEqual(group_perms.count(), 0) + self.assertEqual( + get_users_with_perms( + document, + only_with_perms_in=["change_document"], + ).count(), + 0, + ) + group_perms: QuerySet[Any] = get_groups_with_perms(document) + self.assertEqual(group_perms.count(), 0) + self.assertEqual( + document.custom_fields.all() + .values_list( + "field", ) + .count(), + 0, + ) info = cm.output[0] expected_str = f"Document matched {trigger} from {w}" @@ -3837,6 +3836,7 @@ class TestWorkflows( ) @mock.patch("httpx.post") @mock.patch("django.core.mail.message.EmailMessage.send") + @pytest.mark.usefixtures("fake_progress_manager") def test_workflow_email_consumption_started( self, mock_email_send, @@ -3882,15 +3882,14 @@ class TestWorkflows( self.dirs.scratch_dir / "simple.pdf", ) - with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): - with self.assertLogs("paperless.matching", level="INFO"): - tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ConsumeFolder, - original_file=test_file, - ), - None, - ) + with self.assertLogs("paperless.matching", level="INFO"): + tasks.consume_file( + ConsumableDocument( + source=DocumentSource.ConsumeFolder, + original_file=test_file, + ), + None, + ) mock_email_send.assert_called_once() @@ -4314,6 +4313,7 @@ class TestWorkflows( self.assertIn(expected_str, cm.output[0]) @mock.patch("documents.workflows.webhooks.send_webhook.apply_async") + @pytest.mark.usefixtures("fake_progress_manager") def test_workflow_webhook_action_consumption(self, mock_post) -> None: """ GIVEN: @@ -4354,15 +4354,14 @@ class TestWorkflows( self.dirs.scratch_dir / "simple.pdf", ) - with mock.patch("documents.tasks.ProgressManager", DummyProgressManager): - with self.assertLogs("paperless.matching", level="INFO"): - tasks.consume_file( - ConsumableDocument( - source=DocumentSource.ConsumeFolder, - original_file=test_file, - ), - None, - ) + with self.assertLogs("paperless.matching", level="INFO"): + tasks.consume_file( + ConsumableDocument( + source=DocumentSource.ConsumeFolder, + original_file=test_file, + ), + None, + ) mock_post.assert_called_once() @@ -5424,6 +5423,7 @@ class TestDateWorkflowLocalization( ), ], ) + @pytest.mark.usefixtures("fake_progress_manager") def test_document_consumption_workflow_localization( self, tmp_path: Path, @@ -5460,10 +5460,6 @@ class TestDateWorkflowLocalization( # Temporarily override "now" for the environment so templates using # added/created placeholders behave as if it's a different system date. with ( - mock.patch( - "documents.tasks.ProgressManager", - DummyProgressManager, - ), mock.patch( "django.utils.timezone.now", return_value=self.TEST_DATETIME, diff --git a/src/documents/tests/utils.py b/src/documents/tests/utils.py index 27c139d03..63e907855 100644 --- a/src/documents/tests/utils.py +++ b/src/documents/tests/utils.py @@ -1,126 +1,16 @@ -import time -import warnings -from collections.abc import Callable from collections.abc import Generator from collections.abc import Iterator from contextlib import contextmanager -from os import PathLike from pathlib import Path -from typing import Any from unittest import mock -import httpx -import pytest -from django.apps import apps -from django.db import connection -from django.db.migrations.executor import MigrationExecutor -from django.http import StreamingHttpResponse -from django.test import TransactionTestCase - from documents.consumer import AsnCheckPlugin from documents.consumer import ConsumerPlugin from documents.consumer import ConsumerPreflightPlugin from documents.data_models import ConsumableDocument from documents.data_models import DocumentMetadataOverrides from documents.data_models import DocumentSource -from documents.parsers import ParseError -from documents.plugins.helpers import ProgressStatusOptions - - -def util_call_with_backoff( - method_or_callable: Callable, - args: list | tuple, - *, - skip_on_50x_err=True, -) -> tuple[bool, Any]: - """ - For whatever reason, the images started during the test pipeline like to - segfault sometimes, crash and otherwise fail randomly, when run with the - exact files that usually pass. - - So, this function will retry the given method/function up to 3 times, with larger backoff - periods between each attempt, in hopes the issue resolves itself during - one attempt to parse. - - This will wait the following: - - Attempt 1 - 20s following failure - - Attempt 2 - 40s following failure - - Attempt 3 - 80s following failure - - """ - result = None - succeeded = False - retry_time = 20.0 - retry_count = 0 - status_codes = [] - max_retry_count = 3 - - while retry_count < max_retry_count and not succeeded: - try: - result = method_or_callable(*args) - - succeeded = True - except ParseError as e: # pragma: no cover - cause_exec = e.__cause__ - if cause_exec is not None and isinstance(cause_exec, httpx.HTTPStatusError): - status_codes.append(cause_exec.response.status_code) - warnings.warn( - f"HTTP Exception for {cause_exec.request.url} - {cause_exec}", - ) - else: - warnings.warn(f"Unexpected error: {e}") - except Exception as e: # pragma: no cover - warnings.warn(f"Unexpected error: {e}") - - retry_count = retry_count + 1 - - time.sleep(retry_time) - retry_time = retry_time * 2.0 - - if ( - not succeeded - and status_codes - and skip_on_50x_err - and all(httpx.codes.is_server_error(code) for code in status_codes) - ): - pytest.skip("Repeated HTTP 50x for service") # pragma: no cover - - return succeeded, result - - -def read_streaming_response(response: StreamingHttpResponse) -> bytes: - """Consume a StreamingHttpResponse/FileResponse and close it.""" - content = b"".join(response.streaming_content) - response.close() - return content - - -class FileSystemAssertsMixin: - """ - Utilities for checks various state information of the file system - """ - - def assertIsFile(self, path: PathLike[str] | str) -> None: - self.assertTrue(Path(path).resolve().is_file(), f"File does not exist: {path}") - - def assertIsNotFile(self, path: PathLike[str] | str) -> None: - self.assertFalse(Path(path).resolve().is_file(), f"File does exist: {path}") - - def assertIsDir(self, path: PathLike[str] | str) -> None: - self.assertTrue(Path(path).resolve().is_dir(), f"Dir does not exist: {path}") - - def assertIsNotDir(self, path: PathLike[str] | str) -> None: - self.assertFalse(Path(path).resolve().is_dir(), f"Dir does exist: {path}") - - def assertFileCountInDir(self, path: PathLike[str] | str, count: int) -> None: - path = Path(path).resolve() - self.assertTrue(path.is_dir(), f"Path {path} is not a directory") - files = [x for x in path.iterdir() if x.is_file()] - self.assertEqual( - len(files), - count, - f"Path {path} contains {len(files)} files instead of {count} files", - ) +from paperless_testing.fakes.progress import FakeProgressManager class ConsumeTaskMixin: @@ -158,59 +48,6 @@ class ConsumeTaskMixin: yield (task_kwargs["input_doc"], task_kwargs["overrides"]) -class TestMigrations(TransactionTestCase): - @property - def app(self): - return apps.get_containing_app_config(type(self).__module__).name - - migrate_from = None - dependencies = None - migrate_to = None - - def setUp(self) -> None: - super().setUp() - - assert self.migrate_from and self.migrate_to, ( - f"TestCase '{type(self).__name__}' must define migrate_from and migrate_to properties" - ) - self.migrate_from = [(self.app, self.migrate_from)] - if self.dependencies is not None: - self.migrate_from.extend(self.dependencies) - self.migrate_to = [(self.app, self.migrate_to)] - executor = MigrationExecutor(connection) - old_apps = executor.loader.project_state(self.migrate_from).apps - - # Reverse to the original migration - executor.migrate(self.migrate_from) - - self.setUpBeforeMigration(old_apps) - - self.apps = old_apps - - # Run the migration to test - executor = MigrationExecutor(connection) - executor.loader.build_graph() # reload. - executor.migrate(self.migrate_to) - - self.apps = executor.loader.project_state(self.migrate_to).apps - - def setUpBeforeMigration(self, apps) -> None: - pass - - def tearDown(self) -> None: - """ - Ensure the database schema is restored to the latest migration after - each migration test, so subsequent tests run against HEAD. - """ - try: - executor = MigrationExecutor(connection) - executor.loader.build_graph() - targets = executor.loader.graph.leaf_nodes() - executor.migrate(targets) - finally: - super().tearDown() - - class SampleDirMixin: SAMPLE_DIR = Path(__file__).parent / "samples" @@ -227,7 +64,7 @@ class GetConsumerMixin: mailrule_id: int | None = None, ) -> Generator[ConsumerPlugin, None, None]: # Store this for verification - self.status = DummyProgressManager(filepath.name, None) + self.status = FakeProgressManager(filepath.name, None) doc = ConsumableDocument( source, original_file=filepath, @@ -263,63 +100,3 @@ class GetConsumerMixin: yield reader finally: reader.cleanup() - - -class DummyProgressManager: - """ - A dummy handler for progress management that doesn't actually try to - connect to Redis. Payloads are stored for test assertions if needed. - - Use it with - mock.patch("documents.tasks.ProgressManager", DummyProgressManager) - """ - - def __init__(self, filename: str, task_id: str | None = None) -> None: - self.filename = filename - self.task_id = task_id - self.payloads = [] - - def __enter__(self): - self.open() - return self - - def __exit__(self, exc_type, exc_val, exc_tb) -> None: - self.close() - - def open(self) -> None: - pass - - def close(self) -> None: - pass - - def send_progress( - self, - status: ProgressStatusOptions, - message: str, - current_progress: int, - max_progress: int, - *, - document_id: int | None = None, - owner_id: int | None = None, - users_can_view: list[int] | None = None, - groups_can_view: list[int] | None = None, - ) -> None: - # Ensure the layer is open - self.open() - - payload = { - "type": "status_update", - "data": { - "filename": self.filename, - "task_id": self.task_id, - "current_progress": current_progress, - "max_progress": max_progress, - "status": status, - "message": message, - "document_id": document_id, - "owner_id": owner_id, - "users_can_view": users_can_view or [], - "groups_can_view": groups_can_view or [], - }, - } - self.payloads.append(payload) diff --git a/src/paperless/tests/parsers/test_mail_parser_live.py b/src/paperless/tests/parsers/test_mail_parser_live.py index dd17af314..037c157e0 100644 --- a/src/paperless/tests/parsers/test_mail_parser_live.py +++ b/src/paperless/tests/parsers/test_mail_parser_live.py @@ -10,8 +10,8 @@ from imagehash import average_hash from PIL import Image from pytest_mock import MockerFixture -from documents.tests.utils import util_call_with_backoff from paperless.parsers.mail import MailDocumentParser +from paperless_testing.retry import util_call_with_backoff def extract_text(pdf_path: Path) -> str: diff --git a/src/paperless/tests/parsers/test_tesseract_custom_settings.py b/src/paperless/tests/parsers/test_tesseract_custom_settings.py index ad0dc677f..e5dd3a03b 100644 --- a/src/paperless/tests/parsers/test_tesseract_custom_settings.py +++ b/src/paperless/tests/parsers/test_tesseract_custom_settings.py @@ -3,13 +3,13 @@ import json from django.test import TestCase from django.test import override_settings -from documents.tests.utils import FileSystemAssertsMixin from paperless.models import ApplicationConfiguration from paperless.models import CleanChoices from paperless.models import ColorConvertChoices from paperless.models import ModeChoices from paperless.models import OutputTypeChoices from paperless.parsers.tesseract import RasterisedDocumentParser +from paperless_testing.assertions import FileSystemAssertsMixin from paperless_testing.dirs import DirectoriesMixin diff --git a/src/paperless/tests/parsers/test_tika_liva.py b/src/paperless/tests/parsers/test_tika_liva.py index 87cdd88a5..d7144b3da 100644 --- a/src/paperless/tests/parsers/test_tika_liva.py +++ b/src/paperless/tests/parsers/test_tika_liva.py @@ -3,8 +3,8 @@ from pathlib import Path import pytest -from documents.tests.utils import util_call_with_backoff from paperless.parsers.tika import TikaDocumentParser +from paperless_testing.retry import util_call_with_backoff @pytest.mark.skipif( diff --git a/src/paperless/tests/test_migration_replace_skip_archive_file.py b/src/paperless/tests/test_migration_replace_skip_archive_file.py index e13bfd10c..0bcd5d2d5 100644 --- a/src/paperless/tests/test_migration_replace_skip_archive_file.py +++ b/src/paperless/tests/test_migration_replace_skip_archive_file.py @@ -1,4 +1,4 @@ -from documents.tests.utils import TestMigrations +from paperless_testing.migrations import TestMigrations class TestMigrateSkipArchiveFile(TestMigrations): diff --git a/src/paperless_mail/tests/test_mail.py b/src/paperless_mail/tests/test_mail.py index 62bcabd7c..7634f1522 100644 --- a/src/paperless_mail/tests/test_mail.py +++ b/src/paperless_mail/tests/test_mail.py @@ -27,7 +27,6 @@ from rest_framework.test import APITestCase from documents.models import Correspondent from documents.models import MatchingModel -from documents.tests.utils import FileSystemAssertsMixin from paperless_mail import tasks from paperless_mail.mail import MailAccountHandler from paperless_mail.mail import MailError @@ -40,6 +39,7 @@ from paperless_mail.models import MailRule from paperless_mail.models import ProcessedMail from paperless_mail.tests.factories import MailAccountFactory from paperless_mail.tests.factories import MailRuleFactory +from paperless_testing.assertions import FileSystemAssertsMixin from paperless_testing.dirs import DirectoriesMixin from paperless_testing.factories import CorrespondentFactory from paperless_testing.factories import UserFactory diff --git a/src/paperless_testing/assertions.py b/src/paperless_testing/assertions.py new file mode 100644 index 000000000..42dbceb6f --- /dev/null +++ b/src/paperless_testing/assertions.py @@ -0,0 +1,37 @@ +"""Filesystem assertions for unittest-style tests.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from os import PathLike + + +class FileSystemAssertsMixin: + def assertIsFile(self, path: PathLike[str] | str) -> None: + if not Path(path).resolve().is_file(): + raise AssertionError(f"File does not exist: {path}") + + def assertIsNotFile(self, path: PathLike[str] | str) -> None: + if Path(path).resolve().is_file(): + raise AssertionError(f"File does exist: {path}") + + def assertIsDir(self, path: PathLike[str] | str) -> None: + if not Path(path).resolve().is_dir(): + raise AssertionError(f"Dir does not exist: {path}") + + def assertIsNotDir(self, path: PathLike[str] | str) -> None: + if Path(path).resolve().is_dir(): + raise AssertionError(f"Dir does exist: {path}") + + def assertFileCountInDir(self, path: PathLike[str] | str, count: int) -> None: + path = Path(path).resolve() + if not path.is_dir(): + raise AssertionError(f"Path {path} is not a directory") + found = len([x for x in path.iterdir() if x.is_file()]) + if found != count: + raise AssertionError( + f"Path {path} contains {found} files instead of {count} files", + ) diff --git a/src/paperless_testing/fakes/__init__.py b/src/paperless_testing/fakes/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/paperless_testing/fakes/progress.py b/src/paperless_testing/fakes/progress.py new file mode 100644 index 000000000..3832df9c9 --- /dev/null +++ b/src/paperless_testing/fakes/progress.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from documents.plugins.helpers import ProgressManager + +if TYPE_CHECKING: + from documents.plugins.helpers import WebsocketPayload + + +class FakeProgressManager(ProgressManager): + """ + The real ProgressManager with the channel layer cut out: send_progress still + builds the payload, so it cannot drift, and the payloads are recorded instead + of being sent to Redis. + + Use it through the `fake_progress_manager` fixture, or construct it directly. + """ + + def __init__(self, filename: str | None = None, task_id: str | None = None) -> None: + super().__init__(filename, task_id) + self.payloads: list[WebsocketPayload] = [] + + def open(self) -> None: + pass + + def close(self) -> None: + pass + + def send(self, payload: WebsocketPayload) -> None: + self.payloads.append(payload) diff --git a/src/paperless_testing/http.py b/src/paperless_testing/http.py new file mode 100644 index 000000000..78144ad19 --- /dev/null +++ b/src/paperless_testing/http.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from django.http import StreamingHttpResponse + + +def read_streaming_response(response: StreamingHttpResponse) -> bytes: + """Consume a StreamingHttpResponse/FileResponse and close it.""" + content = b"".join(response.streaming_content) + response.close() + return content diff --git a/src/paperless_testing/migrations.py b/src/paperless_testing/migrations.py new file mode 100644 index 000000000..41c4c5d75 --- /dev/null +++ b/src/paperless_testing/migrations.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING +from typing import Any + +from django.apps import apps +from django.db import connection +from django.db.migrations.executor import MigrationExecutor +from django.test import TransactionTestCase + +if TYPE_CHECKING: + from django.apps.registry import Apps + + +class TestMigrations(TransactionTestCase): + @property + def app(self) -> str: + return apps.get_containing_app_config(type(self).__module__).name + + migrate_from: Any = None + dependencies: list[tuple[str, str]] | None = None + migrate_to: Any = None + + def setUp(self) -> None: + super().setUp() + + assert self.migrate_from and self.migrate_to, ( + f"TestCase '{type(self).__name__}' must define migrate_from and migrate_to properties" + ) + self.migrate_from = [(self.app, self.migrate_from)] + if self.dependencies is not None: + self.migrate_from.extend(self.dependencies) + self.migrate_to = [(self.app, self.migrate_to)] + executor = MigrationExecutor(connection) + old_apps = executor.loader.project_state(self.migrate_from).apps + + # Reverse to the original migration + executor.migrate(self.migrate_from) + + self.setUpBeforeMigration(old_apps) + + self.apps = old_apps + + # Run the migration to test + executor = MigrationExecutor(connection) + executor.loader.build_graph() # reload. + executor.migrate(self.migrate_to) + + self.apps = executor.loader.project_state(self.migrate_to).apps + + def setUpBeforeMigration(self, apps: Apps) -> None: + pass + + def tearDown(self) -> None: + """ + Ensure the database schema is restored to the latest migration after + each migration test, so subsequent tests run against HEAD. + """ + try: + executor = MigrationExecutor(connection) + executor.loader.build_graph() + targets = executor.loader.graph.leaf_nodes() + executor.migrate(targets) + finally: + super().tearDown() diff --git a/src/paperless_testing/retry.py b/src/paperless_testing/retry.py new file mode 100644 index 000000000..67d1f81a4 --- /dev/null +++ b/src/paperless_testing/retry.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import time +import warnings +from typing import TYPE_CHECKING +from typing import Any + +import httpx +import pytest + +from documents.parsers import ParseError + +if TYPE_CHECKING: + from collections.abc import Callable + + +def util_call_with_backoff( + method_or_callable: Callable, + args: list | tuple, + *, + skip_on_50x_err: bool = True, +) -> tuple[bool, Any]: + """ + For whatever reason, the images started during the test pipeline like to + segfault sometimes, crash and otherwise fail randomly, when run with the + exact files that usually pass. + + So, this function will retry the given method/function up to 3 times, with larger backoff + periods between each attempt, in hopes the issue resolves itself during + one attempt to parse. + + This will wait the following: + - Attempt 1 - 20s following failure + - Attempt 2 - 40s following failure + - Attempt 3 - 80s following failure + + """ + result = None + succeeded = False + retry_time = 20.0 + retry_count = 0 + status_codes = [] + max_retry_count = 3 + + while retry_count < max_retry_count and not succeeded: + try: + result = method_or_callable(*args) + + succeeded = True + except ParseError as e: # pragma: no cover + cause_exec = e.__cause__ + if cause_exec is not None and isinstance(cause_exec, httpx.HTTPStatusError): + status_codes.append(cause_exec.response.status_code) + warnings.warn( + f"HTTP Exception for {cause_exec.request.url} - {cause_exec}", + ) + else: + warnings.warn(f"Unexpected error: {e}") + except Exception as e: # pragma: no cover + warnings.warn(f"Unexpected error: {e}") + + retry_count = retry_count + 1 + + time.sleep(retry_time) + retry_time = retry_time * 2.0 + + if ( + not succeeded + and status_codes + and skip_on_50x_err + and all(httpx.codes.is_server_error(code) for code in status_codes) + ): + pytest.skip("Repeated HTTP 50x for service") # pragma: no cover + + return succeeded, result