amorfo77/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2026-09-24 02:10:33 +00:00
Code Issues Projects Releases Wiki Activity

Chore: Move the cross-app test helpers into the shared layer and add a progress fixture (#14221)

Browse Source 
Test modules in paperless, paperless_mail and documents imported filesystem assertions, the migration test base, the retry helper and the streaming-response reader out of documents/tests/utils.py, which kept each app's tests coupled to another app's test package.

They now live in paperless_testing, and the progress manager fake is renamed FakeProgressManager and now subclasses the real ProgressManager, overriding only the transport, so the payload it records is built by the production code. The twenty places that patched documents.tasks.ProgressManager by hand now use a fake_progress_manager fixture.
This commit is contained in:
Trenton H
2026-09-22 08:02:17 -07:00
committed by GitHub
parent 90d23bad9c
commit 03ac4aed7e
32 changed files with 783 additions and 780 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/conftest.py
+13
View File
@@ -20,6 +20,7 @@ if TYPE_CHECKING:
from rest_framework.test import APIClient
from paperless_testing.dirs import PaperlessDirs
from paperless_testing.fakes.progress import FakeProgressManager
@pytest.fixture(scope="session", autouse=True)
@@ -136,3 +137,15 @@ def user_client(rest_api_client: APIClient, regular_user: User) -> APIClient:
rest_api_client.force_authenticate(user=regular_user)
rest_api_client.credentials(HTTP_ACCEPT="application/json; version=10")
return rest_api_client
@pytest.fixture
def fake_progress_manager(
monkeypatch: pytest.MonkeyPatch,
) -> type[FakeProgressManager]:
"""Replace documents.tasks.ProgressManager with the fake, so consuming a file
in a test never tries to reach a broker."""
from paperless_testing.fakes.progress import FakeProgressManager
monkeypatch.setattr("documents.tasks.ProgressManager", FakeProgressManager)
return FakeProgressManager
src/documents/tests/search/test_migration_fulltext_query_field_prefixes.py
+1 -1
View File
@@ -1,6 +1,6 @@
import pytest
from documents.tests.utils import TestMigrations
from paperless_testing.migrations import TestMigrations
pytestmark = pytest.mark.search
src/documents/tests/test_api_app_config.py
+1 -1
View File
@@ -10,11 +10,11 @@ from PIL.PngImagePlugin import PngInfo
from rest_framework import status
from rest_framework.test import APITestCase
from documents.tests.utils import read_streaming_response
from paperless.models import ApplicationConfiguration
from paperless.models import ColorConvertChoices
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import UserFactory
from paperless_testing.http import read_streaming_response
class TestApiAppConfig(DirectoriesMixin, APITestCase):
src/documents/tests/test_api_bulk_download.py
+1 -1
View File
@@ -13,9 +13,9 @@ from documents.models import Correspondent
from documents.models import Document
from documents.models import DocumentType
from documents.tests.utils import SampleDirMixin
from documents.tests.utils import read_streaming_response
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import UserFactory
from paperless_testing.http import read_streaming_response
from paperless_testing.permissions import grant_global
src/documents/tests/test_api_document_versions.py
+1 -1
View File
@@ -16,11 +16,11 @@ from documents.data_models import DocumentSource
from documents.filters import EffectiveContentFilter
from documents.filters import TitleContentFilter
from documents.models import Document
from documents.tests.utils import read_streaming_response
from documents.versioning import annotate_effective_content
from documents.views import DocumentSelectionMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import UserFactory
from paperless_testing.http import read_streaming_response
from paperless_testing.permissions import grant_global
if TYPE_CHECKING:
src/documents/tests/test_api_documents.py
+1 -1
View File
@@ -48,11 +48,11 @@ from documents.models import WorkflowAction
from documents.models import WorkflowTrigger
from documents.signals.handlers import run_workflows
from documents.tests.utils import ConsumeTaskMixin
from documents.tests.utils import read_streaming_response
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import TagFactory
from paperless_testing.factories import UserFactory
from paperless_testing.http import read_streaming_response
from paperless_testing.permissions import grant_all_global
from paperless_testing.permissions import grant_global
from paperless_testing.permissions import grant_object
src/documents/tests/test_barcodes.py
+74 -74
View File
@@ -2,8 +2,8 @@ import shutil
from collections.abc import Generator
from contextlib import contextmanager
from pathlib import Path
from unittest import mock
import pytest
from django.conf import settings
from django.test import TestCase
from django.test import override_settings
@@ -18,11 +18,11 @@ from documents.models import Document
from documents.models import Tag
from documents.plugins.base import StopConsumeTaskError
from documents.tests.utils import ConsumeTaskMixin
from documents.tests.utils import DummyProgressManager
from documents.tests.utils import FileSystemAssertsMixin
from documents.tests.utils import SampleDirMixin
from paperless.models import ApplicationConfiguration
from paperless_testing.assertions import FileSystemAssertsMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.fakes.progress import FakeProgressManager
class GetReaderPluginMixin:
@@ -31,7 +31,7 @@ class GetReaderPluginMixin:
reader = BarcodePlugin(
ConsumableDocument(DocumentSource.ConsumeFolder, original_file=filepath),
DocumentMetadataOverrides(),
DummyProgressManager(filepath.name, None),
FakeProgressManager(filepath.name, None),
self.dirs.scratch_dir,
"task-id",
)
@@ -86,6 +86,7 @@ class TestBarcode(
self.assertDictEqual(separator_page_numbers, {1: False})
@override_settings(CONSUMER_ENABLE_ASN_BARCODE=True)
@pytest.mark.usefixtures("fake_progress_manager")
def test_asn_barcode_duplicate_in_trash_fails(self) -> None:
"""
GIVEN:
@@ -110,15 +111,14 @@ class TestBarcode(
dupe_asn = settings.SCRATCH_DIR / "barcode-39-asn-123-second.pdf"
shutil.copy(test_file, dupe_asn)
with mock.patch("documents.tasks.ProgressManager", DummyProgressManager):
with self.assertRaisesRegex(ConsumerError, r"ASN 123.*trash"):
tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=dupe_asn,
),
None,
)
with self.assertRaisesRegex(ConsumerError, r"ASN 123.*trash"):
tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=dupe_asn,
),
None,
)
@override_settings(
CONSUMER_BARCODE_TIFF_SUPPORT=True,
@@ -606,6 +606,7 @@ class TestBarcodeNewConsume(
TestCase,
):
@override_settings(CONSUMER_ENABLE_BARCODES=True)
@pytest.mark.usefixtures("fake_progress_manager")
def test_consume_barcode_file(self) -> None:
"""
GIVEN:
@@ -624,34 +625,33 @@ class TestBarcodeNewConsume(
overrides = DocumentMetadataOverrides(tag_ids=[1, 2, 9])
with mock.patch("documents.tasks.ProgressManager", DummyProgressManager):
self.assertEqual(
tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=temp_copy,
),
overrides,
self.assertEqual(
tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=temp_copy,
),
{"reason": "Barcode splitting complete!"},
)
# 2 new document consume tasks created
self.assertEqual(self.consume_file_mock.call_count, 2)
overrides,
),
{"reason": "Barcode splitting complete!"},
)
# 2 new document consume tasks created
self.assertEqual(self.consume_file_mock.call_count, 2)
self.assertIsNotFile(temp_copy)
self.assertIsNotFile(temp_copy)
# Check the split files exist
# Check the original_path is set
# Check the source is unchanged
# Check the overrides are unchanged
for (
new_input_doc,
new_doc_overrides,
) in self.get_all_consume_task_call_args():
self.assertIsFile(new_input_doc.original_file)
self.assertEqual(new_input_doc.original_path, temp_copy)
self.assertEqual(new_input_doc.source, DocumentSource.ConsumeFolder)
self.assertEqual(overrides, new_doc_overrides)
# Check the split files exist
# Check the original_path is set
# Check the source is unchanged
# Check the overrides are unchanged
for (
new_input_doc,
new_doc_overrides,
) in self.get_all_consume_task_call_args():
self.assertIsFile(new_input_doc.original_file)
self.assertEqual(new_input_doc.original_path, temp_copy)
self.assertEqual(new_input_doc.source, DocumentSource.ConsumeFolder)
self.assertEqual(overrides, new_doc_overrides)
class TestAsnBarcode(DirectoriesMixin, SampleDirMixin, GetReaderPluginMixin, TestCase):
@@ -660,7 +660,7 @@ class TestAsnBarcode(DirectoriesMixin, SampleDirMixin, GetReaderPluginMixin, Tes
reader = BarcodePlugin(
ConsumableDocument(DocumentSource.ConsumeFolder, original_file=filepath),
DocumentMetadataOverrides(),
DummyProgressManager(filepath.name, None),
FakeProgressManager(filepath.name, None),
self.dirs.scratch_dir,
"task-id",
)
@@ -745,6 +745,7 @@ class TestAsnBarcode(DirectoriesMixin, SampleDirMixin, GetReaderPluginMixin, Tes
self.assertEqual(asn, None)
@override_settings(CONSUMER_ENABLE_ASN_BARCODE=True)
@pytest.mark.usefixtures("fake_progress_manager")
def test_consume_barcode_file_asn_assignment(self) -> None:
"""
GIVEN:
@@ -762,19 +763,18 @@ class TestAsnBarcode(DirectoriesMixin, SampleDirMixin, GetReaderPluginMixin, Tes
dst = settings.SCRATCH_DIR / "barcode-39-asn-123.pdf"
shutil.copy(test_file, dst)
with mock.patch("documents.tasks.ProgressManager", DummyProgressManager):
tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=dst,
),
None,
)
tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=dst,
),
None,
)
document = Document.objects.first()
assert document is not None
document = Document.objects.first()
assert document is not None
self.assertEqual(document.archive_serial_number, 123)
self.assertEqual(document.archive_serial_number, 123)
def test_scan_file_for_qrcode_without_upscale(self) -> None:
"""
@@ -819,7 +819,7 @@ class TestTagBarcode(DirectoriesMixin, SampleDirMixin, GetReaderPluginMixin, Tes
reader = BarcodePlugin(
ConsumableDocument(DocumentSource.ConsumeFolder, original_file=filepath),
DocumentMetadataOverrides(),
DummyProgressManager(filepath.name, None),
FakeProgressManager(filepath.name, None),
self.dirs.scratch_dir,
"task-id",
)
@@ -1024,6 +1024,7 @@ class TestTagBarcode(DirectoriesMixin, SampleDirMixin, GetReaderPluginMixin, Tes
CELERY_TASK_ALWAYS_EAGER=True,
OCR_MODE="auto",
)
@pytest.mark.usefixtures("fake_progress_manager")
def test_consume_barcode_file_tag_split_and_assignment(self) -> None:
"""
GIVEN:
@@ -1042,34 +1043,33 @@ class TestTagBarcode(DirectoriesMixin, SampleDirMixin, GetReaderPluginMixin, Tes
dst = settings.SCRATCH_DIR / "split-by-tag-basic.pdf"
shutil.copy(test_file, dst)
with mock.patch("documents.tasks.ProgressManager", DummyProgressManager):
result = tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=dst,
),
None,
)
result = tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=dst,
),
None,
)
self.assertEqual(result, {"reason": "Barcode splitting complete!"})
self.assertEqual(result, {"reason": "Barcode splitting complete!"})
documents = Document.objects.all().order_by("id")
self.assertEqual(documents.count(), 3)
documents = Document.objects.all().order_by("id")
self.assertEqual(documents.count(), 3)
doc1 = documents[0]
self.assertEqual(doc1.tags.count(), 0)
doc1 = documents[0]
self.assertEqual(doc1.tags.count(), 0)
doc2 = documents[1]
self.assertEqual(doc2.tags.count(), 1)
_tag_1 = doc2.tags.first()
assert _tag_1 is not None
self.assertEqual(_tag_1.name, "invoice")
doc2 = documents[1]
self.assertEqual(doc2.tags.count(), 1)
_tag_1 = doc2.tags.first()
assert _tag_1 is not None
self.assertEqual(_tag_1.name, "invoice")
doc3 = documents[2]
self.assertEqual(doc3.tags.count(), 1)
_tag_2 = doc3.tags.first()
assert _tag_2 is not None
self.assertEqual(_tag_2.name, "receipt")
doc3 = documents[2]
self.assertEqual(doc3.tags.count(), 1)
_tag_2 = doc3.tags.first()
assert _tag_2 is not None
self.assertEqual(_tag_2.name, "receipt")
@override_settings(
CONSUMER_ENABLE_TAG_BARCODE=True,
src/documents/tests/test_consumer.py
+5 -5
View File
@@ -30,12 +30,12 @@ from documents.models import Tag
from documents.parsers import ParseError
from documents.plugins.helpers import ProgressStatusOptions
from documents.tasks import sanity_check
from documents.tests.utils import DummyProgressManager
from documents.tests.utils import FileSystemAssertsMixin
from documents.tests.utils import GetConsumerMixin
from paperless_mail.models import MailRule
from paperless_testing.assertions import FileSystemAssertsMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import UserFactory
from paperless_testing.fakes.progress import FakeProgressManager
class _BaseNewStyleParser:
@@ -777,7 +777,7 @@ class TestConsumer(
)
version_file = self.get_test_file2()
status = DummyProgressManager(version_file.name, None)
status = FakeProgressManager(version_file.name, None)
overrides = DocumentMetadataOverrides(
version_label="v2",
actor_id=actor.pk,
@@ -840,7 +840,7 @@ class TestConsumer(
assert root_doc is not None
version_file = self.get_test_file2()
status = DummyProgressManager(version_file.name, None)
status = FakeProgressManager(version_file.name, None)
overrides = DocumentMetadataOverrides(
filename="valid_pdf_version-upload",
actor_id=999999,
@@ -897,7 +897,7 @@ class TestConsumer(
assert root_doc is not None
def consume_version(version_file: Path) -> Document:
status = DummyProgressManager(version_file.name, None)
status = FakeProgressManager(version_file.name, None)
overrides = DocumentMetadataOverrides()
doc = ConsumableDocument(
DocumentSource.ApiUpload,
src/documents/tests/test_double_sided.py
+10 -14
View File
@@ -2,8 +2,8 @@ import datetime as dt
import os
import shutil
from pathlib import Path
from unittest import mock
import pytest
from django.test import TestCase
from django.test import override_settings
from pdfminer.high_level import extract_text
@@ -15,12 +15,12 @@ from documents.data_models import ConsumableDocument
from documents.data_models import DocumentSource
from documents.double_sided import STAGING_FILE_NAME
from documents.double_sided import TIMEOUT_MINUTES
from documents.tests.utils import DummyProgressManager
from documents.tests.utils import FileSystemAssertsMixin
from documents.tests.utils import SampleDirMixin
from paperless_testing.assertions import FileSystemAssertsMixin
from paperless_testing.dirs import DirectoriesMixin
@pytest.mark.usefixtures("fake_progress_manager")
@override_settings(
CONSUMER_RECURSIVE=True,
CONSUMER_ENABLE_COLLATE_DOUBLE_SIDED=True,
@@ -46,17 +46,13 @@ class TestDoubleSided(
dst = self.double_sided_dir / dstname
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.copy(src, dst)
with mock.patch(
"documents.tasks.ProgressManager",
DummyProgressManager,
):
msg = tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=dst,
),
None,
)
msg = tasks.consume_file(
ConsumableDocument(
source=DocumentSource.ConsumeFolder,
original_file=dst,
),
None,
)
self.assertIsNotFile(dst)
return msg
src/documents/tests/test_file_handling.py
+1 -1
View File
@@ -29,7 +29,7 @@ from documents.models import DocumentType
from documents.models import StoragePath
from documents.serialisers import DocumentSerializer
from documents.tasks import empty_trash
from documents.tests.utils import FileSystemAssertsMixin
from paperless_testing.assertions import FileSystemAssertsMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import UserFactory
src/documents/tests/test_management.py
+1 -1
View File
@@ -20,7 +20,7 @@ if TYPE_CHECKING:
from documents.file_handling import generate_filename
from documents.models import Document
from documents.tasks import update_document_content_maybe_archive_file
from documents.tests.utils import FileSystemAssertsMixin
from paperless_testing.assertions import FileSystemAssertsMixin
from paperless_testing.dirs import DirectoriesMixin
sample_file: Path = Path(__file__).parent / "samples" / "simple.pdf"
src/documents/tests/test_management_exporter.py
+1 -1
View File
@@ -45,9 +45,9 @@ from documents.models import WorkflowTrigger
from documents.sanity_checker import check_sanity
from documents.settings import EXPORTER_FILE_NAME
from documents.settings import EXPORTER_SHARE_LINK_BUNDLE_NAME
from documents.tests.utils import FileSystemAssertsMixin
from documents.tests.utils import SampleDirMixin
from paperless_mail.models import MailAccount
from paperless_testing.assertions import FileSystemAssertsMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.dirs import paperless_environment
from paperless_testing.permissions import grant_object
src/documents/tests/test_management_importer.py
+1 -1
View File
@@ -15,8 +15,8 @@ from documents.management.commands.document_importer import _deserialize_record
from documents.models import Document
from documents.settings import EXPORTER_ARCHIVE_NAME
from documents.settings import EXPORTER_FILE_NAME
from documents.tests.utils import FileSystemAssertsMixin
from documents.tests.utils import SampleDirMixin
from paperless_testing.assertions import FileSystemAssertsMixin
from paperless_testing.dirs import DirectoriesMixin
src/documents/tests/test_management_thumbnails.py
+1 -1
View File
@@ -9,7 +9,7 @@ from django.test import TestCase
from documents.management.commands.document_thumbnails import _process_document
from documents.models import Document
from documents.parsers import get_default_thumbnail
from documents.tests.utils import FileSystemAssertsMixin
from paperless_testing.assertions import FileSystemAssertsMixin
from paperless_testing.dirs import DirectoriesMixin
src/documents/tests/test_migration_saved_view_visibility.py
+1 -1
View File
@@ -1,4 +1,4 @@
from documents.tests.utils import TestMigrations
from paperless_testing.migrations import TestMigrations
SAVED_VIEWS_KEY = "saved_views"
DASHBOARD_VIEWS_VISIBLE_IDS_KEY = "dashboard_views_visible_ids"
src/documents/tests/test_migration_sha256_checksums.py
+1 -1
View File
@@ -7,7 +7,7 @@ from django.conf import settings
from django.db import connection
from django.test import override_settings
from documents.tests.utils import TestMigrations
from paperless_testing.migrations import TestMigrations
def _sha256(data: bytes) -> str:
src/documents/tests/test_migration_share_link_bundle.py
+1 -1
View File
@@ -1,4 +1,4 @@
from documents.tests.utils import TestMigrations
from paperless_testing.migrations import TestMigrations
class TestMigrateShareLinkBundlePermissions(TestMigrations):
src/documents/tests/test_tasks.py
+1 -1
View File
@@ -18,7 +18,7 @@ from documents.models import WorkflowAction
from documents.sanity_checker import SanityCheckFailedException
from documents.sanity_checker import SanityCheckMessages
from documents.tests.test_classifier import dummy_preprocess
from documents.tests.utils import FileSystemAssertsMixin
from paperless_testing.assertions import FileSystemAssertsMixin
from paperless_testing.dirs import DirectoriesMixin
src/documents/tests/test_views.py
+1 -1
View File
@@ -28,12 +28,12 @@ from documents.models import StoragePath
from documents.models import Tag
from documents.models import UiSettings
from documents.signals.handlers import update_llm_suggestions_cache
from documents.tests.utils import read_streaming_response
from paperless.models import ApplicationConfiguration
from paperless_ai.exceptions import LLMProviderError
from paperless_ai.exceptions import LLMTimeoutError
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import UserFactory
from paperless_testing.http import read_streaming_response
from paperless_testing.permissions import grant_global
from paperless_testing.permissions import grant_object
src/documents/tests/test_workflows.py
+438 -442
View File
File diff suppressed because it is too large Load Diff
src/documents/tests/utils.py
+2 -225
View File
@@ -1,126 +1,16 @@
import time
import warnings
from collections.abc import Callable
from collections.abc import Generator
from collections.abc import Iterator
from contextlib import contextmanager
from os import PathLike
from pathlib import Path
from typing import Any
from unittest import mock
import httpx
import pytest
from django.apps import apps
from django.db import connection
from django.db.migrations.executor import MigrationExecutor
from django.http import StreamingHttpResponse
from django.test import TransactionTestCase
from documents.consumer import AsnCheckPlugin
from documents.consumer import ConsumerPlugin
from documents.consumer import ConsumerPreflightPlugin
from documents.data_models import ConsumableDocument
from documents.data_models import DocumentMetadataOverrides
from documents.data_models import DocumentSource
from documents.parsers import ParseError
from documents.plugins.helpers import ProgressStatusOptions
def util_call_with_backoff(
method_or_callable: Callable,
args: list | tuple,
*,
skip_on_50x_err=True,
) -> tuple[bool, Any]:
"""
For whatever reason, the images started during the test pipeline like to
segfault sometimes, crash and otherwise fail randomly, when run with the
exact files that usually pass.
So, this function will retry the given method/function up to 3 times, with larger backoff
periods between each attempt, in hopes the issue resolves itself during
one attempt to parse.
This will wait the following:
- Attempt 1 - 20s following failure
- Attempt 2 - 40s following failure
- Attempt 3 - 80s following failure
"""
result = None
succeeded = False
retry_time = 20.0
retry_count = 0
status_codes = []
max_retry_count = 3
while retry_count < max_retry_count and not succeeded:
try:
result = method_or_callable(*args)
succeeded = True
except ParseError as e: # pragma: no cover
cause_exec = e.__cause__
if cause_exec is not None and isinstance(cause_exec, httpx.HTTPStatusError):
status_codes.append(cause_exec.response.status_code)
warnings.warn(
f"HTTP Exception for {cause_exec.request.url} - {cause_exec}",
)
else:
warnings.warn(f"Unexpected error: {e}")
except Exception as e: # pragma: no cover
warnings.warn(f"Unexpected error: {e}")
retry_count = retry_count + 1
time.sleep(retry_time)
retry_time = retry_time * 2.0
if (
not succeeded
and status_codes
and skip_on_50x_err
and all(httpx.codes.is_server_error(code) for code in status_codes)
):
pytest.skip("Repeated HTTP 50x for service") # pragma: no cover
return succeeded, result
def read_streaming_response(response: StreamingHttpResponse) -> bytes:
"""Consume a StreamingHttpResponse/FileResponse and close it."""
content = b"".join(response.streaming_content)
response.close()
return content
class FileSystemAssertsMixin:
"""
Utilities for checks various state information of the file system
"""
def assertIsFile(self, path: PathLike[str] | str) -> None:
self.assertTrue(Path(path).resolve().is_file(), f"File does not exist: {path}")
def assertIsNotFile(self, path: PathLike[str] | str) -> None:
self.assertFalse(Path(path).resolve().is_file(), f"File does exist: {path}")
def assertIsDir(self, path: PathLike[str] | str) -> None:
self.assertTrue(Path(path).resolve().is_dir(), f"Dir does not exist: {path}")
def assertIsNotDir(self, path: PathLike[str] | str) -> None:
self.assertFalse(Path(path).resolve().is_dir(), f"Dir does exist: {path}")
def assertFileCountInDir(self, path: PathLike[str] | str, count: int) -> None:
path = Path(path).resolve()
self.assertTrue(path.is_dir(), f"Path {path} is not a directory")
files = [x for x in path.iterdir() if x.is_file()]
self.assertEqual(
len(files),
count,
f"Path {path} contains {len(files)} files instead of {count} files",
)
from paperless_testing.fakes.progress import FakeProgressManager
class ConsumeTaskMixin:
@@ -158,59 +48,6 @@ class ConsumeTaskMixin:
yield (task_kwargs["input_doc"], task_kwargs["overrides"])
class TestMigrations(TransactionTestCase):
@property
def app(self):
return apps.get_containing_app_config(type(self).__module__).name
migrate_from = None
dependencies = None
migrate_to = None
def setUp(self) -> None:
super().setUp()
assert self.migrate_from and self.migrate_to, (
f"TestCase '{type(self).__name__}' must define migrate_from and migrate_to properties"
)
self.migrate_from = [(self.app, self.migrate_from)]
if self.dependencies is not None:
self.migrate_from.extend(self.dependencies)
self.migrate_to = [(self.app, self.migrate_to)]
executor = MigrationExecutor(connection)
old_apps = executor.loader.project_state(self.migrate_from).apps
# Reverse to the original migration
executor.migrate(self.migrate_from)
self.setUpBeforeMigration(old_apps)
self.apps = old_apps
# Run the migration to test
executor = MigrationExecutor(connection)
executor.loader.build_graph() # reload.
executor.migrate(self.migrate_to)
self.apps = executor.loader.project_state(self.migrate_to).apps
def setUpBeforeMigration(self, apps) -> None:
pass
def tearDown(self) -> None:
"""
Ensure the database schema is restored to the latest migration after
each migration test, so subsequent tests run against HEAD.
"""
try:
executor = MigrationExecutor(connection)
executor.loader.build_graph()
targets = executor.loader.graph.leaf_nodes()
executor.migrate(targets)
finally:
super().tearDown()
class SampleDirMixin:
SAMPLE_DIR = Path(__file__).parent / "samples"
@@ -227,7 +64,7 @@ class GetConsumerMixin:
mailrule_id: int | None = None,
) -> Generator[ConsumerPlugin, None, None]:
# Store this for verification
self.status = DummyProgressManager(filepath.name, None)
self.status = FakeProgressManager(filepath.name, None)
doc = ConsumableDocument(
source,
original_file=filepath,
@@ -263,63 +100,3 @@ class GetConsumerMixin:
yield reader
finally:
reader.cleanup()
class DummyProgressManager:
"""
A dummy handler for progress management that doesn't actually try to
connect to Redis. Payloads are stored for test assertions if needed.
Use it with
mock.patch("documents.tasks.ProgressManager", DummyProgressManager)
"""
def __init__(self, filename: str, task_id: str | None = None) -> None:
self.filename = filename
self.task_id = task_id
self.payloads = []
def __enter__(self):
self.open()
return self
def __exit__(self, exc_type, exc_val, exc_tb) -> None:
self.close()
def open(self) -> None:
pass
def close(self) -> None:
pass
def send_progress(
self,
status: ProgressStatusOptions,
message: str,
current_progress: int,
max_progress: int,
*,
document_id: int | None = None,
owner_id: int | None = None,
users_can_view: list[int] | None = None,
groups_can_view: list[int] | None = None,
) -> None:
# Ensure the layer is open
self.open()
payload = {
"type": "status_update",
"data": {
"filename": self.filename,
"task_id": self.task_id,
"current_progress": current_progress,
"max_progress": max_progress,
"status": status,
"message": message,
"document_id": document_id,
"owner_id": owner_id,
"users_can_view": users_can_view or [],
"groups_can_view": groups_can_view or [],
},
}
self.payloads.append(payload)
src/paperless/tests/parsers/test_mail_parser_live.py
+1 -1
View File
@@ -10,8 +10,8 @@ from imagehash import average_hash
from PIL import Image
from pytest_mock import MockerFixture
from documents.tests.utils import util_call_with_backoff
from paperless.parsers.mail import MailDocumentParser
from paperless_testing.retry import util_call_with_backoff
def extract_text(pdf_path: Path) -> str:
src/paperless/tests/parsers/test_tesseract_custom_settings.py
+1 -1
View File
@@ -3,13 +3,13 @@ import json
from django.test import TestCase
from django.test import override_settings
from documents.tests.utils import FileSystemAssertsMixin
from paperless.models import ApplicationConfiguration
from paperless.models import CleanChoices
from paperless.models import ColorConvertChoices
from paperless.models import ModeChoices
from paperless.models import OutputTypeChoices
from paperless.parsers.tesseract import RasterisedDocumentParser
from paperless_testing.assertions import FileSystemAssertsMixin
from paperless_testing.dirs import DirectoriesMixin
src/paperless/tests/parsers/test_tika_liva.py
+1 -1
View File
@@ -3,8 +3,8 @@ from pathlib import Path
import pytest
from documents.tests.utils import util_call_with_backoff
from paperless.parsers.tika import TikaDocumentParser
from paperless_testing.retry import util_call_with_backoff
@pytest.mark.skipif(
src/paperless/tests/test_migration_replace_skip_archive_file.py
+1 -1
View File
@@ -1,4 +1,4 @@
from documents.tests.utils import TestMigrations
from paperless_testing.migrations import TestMigrations
class TestMigrateSkipArchiveFile(TestMigrations):
src/paperless_mail/tests/test_mail.py
+1 -1
View File
@@ -27,7 +27,6 @@ from rest_framework.test import APITestCase
from documents.models import Correspondent
from documents.models import MatchingModel
from documents.tests.utils import FileSystemAssertsMixin
from paperless_mail import tasks
from paperless_mail.mail import MailAccountHandler
from paperless_mail.mail import MailError
@@ -40,6 +39,7 @@ from paperless_mail.models import MailRule
from paperless_mail.models import ProcessedMail
from paperless_mail.tests.factories import MailAccountFactory
from paperless_mail.tests.factories import MailRuleFactory
from paperless_testing.assertions import FileSystemAssertsMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import CorrespondentFactory
from paperless_testing.factories import UserFactory
src/paperless_testing/assertions.py
+37
View File
@@ -0,0 +1,37 @@
"""Filesystem assertions for unittest-style tests."""
from __future__ import annotations
from pathlib import Path
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from os import PathLike
class FileSystemAssertsMixin:
def assertIsFile(self, path: PathLike[str] | str) -> None:
if not Path(path).resolve().is_file():
raise AssertionError(f"File does not exist: {path}")
def assertIsNotFile(self, path: PathLike[str] | str) -> None:
if Path(path).resolve().is_file():
raise AssertionError(f"File does exist: {path}")
def assertIsDir(self, path: PathLike[str] | str) -> None:
if not Path(path).resolve().is_dir():
raise AssertionError(f"Dir does not exist: {path}")
def assertIsNotDir(self, path: PathLike[str] | str) -> None:
if Path(path).resolve().is_dir():
raise AssertionError(f"Dir does exist: {path}")
def assertFileCountInDir(self, path: PathLike[str] | str, count: int) -> None:
path = Path(path).resolve()
if not path.is_dir():
raise AssertionError(f"Path {path} is not a directory")
found = len([x for x in path.iterdir() if x.is_file()])
if found != count:
raise AssertionError(
f"Path {path} contains {found} files instead of {count} files",
)
src/paperless_testing/fakes/__init__.py
View File
src/paperless_testing/fakes/progress.py
+31
View File
@@ -0,0 +1,31 @@
from __future__ import annotations
from typing import TYPE_CHECKING
from documents.plugins.helpers import ProgressManager
if TYPE_CHECKING:
from documents.plugins.helpers import WebsocketPayload
class FakeProgressManager(ProgressManager):
"""
The real ProgressManager with the channel layer cut out: send_progress still
builds the payload, so it cannot drift, and the payloads are recorded instead
of being sent to Redis.
Use it through the `fake_progress_manager` fixture, or construct it directly.
"""
def __init__(self, filename: str | None = None, task_id: str | None = None) -> None:
super().__init__(filename, task_id)
self.payloads: list[WebsocketPayload] = []
def open(self) -> None:
pass
def close(self) -> None:
pass
def send(self, payload: WebsocketPayload) -> None:
self.payloads.append(payload)
src/paperless_testing/http.py
+13
View File
@@ -0,0 +1,13 @@
from __future__ import annotations
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from django.http import StreamingHttpResponse
def read_streaming_response(response: StreamingHttpResponse) -> bytes:
"""Consume a StreamingHttpResponse/FileResponse and close it."""
content = b"".join(response.streaming_content)
response.close()
return content
src/paperless_testing/migrations.py
+65
View File
@@ -0,0 +1,65 @@
from __future__ import annotations
from typing import TYPE_CHECKING
from typing import Any
from django.apps import apps
from django.db import connection
from django.db.migrations.executor import MigrationExecutor
from django.test import TransactionTestCase
if TYPE_CHECKING:
from django.apps.registry import Apps
class TestMigrations(TransactionTestCase):
@property
def app(self) -> str:
return apps.get_containing_app_config(type(self).__module__).name
migrate_from: Any = None
dependencies: list[tuple[str, str]] | None = None
migrate_to: Any = None
def setUp(self) -> None:
super().setUp()
assert self.migrate_from and self.migrate_to, (
f"TestCase '{type(self).__name__}' must define migrate_from and migrate_to properties"
)
self.migrate_from = [(self.app, self.migrate_from)]
if self.dependencies is not None:
self.migrate_from.extend(self.dependencies)
self.migrate_to = [(self.app, self.migrate_to)]
executor = MigrationExecutor(connection)
old_apps = executor.loader.project_state(self.migrate_from).apps
# Reverse to the original migration
executor.migrate(self.migrate_from)
self.setUpBeforeMigration(old_apps)
self.apps = old_apps
# Run the migration to test
executor = MigrationExecutor(connection)
executor.loader.build_graph() # reload.
executor.migrate(self.migrate_to)
self.apps = executor.loader.project_state(self.migrate_to).apps
def setUpBeforeMigration(self, apps: Apps) -> None:
pass
def tearDown(self) -> None:
"""
Ensure the database schema is restored to the latest migration after
each migration test, so subsequent tests run against HEAD.
"""
try:
executor = MigrationExecutor(connection)
executor.loader.build_graph()
targets = executor.loader.graph.leaf_nodes()
executor.migrate(targets)
finally:
super().tearDown()
src/paperless_testing/retry.py
+75
View File
@@ -0,0 +1,75 @@
from __future__ import annotations
import time
import warnings
from typing import TYPE_CHECKING
from typing import Any
import httpx
import pytest
from documents.parsers import ParseError
if TYPE_CHECKING:
from collections.abc import Callable
def util_call_with_backoff(
method_or_callable: Callable,
args: list | tuple,
*,
skip_on_50x_err: bool = True,
) -> tuple[bool, Any]:
"""
For whatever reason, the images started during the test pipeline like to
segfault sometimes, crash and otherwise fail randomly, when run with the
exact files that usually pass.
So, this function will retry the given method/function up to 3 times, with larger backoff
periods between each attempt, in hopes the issue resolves itself during
one attempt to parse.
This will wait the following:
- Attempt 1 - 20s following failure
- Attempt 2 - 40s following failure
- Attempt 3 - 80s following failure
"""
result = None
succeeded = False
retry_time = 20.0
retry_count = 0
status_codes = []
max_retry_count = 3
while retry_count < max_retry_count and not succeeded:
try:
result = method_or_callable(*args)
succeeded = True
except ParseError as e: # pragma: no cover
cause_exec = e.__cause__
if cause_exec is not None and isinstance(cause_exec, httpx.HTTPStatusError):
status_codes.append(cause_exec.response.status_code)
warnings.warn(
f"HTTP Exception for {cause_exec.request.url} - {cause_exec}",
)
else:
warnings.warn(f"Unexpected error: {e}")
except Exception as e: # pragma: no cover
warnings.warn(f"Unexpected error: {e}")
retry_count = retry_count + 1
time.sleep(retry_time)
retry_time = retry_time * 2.0
if (
not succeeded
and status_codes
and skip_on_50x_err
and all(httpx.codes.is_server_error(code) for code in status_codes)
):
pytest.skip("Repeated HTTP 50x for service") # pragma: no cover
return succeeded, result