diff --git a/src/conftest.py b/src/conftest.py index 91f5635a2..b88565bdf 100644 --- a/src/conftest.py +++ b/src/conftest.py @@ -77,6 +77,30 @@ def _clear_django_caches() -> None: cache.clear() +@pytest.fixture(autouse=True) +def _keep_connection_open_in_atomic(monkeypatch: pytest.MonkeyPatch) -> None: + """Make ``close_old_connections()`` a no-op inside the test's transaction. + + The Celery signal handlers call it, as a worker would between tasks. Tests + run inside an open atomic block, where Postgres really closes the + connection, taking the test transaction and every later test on the + worker with it. SQLite in-memory ignores the close, which hid this. + """ + from django.db.backends.base.base import BaseDatabaseWrapper + + original = BaseDatabaseWrapper.close_if_unusable_or_obsolete + + def close_unless_atomic(self: BaseDatabaseWrapper) -> None: + if not self.in_atomic_block: + original(self) + + monkeypatch.setattr( + BaseDatabaseWrapper, + "close_if_unusable_or_obsolete", + close_unless_atomic, + ) + + @pytest.fixture def paperless_dirs( tmp_path: Path, diff --git a/src/documents/tests/test_api_custom_fields.py b/src/documents/tests/test_api_custom_fields.py index a1823c501..457d44f87 100644 --- a/src/documents/tests/test_api_custom_fields.py +++ b/src/documents/tests/test_api_custom_fields.py @@ -1062,7 +1062,7 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase): "custom_fields": [ { "field": custom_field_doclink.id, - "value": [2, 3, 4], + "value": [doc2.id, doc3.id, doc4.id], }, ], }, @@ -1073,13 +1073,13 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase): self.assertEqual(CustomFieldInstance.objects.count(), 4) _cf_5 = doc2.custom_fields.first() assert _cf_5 is not None - self.assertEqual(_cf_5.value, [1]) + self.assertEqual(_cf_5.value, [doc1.id]) _cf_6 = doc3.custom_fields.first() assert _cf_6 is not None - self.assertEqual(_cf_6.value, [1]) + self.assertEqual(_cf_6.value, [doc1.id]) _cf_7 = doc4.custom_fields.first() assert _cf_7 is not None - self.assertEqual(_cf_7.value, [1]) + self.assertEqual(_cf_7.value, [doc1.id]) # Add links appends if necessary resp = self.client.patch( @@ -1088,7 +1088,7 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase): "custom_fields": [ { "field": custom_field_doclink.id, - "value": [1, 4], + "value": [doc1.id, doc4.id], }, ], }, @@ -1098,7 +1098,7 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase): self.assertEqual(resp.status_code, status.HTTP_200_OK) _cf_8 = doc4.custom_fields.first() assert _cf_8 is not None - self.assertEqual(_cf_8.value, [1, 3]) + self.assertEqual(_cf_8.value, [doc1.id, doc3.id]) # Remove one of the links, removed on other doc resp = self.client.patch( @@ -1107,7 +1107,7 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase): "custom_fields": [ { "field": custom_field_doclink.id, - "value": [2, 3], + "value": [doc2.id, doc3.id], }, ], }, @@ -1117,13 +1117,13 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase): self.assertEqual(resp.status_code, status.HTTP_200_OK) _cf_9 = doc2.custom_fields.first() assert _cf_9 is not None - self.assertEqual(_cf_9.value, [1]) + self.assertEqual(_cf_9.value, [doc1.id]) _cf_10 = doc3.custom_fields.first() assert _cf_10 is not None - self.assertEqual(_cf_10.value, [1, 4]) + self.assertEqual(_cf_10.value, [doc1.id, doc4.id]) _cf_11 = doc4.custom_fields.first() assert _cf_11 is not None - self.assertEqual(_cf_11.value, [3]) + self.assertEqual(_cf_11.value, [doc3.id]) # Removes the field entirely resp = self.client.patch( @@ -1140,10 +1140,10 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase): self.assertEqual(_cf_12.value, []) _cf_13 = doc3.custom_fields.first() assert _cf_13 is not None - self.assertEqual(_cf_13.value, [4]) + self.assertEqual(_cf_13.value, [doc4.id]) _cf_14 = doc4.custom_fields.first() assert _cf_14 is not None - self.assertEqual(_cf_14.value, [3]) + self.assertEqual(_cf_14.value, [doc3.id]) # If field exists on target doc but value is None doc5 = Document.objects.create( @@ -1170,7 +1170,7 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase): self.assertEqual(resp.status_code, status.HTTP_200_OK) _cf_15 = doc5.custom_fields.first() assert _cf_15 is not None - self.assertEqual(_cf_15.value, [1]) + self.assertEqual(_cf_15.value, [doc1.id]) def test_documentlink_patch_requires_change_permission_on_target_documents( self, diff --git a/src/documents/tests/test_api_filter_by_custom_fields.py b/src/documents/tests/test_api_filter_by_custom_fields.py index 96123fb17..b9637aa3e 100644 --- a/src/documents/tests/test_api_filter_by_custom_fields.py +++ b/src/documents/tests/test_api_filter_by_custom_fields.py @@ -628,22 +628,23 @@ class TestCustomFieldsSearch(DirectoriesMixin, APITestCase): # Subset check (document link field only) # # ==========================================================# def test_document_link_contains(self) -> None: + linked = {self.documents[1].id, self.documents[2].id} # Document link field "contains" performs a subset check. self._assert_query_match_predicate( - ["documentlink_field", "contains", [1, 2]], + ["documentlink_field", "contains", sorted(linked)], lambda document: ( "documentlink_field" in document and document["documentlink_field"] is not None - and set(document["documentlink_field"]) >= {1, 2} + and set(document["documentlink_field"]) >= linked ), ) # The order of IDs don't matter - this is the same as above. self._assert_query_match_predicate( - ["documentlink_field", "contains", [2, 1]], + ["documentlink_field", "contains", sorted(linked, reverse=True)], lambda document: ( "documentlink_field" in document and document["documentlink_field"] is not None - and set(document["documentlink_field"]) >= {1, 2} + and set(document["documentlink_field"]) >= linked ), ) diff --git a/src/documents/tests/test_api_search.py b/src/documents/tests/test_api_search.py index 9c7ff158d..00337602d 100644 --- a/src/documents/tests/test_api_search.py +++ b/src/documents/tests/test_api_search.py @@ -944,7 +944,7 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): self.assertEqual(response.status_code, status.HTTP_200_OK) results = response.data["results"] - self.assertEqual({r["id"] for r in results}, {1, 2}) + self.assertEqual({r["id"] for r in results}, {d1.id, d2.id}) @mock.patch("documents.search._backend.parse_user_query") def test_search_parser_bug_surfaces_as_500_not_400(self, m) -> None: @@ -1801,6 +1801,7 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): def test_search_sorting(self) -> None: u1 = UserFactory(username="user1") u2 = UserFactory(username="user2") + u3 = UserFactory(username="user3") c1 = Correspondent.objects.create(name="corres Ax") c2 = Correspondent.objects.create(name="corres Cx") c3 = Correspondent.objects.create(name="corres Bx") @@ -1826,6 +1827,7 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): content="test", archive_serial_number=1, title="1", + owner=u3, ) Note.objects.create( note="This is a note.", @@ -1878,14 +1880,14 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): search_query("&ordering=-num_notes"), [d1.id, d3.id, d2.id], ) - # owner sort: ORM orders by owner_id (integer); NULLs first in SQLite ASC + # owner sort: ORM orders by owner_id (integer) self.assertListEqual( search_query("&ordering=owner"), - [d3.id, d1.id, d2.id], + [d1.id, d2.id, d3.id], ) self.assertListEqual( search_query("&ordering=-owner"), - [d2.id, d1.id, d3.id], + [d3.id, d2.id, d1.id], ) def test_search_ordering_by_score(self) -> None: diff --git a/src/documents/tests/test_api_status.py b/src/documents/tests/test_api_status.py index 6f200e54f..cea71f136 100644 --- a/src/documents/tests/test_api_status.py +++ b/src/documents/tests/test_api_status.py @@ -5,6 +5,7 @@ from datetime import timedelta from pathlib import Path from unittest import mock +from django.db import connection from django.test import override_settings from django.utils import timezone from rest_framework import status @@ -68,7 +69,7 @@ class TestSystemStatus(APITestCase): self.assertEqual(response.data["install_type"], "bare-metal") self.assertIsNotNone(response.data["storage"]["total"]) self.assertIsNotNone(response.data["storage"]["available"]) - self.assertEqual(response.data["database"]["type"], "sqlite") + self.assertEqual(response.data["database"]["type"], connection.vendor) self.assertIsNotNone(response.data["database"]["url"]) self.assertEqual(response.data["database"]["status"], "OK") self.assertIsNone(response.data["database"]["error"]) diff --git a/src/documents/tests/test_bulk_edit.py b/src/documents/tests/test_bulk_edit.py index b341f5744..ad902e7ff 100644 --- a/src/documents/tests/test_bulk_edit.py +++ b/src/documents/tests/test_bulk_edit.py @@ -1031,9 +1031,7 @@ class TestPDFActions(DirectoriesMixin, TestCase): with self.assertLogs("paperless.bulk_edit", level="ERROR") as cm: bulk_edit.merge(doc_ids) error_str = cm.output[0] - expected_str = ( - "Error merging document 2, it will not be included in the merge" - ) + expected_str = f"Error merging document {self.doc2.id}, it will not be included in the merge" self.assertIn(expected_str, error_str) mock_consume_file.assert_not_called() @@ -1193,7 +1191,7 @@ class TestPDFActions(DirectoriesMixin, TestCase): with self.assertLogs("paperless.bulk_edit", level="ERROR") as cm: bulk_edit.split(doc_ids, pages) error_str = cm.output[0] - expected_str = "Error splitting document 2" + expected_str = f"Error splitting document {self.doc2.id}" self.assertIn(expected_str, error_str) mock_consume_file.assert_not_called() diff --git a/src/documents/tests/test_consumer.py b/src/documents/tests/test_consumer.py index 7c607e1e3..ed4736697 100644 --- a/src/documents/tests/test_consumer.py +++ b/src/documents/tests/test_consumer.py @@ -248,8 +248,8 @@ class TestConsumer( ) self.assertIsNone(document.correspondent) self.assertIsNone(document.document_type) - self.assertEqual(document.filename, "0000001.pdf") - self.assertEqual(document.archive_filename, "0000001.pdf") + self.assertEqual(document.filename, f"{document.pk:07}.pdf") + self.assertEqual(document.archive_filename, f"{document.pk:07}.pdf") self.assertIsFile(document.source_path) @@ -1127,17 +1127,17 @@ class TestConsumer( with self.get_consumer(settings.CONSUMPTION_DIR / "simple.png") as consumer: consumer.run() - doc1 = Document.objects.filter(pk=1).first() + doc1 = Document.objects.latest("pk") with self.get_consumer(settings.CONSUMPTION_DIR / "simple.pdf") as consumer: consumer.run() - doc2 = Document.objects.filter(pk=2).first() + doc2 = Document.objects.latest("pk") with self.get_consumer(settings.CONSUMPTION_DIR / "simple.png.pdf") as consumer: consumer.run() - doc3 = Document.objects.filter(pk=3).first() + doc3 = Document.objects.latest("pk") self.assertEqual(doc1.filename, "simple.png") self.assertEqual(doc1.archive_filename, "simple.pdf") diff --git a/src/documents/tests/test_file_handling.py b/src/documents/tests/test_file_handling.py index 46c54f98f..53048c704 100644 --- a/src/documents/tests/test_file_handling.py +++ b/src/documents/tests/test_file_handling.py @@ -760,6 +760,9 @@ class TestFileHandlingWithArchive(DirectoriesMixin, FileSystemAssertsMixin, Test archive_checksum="B", ) + # Without a format the files are renamed to match the primary key + original = doc.source_path + archive = doc.archive_path self.assertIsFile(original) self.assertIsFile(archive) self.assertIsFile(doc.source_path) @@ -978,6 +981,9 @@ class TestFileHandlingWithArchive(DirectoriesMixin, FileSystemAssertsMixin, Test archive_filename="0000001.pdf", ) + # Without a format the files are renamed to match the primary key + original = doc.source_path + archive = doc.archive_path self.assertIsFile(original) self.assertIsFile(archive) self.assertIsFile(doc.source_path) diff --git a/src/documents/tests/test_management_exporter.py b/src/documents/tests/test_management_exporter.py index ec6a21b77..16ca0a2a3 100644 --- a/src/documents/tests/test_management_exporter.py +++ b/src/documents/tests/test_management_exporter.py @@ -151,6 +151,20 @@ class TestExportImport( super().setUp() + def _install_sample_files(self) -> None: + documents_dir = Path(self.dirs.media_dir) / "documents" + shutil.rmtree(documents_dir) + install_document_samples(documents_dir) + # Thumbnails are located by primary key, which the database sequence + # does not reset between tests on PostgreSQL + for number, document in enumerate( + (self.d1, self.d2, self.d3, self.d4), + start=1, + ): + source = documents_dir / "thumbnails" / f"{number:07}.webp" + if source != document.thumbnail_path: + shutil.copy(source, document.thumbnail_path) + def _get_document_from_manifest(self, manifest, id): f = list( filter( @@ -204,8 +218,7 @@ class TestExportImport( return manifest def test_exporter(self, *, use_filename_format=False) -> None: - shutil.rmtree(Path(self.dirs.media_dir) / "documents") - install_document_samples(Path(self.dirs.media_dir) / "documents") + self._install_sample_files() num_permission_objects = Permission.objects.count() @@ -308,8 +321,7 @@ class TestExportImport( self.assertEqual(messages.total_issue_count, 0) def test_exporter_with_filename_format(self) -> None: - shutil.rmtree(Path(self.dirs.media_dir) / "documents") - install_document_samples(Path(self.dirs.media_dir) / "documents") + self._install_sample_files() with override_settings( FILENAME_FORMAT="{created_year}/{correspondent}/{title}", @@ -317,8 +329,7 @@ class TestExportImport( self.test_exporter(use_filename_format=True) def test_exporter_includes_share_links_and_bundles(self) -> None: - shutil.rmtree(Path(self.dirs.media_dir) / "documents") - install_document_samples(Path(self.dirs.media_dir) / "documents") + self._install_sample_files() share_link = ShareLink.objects.create( slug="share-link-slug", @@ -416,8 +427,7 @@ class TestExportImport( ) def test_update_export_changed_time(self) -> None: - shutil.rmtree(Path(self.dirs.media_dir) / "documents") - install_document_samples(Path(self.dirs.media_dir) / "documents") + self._install_sample_files() self._do_export() self.assertIsFile(self.target / "manifest.json") @@ -452,8 +462,7 @@ class TestExportImport( self.assertEqual(st_mtime_3, st_mtime_4) def test_update_export_changed_checksum(self) -> None: - shutil.rmtree(Path(self.dirs.media_dir) / "documents") - install_document_samples(Path(self.dirs.media_dir) / "documents") + self._install_sample_files() self._do_export() @@ -479,8 +488,7 @@ class TestExportImport( self.assertIsFile(self.target / "manifest.json") def test_update_export_deleted_document(self) -> None: - shutil.rmtree(Path(self.dirs.media_dir) / "documents") - install_document_samples(Path(self.dirs.media_dir) / "documents") + self._install_sample_files() manifest = self._do_export() @@ -511,8 +519,7 @@ class TestExportImport( @override_settings(FILENAME_FORMAT="{title}/{correspondent}") def test_update_export_changed_location(self) -> None: - shutil.rmtree(Path(self.dirs.media_dir) / "documents") - install_document_samples(Path(self.dirs.media_dir) / "documents") + self._install_sample_files() self._do_export(use_filename_format=True) self.assertIsFile(self.target / "wow1" / "c.pdf") @@ -553,8 +560,7 @@ class TestExportImport( - Zipfile is created - Zipfile contains exported files """ - shutil.rmtree(Path(self.dirs.media_dir) / "documents") - install_document_samples(Path(self.dirs.media_dir) / "documents") + self._install_sample_files() args = ["document_exporter", self.target, "--zip"] @@ -582,8 +588,7 @@ class TestExportImport( - Zipfile is created - Zipfile contains exported files """ - shutil.rmtree(Path(self.dirs.media_dir) / "documents") - install_document_samples(Path(self.dirs.media_dir) / "documents") + self._install_sample_files() args = ["document_exporter", self.target, "--zip", "--use-filename-format"] @@ -618,8 +623,7 @@ class TestExportImport( - Zipfile contains exported files - The existing file and directory in target are removed """ - shutil.rmtree(Path(self.dirs.media_dir) / "documents") - install_document_samples(Path(self.dirs.media_dir) / "documents") + self._install_sample_files() # Create stuff in target directory existing_file = self.target / "test.txt" @@ -714,8 +718,7 @@ class TestExportImport( - Manifest.json doesn't contain information about archive files - Documents can be imported again """ - shutil.rmtree(Path(self.dirs.media_dir) / "documents") - install_document_samples(Path(self.dirs.media_dir) / "documents") + self._install_sample_files() manifest = self._do_export() has_archive = False @@ -757,8 +760,7 @@ class TestExportImport( - Manifest.json doesn't contain information about thumbnails - Documents can be imported again """ - shutil.rmtree(Path(self.dirs.media_dir) / "documents") - install_document_samples(Path(self.dirs.media_dir) / "documents") + self._install_sample_files() manifest = self._do_export() has_thumbnail = False @@ -802,8 +804,7 @@ class TestExportImport( - Main manifest.json file doesn't contain information about documents - Documents can be imported again """ - shutil.rmtree(Path(self.dirs.media_dir) / "documents") - install_document_samples(Path(self.dirs.media_dir) / "documents") + self._install_sample_files() manifest = self._do_export(split_manifest=True) has_document = False @@ -827,8 +828,7 @@ class TestExportImport( self.assertEqual(CustomFieldInstance.objects.count(), 1) def _export_import_barcodes(self, *, split_manifest: bool) -> None: - shutil.rmtree(Path(self.dirs.media_dir) / "documents") - install_document_samples(Path(self.dirs.media_dir) / "documents") + self._install_sample_files() DocumentBarcodeFactory(document=self.d1, value="https://example.com") DocumentBarcodeFactory(document=self.d2, page=2, value="DE8937") @@ -863,8 +863,7 @@ class TestExportImport( THEN: - Documents can be imported again """ - shutil.rmtree(Path(self.dirs.media_dir) / "documents") - install_document_samples(Path(self.dirs.media_dir) / "documents") + self._install_sample_files() self._do_export(use_folder_prefix=True) @@ -890,8 +889,7 @@ class TestExportImport( THEN: - Documents can be imported again """ - shutil.rmtree(Path(self.dirs.media_dir) / "documents") - install_document_samples(Path(self.dirs.media_dir) / "documents") + self._install_sample_files() self._do_export(use_folder_prefix=True, split_manifest=True) @@ -917,8 +915,7 @@ class TestExportImport( - ContentType & Permission objects are not deleted, db transaction rolled back """ - shutil.rmtree(Path(self.dirs.media_dir) / "documents") - install_document_samples(Path(self.dirs.media_dir) / "documents") + self._install_sample_files() num_content_type_objects = ContentType.objects.count() num_permission_objects = Permission.objects.count() @@ -954,8 +951,7 @@ class TestExportImport( self.assertEqual(Permission.objects.count(), num_permission_objects + 1) def test_exporter_with_auditlog_disabled(self) -> None: - shutil.rmtree(Path(self.dirs.media_dir) / "documents") - install_document_samples(Path(self.dirs.media_dir) / "documents") + self._install_sample_files() with override_settings( AUDIT_LOG_ENABLED=False, @@ -974,8 +970,7 @@ class TestExportImport( - The soft-deleted document, note, and custom field instance survive the round-trip with deleted_at preserved """ - shutil.rmtree(Path(self.dirs.media_dir) / "documents") - install_document_samples(Path(self.dirs.media_dir) / "documents") + self._install_sample_files() # d1 has self.note and self.cfi1 attached via setUp self.d1.delete() diff --git a/src/documents/tests/test_management_fuzzy.py b/src/documents/tests/test_management_fuzzy.py index fdfe459c7..3ca579aee 100644 --- a/src/documents/tests/test_management_fuzzy.py +++ b/src/documents/tests/test_management_fuzzy.py @@ -19,6 +19,10 @@ class TestFuzzyMatchCommand(TestCase): call_command( "document_fuzzy_match", "--no-progress-bar", + # The default scales with the CPU count and forks, which closes + # the test's database connection on PostgreSQL + "--processes", + "1", *args, stdout=stdout, stderr=stderr, diff --git a/src/documents/tests/test_management_retagger.py b/src/documents/tests/test_management_retagger.py index d909946a8..b280b78e1 100644 --- a/src/documents/tests/test_management_retagger.py +++ b/src/documents/tests/test_management_retagger.py @@ -425,26 +425,29 @@ class TestRetaggerStoragePath(DirectoriesMixin): @pytest.mark.management @pytest.mark.django_db class TestRetaggerIdRange(DirectoriesMixin): - @pytest.mark.usefixtures("documents") @pytest.mark.parametrize( - ("id_range_args", "expected_count"), + ("range_end", "expected_count"), [ - pytest.param(["1", "2"], 1, id="narrow_range_limits_scope"), - pytest.param(["1", "9999"], 2, id="wide_range_tags_all_matches"), + pytest.param("second", 1, id="narrow_range_limits_scope"), + pytest.param("extra", 2, id="wide_range_tags_all_matches"), ], ) def test_id_range_limits_scope( self, tags: TagTuple, - id_range_args: list[str], + documents: DocumentTuple, + range_end: str, expected_count: int, ) -> None: - DocumentFactory(content="NOT the first document") + first, second, *_ = documents + extra = DocumentFactory(content="NOT the first document") + end = {"second": second, "extra": extra}[range_end] call_command( "document_retagger", "--tags", "--id-range", - *id_range_args, + str(first.pk), + str(end.pk), skip_checks=True, ) tag_first, *_ = tags diff --git a/src/documents/tests/test_workflows.py b/src/documents/tests/test_workflows.py index 820882b49..020abe79e 100644 --- a/src/documents/tests/test_workflows.py +++ b/src/documents/tests/test_workflows.py @@ -174,8 +174,8 @@ class TestWorkflows( w.save() self.assertEqual(w.__str__(), "Workflow: Workflow 1") - self.assertEqual(trigger.__str__(), "WorkflowTrigger 1") - self.assertEqual(action.__str__(), "WorkflowAction 1") + self.assertEqual(trigger.__str__(), f"WorkflowTrigger {trigger.pk}") + self.assertEqual(action.__str__(), f"WorkflowAction {action.pk}") test_file = shutil.copy( self.SIMPLE_PDF, @@ -433,7 +433,7 @@ class TestWorkflows( list(document.tags.all()), [self.t1, self.t2, self.t3], ) - self.assertEqual( + self.assertCountEqual( list( get_users_with_perms( document, diff --git a/src/paperless_ai/tests/test_chat.py b/src/paperless_ai/tests/test_chat.py index 696aa250f..90a3cb94d 100644 --- a/src/paperless_ai/tests/test_chat.py +++ b/src/paperless_ai/tests/test_chat.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +from contextlib import nullcontext from typing import TYPE_CHECKING from typing import Any from unittest.mock import MagicMock @@ -37,6 +38,13 @@ def patch_embed_model(): llama_settings.Settings.embed_model = None +@pytest.fixture(autouse=True) +def keep_db_connection(): + # Closing the connection would roll back the test's transaction on PostgreSQL + with patch("paperless_ai.chat.db_connection_released", nullcontext): + yield + + @pytest.fixture(autouse=True) def patch_embed_nodes(): with patch(