mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-11-19 20:25:21 +00:00
Merge branch 'dev' into feature/ocr-templates
This commit is contained in:
33 files changed
+1399
-706
No files matched your search
@@ -15,6 +15,7 @@ from paperless.models import ColorConvertChoices
|
||||
from paperless_testing.dirs import DirectoriesMixin
|
||||
from paperless_testing.factories import UserFactory
|
||||
from paperless_testing.http import read_streaming_response
|
||||
from paperless_testing.permissions import grant_global
|
||||
|
||||
|
||||
class TestApiAppConfig(DirectoriesMixin, APITestCase):
|
||||
@@ -26,6 +27,67 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
|
||||
user = UserFactory(username="temp_admin", superuser=True)
|
||||
self.client.force_authenticate(user=user)
|
||||
|
||||
def test_api_read_config_requires_view_permission(self) -> None:
|
||||
for permissions in (
|
||||
(),
|
||||
("change_applicationconfiguration",),
|
||||
("view_document",),
|
||||
):
|
||||
user = UserFactory()
|
||||
grant_global(user, *permissions)
|
||||
self.client.force_authenticate(user=user)
|
||||
|
||||
for endpoint in (self.ENDPOINT, f"{self.ENDPOINT}1/"):
|
||||
for method in ("get", "head", "options"):
|
||||
with self.subTest(
|
||||
permissions=permissions,
|
||||
endpoint=endpoint,
|
||||
method=method,
|
||||
):
|
||||
response = getattr(self.client, method)(endpoint)
|
||||
self.assertEqual(
|
||||
response.status_code,
|
||||
status.HTTP_403_FORBIDDEN,
|
||||
)
|
||||
|
||||
def test_api_read_config_with_view_permission(self) -> None:
|
||||
user = UserFactory()
|
||||
grant_global(user, "view_applicationconfiguration")
|
||||
self.client.force_authenticate(user=user)
|
||||
|
||||
for endpoint in (self.ENDPOINT, f"{self.ENDPOINT}1/"):
|
||||
for method in ("get", "head", "options"):
|
||||
with self.subTest(endpoint=endpoint, method=method):
|
||||
response = getattr(self.client, method)(endpoint)
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
|
||||
def test_api_view_config_permission_does_not_allow_writes(self) -> None:
|
||||
user = UserFactory()
|
||||
grant_global(user, "view_applicationconfiguration")
|
||||
self.client.force_authenticate(user=user)
|
||||
|
||||
for method in ("put", "patch", "delete"):
|
||||
with self.subTest(method=method):
|
||||
response = getattr(self.client, method)(f"{self.ENDPOINT}1/")
|
||||
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
|
||||
|
||||
def test_api_change_config_permission_allows_updates(self) -> None:
|
||||
user = UserFactory()
|
||||
grant_global(user, "change_applicationconfiguration")
|
||||
self.client.force_authenticate(user=user)
|
||||
|
||||
response = self.client.patch(
|
||||
f"{self.ENDPOINT}1/",
|
||||
{"app_title": "Updated title"},
|
||||
format="json",
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(
|
||||
ApplicationConfiguration.objects.get().app_title,
|
||||
"Updated title",
|
||||
)
|
||||
|
||||
def test_api_get_config(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
|
||||
@@ -1223,8 +1223,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
|
||||
self.assertCountEqual(
|
||||
response.data["selected_document_types"],
|
||||
[
|
||||
{"id": self.c1.id, "document_count": 1},
|
||||
{"id": self.c2.id, "document_count": 0},
|
||||
{"id": self.dt1.id, "document_count": 1},
|
||||
{"id": self.dt2.id, "document_count": 0},
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@@ -49,6 +49,7 @@ from documents.models import WorkflowTrigger
|
||||
from documents.search import get_backend
|
||||
from documents.signals.handlers import run_workflows
|
||||
from documents.tests.utils import ConsumeTaskMixin
|
||||
from documents.views import DocumentViewSet
|
||||
from paperless_testing.dirs import DirectoriesMixin
|
||||
from paperless_testing.factories import DocumentFactory
|
||||
from paperless_testing.factories import TagFactory
|
||||
@@ -1755,12 +1756,113 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
|
||||
}
|
||||
self.assertEqual(counts_by_field[custom_field.id], 1)
|
||||
|
||||
def test_selection_data_counts_with_shared_by_filter(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Several documents owned by one user with the same correspondent
|
||||
and document type, only one of which is shared
|
||||
WHEN:
|
||||
- Requesting the document list filtered by shared_by__id with
|
||||
include_selection_data=true
|
||||
THEN:
|
||||
- Counts reflect only the shared document, not every document the
|
||||
owner has with that correspondent or document type, regardless of
|
||||
the batch size
|
||||
"""
|
||||
owner = User.objects.create_user(username="owner")
|
||||
other = User.objects.create_user(username="other")
|
||||
correspondent = Correspondent.objects.create(name="c1")
|
||||
doc_type = DocumentType.objects.create(name="dt1")
|
||||
tag = Tag.objects.create(name="tag")
|
||||
docs = [
|
||||
Document.objects.create(
|
||||
checksum=str(i),
|
||||
owner=owner,
|
||||
correspondent=correspondent,
|
||||
document_type=doc_type,
|
||||
)
|
||||
for i in range(3)
|
||||
]
|
||||
for doc in docs:
|
||||
doc.tags.add(tag)
|
||||
grant_object(other, docs[0], "view_document")
|
||||
|
||||
for batch_size in (10_000, 1):
|
||||
with (
|
||||
self.subTest(batch_size=batch_size),
|
||||
mock.patch.object(
|
||||
DocumentViewSet,
|
||||
"selection_data_batch_size",
|
||||
batch_size,
|
||||
),
|
||||
):
|
||||
response = self.client.get(
|
||||
f"/api/documents/?shared_by__id={owner.id}"
|
||||
"&include_selection_data=true",
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(response.data["count"], 1)
|
||||
selection_data = response.data["selection_data"]
|
||||
for key, obj in (
|
||||
("selected_correspondents", correspondent),
|
||||
("selected_document_types", doc_type),
|
||||
("selected_tags", tag),
|
||||
):
|
||||
counts = {
|
||||
item["id"]: item["document_count"]
|
||||
for item in selection_data[key]
|
||||
}
|
||||
self.assertEqual(counts[obj.id], 1, key)
|
||||
|
||||
def test_selection_data_counts_documents_once_when_filter_joins(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A document matching a multi-valued filter through two tags
|
||||
WHEN:
|
||||
- Requesting the document list with include_selection_data=true
|
||||
THEN:
|
||||
- The document is counted once per correspondent, not once per
|
||||
matching tag, regardless of the batch size
|
||||
"""
|
||||
correspondent = Correspondent.objects.create(name="c1")
|
||||
tag_a = Tag.objects.create(name="a")
|
||||
tag_b = Tag.objects.create(name="b")
|
||||
both = Document.objects.create(checksum="1", correspondent=correspondent)
|
||||
both.tags.add(tag_a, tag_b)
|
||||
only_a = Document.objects.create(checksum="2", correspondent=correspondent)
|
||||
only_a.tags.add(tag_a)
|
||||
|
||||
for batch_size in (10_000, 1):
|
||||
with (
|
||||
self.subTest(batch_size=batch_size),
|
||||
mock.patch.object(
|
||||
DocumentViewSet,
|
||||
"selection_data_batch_size",
|
||||
batch_size,
|
||||
),
|
||||
):
|
||||
response = self.client.get(
|
||||
f"/api/documents/?tags__id__in={tag_a.id},{tag_b.id}"
|
||||
"&include_selection_data=true",
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
counts = {
|
||||
item["id"]: item["document_count"]
|
||||
for item in response.data["selection_data"][
|
||||
"selected_correspondents"
|
||||
]
|
||||
}
|
||||
self.assertEqual(counts[correspondent.id], 2)
|
||||
|
||||
def test_statistics(self) -> None:
|
||||
doc1 = Document.objects.create(
|
||||
title="none1",
|
||||
checksum="A",
|
||||
mime_type="application/pdf",
|
||||
content="abc",
|
||||
archive_serial_number=42,
|
||||
)
|
||||
Document.objects.create(
|
||||
title="none2",
|
||||
@@ -1804,6 +1906,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
|
||||
self.assertEqual(response.data["correspondent_count"], 2)
|
||||
self.assertEqual(response.data["document_type_count"], 1)
|
||||
self.assertEqual(response.data["storage_path_count"], 2)
|
||||
self.assertEqual(response.data["current_asn"], 42)
|
||||
|
||||
def test_statistics_excludes_document_versions(self) -> None:
|
||||
root = Document.objects.create(
|
||||
@@ -1811,6 +1914,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
|
||||
checksum="A",
|
||||
mime_type="application/pdf",
|
||||
content="root",
|
||||
archive_serial_number=42,
|
||||
)
|
||||
version = Document.objects.create(
|
||||
title="version",
|
||||
@@ -1819,6 +1923,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
|
||||
content="version",
|
||||
root_document=root,
|
||||
version_index=1,
|
||||
archive_serial_number=99,
|
||||
)
|
||||
tag_inbox = Tag.objects.create(name="t1", is_inbox_tag=True)
|
||||
version.tags.add(tag_inbox)
|
||||
@@ -1829,6 +1934,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
|
||||
self.assertEqual(response.data["documents_total"], 1)
|
||||
self.assertEqual(response.data["documents_inbox"], 0)
|
||||
self.assertEqual(response.data["character_count"], 4)
|
||||
self.assertEqual(response.data["current_asn"], 42)
|
||||
self.assertEqual(
|
||||
response.data["document_file_type_counts"][0]["mime_type_count"],
|
||||
1,
|
||||
@@ -1842,6 +1948,45 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
|
||||
self.assertEqual(response.data["documents_inbox"], None)
|
||||
self.assertEqual(response.data["inbox_tags"], None)
|
||||
|
||||
def test_statistics_current_asn_respects_document_permissions(self) -> None:
|
||||
viewer = UserFactory()
|
||||
grant_global(viewer, "view_document")
|
||||
DocumentFactory(owner=viewer, archive_serial_number=12)
|
||||
shared = DocumentFactory(owner=self.user, archive_serial_number=23)
|
||||
grant_object(viewer, shared, "view_document")
|
||||
DocumentFactory(owner=self.user, archive_serial_number=99)
|
||||
|
||||
self.client.force_authenticate(user=viewer)
|
||||
response = self.client.get("/api/statistics/")
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(response.data["current_asn"], 23)
|
||||
|
||||
def test_statistics_current_asn_without_visible_documents(self) -> None:
|
||||
DocumentFactory(owner=self.user, archive_serial_number=99)
|
||||
|
||||
for permissions in ((), ("view_document",)):
|
||||
viewer = UserFactory()
|
||||
grant_global(viewer, *permissions)
|
||||
self.client.force_authenticate(user=viewer)
|
||||
|
||||
with self.subTest(permissions=permissions):
|
||||
response = self.client.get("/api/statistics/")
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(response.data["current_asn"], 0)
|
||||
|
||||
def test_statistics_current_asn_with_global_statistics_permission(self) -> None:
|
||||
viewer = UserFactory()
|
||||
grant_global(viewer, "view_global_statistics")
|
||||
DocumentFactory(owner=self.user, archive_serial_number=99)
|
||||
|
||||
self.client.force_authenticate(user=viewer)
|
||||
response = self.client.get("/api/statistics/")
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(response.data["current_asn"], 99)
|
||||
|
||||
def test_statistics_multiple_users(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
|
||||
@@ -25,6 +25,7 @@ from documents.models import Tag
|
||||
from documents.models import Workflow
|
||||
from documents.search import get_backend
|
||||
from documents.search import reset_backend
|
||||
from documents.views import DocumentViewSet
|
||||
from paperless_mail.models import MailAccount
|
||||
from paperless_mail.models import MailRule
|
||||
from paperless_testing.dirs import DirectoriesMixin
|
||||
@@ -345,38 +346,39 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
|
||||
storage_path = StoragePath.objects.create(name="sp1")
|
||||
tag = Tag.objects.create(name="tag")
|
||||
|
||||
matching_doc = Document.objects.create(
|
||||
title="bank statement",
|
||||
content="bank content",
|
||||
checksum="A",
|
||||
correspondent=correspondent,
|
||||
document_type=doc_type,
|
||||
storage_path=storage_path,
|
||||
)
|
||||
matching_doc.tags.add(tag)
|
||||
for checksum, title in (
|
||||
("A", "bank statement"),
|
||||
("B", "bank account"),
|
||||
("C", "credit card"),
|
||||
):
|
||||
doc = Document.objects.create(
|
||||
title=title,
|
||||
checksum=checksum,
|
||||
correspondent=correspondent,
|
||||
document_type=doc_type,
|
||||
storage_path=storage_path,
|
||||
)
|
||||
doc.tags.add(tag)
|
||||
get_backend().add_or_update(doc)
|
||||
|
||||
get_backend().add_or_update(matching_doc)
|
||||
|
||||
response = self.client.get(
|
||||
"/api/documents/?query=bank&include_selection_data=true",
|
||||
)
|
||||
with mock.patch.object(DocumentViewSet, "selection_data_batch_size", 1):
|
||||
response = self.client.get(
|
||||
"/api/documents/?query=bank&page_size=1&include_selection_data=true",
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
self.assertIn("selection_data", response.data)
|
||||
|
||||
selected_correspondent = next(
|
||||
item
|
||||
for item in response.data["selection_data"]["selected_correspondents"]
|
||||
if item["id"] == correspondent.id
|
||||
)
|
||||
selected_tag = next(
|
||||
item
|
||||
for item in response.data["selection_data"]["selected_tags"]
|
||||
if item["id"] == tag.id
|
||||
)
|
||||
|
||||
self.assertEqual(selected_correspondent["document_count"], 1)
|
||||
self.assertEqual(selected_tag["document_count"], 1)
|
||||
self.assertEqual(response.data["count"], 2)
|
||||
self.assertEqual(len(response.data["results"]), 1)
|
||||
for key, pk in (
|
||||
("selected_correspondents", correspondent.pk),
|
||||
("selected_document_types", doc_type.pk),
|
||||
("selected_storage_paths", storage_path.pk),
|
||||
("selected_tags", tag.pk),
|
||||
):
|
||||
self.assertEqual(
|
||||
response.data["selection_data"][key],
|
||||
[{"id": pk, "document_count": 2}],
|
||||
)
|
||||
|
||||
def test_search_custom_field_ordering(self) -> None:
|
||||
custom_field = CustomField.objects.create(
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
from django.core.exceptions import FieldDoesNotExist
|
||||
from django.db import connection
|
||||
from django.test.utils import CaptureQueriesContext
|
||||
|
||||
from documents.management.commands.document_importer import _deserialize_record
|
||||
from documents.models import Tag
|
||||
from documents.tag_hierarchy import TagHierarchy
|
||||
from paperless_testing.factories import TagFactory
|
||||
from paperless_testing.migrations import TestMigrations
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from django.apps.registry import Apps
|
||||
from pytest_django.fixtures import DjangoAssertNumQueries
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
class TestTagAdjacency:
|
||||
def test_held_instance_uses_current_parent_edges(self) -> None:
|
||||
"""Descendant lookups reflect parent changes made after loading a tag."""
|
||||
first = TagFactory(name="First")
|
||||
second = TagFactory(name="Second")
|
||||
child = TagFactory(name="Child", tn_parent=first)
|
||||
Tag.objects.filter(pk=child.pk).update(tn_parent=second)
|
||||
assert first.get_descendants_pks() == set()
|
||||
assert second.get_descendants_pks() == {child.pk}
|
||||
|
||||
def test_queryset_delete_preserves_cascade(self) -> None:
|
||||
"""Deleting a parent through a queryset deletes only its subtree."""
|
||||
root = TagFactory(name="Root")
|
||||
child = TagFactory(name="Child", tn_parent=root)
|
||||
TagFactory(name="Grandchild", tn_parent=child)
|
||||
survivor = TagFactory(name="Other root")
|
||||
Tag.objects.filter(pk=root.pk).delete()
|
||||
assert list(Tag.objects.values_list("pk", flat=True)) == [survivor.pk]
|
||||
|
||||
def test_flat_writes_do_not_update_other_tags(self) -> None:
|
||||
"""Creating and renaming flat tags never rewrite unrelated tag rows."""
|
||||
TagFactory.create_batch(100)
|
||||
with CaptureQueriesContext(connection) as queries:
|
||||
new = TagFactory(name="New")
|
||||
assert not any(q["sql"].startswith("UPDATE") for q in queries)
|
||||
with CaptureQueriesContext(connection) as queries:
|
||||
new.name = "Renamed"
|
||||
new.save()
|
||||
tag_updates = [
|
||||
q["sql"] for q in queries if q["sql"].startswith('UPDATE "documents_tag"')
|
||||
]
|
||||
assert len(tag_updates) == 1
|
||||
assert 'WHERE "documents_tag"."id" =' in tag_updates[0]
|
||||
|
||||
def test_batch_snapshot_is_one_query_and_includes_overlapping_roots(
|
||||
self,
|
||||
django_assert_num_queries: "DjangoAssertNumQueries",
|
||||
) -> None:
|
||||
"""One edge query supports multiple traversals with overlapping roots."""
|
||||
root = TagFactory(name="Root")
|
||||
child = TagFactory(name="Child", tn_parent=root)
|
||||
leaf = TagFactory(name="Leaf", tn_parent=child)
|
||||
with django_assert_num_queries(1):
|
||||
hierarchy = Tag.get_hierarchy()
|
||||
assert hierarchy.with_ancestors([leaf.pk]) == {root.pk, child.pk, leaf.pk}
|
||||
assert hierarchy.with_descendants([root.pk, child.pk]) == {
|
||||
root.pk,
|
||||
child.pk,
|
||||
leaf.pk,
|
||||
}
|
||||
assert hierarchy.height(None) == 0
|
||||
|
||||
def test_legacy_export_fields_are_ignored_but_parent_is_preserved(self) -> None:
|
||||
"""Legacy bookkeeping is ignored while parent links and field checks survive."""
|
||||
model, tag, m2m = _deserialize_record(
|
||||
{
|
||||
"model": "documents.tag",
|
||||
"pk": 2,
|
||||
"fields": {
|
||||
"name": "Child",
|
||||
"tn_parent": 1,
|
||||
"tn_order": 3,
|
||||
"tn_siblings_pks": "4,5",
|
||||
"tn_descendants_pks": "6",
|
||||
},
|
||||
},
|
||||
)
|
||||
assert model is Tag
|
||||
assert tag.tn_parent_id == 1
|
||||
assert tag.name == "Child"
|
||||
assert not m2m
|
||||
with pytest.raises(FieldDoesNotExist):
|
||||
_deserialize_record(
|
||||
{"model": "documents.tag", "pk": 3, "fields": {"tn_typo": "bad"}},
|
||||
)
|
||||
|
||||
def test_malformed_cycles_terminate(self) -> None:
|
||||
"""Traversal terminates even when stored parent edges contain a cycle."""
|
||||
hierarchy = TagHierarchy([(1, 2), (2, 1)])
|
||||
assert hierarchy.with_descendants([1]) == {1, 2}
|
||||
assert set(hierarchy.ancestors(2)) == {1, 2}
|
||||
|
||||
|
||||
class TestTagAdjacencyMigration(TestMigrations):
|
||||
migrate_from = "0027_documentbarcode"
|
||||
migrate_to = "0028_replace_tag_tree_fields"
|
||||
|
||||
def setUpBeforeMigration(self, apps: "Apps") -> None:
|
||||
"""Seed parent links using the model from the pre-migration state."""
|
||||
tag = apps.get_model("documents", "Tag")
|
||||
root = tag.objects.create(name="Root")
|
||||
child = tag.objects.create(name="Child", tn_parent=root)
|
||||
self.edges = [(root.pk, None), (child.pk, root.pk)]
|
||||
|
||||
def test_parent_edges_survive(self) -> None:
|
||||
"""Removing derived tree columns preserves the existing parent links."""
|
||||
tag = self.apps.get_model("documents", "Tag")
|
||||
assert (
|
||||
list(tag.objects.order_by("pk").values_list("pk", "tn_parent_id"))
|
||||
== self.edges
|
||||
)
|
||||
assert {
|
||||
field.name for field in tag._meta.fields if field.name.startswith("tn_")
|
||||
} == {"tn_parent"}
|
||||
Reference in new issue
Block a user