Chore: use document type ids in the bulk edit selection_data test

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

docker/rootfs/etc/s6-overlay/s6-rc.d/init-compile-bytecode/run

@@ -39,6 +39,7 @@ declare -a scope=(
 	"${site_packages}/allauth"
 	"${site_packages}/drf_spectacular"
 	"${site_packages}/drf_spectacular_sidecar"
+	"${site_packages}/treenode"
 	"${site_packages}/compression_middleware"
 )
 

pyproject.toml

@@ -38,6 +38,7 @@ dependencies = [
   "django-multiselectfield~=1.0.1",
   "django-rich~=2.2.0",
   "django-soft-delete~=1.0.18",
+  "django-treenode>=0.24",
   "djangorestframework~=3.16",
   "drf-spectacular~=0.30",
   "drf-spectacular-sidecar>=2026.7.1,<2026.10",

src/documents/admin.py

@@ -1,6 +1,7 @@
 from django.conf import settings
 from django.contrib import admin
 from guardian.admin import GuardedModelAdmin
+from treenode.admin import TreeNodeModelAdmin
 
 from documents.models import Correspondent
 from documents.models import CustomField
@@ -28,7 +29,7 @@ class CorrespondentAdmin(GuardedModelAdmin):
     list_editable = ("match", "matching_algorithm")
 
 
-class TagAdmin(GuardedModelAdmin):
+class TagAdmin(GuardedModelAdmin, TreeNodeModelAdmin):
     list_display = ("name", "color", "match", "matching_algorithm")
     list_filter = ("matching_algorithm",)
     list_editable = ("color", "match", "matching_algorithm")

src/documents/bulk_edit.py

@@ -255,13 +255,19 @@ def modify_tags(
     affected_docs = list(qs.values_list("pk", flat=True))
     DocumentTagRelationship = Document.tags.through
 
-    hierarchy = Tag.get_hierarchy()
-    expanded_add_tags = hierarchy.with_ancestors(
-        Tag.objects.filter(pk__in=add_tags).values_list("pk", flat=True),
-    )
-    expanded_remove_tags = hierarchy.with_descendants(
-        Tag.objects.filter(pk__in=remove_tags).values_list("pk", flat=True),
-    )
+    # add with all ancestors
+    expanded_add_tags: set[int] = set()
+    add_tag_objects = Tag.objects.filter(pk__in=add_tags)
+    for t in add_tag_objects:
+        expanded_add_tags.add(int(t.id))
+        expanded_add_tags.update(int(pk) for pk in t.get_ancestors_pks())
+
+    # remove with all descendants
+    expanded_remove_tags: set[int] = set()
+    remove_tag_objects = Tag.objects.filter(pk__in=remove_tags)
+    for t in remove_tag_objects:
+        expanded_remove_tags.add(int(t.id))
+        expanded_remove_tags.update(int(pk) for pk in t.get_descendants_pks())
 
     with transaction.atomic():
         if expanded_remove_tags:

src/documents/management/commands/document_importer.py

@@ -108,24 +108,6 @@ def _deserialize_record(
         ) from e
 
     for field_name, field_value in record.get("fields", {}).items():
-        # Older exports include django-treenode's derived bookkeeping. The
-        # parent FK is authoritative; accept only these known removed fields.
-        if Model is Tag and field_name in {
-            "tn_ancestors_count",
-            "tn_ancestors_pks",
-            "tn_children_count",
-            "tn_children_pks",
-            "tn_depth",
-            "tn_descendants_count",
-            "tn_descendants_pks",
-            "tn_index",
-            "tn_level",
-            "tn_order",
-            "tn_priority",
-            "tn_siblings_count",
-            "tn_siblings_pks",
-        }:
-            continue
         field = Model._meta.get_field(field_name)
         remote = field.remote_field
 

src/documents/migrations/0028_replace_tag_tree_fields.py

@@ -1,64 +0,0 @@
-# Generated by Django 5.2.17 on 2026-10-28 22:54
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("documents", "0027_documentbarcode"),
-    ]
-
-    operations = [
-        migrations.RemoveField(
-            model_name="tag",
-            name="tn_ancestors_count",
-        ),
-        migrations.RemoveField(
-            model_name="tag",
-            name="tn_ancestors_pks",
-        ),
-        migrations.RemoveField(
-            model_name="tag",
-            name="tn_children_count",
-        ),
-        migrations.RemoveField(
-            model_name="tag",
-            name="tn_children_pks",
-        ),
-        migrations.RemoveField(
-            model_name="tag",
-            name="tn_depth",
-        ),
-        migrations.RemoveField(
-            model_name="tag",
-            name="tn_descendants_count",
-        ),
-        migrations.RemoveField(
-            model_name="tag",
-            name="tn_descendants_pks",
-        ),
-        migrations.RemoveField(
-            model_name="tag",
-            name="tn_index",
-        ),
-        migrations.RemoveField(
-            model_name="tag",
-            name="tn_level",
-        ),
-        migrations.RemoveField(
-            model_name="tag",
-            name="tn_order",
-        ),
-        migrations.RemoveField(
-            model_name="tag",
-            name="tn_priority",
-        ),
-        migrations.RemoveField(
-            model_name="tag",
-            name="tn_siblings_count",
-        ),
-        migrations.RemoveField(
-            model_name="tag",
-            name="tn_siblings_pks",
-        ),
-    ]

src/documents/models.py

@@ -14,6 +14,7 @@ from django.db import models
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from multiselectfield import MultiSelectField
+from treenode.models import TreeNodeModel
 
 if settings.AUDIT_LOG_ENABLED:
     from auditlog.registry import auditlog
@@ -99,17 +100,7 @@ class Correspondent(MatchingModel):
         verbose_name_plural = _("correspondents")
 
 
-class Tag(MatchingModel):
-    # Keep the existing column and reverse relation during the transition.
-    tn_parent = models.ForeignKey(
-        "self",
-        blank=True,
-        null=True,
-        on_delete=models.CASCADE,
-        related_name="tn_children",
-        verbose_name=_("Parent"),
-    )
-
+class Tag(MatchingModel, TreeNodeModel):
     color = models.CharField(_("color"), max_length=7, default="#a6cee3")
     # Maximum allowed nesting depth for tags (root = 1, max depth = 5)
     MAX_NESTING_DEPTH: Final[int] = 5
@@ -123,51 +114,27 @@ class Tag(MatchingModel):
         ),
     )
 
-    class Meta(MatchingModel.Meta):
+    class Meta(MatchingModel.Meta, TreeNodeModel.Meta):
         verbose_name = _("tag")
         verbose_name_plural = _("tags")
 
-    @classmethod
-    def get_hierarchy(cls):
-        from documents.tag_hierarchy import TagHierarchy
-
-        return TagHierarchy(cls.objects.values_list("pk", "tn_parent_id"))
-
-    @property
-    def parent_pk(self):
-        return self.tn_parent_id
-
-    def get_parent(self):
-        return self.tn_parent
-
-    def get_ancestors_pks(self):
-        return self.get_hierarchy().ancestors(self.tn_parent_id)
-
-    def get_ancestors(self):
-        ids = self.get_ancestors_pks()
-        tags = self.__class__.objects.in_bulk(ids)
-        return [tags[pk] for pk in ids]
-
-    def get_descendants_pks(self):
-        return self.get_hierarchy().descendants([self.pk])
-
-    def get_descendants(self):
-        return self.__class__.objects.filter(pk__in=self.get_descendants_pks())
-
-    def get_children_queryset(self):
-        return self.__class__.objects.filter(tn_parent_id=self.pk)
-
     def clean(self) -> None:
-        hierarchy = self.get_hierarchy()
-        if self.pk is not None and self.tn_parent_id == self.pk:
-            raise ValidationError({"tn_parent": _("Cannot set itself as parent.")})
-        ancestors = hierarchy.ancestors(self.tn_parent_id)
-        if self.pk is not None and self.pk in ancestors:
-            raise ValidationError(
-                {"tn_parent": _("Cannot set parent to a descendant.")},
-            )
-        if len(ancestors) + 1 + hierarchy.height(self.pk) > self.MAX_NESTING_DEPTH:
-            raise ValidationError({"tn_parent": _("Maximum nesting depth exceeded.")})
+        # Prevent self-parenting and assigning a descendant as parent
+        parent = self.get_parent()
+        if parent == self:
+            raise ValidationError({"parent": _("Cannot set itself as parent.")})
+        if parent and self.pk is not None and self.is_ancestor_of(parent):
+            raise ValidationError({"parent": _("Cannot set parent to a descendant.")})
+
+        # Enforce maximum nesting depth
+        new_parent_depth = 0
+        if parent:
+            new_parent_depth = parent.get_ancestors_count() + 1
+
+        height = 0 if self.pk is None else self.get_depth()
+        deepest_new_depth = (new_parent_depth + 1) + height
+        if deepest_new_depth > self.MAX_NESTING_DEPTH:
+            raise ValidationError({"parent": _("Maximum nesting depth exceeded.")})
 
         return super().clean()
 
@@ -572,7 +539,10 @@ class Document(SoftDeleteModel, ModelWithOwner):  # type: ignore[django-manager-
         return self.created
 
     def add_nested_tags(self, tags) -> None:
-        tag_ids = Tag.get_hierarchy().with_ancestors(tag.pk for tag in tags)
+        tag_ids = set()
+        for tag in tags:
+            tag_ids.add(tag.id)
+            tag_ids.update(tag.get_ancestors_pks())
 
         tags_to_add = self.tags.model.objects.filter(id__in=tag_ids)
         self.tags.add(*tags_to_add)

src/documents/permissions.py

@@ -576,12 +576,9 @@ def annotate_document_count_by_ids(
                        or CustomFieldInstance)
         related_object_field: field on the relation pointing back to queryset pk
         document_ids: the document ids to count against -- a concrete list/set,
-                       or a simple (already resolved) queryset of ids. Callers
-                       that need this filtered by a complex condition (e.g. a
-                       permission check) should resolve it to a concrete list
-                       first if the same ids will be reused across multiple
-                       calls, rather than passing the complex queryset itself
-                       into each -- see ``_get_selection_data_for_queryset``.
+                       or a simple (already resolved) queryset of ids. Keep
+                       the list small: it is embedded in the statement as
+                       literal parameters.
         target_field: field on the relation pointing to Document id
     """
 

src/documents/serialisers.py

@@ -589,7 +589,7 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
 
     text_color = serializers.SerializerMethodField()
 
-    # Preserve the public API parent field.
+    # map to treenode's tn_parent
     parent = serializers.PrimaryKeyRelatedField(
         queryset=Tag.objects.all(),
         allow_null=True,
@@ -692,9 +692,7 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
                 self.instance.clean()
             except ValidationError as e:
                 logger.debug("Tag parent validation failed: %s", e)
-                raise serializers.ValidationError(
-                    {"parent": e.message_dict["tn_parent"]},
-                ) from e
+                raise e
             finally:
                 self.instance.tn_parent = original_parent
         else:
@@ -704,9 +702,7 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
                 temp.clean()
             except ValidationError as e:
                 logger.debug("Tag parent validation failed: %s", e)
-                raise serializers.ValidationError(
-                    {"parent": e.message_dict["tn_parent"]},
-                ) from e
+                raise e
 
         return super().validate(attrs)
 
@@ -1202,35 +1198,61 @@ class DocumentSerializer(
                             doc_id,
                         )
         if "tags" in validated_data:
-            hierarchy = Tag.get_hierarchy()
-            previous_ids = set(instance.tags.values_list("pk", flat=True))
-            requested_ids = {tag.pk for tag in validated_data["tags"]}
-            required_ids = hierarchy.with_ancestors(requested_ids - previous_ids)
-            blocked_ids = hierarchy.with_descendants(previous_ids - requested_ids)
-            blocked_ids.difference_update(required_ids)
-            final_ids = hierarchy.with_ancestors(requested_ids) - blocked_ids
-            validated_data["tags"] = list(Tag.objects.filter(pk__in=final_ids))
+            # Respect tag hierarchy on updates:
+            # - Adding a child adds its ancestors
+            # - Removing a parent removes all its descendants
+            prev_tags = set(instance.tags.all())
+            requested_tags = set(validated_data["tags"])
 
+            # Tags newly added in this update and the ancestors they require
+            added_tags = requested_tags - prev_tags
+            required_by_add_tags = set(added_tags)
+            for t in added_tags:
+                required_by_add_tags.update(t.get_ancestors())
+
+            # Tags being removed in this update and all descendants, except
+            # those required by a tag that is being added in this same update
+            removed_tags = prev_tags - requested_tags
+            blocked_tags = set(removed_tags)
+            for t in removed_tags:
+                blocked_tags.update(t.get_descendants())
+            blocked_tags.difference_update(required_by_add_tags)
+
+            # Add all parent tags
+            final_tags = set(requested_tags)
+            for t in requested_tags:
+                final_tags.update(t.get_ancestors())
+
+            # Drop removed parents and their descendants
+            final_tags.difference_update(blocked_tags)
+
+            validated_data["tags"] = list(final_tags)
         if validated_data.get("remove_inbox_tags"):
-            hierarchy = Tag.get_hierarchy()
+            current_tag_ids = {t.pk for t in instance.tags.all()}
             tags = (
                 validated_data["tags"]
                 if "tags" in validated_data
                 else list(instance.tags.all())
             )
-            current_ids = set(instance.tags.values_list("pk", flat=True))
-            keep_ids = hierarchy.with_ancestors(
-                tag.pk for tag in tags if tag.pk not in current_ids
-            )
-            inbox_ids = (
+
+            # Tags newly added in this update, plus their ancestors, are kept
+            keep_ids: set[int] = set()
+            for tag in tags:
+                if tag.pk not in current_tag_ids:
+                    keep_ids.add(tag.pk)
+                    keep_ids.update(int(pk) for pk in tag.get_ancestors_pks())
+
+            # Remove inbox tags and their descendants, except those being kept
+            remove_ids: set[int] = set()
+            for inbox_tag in (
                 Tag.objects.filter(is_inbox_tag=True)
-                .exclude(
-                    pk__in=keep_ids,
-                )
-                .values_list("pk", flat=True)
-            )
-            remove_ids = hierarchy.with_descendants(inbox_ids) - keep_ids
-            validated_data["tags"] = [tag for tag in tags if tag.pk not in remove_ids]
+                .exclude(pk__in=keep_ids)
+                .only("pk", "tn_descendants_pks")
+            ):
+                remove_ids.add(inbox_tag.pk)
+                remove_ids.update(int(pk) for pk in inbox_tag.get_descendants_pks())
+
+            validated_data["tags"] = [t for t in tags if t.pk not in remove_ids]
 
         if settings.AUDIT_LOG_ENABLED:
             with set_actor(self.user):

src/documents/tag_hierarchy.py

@@ -1,61 +0,0 @@
-"""An operation-local snapshot of tag edges, with no persistent derived fields.
-
-Never cache this globally: tasks, bulk updates and other processes can change
-parents. Callers traversing multiple tags should share one snapshot.
-"""
-
-from collections import defaultdict
-from collections.abc import Iterable
-
-
-class TagHierarchy:
-    def __init__(self, edges: Iterable[tuple[int, int | None]]) -> None:
-        self.parents = dict(edges)
-        self.children: dict[int | None, list[int]] = defaultdict(list)
-        for pk, parent in self.parents.items():
-            self.children[parent].append(pk)
-
-    def ancestors(self, parent_id: int | None) -> list[int]:
-        result = []
-        seen = set()
-        while parent_id is not None and parent_id not in seen:
-            seen.add(parent_id)
-            result.append(parent_id)
-            parent_id = self.parents.get(parent_id)
-        return result[::-1]
-
-    def with_ancestors(self, roots: Iterable[int]) -> set[int]:
-        result = set(roots)
-        for pk in list(result):
-            result.update(self.ancestors(self.parents.get(pk)))
-        return result
-
-    def with_descendants(self, roots: Iterable[int]) -> set[int]:
-        roots = set(roots)
-        return roots | self.descendants(roots)
-
-    def descendants(self, roots: Iterable[int]) -> set[int]:
-        roots = {pk for pk in roots if pk is not None}
-        seen = set(roots)
-        pending = list(roots)
-        while pending:
-            for child in self.children.get(pending.pop(), ()):
-                if child not in seen:
-                    seen.add(child)
-                    pending.append(child)
-        return seen - roots
-
-    def height(self, pk: int | None) -> int:
-        if pk is None:
-            return 0
-        seen = {pk}
-        pending = [(pk, 0)]
-        height = 0
-        while pending:
-            parent, depth = pending.pop()
-            height = max(height, depth)
-            for child in self.children.get(parent, ()):
-                if child not in seen:
-                    seen.add(child)
-                    pending.append((child, depth + 1))
-        return height

src/documents/tests/test_api_bulk_edit.py

@@ -1141,8 +1141,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertCountEqual(
             response.data["selected_document_types"],
             [
-                {"id": self.c1.id, "document_count": 1},
-                {"id": self.c2.id, "document_count": 0},
+                {"id": self.dt1.id, "document_count": 1},
+                {"id": self.dt2.id, "document_count": 0},
             ],
         )
 

src/documents/tests/test_api_documents.py

@@ -49,6 +49,7 @@ from documents.models import WorkflowTrigger
 from documents.search import get_backend
 from documents.signals.handlers import run_workflows
 from documents.tests.utils import ConsumeTaskMixin
+from documents.views import DocumentViewSet
 from paperless_testing.dirs import DirectoriesMixin
 from paperless_testing.factories import DocumentFactory
 from paperless_testing.factories import TagFactory
@@ -1755,6 +1756,106 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         }
         self.assertEqual(counts_by_field[custom_field.id], 1)
 
+    def test_selection_data_counts_with_shared_by_filter(self) -> None:
+        """
+        GIVEN:
+            - Several documents owned by one user with the same correspondent
+              and document type, only one of which is shared
+        WHEN:
+            - Requesting the document list filtered by shared_by__id with
+              include_selection_data=true
+        THEN:
+            - Counts reflect only the shared document, not every document the
+              owner has with that correspondent or document type, however the
+              link-table lookups are batched
+        """
+        owner = User.objects.create_user(username="owner")
+        other = User.objects.create_user(username="other")
+        correspondent = Correspondent.objects.create(name="c1")
+        doc_type = DocumentType.objects.create(name="dt1")
+        tag = Tag.objects.create(name="tag")
+        docs = [
+            Document.objects.create(
+                checksum=str(i),
+                owner=owner,
+                correspondent=correspondent,
+                document_type=doc_type,
+            )
+            for i in range(3)
+        ]
+        for doc in docs:
+            doc.tags.add(tag)
+        grant_object(other, docs[0], "view_document")
+
+        for batch_size in (10_000, 1):
+            with (
+                self.subTest(link_batch_size=batch_size),
+                mock.patch.object(
+                    DocumentViewSet,
+                    "selection_data_link_batch_size",
+                    batch_size,
+                ),
+            ):
+                response = self.client.get(
+                    f"/api/documents/?shared_by__id={owner.id}"
+                    "&include_selection_data=true",
+                )
+
+                self.assertEqual(response.status_code, status.HTTP_200_OK)
+                self.assertEqual(response.data["count"], 1)
+                selection_data = response.data["selection_data"]
+                for key, obj in (
+                    ("selected_correspondents", correspondent),
+                    ("selected_document_types", doc_type),
+                    ("selected_tags", tag),
+                ):
+                    counts = {
+                        item["id"]: item["document_count"]
+                        for item in selection_data[key]
+                    }
+                    self.assertEqual(counts[obj.id], 1, key)
+
+    def test_selection_data_counts_documents_once_when_filter_joins(self) -> None:
+        """
+        GIVEN:
+            - A document matching a multi-valued filter through two tags
+        WHEN:
+            - Requesting the document list with include_selection_data=true
+        THEN:
+            - The document is counted once per correspondent, not once per
+              matching tag, however the link-table lookups are batched
+        """
+        correspondent = Correspondent.objects.create(name="c1")
+        tag_a = Tag.objects.create(name="a")
+        tag_b = Tag.objects.create(name="b")
+        both = Document.objects.create(checksum="1", correspondent=correspondent)
+        both.tags.add(tag_a, tag_b)
+        only_a = Document.objects.create(checksum="2", correspondent=correspondent)
+        only_a.tags.add(tag_a)
+
+        for batch_size in (10_000, 1):
+            with (
+                self.subTest(link_batch_size=batch_size),
+                mock.patch.object(
+                    DocumentViewSet,
+                    "selection_data_link_batch_size",
+                    batch_size,
+                ),
+            ):
+                response = self.client.get(
+                    f"/api/documents/?tags__id__in={tag_a.id},{tag_b.id}"
+                    "&include_selection_data=true",
+                )
+
+                self.assertEqual(response.status_code, status.HTTP_200_OK)
+                counts = {
+                    item["id"]: item["document_count"]
+                    for item in response.data["selection_data"][
+                        "selected_correspondents"
+                    ]
+                }
+                self.assertEqual(counts[correspondent.id], 2)
+
     def test_statistics(self) -> None:
         doc1 = Document.objects.create(
             title="none1",

src/documents/tests/test_tag_adjacency.py

@@ -1,104 +0,0 @@
-from django.core.exceptions import FieldDoesNotExist
-from django.db import connection
-from django.test import TestCase
-from django.test.utils import CaptureQueriesContext
-
-from documents.management.commands.document_importer import _deserialize_record
-from documents.models import Tag
-from documents.tag_hierarchy import TagHierarchy
-from paperless_testing.migrations import TestMigrations
-
-
-class TestTagAdjacency(TestCase):
-    def test_held_instance_uses_current_parent_edges(self):
-        first = Tag.objects.create(name="First")
-        second = Tag.objects.create(name="Second")
-        child = Tag.objects.create(name="Child", tn_parent=first)
-        Tag.objects.filter(pk=child.pk).update(tn_parent=second)
-        assert first.get_descendants_pks() == set()
-        assert second.get_descendants_pks() == {child.pk}
-
-    def test_queryset_delete_preserves_cascade(self):
-        root = Tag.objects.create(name="Root")
-        child = Tag.objects.create(name="Child", tn_parent=root)
-        Tag.objects.create(name="Grandchild", tn_parent=child)
-        survivor = Tag.objects.create(name="Other root")
-        Tag.objects.filter(pk=root.pk).delete()
-        assert list(Tag.objects.values_list("pk", flat=True)) == [survivor.pk]
-
-    def test_flat_writes_do_not_update_other_tags(self):
-        Tag.objects.bulk_create([Tag(name=f"Tag {i}") for i in range(100)])
-        with CaptureQueriesContext(connection) as queries:
-            new = Tag.objects.create(name="New")
-        assert not any(q["sql"].startswith("UPDATE") for q in queries)
-        with CaptureQueriesContext(connection) as queries:
-            new.name = "Renamed"
-            new.save()
-        tag_updates = [
-            q["sql"] for q in queries if q["sql"].startswith('UPDATE "documents_tag"')
-        ]
-        assert len(tag_updates) == 1
-        assert 'WHERE "documents_tag"."id" =' in tag_updates[0]
-
-    def test_batch_snapshot_is_one_query_and_includes_overlapping_roots(self):
-        root = Tag.objects.create(name="Root")
-        child = Tag.objects.create(name="Child", tn_parent=root)
-        leaf = Tag.objects.create(name="Leaf", tn_parent=child)
-        with self.assertNumQueries(1):
-            hierarchy = Tag.get_hierarchy()
-            assert hierarchy.with_ancestors([leaf.pk]) == {root.pk, child.pk, leaf.pk}
-            assert hierarchy.with_descendants([root.pk, child.pk]) == {
-                root.pk,
-                child.pk,
-                leaf.pk,
-            }
-        assert hierarchy.height(None) == 0
-
-    def test_legacy_export_fields_are_ignored_but_parent_is_preserved(self):
-        model, tag, m2m = _deserialize_record(
-            {
-                "model": "documents.tag",
-                "pk": 2,
-                "fields": {
-                    "name": "Child",
-                    "tn_parent": 1,
-                    "tn_order": 3,
-                    "tn_siblings_pks": "4,5",
-                    "tn_descendants_pks": "6",
-                },
-            },
-        )
-        assert model is Tag
-        assert tag.tn_parent_id == 1
-        assert tag.name == "Child"
-        assert not m2m
-        with self.assertRaises(FieldDoesNotExist):
-            _deserialize_record(
-                {"model": "documents.tag", "pk": 3, "fields": {"tn_typo": "bad"}},
-            )
-
-    def test_malformed_cycles_terminate(self):
-        hierarchy = TagHierarchy([(1, 2), (2, 1)])
-        assert hierarchy.with_descendants([1]) == {1, 2}
-        assert set(hierarchy.ancestors(2)) == {1, 2}
-
-
-class TestTagAdjacencyMigration(TestMigrations):
-    migrate_from = "0027_documentbarcode"
-    migrate_to = "0028_replace_tag_tree_fields"
-
-    def setUpBeforeMigration(self, apps):
-        tag = apps.get_model("documents", "Tag")
-        root = tag.objects.create(name="Root")
-        child = tag.objects.create(name="Child", tn_parent=root)
-        self.edges = [(root.pk, None), (child.pk, root.pk)]
-
-    def test_parent_edges_survive(self):
-        tag = self.apps.get_model("documents", "Tag")
-        assert (
-            list(tag.objects.order_by("pk").values_list("pk", "tn_parent_id"))
-            == self.edges
-        )
-        assert {
-            field.name for field in tag._meta.fields if field.name.startswith("tn_")
-        } == {"tn_parent"}

src/documents/views.py

@@ -5,6 +5,7 @@ import platform
 import re
 import tempfile
 import zipfile
+from collections import Counter
 from collections import deque
 from collections.abc import Callable
 from datetime import datetime
@@ -168,7 +169,6 @@ from documents.permissions import PaperlessNotePermissions
 from documents.permissions import PaperlessObjectPermissions
 from documents.permissions import TrashPermissions
 from documents.permissions import ViewDocumentsPermissions
-from documents.permissions import annotate_document_count_by_ids
 from documents.permissions import annotate_document_count_for_related_queryset
 from documents.permissions import get_document_count_filter_for_user
 from documents.permissions import get_permission_grantees
@@ -627,7 +627,7 @@ class TagViewSet(PermissionsAwareDocumentCountMixin, ModelViewSet[Tag]):
         queryset = queryset.order_by(*ordering)
 
         all_tags = list(queryset)
-        descendant_pks = Tag.get_hierarchy().descendants(t.pk for t in all_tags)
+        descendant_pks = {pk for tag in all_tags for pk in tag.get_descendants_pks()}
 
         if descendant_pks:
             user = getattr(getattr(self, "request", None), "user", None)
@@ -1022,71 +1022,83 @@ class DocumentViewSet(
         "custom_field_",
     )
 
-    def _get_selection_data_for_queryset(self, queryset):
-        # Resolve once instead of once per model below. `queryset` can carry an
-        # arbitrarily expensive WHERE clause (user filters plus the permission
-        # filter); re-embedding it as a subquery inside 5 separate Count(...)
-        # calls forces the database to re-evaluate that whole thing 5 times, and
-        # -- for FK relations especially -- can defeat semi-join planning
-        # entirely at scale. A concrete id list is cheap to reuse.
-        # order_by() drops the default/user ordering -- irrelevant for a plain
-        # id list, but left in place it forces a sort over the full filtered
-        # set before the ids can even be collected.
-        document_ids = list(queryset.order_by().values_list("pk", flat=True))
+    # Matching document ids are looked up in the link tables in batches of this
+    # size: one literal IN (...) over every match is a multi-MB statement per
+    # relation and, on SQLite, exceeds the bound-parameter limit.
+    selection_data_link_batch_size = 10_000
 
-        correspondents = Correspondent.objects.annotate(
-            document_count=Count(
-                "documents",
-                filter=Q(documents__id__in=document_ids),
-                distinct=True,
-            ),
-        )
-        document_types = DocumentType.objects.annotate(
-            document_count=Count(
-                "documents",
-                filter=Q(documents__id__in=document_ids),
-                distinct=True,
-            ),
-        )
-        storage_paths = StoragePath.objects.annotate(
-            document_count=Count(
-                "documents",
-                filter=Q(documents__id__in=document_ids),
-                distinct=True,
-            ),
-        )
-        # Tag and CustomField reach Document through an M2M/through-model table;
-        # a plain Count(filter=...) there is a much more expensive plan than the
-        # FK relations above once the bridge table is large -- see
-        # annotate_document_count_by_ids() for why.
-        tags = annotate_document_count_by_ids(
-            Tag.objects.all(),
-            through_model=Document.tags.through,
-            related_object_field="tag_id",
-            document_ids=document_ids,
-        )
-        custom_fields = annotate_document_count_by_ids(
-            CustomField.objects.all(),
-            through_model=CustomFieldInstance,
-            related_object_field="field_id",
-            document_ids=document_ids,
+    def _get_selection_data_for_queryset(self, queryset):
+        # The filtered queryset is evaluated once, as the page's own filter is,
+        # and never wrapped in a subquery: re-deriving a join or a HAVING
+        # (shared_by__id) per statement is very slow on some databases, and
+        # regrouping the queryset by another column would miscount.
+        # order_by() drops the default/user ordering, which only costs a sort.
+        correspondents: Counter = Counter()
+        document_types: Counter = Counter()
+        storage_paths: Counter = Counter()
+        document_ids: list[int] = []
+        seen: set[int] = set()
+        rows = queryset.order_by().values_list(
+            "pk",
+            "correspondent_id",
+            "document_type_id",
+            "storage_path_id",
         )
+        for pk, correspondent, document_type, storage_path in rows:
+            # A filter can join, repeating a document row.
+            if pk in seen:
+                continue
+            seen.add(pk)
+            document_ids.append(pk)
+            # Documents without a value count under None, which selection()
+            # never looks up.
+            correspondents[correspondent] += 1
+            document_types[document_type] += 1
+            storage_paths[storage_path] += 1
+
+        def link_table_counts(link_model, field_name: str) -> Counter:
+            result: Counter = Counter()
+            batch_size = self.selection_data_link_batch_size
+            for start in range(0, len(document_ids), batch_size):
+                result.update(
+                    dict(
+                        link_model.objects.filter(
+                            document_id__in=document_ids[start : start + batch_size],
+                        )
+                        .values_list(field_name)
+                        .annotate(document_count=Count("document_id")),
+                    ),
+                )
+            return result
+
+        def selection(model, counts: Counter) -> list[dict[str, int]]:
+            # Every object is listed, with 0 for those with no matching document.
+            return [
+                {"id": pk, "document_count": counts.get(pk, 0)}
+                for pk in model.objects.values_list("pk", flat=True)
+            ]
+
         return {
-            "selected_correspondents": [
-                {"id": t.id, "document_count": t.document_count} for t in correspondents
-            ],
-            "selected_tags": [
-                {"id": t.id, "document_count": t.document_count} for t in tags
-            ],
-            "selected_document_types": [
-                {"id": t.id, "document_count": t.document_count} for t in document_types
-            ],
-            "selected_storage_paths": [
-                {"id": t.id, "document_count": t.document_count} for t in storage_paths
-            ],
-            "selected_custom_fields": [
-                {"id": t.id, "document_count": t.document_count} for t in custom_fields
-            ],
+            "selected_correspondents": selection(
+                Correspondent,
+                correspondents,
+            ),
+            "selected_tags": selection(
+                Tag,
+                link_table_counts(Document.tags.through, "tag_id"),
+            ),
+            "selected_document_types": selection(
+                DocumentType,
+                document_types,
+            ),
+            "selected_storage_paths": selection(
+                StoragePath,
+                storage_paths,
+            ),
+            "selected_custom_fields": selection(
+                CustomField,
+                link_table_counts(CustomFieldInstance, "field_id"),
+            ),
         }
 
     @classmethod
@@ -5060,7 +5072,12 @@ class BulkEditObjectsView(PassUserMixin):
             if object_type == "tags":
                 editable_ids = set(user_permitted_objects.values_list("pk", flat=True))
                 all_ids = set(objs.values_list("pk", flat=True))
-                all_ids.update(Tag.get_hierarchy().descendants(all_ids) & editable_ids)
+                for tag in objs:
+                    all_ids.update(
+                        descendant.pk
+                        for descendant in tag.get_descendants()
+                        if descendant.pk in editable_ids
+                    )
                 objs = object_class.objects.filter(pk__in=all_ids)
             objs = objs.select_related("owner")
             object_ids = list(objs.values_list("pk", flat=True))

src/documents/workflows/mutations.py

@@ -6,7 +6,6 @@ from guardian.shortcuts import remove_perm
 from documents.data_models import DocumentMetadataOverrides
 from documents.models import CustomFieldInstance
 from documents.models import Document
-from documents.models import Tag
 from documents.models import WorkflowAction
 from documents.permissions import PermissionsSpec
 from documents.permissions import set_permissions_for_objects
@@ -135,9 +134,10 @@ def apply_assignment_to_overrides(
     if action.has_assign_tags:
         if overrides.tag_ids is None:
             overrides.tag_ids = []
-        tag_ids_to_add = Tag.get_hierarchy().with_ancestors(
-            tag.pk for tag in action.assign_tags.all()
-        )
+        tag_ids_to_add: set[int] = set()
+        for tag in action.assign_tags.all():
+            tag_ids_to_add.add(tag.pk)
+            tag_ids_to_add.update(int(pk) for pk in tag.get_ancestors_pks())
 
         overrides.tag_ids = list(set(overrides.tag_ids) | tag_ids_to_add)
 
@@ -216,9 +216,10 @@ def apply_removal_to_document(
     if action.remove_all_tags:
         Document.objects.get(pk=document.pk).tags.clear()
     else:
-        tag_ids_to_remove = Tag.get_hierarchy().with_descendants(
-            tag.pk for tag in action.remove_tags.all()
-        )
+        tag_ids_to_remove: set[int] = set()
+        for tag in action.remove_tags.all():
+            tag_ids_to_remove.add(tag.pk)
+            tag_ids_to_remove.update(int(pk) for pk in tag.get_descendants_pks())
 
         if tag_ids_to_remove:
             Document.objects.get(pk=document.pk).tags.remove(*tag_ids_to_remove)
@@ -296,9 +297,10 @@ def apply_removal_to_overrides(
     if action.remove_all_tags:
         overrides.tag_ids = None
     elif overrides.tag_ids:
-        tag_ids_to_remove = Tag.get_hierarchy().with_descendants(
-            tag.pk for tag in action.remove_tags.all()
-        )
+        tag_ids_to_remove: set[int] = set()
+        for tag in action.remove_tags.all():
+            tag_ids_to_remove.add(tag.pk)
+            tag_ids_to_remove.update(int(pk) for pk in tag.get_descendants_pks())
 
         overrides.tag_ids = [t for t in overrides.tag_ids if t not in tag_ids_to_remove]
 

src/paperless/settings/__init__.py

@@ -157,6 +157,7 @@ INSTALLED_APPS = [
     "allauth.headless",
     "drf_spectacular",
     "drf_spectacular_sidecar",
+    "treenode",
     *env_apps,
 ]
 

uv.lock

@@ -1210,6 +1210,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/99/c3/305a9f7611c0dc96d2a638bc3a67bdcde5a7faa5f4669188db018b8af6fc/django_stubs_ext-6.1.1-py3-none-any.whl", hash = "sha256:64aebb7ce2c327a4a2b75207d67034e879fd7edf027c93487be69da363dc79fa", size = 10853, upload-time = "2026-09-14T08:45:51.289Z" },
 ]
 
+[[package]]
+name = "django-treenode"
+version = "0.25.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/cb/2b/09f04b44b3583a911ae4f5d0e3360a23d529704c9e9115fce2520b7d4420/django_treenode-0.25.0.tar.gz", hash = "sha256:d14d0eb3571c227ce11147210d62a8f87b568bbb30fa76a5be9b9e6d514157c3", size = 29951, upload-time = "2026-08-26T00:20:13.489Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e1/ac/2140e8c20e2a47457ec23302273527d3430bf5c09edb8940d74c0725c138/django_treenode-0.25.0-py3-none-any.whl", hash = "sha256:f47caab5440420870b6287602b3ee401ca354416d44b24ab97579be5ea658927", size = 22343, upload-time = "2026-08-26T00:20:12.078Z" },
+]
+
 [[package]]
 name = "djangorestframework"
 version = "3.18.3"
@@ -3098,6 +3107,7 @@ dependencies = [
     { name = "django-multiselectfield" },
     { name = "django-rich" },
     { name = "django-soft-delete" },
+    { name = "django-treenode" },
     { name = "djangorestframework" },
     { name = "drf-spectacular" },
     { name = "drf-spectacular-sidecar" },
@@ -3248,6 +3258,7 @@ requires-dist = [
     { name = "django-multiselectfield", specifier = "~=1.0.1" },
     { name = "django-rich", specifier = "~=2.2.0" },
     { name = "django-soft-delete", specifier = "~=1.0.18" },
+    { name = "django-treenode", specifier = ">=0.24" },
     { name = "djangorestframework", specifier = "~=3.16" },
     { name = "drf-spectacular", specifier = "~=0.30" },
     { name = "drf-spectacular-sidecar", specifier = ">=2026.7.1,<2026.10" },