Performance: replace django-treenode with adjacency-list tags

Co-Authored-By: Codex <267193182+codex@users.noreply.github.com>

docker/rootfs/etc/s6-overlay/s6-rc.d/init-compile-bytecode/run

@@ -39,7 +39,6 @@ declare -a scope=(
 	"${site_packages}/allauth"
 	"${site_packages}/drf_spectacular"
 	"${site_packages}/drf_spectacular_sidecar"
-	"${site_packages}/treenode"
 	"${site_packages}/compression_middleware"
 )
 

pyproject.toml

@@ -38,7 +38,6 @@ dependencies = [
   "django-multiselectfield~=1.0.1",
   "django-rich~=2.2.0",
   "django-soft-delete~=1.0.18",
-  "django-treenode>=0.24",
   "djangorestframework~=3.16",
   "drf-spectacular~=0.30",
   "drf-spectacular-sidecar>=2026.7.1,<2026.10",

src/documents/admin.py

@@ -1,7 +1,6 @@
 from django.conf import settings
 from django.contrib import admin
 from guardian.admin import GuardedModelAdmin
-from treenode.admin import TreeNodeModelAdmin
 
 from documents.models import Correspondent
 from documents.models import CustomField
@@ -29,7 +28,7 @@ class CorrespondentAdmin(GuardedModelAdmin):
     list_editable = ("match", "matching_algorithm")
 
 
-class TagAdmin(GuardedModelAdmin, TreeNodeModelAdmin):
+class TagAdmin(GuardedModelAdmin):
     list_display = ("name", "color", "match", "matching_algorithm")
     list_filter = ("matching_algorithm",)
     list_editable = ("color", "match", "matching_algorithm")

src/documents/bulk_edit.py

@@ -255,19 +255,13 @@ def modify_tags(
     affected_docs = list(qs.values_list("pk", flat=True))
     DocumentTagRelationship = Document.tags.through
 
-    # add with all ancestors
+    hierarchy = Tag.get_hierarchy()
-    expanded_add_tags: set[int] = set()
+    expanded_add_tags = hierarchy.with_ancestors(
-    add_tag_objects = Tag.objects.filter(pk__in=add_tags)
+        Tag.objects.filter(pk__in=add_tags).values_list("pk", flat=True),
-    for t in add_tag_objects:
+    )
-        expanded_add_tags.add(int(t.id))
+    expanded_remove_tags = hierarchy.with_descendants(
-        expanded_add_tags.update(int(pk) for pk in t.get_ancestors_pks())
+        Tag.objects.filter(pk__in=remove_tags).values_list("pk", flat=True),
-
+    )
-    # remove with all descendants
-    expanded_remove_tags: set[int] = set()
-    remove_tag_objects = Tag.objects.filter(pk__in=remove_tags)
-    for t in remove_tag_objects:
-        expanded_remove_tags.add(int(t.id))
-        expanded_remove_tags.update(int(pk) for pk in t.get_descendants_pks())
 
     with transaction.atomic():
         if expanded_remove_tags:

src/documents/management/commands/document_importer.py

@@ -108,6 +108,24 @@ def _deserialize_record(
         ) from e
 
     for field_name, field_value in record.get("fields", {}).items():
+        # Older exports include django-treenode's derived bookkeeping. The
+        # parent FK is authoritative; accept only these known removed fields.
+        if Model is Tag and field_name in {
+            "tn_ancestors_count",
+            "tn_ancestors_pks",
+            "tn_children_count",
+            "tn_children_pks",
+            "tn_depth",
+            "tn_descendants_count",
+            "tn_descendants_pks",
+            "tn_index",
+            "tn_level",
+            "tn_order",
+            "tn_priority",
+            "tn_siblings_count",
+            "tn_siblings_pks",
+        }:
+            continue
         field = Model._meta.get_field(field_name)
         remote = field.remote_field
 

src/documents/migrations/0028_replace_tag_tree_fields.py

@@ -0,0 +1,64 @@
+# Generated by Django 5.2.17 on 2026-10-28 22:54
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("documents", "0027_documentbarcode"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="tag",
+            name="tn_ancestors_count",
+        ),
+        migrations.RemoveField(
+            model_name="tag",
+            name="tn_ancestors_pks",
+        ),
+        migrations.RemoveField(
+            model_name="tag",
+            name="tn_children_count",
+        ),
+        migrations.RemoveField(
+            model_name="tag",
+            name="tn_children_pks",
+        ),
+        migrations.RemoveField(
+            model_name="tag",
+            name="tn_depth",
+        ),
+        migrations.RemoveField(
+            model_name="tag",
+            name="tn_descendants_count",
+        ),
+        migrations.RemoveField(
+            model_name="tag",
+            name="tn_descendants_pks",
+        ),
+        migrations.RemoveField(
+            model_name="tag",
+            name="tn_index",
+        ),
+        migrations.RemoveField(
+            model_name="tag",
+            name="tn_level",
+        ),
+        migrations.RemoveField(
+            model_name="tag",
+            name="tn_order",
+        ),
+        migrations.RemoveField(
+            model_name="tag",
+            name="tn_priority",
+        ),
+        migrations.RemoveField(
+            model_name="tag",
+            name="tn_siblings_count",
+        ),
+        migrations.RemoveField(
+            model_name="tag",
+            name="tn_siblings_pks",
+        ),
+    ]

src/documents/models.py

@@ -14,7 +14,6 @@ from django.db import models
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 from multiselectfield import MultiSelectField
-from treenode.models import TreeNodeModel
 
 if settings.AUDIT_LOG_ENABLED:
     from auditlog.registry import auditlog
@@ -100,7 +99,17 @@ class Correspondent(MatchingModel):
         verbose_name_plural = _("correspondents")
 
 
-class Tag(MatchingModel, TreeNodeModel):
+class Tag(MatchingModel):
+    # Keep the existing column and reverse relation during the transition.
+    tn_parent = models.ForeignKey(
+        "self",
+        blank=True,
+        null=True,
+        on_delete=models.CASCADE,
+        related_name="tn_children",
+        verbose_name=_("Parent"),
+    )
+
     color = models.CharField(_("color"), max_length=7, default="#a6cee3")
     # Maximum allowed nesting depth for tags (root = 1, max depth = 5)
     MAX_NESTING_DEPTH: Final[int] = 5
@@ -114,27 +123,51 @@ class Tag(MatchingModel, TreeNodeModel):
         ),
     )
 
-    class Meta(MatchingModel.Meta, TreeNodeModel.Meta):
+    class Meta(MatchingModel.Meta):
         verbose_name = _("tag")
         verbose_name_plural = _("tags")
 
+    @classmethod
+    def get_hierarchy(cls):
+        from documents.tag_hierarchy import TagHierarchy
+
+        return TagHierarchy(cls.objects.values_list("pk", "tn_parent_id"))
+
+    @property
+    def parent_pk(self):
+        return self.tn_parent_id
+
+    def get_parent(self):
+        return self.tn_parent
+
+    def get_ancestors_pks(self):
+        return self.get_hierarchy().ancestors(self.tn_parent_id)
+
+    def get_ancestors(self):
+        ids = self.get_ancestors_pks()
+        tags = self.__class__.objects.in_bulk(ids)
+        return [tags[pk] for pk in ids]
+
+    def get_descendants_pks(self):
+        return self.get_hierarchy().descendants([self.pk])
+
+    def get_descendants(self):
+        return self.__class__.objects.filter(pk__in=self.get_descendants_pks())
+
+    def get_children_queryset(self):
+        return self.__class__.objects.filter(tn_parent_id=self.pk)
+
     def clean(self) -> None:
-        # Prevent self-parenting and assigning a descendant as parent
+        hierarchy = self.get_hierarchy()
-        parent = self.get_parent()
+        if self.pk is not None and self.tn_parent_id == self.pk:
-        if parent == self:
+            raise ValidationError({"tn_parent": _("Cannot set itself as parent.")})
-            raise ValidationError({"parent": _("Cannot set itself as parent.")})
+        ancestors = hierarchy.ancestors(self.tn_parent_id)
-        if parent and self.pk is not None and self.is_ancestor_of(parent):
+        if self.pk is not None and self.pk in ancestors:
-            raise ValidationError({"parent": _("Cannot set parent to a descendant.")})
+            raise ValidationError(
-
+                {"tn_parent": _("Cannot set parent to a descendant.")},
-        # Enforce maximum nesting depth
+            )
-        new_parent_depth = 0
+        if len(ancestors) + 1 + hierarchy.height(self.pk) > self.MAX_NESTING_DEPTH:
-        if parent:
+            raise ValidationError({"tn_parent": _("Maximum nesting depth exceeded.")})
-            new_parent_depth = parent.get_ancestors_count() + 1
-
-        height = 0 if self.pk is None else self.get_depth()
-        deepest_new_depth = (new_parent_depth + 1) + height
-        if deepest_new_depth > self.MAX_NESTING_DEPTH:
-            raise ValidationError({"parent": _("Maximum nesting depth exceeded.")})
 
         return super().clean()
 
@@ -539,10 +572,7 @@ class Document(SoftDeleteModel, ModelWithOwner):  # type: ignore[django-manager-
         return self.created
 
     def add_nested_tags(self, tags) -> None:
-        tag_ids = set()
+        tag_ids = Tag.get_hierarchy().with_ancestors(tag.pk for tag in tags)
-        for tag in tags:
-            tag_ids.add(tag.id)
-            tag_ids.update(tag.get_ancestors_pks())
 
         tags_to_add = self.tags.model.objects.filter(id__in=tag_ids)
         self.tags.add(*tags_to_add)

src/documents/permissions.py

@@ -576,9 +576,12 @@ def annotate_document_count_by_ids(
                        or CustomFieldInstance)
         related_object_field: field on the relation pointing back to queryset pk
         document_ids: the document ids to count against -- a concrete list/set,
-                       or a simple (already resolved) queryset of ids. Keep
+                       or a simple (already resolved) queryset of ids. Callers
-                       the list small: it is embedded in the statement as
+                       that need this filtered by a complex condition (e.g. a
-                       literal parameters.
+                       permission check) should resolve it to a concrete list
+                       first if the same ids will be reused across multiple
+                       calls, rather than passing the complex queryset itself
+                       into each -- see ``_get_selection_data_for_queryset``.
         target_field: field on the relation pointing to Document id
     """
 

src/documents/serialisers.py

@@ -589,7 +589,7 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
 
     text_color = serializers.SerializerMethodField()
 
-    # map to treenode's tn_parent
+    # Preserve the public API parent field.
     parent = serializers.PrimaryKeyRelatedField(
         queryset=Tag.objects.all(),
         allow_null=True,
@@ -692,7 +692,9 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
                 self.instance.clean()
             except ValidationError as e:
                 logger.debug("Tag parent validation failed: %s", e)
-                raise e
+                raise serializers.ValidationError(
+                    {"parent": e.message_dict["tn_parent"]},
+                ) from e
             finally:
                 self.instance.tn_parent = original_parent
         else:
@@ -702,7 +704,9 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
                 temp.clean()
             except ValidationError as e:
                 logger.debug("Tag parent validation failed: %s", e)
-                raise e
+                raise serializers.ValidationError(
+                    {"parent": e.message_dict["tn_parent"]},
+                ) from e
 
         return super().validate(attrs)
 
@@ -1198,61 +1202,35 @@ class DocumentSerializer(
                             doc_id,
                         )
         if "tags" in validated_data:
-            # Respect tag hierarchy on updates:
+            hierarchy = Tag.get_hierarchy()
-            # - Adding a child adds its ancestors
+            previous_ids = set(instance.tags.values_list("pk", flat=True))
-            # - Removing a parent removes all its descendants
+            requested_ids = {tag.pk for tag in validated_data["tags"]}
-            prev_tags = set(instance.tags.all())
+            required_ids = hierarchy.with_ancestors(requested_ids - previous_ids)
-            requested_tags = set(validated_data["tags"])
+            blocked_ids = hierarchy.with_descendants(previous_ids - requested_ids)
+            blocked_ids.difference_update(required_ids)
+            final_ids = hierarchy.with_ancestors(requested_ids) - blocked_ids
+            validated_data["tags"] = list(Tag.objects.filter(pk__in=final_ids))
 
-            # Tags newly added in this update and the ancestors they require
-            added_tags = requested_tags - prev_tags
-            required_by_add_tags = set(added_tags)
-            for t in added_tags:
-                required_by_add_tags.update(t.get_ancestors())
-
-            # Tags being removed in this update and all descendants, except
-            # those required by a tag that is being added in this same update
-            removed_tags = prev_tags - requested_tags
-            blocked_tags = set(removed_tags)
-            for t in removed_tags:
-                blocked_tags.update(t.get_descendants())
-            blocked_tags.difference_update(required_by_add_tags)
-
-            # Add all parent tags
-            final_tags = set(requested_tags)
-            for t in requested_tags:
-                final_tags.update(t.get_ancestors())
-
-            # Drop removed parents and their descendants
-            final_tags.difference_update(blocked_tags)
-
-            validated_data["tags"] = list(final_tags)
         if validated_data.get("remove_inbox_tags"):
-            current_tag_ids = {t.pk for t in instance.tags.all()}
+            hierarchy = Tag.get_hierarchy()
             tags = (
                 validated_data["tags"]
                 if "tags" in validated_data
                 else list(instance.tags.all())
             )
-
+            current_ids = set(instance.tags.values_list("pk", flat=True))
-            # Tags newly added in this update, plus their ancestors, are kept
+            keep_ids = hierarchy.with_ancestors(
-            keep_ids: set[int] = set()
+                tag.pk for tag in tags if tag.pk not in current_ids
-            for tag in tags:
+            )
-                if tag.pk not in current_tag_ids:
+            inbox_ids = (
-                    keep_ids.add(tag.pk)
-                    keep_ids.update(int(pk) for pk in tag.get_ancestors_pks())
-
-            # Remove inbox tags and their descendants, except those being kept
-            remove_ids: set[int] = set()
-            for inbox_tag in (
                 Tag.objects.filter(is_inbox_tag=True)
-                .exclude(pk__in=keep_ids)
+                .exclude(
-                .only("pk", "tn_descendants_pks")
+                    pk__in=keep_ids,
-            ):
+                )
-                remove_ids.add(inbox_tag.pk)
+                .values_list("pk", flat=True)
-                remove_ids.update(int(pk) for pk in inbox_tag.get_descendants_pks())
+            )
-
+            remove_ids = hierarchy.with_descendants(inbox_ids) - keep_ids
-            validated_data["tags"] = [t for t in tags if t.pk not in remove_ids]
+            validated_data["tags"] = [tag for tag in tags if tag.pk not in remove_ids]
 
         if settings.AUDIT_LOG_ENABLED:
             with set_actor(self.user):

src/documents/tag_hierarchy.py

@@ -0,0 +1,61 @@
+"""An operation-local snapshot of tag edges, with no persistent derived fields.
+
+Never cache this globally: tasks, bulk updates and other processes can change
+parents. Callers traversing multiple tags should share one snapshot.
+"""
+
+from collections import defaultdict
+from collections.abc import Iterable
+
+
+class TagHierarchy:
+    def __init__(self, edges: Iterable[tuple[int, int | None]]) -> None:
+        self.parents = dict(edges)
+        self.children: dict[int | None, list[int]] = defaultdict(list)
+        for pk, parent in self.parents.items():
+            self.children[parent].append(pk)
+
+    def ancestors(self, parent_id: int | None) -> list[int]:
+        result = []
+        seen = set()
+        while parent_id is not None and parent_id not in seen:
+            seen.add(parent_id)
+            result.append(parent_id)
+            parent_id = self.parents.get(parent_id)
+        return result[::-1]
+
+    def with_ancestors(self, roots: Iterable[int]) -> set[int]:
+        result = set(roots)
+        for pk in list(result):
+            result.update(self.ancestors(self.parents.get(pk)))
+        return result
+
+    def with_descendants(self, roots: Iterable[int]) -> set[int]:
+        roots = set(roots)
+        return roots | self.descendants(roots)
+
+    def descendants(self, roots: Iterable[int]) -> set[int]:
+        roots = {pk for pk in roots if pk is not None}
+        seen = set(roots)
+        pending = list(roots)
+        while pending:
+            for child in self.children.get(pending.pop(), ()):
+                if child not in seen:
+                    seen.add(child)
+                    pending.append(child)
+        return seen - roots
+
+    def height(self, pk: int | None) -> int:
+        if pk is None:
+            return 0
+        seen = {pk}
+        pending = [(pk, 0)]
+        height = 0
+        while pending:
+            parent, depth = pending.pop()
+            height = max(height, depth)
+            for child in self.children.get(parent, ()):
+                if child not in seen:
+                    seen.add(child)
+                    pending.append((child, depth + 1))
+        return height

src/documents/tests/test_api_bulk_edit.py

@@ -1141,8 +1141,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
         self.assertCountEqual(
             response.data["selected_document_types"],
             [
-                {"id": self.dt1.id, "document_count": 1},
+                {"id": self.c1.id, "document_count": 1},
-                {"id": self.dt2.id, "document_count": 0},
+                {"id": self.c2.id, "document_count": 0},
             ],
         )
 

src/documents/tests/test_api_documents.py

@@ -49,7 +49,6 @@ from documents.models import WorkflowTrigger
 from documents.search import get_backend
 from documents.signals.handlers import run_workflows
 from documents.tests.utils import ConsumeTaskMixin
-from documents.views import DocumentViewSet
 from paperless_testing.dirs import DirectoriesMixin
 from paperless_testing.factories import DocumentFactory
 from paperless_testing.factories import TagFactory
@@ -1756,106 +1755,6 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         }
         self.assertEqual(counts_by_field[custom_field.id], 1)
 
-    def test_selection_data_counts_with_shared_by_filter(self) -> None:
-        """
-        GIVEN:
-            - Several documents owned by one user with the same correspondent
-              and document type, only one of which is shared
-        WHEN:
-            - Requesting the document list filtered by shared_by__id with
-              include_selection_data=true
-        THEN:
-            - Counts reflect only the shared document, not every document the
-              owner has with that correspondent or document type, however the
-              link-table lookups are batched
-        """
-        owner = User.objects.create_user(username="owner")
-        other = User.objects.create_user(username="other")
-        correspondent = Correspondent.objects.create(name="c1")
-        doc_type = DocumentType.objects.create(name="dt1")
-        tag = Tag.objects.create(name="tag")
-        docs = [
-            Document.objects.create(
-                checksum=str(i),
-                owner=owner,
-                correspondent=correspondent,
-                document_type=doc_type,
-            )
-            for i in range(3)
-        ]
-        for doc in docs:
-            doc.tags.add(tag)
-        grant_object(other, docs[0], "view_document")
-
-        for batch_size in (10_000, 1):
-            with (
-                self.subTest(link_batch_size=batch_size),
-                mock.patch.object(
-                    DocumentViewSet,
-                    "selection_data_link_batch_size",
-                    batch_size,
-                ),
-            ):
-                response = self.client.get(
-                    f"/api/documents/?shared_by__id={owner.id}"
-                    "&include_selection_data=true",
-                )
-
-                self.assertEqual(response.status_code, status.HTTP_200_OK)
-                self.assertEqual(response.data["count"], 1)
-                selection_data = response.data["selection_data"]
-                for key, obj in (
-                    ("selected_correspondents", correspondent),
-                    ("selected_document_types", doc_type),
-                    ("selected_tags", tag),
-                ):
-                    counts = {
-                        item["id"]: item["document_count"]
-                        for item in selection_data[key]
-                    }
-                    self.assertEqual(counts[obj.id], 1, key)
-
-    def test_selection_data_counts_documents_once_when_filter_joins(self) -> None:
-        """
-        GIVEN:
-            - A document matching a multi-valued filter through two tags
-        WHEN:
-            - Requesting the document list with include_selection_data=true
-        THEN:
-            - The document is counted once per correspondent, not once per
-              matching tag, however the link-table lookups are batched
-        """
-        correspondent = Correspondent.objects.create(name="c1")
-        tag_a = Tag.objects.create(name="a")
-        tag_b = Tag.objects.create(name="b")
-        both = Document.objects.create(checksum="1", correspondent=correspondent)
-        both.tags.add(tag_a, tag_b)
-        only_a = Document.objects.create(checksum="2", correspondent=correspondent)
-        only_a.tags.add(tag_a)
-
-        for batch_size in (10_000, 1):
-            with (
-                self.subTest(link_batch_size=batch_size),
-                mock.patch.object(
-                    DocumentViewSet,
-                    "selection_data_link_batch_size",
-                    batch_size,
-                ),
-            ):
-                response = self.client.get(
-                    f"/api/documents/?tags__id__in={tag_a.id},{tag_b.id}"
-                    "&include_selection_data=true",
-                )
-
-                self.assertEqual(response.status_code, status.HTTP_200_OK)
-                counts = {
-                    item["id"]: item["document_count"]
-                    for item in response.data["selection_data"][
-                        "selected_correspondents"
-                    ]
-                }
-                self.assertEqual(counts[correspondent.id], 2)
-
     def test_statistics(self) -> None:
         doc1 = Document.objects.create(
             title="none1",

src/documents/tests/test_tag_adjacency.py

@@ -0,0 +1,104 @@
+from django.core.exceptions import FieldDoesNotExist
+from django.db import connection
+from django.test import TestCase
+from django.test.utils import CaptureQueriesContext
+
+from documents.management.commands.document_importer import _deserialize_record
+from documents.models import Tag
+from documents.tag_hierarchy import TagHierarchy
+from paperless_testing.migrations import TestMigrations
+
+
+class TestTagAdjacency(TestCase):
+    def test_held_instance_uses_current_parent_edges(self):
+        first = Tag.objects.create(name="First")
+        second = Tag.objects.create(name="Second")
+        child = Tag.objects.create(name="Child", tn_parent=first)
+        Tag.objects.filter(pk=child.pk).update(tn_parent=second)
+        assert first.get_descendants_pks() == set()
+        assert second.get_descendants_pks() == {child.pk}
+
+    def test_queryset_delete_preserves_cascade(self):
+        root = Tag.objects.create(name="Root")
+        child = Tag.objects.create(name="Child", tn_parent=root)
+        Tag.objects.create(name="Grandchild", tn_parent=child)
+        survivor = Tag.objects.create(name="Other root")
+        Tag.objects.filter(pk=root.pk).delete()
+        assert list(Tag.objects.values_list("pk", flat=True)) == [survivor.pk]
+
+    def test_flat_writes_do_not_update_other_tags(self):
+        Tag.objects.bulk_create([Tag(name=f"Tag {i}") for i in range(100)])
+        with CaptureQueriesContext(connection) as queries:
+            new = Tag.objects.create(name="New")
+        assert not any(q["sql"].startswith("UPDATE") for q in queries)
+        with CaptureQueriesContext(connection) as queries:
+            new.name = "Renamed"
+            new.save()
+        tag_updates = [
+            q["sql"] for q in queries if q["sql"].startswith('UPDATE "documents_tag"')
+        ]
+        assert len(tag_updates) == 1
+        assert 'WHERE "documents_tag"."id" =' in tag_updates[0]
+
+    def test_batch_snapshot_is_one_query_and_includes_overlapping_roots(self):
+        root = Tag.objects.create(name="Root")
+        child = Tag.objects.create(name="Child", tn_parent=root)
+        leaf = Tag.objects.create(name="Leaf", tn_parent=child)
+        with self.assertNumQueries(1):
+            hierarchy = Tag.get_hierarchy()
+            assert hierarchy.with_ancestors([leaf.pk]) == {root.pk, child.pk, leaf.pk}
+            assert hierarchy.with_descendants([root.pk, child.pk]) == {
+                root.pk,
+                child.pk,
+                leaf.pk,
+            }
+        assert hierarchy.height(None) == 0
+
+    def test_legacy_export_fields_are_ignored_but_parent_is_preserved(self):
+        model, tag, m2m = _deserialize_record(
+            {
+                "model": "documents.tag",
+                "pk": 2,
+                "fields": {
+                    "name": "Child",
+                    "tn_parent": 1,
+                    "tn_order": 3,
+                    "tn_siblings_pks": "4,5",
+                    "tn_descendants_pks": "6",
+                },
+            },
+        )
+        assert model is Tag
+        assert tag.tn_parent_id == 1
+        assert tag.name == "Child"
+        assert not m2m
+        with self.assertRaises(FieldDoesNotExist):
+            _deserialize_record(
+                {"model": "documents.tag", "pk": 3, "fields": {"tn_typo": "bad"}},
+            )
+
+    def test_malformed_cycles_terminate(self):
+        hierarchy = TagHierarchy([(1, 2), (2, 1)])
+        assert hierarchy.with_descendants([1]) == {1, 2}
+        assert set(hierarchy.ancestors(2)) == {1, 2}
+
+
+class TestTagAdjacencyMigration(TestMigrations):
+    migrate_from = "0027_documentbarcode"
+    migrate_to = "0028_replace_tag_tree_fields"
+
+    def setUpBeforeMigration(self, apps):
+        tag = apps.get_model("documents", "Tag")
+        root = tag.objects.create(name="Root")
+        child = tag.objects.create(name="Child", tn_parent=root)
+        self.edges = [(root.pk, None), (child.pk, root.pk)]
+
+    def test_parent_edges_survive(self):
+        tag = self.apps.get_model("documents", "Tag")
+        assert (
+            list(tag.objects.order_by("pk").values_list("pk", "tn_parent_id"))
+            == self.edges
+        )
+        assert {
+            field.name for field in tag._meta.fields if field.name.startswith("tn_")
+        } == {"tn_parent"}

src/documents/views.py

@@ -5,7 +5,6 @@ import platform
 import re
 import tempfile
 import zipfile
-from collections import Counter
 from collections import deque
 from collections.abc import Callable
 from datetime import datetime
@@ -169,6 +168,7 @@ from documents.permissions import PaperlessNotePermissions
 from documents.permissions import PaperlessObjectPermissions
 from documents.permissions import TrashPermissions
 from documents.permissions import ViewDocumentsPermissions
+from documents.permissions import annotate_document_count_by_ids
 from documents.permissions import annotate_document_count_for_related_queryset
 from documents.permissions import get_document_count_filter_for_user
 from documents.permissions import get_permission_grantees
@@ -627,7 +627,7 @@ class TagViewSet(PermissionsAwareDocumentCountMixin, ModelViewSet[Tag]):
         queryset = queryset.order_by(*ordering)
 
         all_tags = list(queryset)
-        descendant_pks = {pk for tag in all_tags for pk in tag.get_descendants_pks()}
+        descendant_pks = Tag.get_hierarchy().descendants(t.pk for t in all_tags)
 
         if descendant_pks:
             user = getattr(getattr(self, "request", None), "user", None)
@@ -1022,83 +1022,71 @@ class DocumentViewSet(
         "custom_field_",
     )
 
-    # Matching document ids are looked up in the link tables in batches of this
-    # size: one literal IN (...) over every match is a multi-MB statement per
-    # relation and, on SQLite, exceeds the bound-parameter limit.
-    selection_data_link_batch_size = 10_000
-
     def _get_selection_data_for_queryset(self, queryset):
-        # The filtered queryset is evaluated once, as the page's own filter is,
+        # Resolve once instead of once per model below. `queryset` can carry an
-        # and never wrapped in a subquery: re-deriving a join or a HAVING
+        # arbitrarily expensive WHERE clause (user filters plus the permission
-        # (shared_by__id) per statement is very slow on some databases, and
+        # filter); re-embedding it as a subquery inside 5 separate Count(...)
-        # regrouping the queryset by another column would miscount.
+        # calls forces the database to re-evaluate that whole thing 5 times, and
-        # order_by() drops the default/user ordering, which only costs a sort.
+        # -- for FK relations especially -- can defeat semi-join planning
-        correspondents: Counter = Counter()
+        # entirely at scale. A concrete id list is cheap to reuse.
-        document_types: Counter = Counter()
+        # order_by() drops the default/user ordering -- irrelevant for a plain
-        storage_paths: Counter = Counter()
+        # id list, but left in place it forces a sort over the full filtered
-        document_ids: list[int] = []
+        # set before the ids can even be collected.
-        seen: set[int] = set()
+        document_ids = list(queryset.order_by().values_list("pk", flat=True))
-        rows = queryset.order_by().values_list(
+
-            "pk",
+        correspondents = Correspondent.objects.annotate(
-            "correspondent_id",
+            document_count=Count(
-            "document_type_id",
+                "documents",
-            "storage_path_id",
+                filter=Q(documents__id__in=document_ids),
+                distinct=True,
+            ),
+        )
+        document_types = DocumentType.objects.annotate(
+            document_count=Count(
+                "documents",
+                filter=Q(documents__id__in=document_ids),
+                distinct=True,
+            ),
+        )
+        storage_paths = StoragePath.objects.annotate(
+            document_count=Count(
+                "documents",
+                filter=Q(documents__id__in=document_ids),
+                distinct=True,
+            ),
+        )
+        # Tag and CustomField reach Document through an M2M/through-model table;
+        # a plain Count(filter=...) there is a much more expensive plan than the
+        # FK relations above once the bridge table is large -- see
+        # annotate_document_count_by_ids() for why.
+        tags = annotate_document_count_by_ids(
+            Tag.objects.all(),
+            through_model=Document.tags.through,
+            related_object_field="tag_id",
+            document_ids=document_ids,
+        )
+        custom_fields = annotate_document_count_by_ids(
+            CustomField.objects.all(),
+            through_model=CustomFieldInstance,
+            related_object_field="field_id",
+            document_ids=document_ids,
         )
-        for pk, correspondent, document_type, storage_path in rows:
-            # A filter can join, repeating a document row.
-            if pk in seen:
-                continue
-            seen.add(pk)
-            document_ids.append(pk)
-            # Documents without a value count under None, which selection()
-            # never looks up.
-            correspondents[correspondent] += 1
-            document_types[document_type] += 1
-            storage_paths[storage_path] += 1
-
-        def link_table_counts(link_model, field_name: str) -> Counter:
-            result: Counter = Counter()
-            batch_size = self.selection_data_link_batch_size
-            for start in range(0, len(document_ids), batch_size):
-                result.update(
-                    dict(
-                        link_model.objects.filter(
-                            document_id__in=document_ids[start : start + batch_size],
-                        )
-                        .values_list(field_name)
-                        .annotate(document_count=Count("document_id")),
-                    ),
-                )
-            return result
-
-        def selection(model, counts: Counter) -> list[dict[str, int]]:
-            # Every object is listed, with 0 for those with no matching document.
-            return [
-                {"id": pk, "document_count": counts.get(pk, 0)}
-                for pk in model.objects.values_list("pk", flat=True)
-            ]
-
         return {
-            "selected_correspondents": selection(
+            "selected_correspondents": [
-                Correspondent,
+                {"id": t.id, "document_count": t.document_count} for t in correspondents
-                correspondents,
+            ],
-            ),
+            "selected_tags": [
-            "selected_tags": selection(
+                {"id": t.id, "document_count": t.document_count} for t in tags
-                Tag,
+            ],
-                link_table_counts(Document.tags.through, "tag_id"),
+            "selected_document_types": [
-            ),
+                {"id": t.id, "document_count": t.document_count} for t in document_types
-            "selected_document_types": selection(
+            ],
-                DocumentType,
+            "selected_storage_paths": [
-                document_types,
+                {"id": t.id, "document_count": t.document_count} for t in storage_paths
-            ),
+            ],
-            "selected_storage_paths": selection(
+            "selected_custom_fields": [
-                StoragePath,
+                {"id": t.id, "document_count": t.document_count} for t in custom_fields
-                storage_paths,
+            ],
-            ),
-            "selected_custom_fields": selection(
-                CustomField,
-                link_table_counts(CustomFieldInstance, "field_id"),
-            ),
         }
 
     @classmethod
@@ -5072,12 +5060,7 @@ class BulkEditObjectsView(PassUserMixin):
             if object_type == "tags":
                 editable_ids = set(user_permitted_objects.values_list("pk", flat=True))
                 all_ids = set(objs.values_list("pk", flat=True))
-                for tag in objs:
+                all_ids.update(Tag.get_hierarchy().descendants(all_ids) & editable_ids)
-                    all_ids.update(
-                        descendant.pk
-                        for descendant in tag.get_descendants()
-                        if descendant.pk in editable_ids
-                    )
                 objs = object_class.objects.filter(pk__in=all_ids)
             objs = objs.select_related("owner")
             object_ids = list(objs.values_list("pk", flat=True))

src/documents/workflows/mutations.py

@@ -6,6 +6,7 @@ from guardian.shortcuts import remove_perm
 from documents.data_models import DocumentMetadataOverrides
 from documents.models import CustomFieldInstance
 from documents.models import Document
+from documents.models import Tag
 from documents.models import WorkflowAction
 from documents.permissions import PermissionsSpec
 from documents.permissions import set_permissions_for_objects
@@ -134,10 +135,9 @@ def apply_assignment_to_overrides(
     if action.has_assign_tags:
         if overrides.tag_ids is None:
             overrides.tag_ids = []
-        tag_ids_to_add: set[int] = set()
+        tag_ids_to_add = Tag.get_hierarchy().with_ancestors(
-        for tag in action.assign_tags.all():
+            tag.pk for tag in action.assign_tags.all()
-            tag_ids_to_add.add(tag.pk)
+        )
-            tag_ids_to_add.update(int(pk) for pk in tag.get_ancestors_pks())
 
         overrides.tag_ids = list(set(overrides.tag_ids) | tag_ids_to_add)
 
@@ -216,10 +216,9 @@ def apply_removal_to_document(
     if action.remove_all_tags:
         Document.objects.get(pk=document.pk).tags.clear()
     else:
-        tag_ids_to_remove: set[int] = set()
+        tag_ids_to_remove = Tag.get_hierarchy().with_descendants(
-        for tag in action.remove_tags.all():
+            tag.pk for tag in action.remove_tags.all()
-            tag_ids_to_remove.add(tag.pk)
+        )
-            tag_ids_to_remove.update(int(pk) for pk in tag.get_descendants_pks())
 
         if tag_ids_to_remove:
             Document.objects.get(pk=document.pk).tags.remove(*tag_ids_to_remove)
@@ -297,10 +296,9 @@ def apply_removal_to_overrides(
     if action.remove_all_tags:
         overrides.tag_ids = None
     elif overrides.tag_ids:
-        tag_ids_to_remove: set[int] = set()
+        tag_ids_to_remove = Tag.get_hierarchy().with_descendants(
-        for tag in action.remove_tags.all():
+            tag.pk for tag in action.remove_tags.all()
-            tag_ids_to_remove.add(tag.pk)
+        )
-            tag_ids_to_remove.update(int(pk) for pk in tag.get_descendants_pks())
 
         overrides.tag_ids = [t for t in overrides.tag_ids if t not in tag_ids_to_remove]
 

src/paperless/settings/__init__.py

@@ -157,7 +157,6 @@ INSTALLED_APPS = [
     "allauth.headless",
     "drf_spectacular",
     "drf_spectacular_sidecar",
-    "treenode",
     *env_apps,
 ]
 

uv.lock

@@ -1210,15 +1210,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/99/c3/305a9f7611c0dc96d2a638bc3a67bdcde5a7faa5f4669188db018b8af6fc/django_stubs_ext-6.1.1-py3-none-any.whl", hash = "sha256:64aebb7ce2c327a4a2b75207d67034e879fd7edf027c93487be69da363dc79fa", size = 10853, upload-time = "2026-09-14T08:45:51.289Z" },
 ]
 
-[[package]]
-name = "django-treenode"
-version = "0.25.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/cb/2b/09f04b44b3583a911ae4f5d0e3360a23d529704c9e9115fce2520b7d4420/django_treenode-0.25.0.tar.gz", hash = "sha256:d14d0eb3571c227ce11147210d62a8f87b568bbb30fa76a5be9b9e6d514157c3", size = 29951, upload-time = "2026-08-26T00:20:13.489Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/e1/ac/2140e8c20e2a47457ec23302273527d3430bf5c09edb8940d74c0725c138/django_treenode-0.25.0-py3-none-any.whl", hash = "sha256:f47caab5440420870b6287602b3ee401ca354416d44b24ab97579be5ea658927", size = 22343, upload-time = "2026-08-26T00:20:12.078Z" },
-]
-
 [[package]]
 name = "djangorestframework"
 version = "3.18.3"
@@ -3107,7 +3098,6 @@ dependencies = [
     { name = "django-multiselectfield" },
     { name = "django-rich" },
     { name = "django-soft-delete" },
-    { name = "django-treenode" },
     { name = "djangorestframework" },
     { name = "drf-spectacular" },
     { name = "drf-spectacular-sidecar" },
@@ -3258,7 +3248,6 @@ requires-dist = [
     { name = "django-multiselectfield", specifier = "~=1.0.1" },
     { name = "django-rich", specifier = "~=2.2.0" },
     { name = "django-soft-delete", specifier = "~=1.0.18" },
-    { name = "django-treenode", specifier = ">=0.24" },
     { name = "djangorestframework", specifier = "~=3.16" },
     { name = "drf-spectacular", specifier = "~=0.30" },
     { name = "drf-spectacular-sidecar", specifier = ">=2026.7.1,<2026.10" },