Fix: skip vector store document id filter for unrestricted chat users (#13937)

* Fix: skip vector store document id filter for unrestricted chat users

ChatStreamingView built an IN filter from every permitted document id
for the "chat over all documents" case, which exceeds the vector
store's SQLite bound-parameter safety limit on installs with more
than ~32700 documents, silently returning no context. For a user who
can see every document (an active superuser), that filter never
narrows anything, so skip it and let the retriever search the whole
index instead.

* Minor improvements from a Claude review

* When a user is unrestricted chatting, still exclude trashed documents using a 'NOT IN' SQL statement.  Wire that up where we need it

* Update src/paperless_ai/chat.py

Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com>

src/documents/tests/test_api_chat.py

@@ -1,11 +1,18 @@
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
 from unittest import mock
 
+import pytest
+from django.contrib.auth.models import Permission
 from django.contrib.auth.models import User
 from rest_framework import status
+from rest_framework.test import APIClient
 from rest_framework.test import APITestCase
 
+if TYPE_CHECKING:
+    from pytest_mock import MockerFixture
+
 
 class TestChatStreamingViewInputValidation(APITestCase):
     def setUp(self) -> None:
@@ -42,3 +49,73 @@ class TestChatStreamingViewInputValidation(APITestCase):
                 format="json",
             )
         assert resp.status_code == status.HTTP_400_BAD_REQUEST
+
+
+@pytest.mark.django_db
+class TestChatStreamingViewUnrestrictedFlag:
+    """The document id filter may only be skipped (``unrestricted=True``) for
+    an active superuser, never for a regular user -- regardless of what
+    permissions that user holds.
+    """
+
+    @pytest.fixture
+    def mocked_stream_chat(self, mocker: MockerFixture) -> mock.MagicMock:
+        """AI enabled, with stream_chat_with_documents patched so the view
+        never touches the real vector store; returns the patched callable so
+        tests can inspect how it was called.
+        """
+        mocker.patch("documents.views.AIConfig").return_value.ai_enabled = True
+        return mocker.patch(
+            "documents.views.stream_chat_with_documents",
+            return_value=iter(()),
+        )
+
+    @pytest.fixture
+    def viewer_client(self, user_client: APIClient, regular_user: User) -> APIClient:
+        """The conftest regular-user client, granted the global
+        view_document permission -- the minimum ViewDocumentsPermissions
+        needs to reach the view at all. Model-level only: says nothing
+        about which documents (if any) this user can actually see.
+        """
+        regular_user.user_permissions.add(
+            *Permission.objects.filter(codename="view_document"),
+        )
+        return user_client
+
+    @pytest.mark.parametrize(
+        ("client_fixture", "expected_unrestricted"),
+        [
+            pytest.param("admin_client", True, id="superuser_is_unrestricted"),
+            pytest.param("viewer_client", False, id="regular_user_is_restricted"),
+        ],
+    )
+    def test_unrestricted_only_for_superuser(
+        self,
+        request: pytest.FixtureRequest,
+        mocked_stream_chat: mock.MagicMock,
+        client_fixture: str,
+        *,
+        expected_unrestricted: bool,
+    ) -> None:
+        """
+        GIVEN:
+            - A superuser, or a regular user holding the global
+              view_document permission (but no object-level document access)
+        WHEN:
+            - They post a chat question with no document_id
+        THEN:
+            - stream_chat_with_documents is called with unrestricted=True
+              only for the superuser; the regular user is always
+              unrestricted=False, regardless of their permissions
+        """
+        client: APIClient = request.getfixturevalue(client_fixture)
+
+        client.post(
+            "/api/documents/chat/",
+            data={"q": "What's in these documents?"},
+            format="json",
+        )
+
+        assert (
+            mocked_stream_chat.call_args.kwargs["unrestricted"] is expected_unrestricted
+        )

src/documents/views.py

@@ -180,6 +180,7 @@ from documents.permissions import has_system_status_permission
 from documents.permissions import permitted_document_ids
 from documents.permissions import permitted_object_ids
 from documents.permissions import set_permissions_for_object
+from documents.permissions import user_is_unrestricted
 from documents.plugins.date_parsing import get_date_parser
 from documents.schema import generate_object_with_permissions_schema
 from documents.search import SearchHit
@@ -2329,10 +2330,12 @@ class ChatStreamingView(GenericAPIView[Any]):
                 return HttpResponseForbidden("Insufficient permissions")
 
             documents = Document.objects.filter(pk=document.pk)
+            unrestricted = False
         else:
             documents = Document.objects.filter(
                 id__in=permitted_document_ids(request.user),
             )
+            unrestricted = user_is_unrestricted(request.user)
 
         output_language = get_llm_output_language(
             ai_config=ai_config,
@@ -2343,6 +2346,7 @@ class ChatStreamingView(GenericAPIView[Any]):
             stream_chat_with_documents(
                 query_str=question,
                 documents=documents,
+                unrestricted=unrestricted,
                 output_language=output_language,
             ),
             content_type="text/event-stream",