From a690885a8a300920aad30132bd0585b0d8114949 Mon Sep 17 00:00:00 2001 From: stumpylog <797416+stumpylog@users.noreply.github.com> Date: Tue, 17 Nov 2026 07:36:50 -0800 Subject: [PATCH] Fix: stream AI chat answers under ASGI --- src/documents/tests/test_api_chat.py | 83 +++++++++++++++++++++++++++- src/documents/views.py | 36 +++++++++--- 2 files changed, 110 insertions(+), 9 deletions(-) diff --git a/src/documents/tests/test_api_chat.py b/src/documents/tests/test_api_chat.py index 11703dfd98..d8521d387e 100644 --- a/src/documents/tests/test_api_chat.py +++ b/src/documents/tests/test_api_chat.py @@ -4,14 +4,18 @@ from typing import TYPE_CHECKING from unittest import mock import pytest +from asgiref.sync import async_to_sync from rest_framework import status from rest_framework.test import APIClient from rest_framework.test import APITestCase +from documents.views import _iterate_in_sync_thread from paperless_testing.factories import UserFactory from paperless_testing.permissions import grant_global if TYPE_CHECKING: + from collections.abc import Generator + from django.contrib.auth.models import User from pytest_mock import MockerFixture @@ -62,7 +66,7 @@ class TestChatStreamingViewInputValidation(APITestCase): ), mock.patch( "paperless_ai.chat.stream_chat_with_documents", - return_value=iter(chunks), + return_value=(c for c in chunks), ), ): resp = self.client.post( @@ -74,7 +78,11 @@ class TestChatStreamingViewInputValidation(APITestCase): assert resp.status_code == status.HTTP_200_OK assert not resp.has_header("Content-Encoding") - assert list(resp.streaming_content) == [c.encode() for c in chunks] + + async def read_parts() -> list[bytes]: + return [part async for part in resp.streaming_content] + + assert async_to_sync(read_parts)() == [c.encode() for c in chunks] def test_missing_question_is_rejected(self) -> None: with mock.patch( @@ -155,3 +163,74 @@ class TestChatStreamingViewUnrestrictedFlag: assert ( mocked_stream_chat.call_args.kwargs["unrestricted"] is expected_unrestricted ) + + +class TestIterateInSyncThread: + @pytest.fixture + def events(self) -> list[str]: + return [] + + @pytest.fixture + def chunks(self, events: list[str]) -> Generator[str, None, None]: + def generate() -> Generator[str, None, None]: + try: + for chunk in ("first", "second"): + events.append(f"produced {chunk}") + yield chunk + finally: + events.append("closed") + + return generate() + + def test_yields_each_chunk_before_pulling_the_next( + self, + chunks: Generator[str, None, None], + events: list[str], + ) -> None: + """ + GIVEN: + - A sync generator wrapped for async iteration + WHEN: + - The wrapper is consumed + THEN: + - Each chunk is delivered before the next one is generated, + rather than the whole generator being drained up front + """ + + async def consume() -> None: + async for chunk in _iterate_in_sync_thread(chunks): + events.append(f"received {chunk}") + + async_to_sync(consume)() + + assert events == [ + "produced first", + "received first", + "produced second", + "received second", + "closed", + ] + + def test_closing_early_closes_the_source_generator( + self, + chunks: Generator[str, None, None], + events: list[str], + ) -> None: + """ + GIVEN: + - A sync generator wrapped for async iteration + WHEN: + - The consumer closes the wrapper after the first chunk, as + happens when the client disconnects + THEN: + - The source generator is closed, so its cleanup runs + """ + + async def consume_one() -> None: + wrapper = _iterate_in_sync_thread(chunks) + await anext(wrapper) + await wrapper.aclose() + + async_to_sync(consume_one)() + + assert events == ["produced first", "closed"] diff --git a/src/documents/views.py b/src/documents/views.py index 042b8e2c47..3cf9a9caf6 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -7,7 +7,9 @@ import tempfile import zipfile from collections import Counter from collections import deque +from collections.abc import AsyncIterator from collections.abc import Callable +from collections.abc import Generator from datetime import datetime from datetime import timedelta from http import HTTPStatus @@ -27,6 +29,7 @@ from urllib.parse import urlparse import httpx import magic import pathvalidate +from asgiref.sync import sync_to_async from django.conf import settings from django.contrib.auth.models import Group from django.contrib.auth.models import User @@ -2416,6 +2419,24 @@ class DocumentViewSet( ) +async def _iterate_in_sync_thread( + chunks: Generator[str, None, None], +) -> AsyncIterator[str]: + """Serve a blocking sync generator to an ASGI server chunk by chunk. + + Under ASGI, Django drains a sync iterator with ``sync_to_async(list)`` + before sending anything. Pull each chunk in the request's sync thread + instead, where the generator's ORM access belongs. + """ + sentinel = object() # StopIteration can't cross an asyncio Future + next_chunk = sync_to_async(next) + try: + while (chunk := await next_chunk(chunks, sentinel)) is not sentinel: + yield chunk + finally: + await sync_to_async(chunks.close)() + + class ChatStreamingSerializer(serializers.Serializer[dict[str, Any]]): q = serializers.CharField(required=True, max_length=4000) document_id = serializers.IntegerField(required=False, allow_null=True) @@ -2474,16 +2495,17 @@ class ChatStreamingView(GenericAPIView[Any]): user=request.user, ) - response = StreamingHttpResponse( - stream_chat_with_documents( - query_str=question, - documents=documents, - unrestricted=unrestricted, - output_language=output_language, + return StreamingHttpResponse( + _iterate_in_sync_thread( + stream_chat_with_documents( + query_str=question, + documents=documents, + unrestricted=unrestricted, + output_language=output_language, + ), ), content_type="text/event-stream", ) - return response @extend_schema_view(