Fix: stream AI chat natively async so a disconnect cancels the LLM request

The chat answer was a sync generator wrapped for ASGI, pulling each chunk
through sync_to_async(next). A client disconnect cancelled the response
task, but the worker thread waiting on the LLM could not be interrupted,
so the upstream request kept generating the whole answer for nobody.

Make stream_chat_with_documents an async generator. All the blocking setup
(ORM queries, the index read lock and its thread-bound sqlite connection,
the query embedding, building the LLM client) moves into one sync
_prepare_chat called once through sync_to_async, which also hands the DB
connection back before the answer streams. The answer then streams through
llama-index's asynthesize over the pinned async httpx clients, so a
disconnect cancels the LLM request directly. The view returns the async
generator as is, replacing the sync-to-async wrapper.

The async pinned transport resolved hostnames with a blocking getaddrinfo
on the event loop, which would stall the whole worker on slow DNS now that
chat uses it in the web server. Resolve in a thread instead, and add the
async counterparts of the existing transport tests.
This commit is contained in:
stumpylog committed 2026-11-17 10:04:56 -08:00
1 parent a690885a8a
commit 179307d57a
6 files changed
+188 -158

No files matched your search

+7 -74
View File
@@ -9,12 +9,11 @@ from rest_framework import status
from rest_framework.test import APIClient
from rest_framework.test import APITestCase
from documents.views import _iterate_in_sync_thread
from paperless_testing.factories import UserFactory
from paperless_testing.permissions import grant_global
if TYPE_CHECKING:
from collections.abc import Generator
from collections.abc import AsyncIterator
from django.contrib.auth.models import User
from pytest_mock import MockerFixture
@@ -59,6 +58,11 @@ class TestChatStreamingViewInputValidation(APITestCase):
reaches the Django request the middleware sees.
"""
chunks = [f"token{i} " for i in range(40)]
async def answer() -> AsyncIterator[str]:
for chunk in chunks:
yield chunk
with (
mock.patch(
"documents.views.AIConfig",
@@ -66,7 +70,7 @@ class TestChatStreamingViewInputValidation(APITestCase):
),
mock.patch(
"paperless_ai.chat.stream_chat_with_documents",
return_value=(c for c in chunks),
return_value=answer(),
),
):
resp = self.client.post(
@@ -163,74 +167,3 @@ class TestChatStreamingViewUnrestrictedFlag:
assert (
mocked_stream_chat.call_args.kwargs["unrestricted"] is expected_unrestricted
)
class TestIterateInSyncThread:
@pytest.fixture
def events(self) -> list[str]:
return []
@pytest.fixture
def chunks(self, events: list[str]) -> Generator[str, None, None]:
def generate() -> Generator[str, None, None]:
try:
for chunk in ("first", "second"):
events.append(f"produced {chunk}")
yield chunk
finally:
events.append("closed")
return generate()
def test_yields_each_chunk_before_pulling_the_next(
self,
chunks: Generator[str, None, None],
events: list[str],
) -> None:
"""
GIVEN:
- A sync generator wrapped for async iteration
WHEN:
- The wrapper is consumed
THEN:
- Each chunk is delivered before the next one is generated,
rather than the whole generator being drained up front
"""
async def consume() -> None:
async for chunk in _iterate_in_sync_thread(chunks):
events.append(f"received {chunk}")
async_to_sync(consume)()
assert events == [
"produced first",
"received first",
"produced second",
"received second",
"closed",
]
def test_closing_early_closes_the_source_generator(
self,
chunks: Generator[str, None, None],
events: list[str],
) -> None:
"""
GIVEN:
- A sync generator wrapped for async iteration
WHEN:
- The consumer closes the wrapper after the first chunk, as
happens when the client disconnects
THEN:
- The source generator is closed, so its cleanup runs
"""
async def consume_one() -> None:
wrapper = _iterate_in_sync_thread(chunks)
await anext(wrapper)
await wrapper.aclose()
async_to_sync(consume_one)()
assert events == ["produced first", "closed"]