diff --git a/src/documents/tests/test_api_chat.py b/src/documents/tests/test_api_chat.py index d8521d387e..abcc965647 100644 --- a/src/documents/tests/test_api_chat.py +++ b/src/documents/tests/test_api_chat.py @@ -9,12 +9,11 @@ from rest_framework import status from rest_framework.test import APIClient from rest_framework.test import APITestCase -from documents.views import _iterate_in_sync_thread from paperless_testing.factories import UserFactory from paperless_testing.permissions import grant_global if TYPE_CHECKING: - from collections.abc import Generator + from collections.abc import AsyncIterator from django.contrib.auth.models import User from pytest_mock import MockerFixture @@ -59,6 +58,11 @@ class TestChatStreamingViewInputValidation(APITestCase): reaches the Django request the middleware sees. """ chunks = [f"token{i} " for i in range(40)] + + async def answer() -> AsyncIterator[str]: + for chunk in chunks: + yield chunk + with ( mock.patch( "documents.views.AIConfig", @@ -66,7 +70,7 @@ class TestChatStreamingViewInputValidation(APITestCase): ), mock.patch( "paperless_ai.chat.stream_chat_with_documents", - return_value=(c for c in chunks), + return_value=answer(), ), ): resp = self.client.post( @@ -163,74 +167,3 @@ class TestChatStreamingViewUnrestrictedFlag: assert ( mocked_stream_chat.call_args.kwargs["unrestricted"] is expected_unrestricted ) - - -class TestIterateInSyncThread: - @pytest.fixture - def events(self) -> list[str]: - return [] - - @pytest.fixture - def chunks(self, events: list[str]) -> Generator[str, None, None]: - def generate() -> Generator[str, None, None]: - try: - for chunk in ("first", "second"): - events.append(f"produced {chunk}") - yield chunk - finally: - events.append("closed") - - return generate() - - def test_yields_each_chunk_before_pulling_the_next( - self, - chunks: Generator[str, None, None], - events: list[str], - ) -> None: - """ - GIVEN: - - A sync generator wrapped for async iteration - WHEN: - - The wrapper is consumed - THEN: - - Each chunk is delivered before the next one is generated, - rather than the whole generator being drained up front - """ - - async def consume() -> None: - async for chunk in _iterate_in_sync_thread(chunks): - events.append(f"received {chunk}") - - async_to_sync(consume)() - - assert events == [ - "produced first", - "received first", - "produced second", - "received second", - "closed", - ] - - def test_closing_early_closes_the_source_generator( - self, - chunks: Generator[str, None, None], - events: list[str], - ) -> None: - """ - GIVEN: - - A sync generator wrapped for async iteration - WHEN: - - The consumer closes the wrapper after the first chunk, as - happens when the client disconnects - THEN: - - The source generator is closed, so its cleanup runs - """ - - async def consume_one() -> None: - wrapper = _iterate_in_sync_thread(chunks) - await anext(wrapper) - await wrapper.aclose() - - async_to_sync(consume_one)() - - assert events == ["produced first", "closed"] diff --git a/src/documents/views.py b/src/documents/views.py index 3cf9a9caf6..85763e95b1 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -7,9 +7,7 @@ import tempfile import zipfile from collections import Counter from collections import deque -from collections.abc import AsyncIterator from collections.abc import Callable -from collections.abc import Generator from datetime import datetime from datetime import timedelta from http import HTTPStatus @@ -29,7 +27,6 @@ from urllib.parse import urlparse import httpx import magic import pathvalidate -from asgiref.sync import sync_to_async from django.conf import settings from django.contrib.auth.models import Group from django.contrib.auth.models import User @@ -2419,24 +2416,6 @@ class DocumentViewSet( ) -async def _iterate_in_sync_thread( - chunks: Generator[str, None, None], -) -> AsyncIterator[str]: - """Serve a blocking sync generator to an ASGI server chunk by chunk. - - Under ASGI, Django drains a sync iterator with ``sync_to_async(list)`` - before sending anything. Pull each chunk in the request's sync thread - instead, where the generator's ORM access belongs. - """ - sentinel = object() # StopIteration can't cross an asyncio Future - next_chunk = sync_to_async(next) - try: - while (chunk := await next_chunk(chunks, sentinel)) is not sentinel: - yield chunk - finally: - await sync_to_async(chunks.close)() - - class ChatStreamingSerializer(serializers.Serializer[dict[str, Any]]): q = serializers.CharField(required=True, max_length=4000) document_id = serializers.IntegerField(required=False, allow_null=True) @@ -2496,13 +2475,11 @@ class ChatStreamingView(GenericAPIView[Any]): ) return StreamingHttpResponse( - _iterate_in_sync_thread( - stream_chat_with_documents( - query_str=question, - documents=documents, - unrestricted=unrestricted, - output_language=output_language, - ), + stream_chat_with_documents( + query_str=question, + documents=documents, + unrestricted=unrestricted, + output_language=output_language, ), content_type="text/event-stream", ) diff --git a/src/paperless/network.py b/src/paperless/network.py index b7006bf5af..018994c62f 100644 --- a/src/paperless/network.py +++ b/src/paperless/network.py @@ -1,3 +1,4 @@ +import asyncio import ipaddress import socket from collections.abc import Collection @@ -175,7 +176,9 @@ class PinnedHostAsyncHTTPTransport(httpx.AsyncHTTPTransport): self.allow_internal = allow_internal async def handle_async_request(self, request: httpx.Request) -> httpx.Response: - request = _rewrite_request_to_pinned_ip( + # Resolving the host blocks, so keep it off the event loop + request = await asyncio.to_thread( + _rewrite_request_to_pinned_ip, request, allow_internal=self.allow_internal, ) diff --git a/src/paperless/tests/test_network.py b/src/paperless/tests/test_network.py index 3fca931fe4..a88260aa56 100644 --- a/src/paperless/tests/test_network.py +++ b/src/paperless/tests/test_network.py @@ -1,11 +1,19 @@ +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING from unittest import mock import httpx import pytest +from paperless.network import PinnedHostAsyncHTTPTransport from paperless.network import PinnedHostHTTPTransport from paperless.network import is_public_ip +if TYPE_CHECKING: + from pytest_mock import MockerFixture + def test_pinned_host_transport_blocks_internal_rebinding(): transport = PinnedHostHTTPTransport(allow_internal=False) @@ -51,6 +59,58 @@ def test_pinned_host_transport_rewrites_to_vetted_ip(): assert response.status_code == 200 +class TestPinnedHostAsyncHTTPTransport: + def test_blocks_internal_rebinding(self, mocker: MockerFixture) -> None: + """ + GIVEN: + - A hostname that resolves to a loopback address + WHEN: + - The async transport sends a request to it + THEN: + - The request is refused as a non-public address + """ + mocker.patch( + "paperless.network.resolve_hostname_ips", + return_value=["127.0.0.1"], + ) + transport = PinnedHostAsyncHTTPTransport(allow_internal=False) + request = httpx.Request("GET", "http://example.com/test") + + with pytest.raises(httpx.ConnectError, match="non-public address"): + asyncio.run(transport.handle_async_request(request)) + + def test_rewrites_to_vetted_ip(self, mocker: MockerFixture) -> None: + """ + GIVEN: + - A hostname that resolves to a public address + WHEN: + - The async transport sends a request to it + THEN: + - The connection goes to that address, keeping the original Host + header and TLS SNI hostname + """ + mocker.patch( + "paperless.network.resolve_hostname_ips", + return_value=["93.184.216.34"], + ) + sent = mocker.patch.object( + httpx.AsyncHTTPTransport, + "handle_async_request", + autospec=True, + side_effect=lambda _self, req: httpx.Response(200, request=req), + ) + transport = PinnedHostAsyncHTTPTransport(allow_internal=False) + request = httpx.Request("GET", "https://example.com:8443/test") + + response = asyncio.run(transport.handle_async_request(request)) + + assert response.status_code == 200 + rewritten = sent.call_args.args[1] + assert str(rewritten.url) == "https://93.184.216.34:8443/test" + assert rewritten.headers["Host"] == "example.com:8443" + assert rewritten.extensions["sni_hostname"] == "example.com" + + @pytest.mark.parametrize( "address", [ diff --git a/src/paperless_ai/chat.py b/src/paperless_ai/chat.py index bf1c720bc1..edcf969a50 100644 --- a/src/paperless_ai/chat.py +++ b/src/paperless_ai/chat.py @@ -1,7 +1,10 @@ import json import logging -import sys +from collections.abc import AsyncIterator +from typing import TYPE_CHECKING +from asgiref.sync import sync_to_async +from django.db import connections from django.db.models import QuerySet from documents.models import Document @@ -17,6 +20,10 @@ from paperless_ai.prompts.context import ChatQaPromptContext from paperless_ai.prompts.context import ChatRefinePromptContext from paperless_ai.prompts.render import render_prompt +if TYPE_CHECKING: + from llama_index.core.response_synthesizers import BaseSynthesizer + from llama_index.core.schema import NodeWithScore + logger = logging.getLogger("paperless_ai.chat") CHAT_METADATA_DELIMITER = "\n\n__PAPERLESS_CHAT_METADATA__" @@ -103,35 +110,67 @@ def _format_chat_metadata_trailer(references: list[dict[str, int | str]]) -> str ) -def stream_chat_with_documents( +async def stream_chat_with_documents( query_str: str, documents: QuerySet[Document], *, unrestricted: bool = False, output_language: str | None = None, -): +) -> AsyncIterator[str]: try: - yield from _stream_chat_with_documents( + async for chunk in _stream_chat_with_documents( query_str, documents, unrestricted=unrestricted, output_language=output_language, - ) + ): + yield chunk except Exception as e: logger.exception("Failed to stream document chat response: %s", e) yield CHAT_ERROR_MESSAGE -def _stream_chat_with_documents( +async def _stream_chat_with_documents( query_str: str, documents: QuerySet[Document], *, unrestricted: bool = False, output_language: str | None = None, -): - if not documents.exists(): +) -> AsyncIterator[str]: + prepared = await sync_to_async(_prepare_chat)( + query_str, + documents, + unrestricted=unrestricted, + output_language=output_language, + ) + if prepared is None: yield CHAT_NO_CONTENT_MESSAGE return + response_synthesizer, top_nodes, references = prepared + + logger.debug("Document chat query: %s", query_str) + response_stream = await response_synthesizer.asynthesize(query_str, top_nodes) + async for chunk in response_stream.async_response_gen(): + yield chunk + + if references: + yield _format_chat_metadata_trailer(references) + + +def _prepare_chat( + query_str: str, + documents: QuerySet[Document], + *, + unrestricted: bool, + output_language: str | None, +) -> tuple["BaseSynthesizer", list["NodeWithScore"], list[dict[str, int | str]]] | None: + """Everything before the answer streams, all of it blocking: ORM queries, + the index read lock and its thread-bound sqlite connection, the query + embedding, and building the LLM client. Returns None when there is no + content to answer from. + """ + if not documents.exists(): + return None from llama_index.core.prompts import PromptTemplate from llama_index.core.response_synthesizers import get_response_synthesizer @@ -150,8 +189,6 @@ def _stream_chat_with_documents( # Hold the shared read lock only while reading the index, so the compaction # swap cannot replace the database file while this connection is open. - # Everything after retrieval works on the nodes already in memory, so a slow - # LLM stream never holds the lock. with read_store() as store: if permitted_ids is not None: filters = _permitted_documents_filter(store, permitted_ids) @@ -179,35 +216,26 @@ def _stream_chat_with_documents( if not top_nodes: logger.warning("No nodes found for the given documents.") - yield CHAT_NO_CONTENT_MESSAGE - return - - client = AIClient() + return None references = _get_document_references(documents, top_nodes) - prompt_template = PromptTemplate(template=_build_chat_prompt(output_language)) - refine_template = PromptTemplate(template=_build_refine_prompt(output_language)) + client = AIClient() response_synthesizer = get_response_synthesizer( llm=client.llm, prompt_helper=get_rag_prompt_helper( chunk_size=config.llm_embedding_chunk_size, context_size=config.llm_context_size, ), - text_qa_template=prompt_template, - refine_template=refine_template, + text_qa_template=PromptTemplate(template=_build_chat_prompt(output_language)), + refine_template=PromptTemplate( + template=_build_refine_prompt(output_language), + ), streaming=True, ) - logger.debug("Document chat query: %s", query_str) - # Release the pooled DB connection for the slow streaming LLM response - # so it is not pinned for the whole stream; see paperless_ai.db and - # #12976. - with db_connection_released(): - response_stream = response_synthesizer.synthesize(query_str, top_nodes) - for chunk in response_stream.response_gen: - yield chunk - sys.stdout.flush() + # No ORM access happens while the answer streams, so hand the pooled DB + # connection back now rather than pin it for the whole stream. See #12976. + connections.close_all() - if references: - yield _format_chat_metadata_trailer(references) + return response_synthesizer, top_nodes, references diff --git a/src/paperless_ai/tests/test_chat.py b/src/paperless_ai/tests/test_chat.py index 358dec8d22..7ae034ba65 100644 --- a/src/paperless_ai/tests/test_chat.py +++ b/src/paperless_ai/tests/test_chat.py @@ -4,10 +4,12 @@ import json from contextlib import contextmanager from typing import TYPE_CHECKING from typing import Any +from unittest.mock import AsyncMock from unittest.mock import MagicMock from unittest.mock import patch import pytest +from asgiref.sync import async_to_sync from django.db.models.signals import post_init from django.utils import timezone from llama_index.core import settings as llama_settings @@ -26,6 +28,7 @@ from paperless_ai.chat import stream_chat_with_documents from paperless_testing.factories import DocumentFactory if TYPE_CHECKING: + from collections.abc import AsyncIterator from collections.abc import Iterator from pathlib import Path @@ -66,6 +69,26 @@ def assert_chat_output( } +def collect(chunks: AsyncIterator[str]) -> list[str]: + """Drain the chat stream from sync test code. Its blocking setup then runs + on the test's own thread, inside the test's DB transaction.""" + + async def drain() -> list[str]: + return [chunk async for chunk in chunks] + + return async_to_sync(drain)() + + +def fake_response_stream(chunks: list[str]) -> MagicMock: + """An AsyncStreamingResponse stand-in yielding ``chunks``.""" + + async def response_gen() -> AsyncIterator[str]: + for chunk in chunks: + yield chunk + + return MagicMock(async_response_gen=response_gen) + + def _fake_documents_queryset(pks: list[int]) -> MagicMock: qs = MagicMock() qs.exists.return_value = bool(pks) @@ -170,19 +193,19 @@ def test_stream_chat_with_one_document_retrieval( ), ] - mock_response_stream = MagicMock() - mock_response_stream.response_gen = iter(["chunk1", "chunk2"]) mock_synthesizer = mock_get_response_synthesizer.return_value - mock_synthesizer.synthesize.return_value = mock_response_stream + mock_synthesizer.asynthesize = AsyncMock( + return_value=fake_response_stream(["chunk1", "chunk2"]), + ) with patch( "llama_index.core.retrievers.VectorIndexRetriever", return_value=mock_retriever_instance, ): - output = list(stream_chat_with_documents("What is this?", documents)) + output = collect(stream_chat_with_documents("What is this?", documents)) mock_retriever_instance.retrieve.assert_called_once_with("What is this?") - mock_synthesizer.synthesize.assert_called_once_with( + mock_synthesizer.asynthesize.assert_awaited_once_with( "What is this?", mock_retriever_instance.retrieve.return_value, ) @@ -236,20 +259,19 @@ def test_stream_chat_with_multiple_documents_retrieval(patch_embed_nodes) -> Non MagicMock(metadata={"document_id": str(doc2.pk), "title": "Document 2"}), ] - mock_response_stream = MagicMock() - mock_response_stream.response_gen = iter(["chunk1", "chunk2"]) - mock_synthesizer = mock_get_response_synthesizer.return_value - mock_synthesizer.synthesize.return_value = mock_response_stream + mock_synthesizer.asynthesize = AsyncMock( + return_value=fake_response_stream(["chunk1", "chunk2"]), + ) with patch( "llama_index.core.retrievers.VectorIndexRetriever", return_value=mock_retriever_instance, ): - output = list(stream_chat_with_documents("What's up?", documents)) + output = collect(stream_chat_with_documents("What's up?", documents)) mock_retriever_instance.retrieve.assert_called_once_with("What's up?") - mock_synthesizer.synthesize.assert_called_once_with( + mock_synthesizer.asynthesize.assert_awaited_once_with( "What's up?", mock_retriever_instance.retrieve.return_value, ) @@ -266,7 +288,9 @@ def test_stream_chat_with_multiple_documents_retrieval(patch_embed_nodes) -> Non def test_stream_chat_empty_document_list() -> None: with patch("paperless_ai.chat.load_or_build_index") as mock_load_index: - output = list(stream_chat_with_documents("Any info?", Document.objects.none())) + output = collect( + stream_chat_with_documents("Any info?", Document.objects.none()), + ) mock_load_index.assert_not_called() assert output == ["Sorry, I couldn't find any content to answer your question."] @@ -286,7 +310,7 @@ def test_stream_chat_no_matching_nodes() -> None: mock_index.vector_store.get_nodes.return_value = [] mock_load_index.return_value = mock_index - output = list( + output = collect( stream_chat_with_documents("Any info?", _fake_documents_queryset([1])), ) @@ -317,7 +341,7 @@ def test_stream_chat_unexpected_failure_returns_generic_error(caplog) -> None: ) mock_retriever_cls.return_value = mock_retriever - output = list( + output = collect( stream_chat_with_documents("Any info?", _fake_documents_queryset([1])), ) @@ -367,7 +391,7 @@ class TestStreamChatRetrieval: ) -> None: doc = DocumentFactory.create(content="hello world") # Nothing indexed for this document yet. - out = list( + out = collect( chat.stream_chat_with_documents( "question?", Document.objects.filter(pk=doc.pk), @@ -390,7 +414,7 @@ class TestStreamChatRetrieval: indexing.llm_index_add_or_update_document(included) indexing.llm_index_add_or_update_document(excluded) - list( + collect( chat.stream_chat_with_documents( "question?", Document.objects.filter(pk=included.pk), @@ -419,7 +443,7 @@ class TestStreamChatRetrieval: document = DocumentFactory.create(content="indexed document content") indexing.llm_index_add_or_update_document(document) - list( + collect( chat.stream_chat_with_documents( "question?", Document.objects.filter(pk=document.pk), @@ -455,7 +479,7 @@ class TestStreamChatRetrieval: deleted_at=timezone.now(), ) - list( + collect( chat.stream_chat_with_documents( "question?", Document.objects.filter(pk=kept.pk), @@ -487,7 +511,7 @@ class TestStreamChatRetrieval: indexing.llm_index_add_or_update_document(doc) permitted = Document.objects.filter(pk__in=[docs[0].pk, docs[1].pk]) - list(chat.stream_chat_with_documents("question?", permitted)) + collect(chat.stream_chat_with_documents("question?", permitted)) metadata_filter = captured_filters[0].filters[0] assert metadata_filter.operator == FilterOperator.NIN @@ -520,16 +544,21 @@ class TestStreamChatRetrieval: synthesizer = mocker.patch( "llama_index.core.response_synthesizers.get_response_synthesizer", ).return_value - synthesizer.synthesize.return_value.response_gen = iter(["chunk"]) + synthesizer.asynthesize = AsyncMock( + return_value=fake_response_stream(["chunk"]), + ) - list( + collect( chat.stream_chat_with_documents( "question?", Document.objects.filter(pk=permitted_doc.pk), ), ) - synthesizer.synthesize.assert_called_once_with("question?", [permitted_node]) + synthesizer.asynthesize.assert_awaited_once_with( + "question?", + [permitted_node], + ) @pytest.mark.django_db def test_get_document_references_only_queries_referenced_documents( @@ -612,9 +641,9 @@ class TestChatReadLock: ] ) - def synthesize(*_args: object) -> object: + def asynthesize(*_args: object) -> MagicMock: events.append("synthesize") - return mocker.MagicMock(response_gen=iter(["chunk"])) + return fake_response_stream(["chunk"]) mocker.patch("paperless_ai.chat.read_store", fake_read_store) mocker.patch("paperless_ai.chat.AIClient") @@ -625,9 +654,9 @@ class TestChatReadLock: ) mocker.patch( "llama_index.core.response_synthesizers.get_response_synthesizer", - ).return_value.synthesize.side_effect = synthesize + ).return_value.asynthesize = AsyncMock(side_effect=asynthesize) - list( + collect( stream_chat_with_documents( "What is this?", Document.objects.filter(pk=document.pk),