Fix: Use FileResponse for file API responses (#12638)

* Updates code to use a FileResponse for streaming and unlink the file, but keep a handle to it

* Transitions the rest of the code to use FileResponse instead of a basic response, fixes up tests which assumed .content exists

* While here, let's add schema for it

src/documents/tests/test_api_bulk_download.py

@@ -15,6 +15,7 @@ from documents.models import Document
 from documents.models import DocumentType
 from documents.tests.utils import DirectoriesMixin
 from documents.tests.utils import SampleDirMixin
+from documents.tests.utils import read_streaming_response
 
 
 class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
@@ -68,7 +69,7 @@ class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(response["Content-Type"], "application/zip")
 
-        with zipfile.ZipFile(io.BytesIO(response.content)) as zipf:
+        with zipfile.ZipFile(io.BytesIO(read_streaming_response(response))) as zipf:
             self.assertEqual(len(zipf.filelist), 2)
             self.assertIn("2021-01-01 document A.pdf", zipf.namelist())
             self.assertIn("2020-03-21 document B.jpg", zipf.namelist())
@@ -89,7 +90,7 @@ class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(response["Content-Type"], "application/zip")
 
-        with zipfile.ZipFile(io.BytesIO(response.content)) as zipf:
+        with zipfile.ZipFile(io.BytesIO(read_streaming_response(response))) as zipf:
             self.assertEqual(len(zipf.filelist), 2)
             self.assertIn("2021-01-01 document A.pdf", zipf.namelist())
             self.assertIn("2020-03-21 document B.pdf", zipf.namelist())
@@ -110,7 +111,7 @@ class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(response["Content-Type"], "application/zip")
 
-        with zipfile.ZipFile(io.BytesIO(response.content)) as zipf:
+        with zipfile.ZipFile(io.BytesIO(read_streaming_response(response))) as zipf:
             self.assertEqual(len(zipf.filelist), 3)
             self.assertIn("originals/2021-01-01 document A.pdf", zipf.namelist())
             self.assertIn("archive/2020-03-21 document B.pdf", zipf.namelist())
@@ -144,7 +145,7 @@ class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(response["Content-Type"], "application/zip")
 
-        with zipfile.ZipFile(io.BytesIO(response.content)) as zipf:
+        with zipfile.ZipFile(io.BytesIO(read_streaming_response(response))) as zipf:
             self.assertEqual(len(zipf.filelist), 2)
 
             self.assertIn("2021-01-01 document A.pdf", zipf.namelist())
@@ -157,13 +158,14 @@ class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
                 self.assertEqual(f.read(), zipf.read("2021-01-01 document A_01.pdf"))
 
     def test_compression(self) -> None:
-        self.client.post(
+        response = self.client.post(
             self.ENDPOINT,
             json.dumps(
                 {"documents": [self.doc2.id, self.doc2b.id], "compression": "lzma"},
             ),
             content_type="application/json",
         )
+        response.close()
 
     @override_settings(FILENAME_FORMAT="{correspondent}/{title}")
     def test_formatted_download_originals(self) -> None:
@@ -203,7 +205,7 @@ class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(response["Content-Type"], "application/zip")
 
-        with zipfile.ZipFile(io.BytesIO(response.content)) as zipf:
+        with zipfile.ZipFile(io.BytesIO(read_streaming_response(response))) as zipf:
             self.assertEqual(len(zipf.filelist), 2)
             self.assertIn("a space name/Title 2 - Doc 3.jpg", zipf.namelist())
             self.assertIn("test/This is Doc 2.pdf", zipf.namelist())
@@ -249,7 +251,7 @@ class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(response["Content-Type"], "application/zip")
 
-        with zipfile.ZipFile(io.BytesIO(response.content)) as zipf:
+        with zipfile.ZipFile(io.BytesIO(read_streaming_response(response))) as zipf:
             self.assertEqual(len(zipf.filelist), 2)
             self.assertIn("somewhere/This is Doc 2.pdf", zipf.namelist())
             self.assertIn("somewhere/Title 2 - Doc 3.pdf", zipf.namelist())
@@ -298,7 +300,7 @@ class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(response["Content-Type"], "application/zip")
 
-        with zipfile.ZipFile(io.BytesIO(response.content)) as zipf:
+        with zipfile.ZipFile(io.BytesIO(read_streaming_response(response))) as zipf:
             self.assertEqual(len(zipf.filelist), 3)
             self.assertIn("originals/bill/This is Doc 2.pdf", zipf.namelist())
             self.assertIn("archive/statement/Title 2 - Doc 3.pdf", zipf.namelist())

src/documents/tests/test_api_document_versions.py

@@ -18,6 +18,7 @@ from documents.filters import EffectiveContentFilter
 from documents.filters import TitleContentFilter
 from documents.models import Document
 from documents.tests.utils import DirectoriesMixin
+from documents.tests.utils import read_streaming_response
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -449,19 +450,19 @@ class TestDocumentVersioningApi(DirectoriesMixin, APITestCase):
             f"/api/documents/{root.id}/download/?version={version.id}",
         )
         self.assertEqual(resp.status_code, status.HTTP_200_OK)
-        self.assertEqual(resp.content, b"version")
+        self.assertEqual(read_streaming_response(resp), b"version")
 
         resp = self.client.get(
             f"/api/documents/{root.id}/preview/?version={version.id}",
         )
         self.assertEqual(resp.status_code, status.HTTP_200_OK)
-        self.assertEqual(resp.content, b"version")
+        self.assertEqual(read_streaming_response(resp), b"version")
 
         resp = self.client.get(
             f"/api/documents/{root.id}/thumb/?version={version.id}",
         )
         self.assertEqual(resp.status_code, status.HTTP_200_OK)
-        self.assertEqual(resp.content, b"thumb")
+        self.assertEqual(read_streaming_response(resp), b"thumb")
 
     def test_metadata_version_param_uses_version(self) -> None:
         root = Document.objects.create(

src/documents/tests/test_api_documents.py

@@ -49,6 +49,7 @@ from documents.models import WorkflowTrigger
 from documents.signals.handlers import run_workflows
 from documents.tests.utils import ConsumeTaskMixin
 from documents.tests.utils import DirectoriesMixin
+from documents.tests.utils import read_streaming_response
 
 
 class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
@@ -323,19 +324,16 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
             f.write(content_thumbnail)
 
         response = self.client.get(f"/api/documents/{doc.pk}/download/")
-
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertEqual(response.content, content)
+        self.assertEqual(read_streaming_response(response), content)
 
         response = self.client.get(f"/api/documents/{doc.pk}/preview/")
-
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertEqual(response.content, content)
+        self.assertEqual(read_streaming_response(response), content)
 
         response = self.client.get(f"/api/documents/{doc.pk}/thumb/")
-
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertEqual(response.content, content_thumbnail)
+        self.assertEqual(read_streaming_response(response), content_thumbnail)
 
     def test_document_actions_with_perms(self) -> None:
         """
@@ -386,12 +384,15 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
 
         response = self.client.get(f"/api/documents/{doc.pk}/download/")
         self.assertEqual(response.status_code, status.HTTP_200_OK)
+        response.close()
 
         response = self.client.get(f"/api/documents/{doc.pk}/preview/")
         self.assertEqual(response.status_code, status.HTTP_200_OK)
+        response.close()
 
         response = self.client.get(f"/api/documents/{doc.pk}/thumb/")
         self.assertEqual(response.status_code, status.HTTP_200_OK)
+        response.close()
 
     @override_settings(FILENAME_FORMAT="")
     def test_download_with_archive(self) -> None:
@@ -412,28 +413,24 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
             f.write(content_archive)
 
         response = self.client.get(f"/api/documents/{doc.pk}/download/")
-
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertEqual(response.content, content_archive)
+        self.assertEqual(read_streaming_response(response), content_archive)
 
         response = self.client.get(
             f"/api/documents/{doc.pk}/download/?original=true",
         )
-
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertEqual(response.content, content)
+        self.assertEqual(read_streaming_response(response), content)
 
         response = self.client.get(f"/api/documents/{doc.pk}/preview/")
-
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertEqual(response.content, content_archive)
+        self.assertEqual(read_streaming_response(response), content_archive)
 
         response = self.client.get(
             f"/api/documents/{doc.pk}/preview/?original=true",
         )
-
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertEqual(response.content, content)
+        self.assertEqual(read_streaming_response(response), content)
 
     @override_settings(FILENAME_FORMAT="")
     def test_download_follow_formatting(self) -> None:
@@ -456,18 +453,21 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
         # Without follow_formatting, should use public filename
         response = self.client.get(f"/api/documents/{doc.pk}/download/")
         self.assertIn("none.pdf", response["Content-Disposition"])
+        response.close()
 
         # With follow_formatting, should use actual filename on disk
         response = self.client.get(
             f"/api/documents/{doc.pk}/download/?follow_formatting=true",
         )
         self.assertIn("archived.pdf", response["Content-Disposition"])
+        response.close()
 
         # With follow_formatting and original, should use source filename
         response = self.client.get(
             f"/api/documents/{doc.pk}/download/?original=true&follow_formatting=true",
         )
         self.assertIn("my_document.pdf", response["Content-Disposition"])
+        response.close()
 
     def test_document_actions_not_existing_file(self) -> None:
         doc = Document.objects.create(

src/documents/tests/test_api_schema.py

@@ -253,3 +253,47 @@ class TestShareLinkBundleRebuildSchema:
         else:
             props = resp_400.get("properties", {})
         assert "detail" in props, "rebuild 400 response must have a 'detail' field"
+
+
+class TestBulkDownloadSchema:
+    """bulk_download_create: POST accepts BulkDownloadSerializer, returns application/zip, documents 403."""
+
+    def test_bulk_download_path_exists(self, api_schema: SchemaGenerator) -> None:
+        assert "/api/documents/bulk_download/" in api_schema["paths"]
+
+    def test_bulk_download_operation_id(self, api_schema: SchemaGenerator) -> None:
+        op = api_schema["paths"]["/api/documents/bulk_download/"]["post"]
+        assert op["operationId"] == "bulk_download"
+
+    def test_bulk_download_request_body_is_json(
+        self,
+        api_schema: SchemaGenerator,
+    ) -> None:
+        op = api_schema["paths"]["/api/documents/bulk_download/"]["post"]
+        assert "requestBody" in op
+        assert "application/json" in op["requestBody"]["content"]
+
+    def test_bulk_download_request_references_serializer(
+        self,
+        api_schema: SchemaGenerator,
+    ) -> None:
+        op = api_schema["paths"]["/api/documents/bulk_download/"]["post"]
+        schema_ref = (
+            op["requestBody"]["content"]["application/json"]
+            .get("schema", {})
+            .get("$ref", "")
+        )
+        component_name = schema_ref.split("/")[-1]
+        assert component_name == "BulkDownloadRequest"
+
+    def test_bulk_download_response_200_is_zip(
+        self,
+        api_schema: SchemaGenerator,
+    ) -> None:
+        op = api_schema["paths"]["/api/documents/bulk_download/"]["post"]
+        assert "200" in op["responses"]
+        assert "application/zip" in op["responses"]["200"]["content"]
+
+    def test_bulk_download_response_403(self, api_schema: SchemaGenerator) -> None:
+        op = api_schema["paths"]["/api/documents/bulk_download/"]["post"]
+        assert "403" in op["responses"]

src/documents/tests/test_views.py

@@ -27,6 +27,7 @@ from documents.models import StoragePath
 from documents.models import Tag
 from documents.signals.handlers import update_llm_suggestions_cache
 from documents.tests.utils import DirectoriesMixin
+from documents.tests.utils import read_streaming_response
 from paperless.models import ApplicationConfiguration
 
 
@@ -157,7 +158,7 @@ class TestViews(DirectoriesMixin, TestCase):
         # Valid
         response = self.client.get(f"/share/{sl1.slug}")
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertEqual(response.content, content)
+        self.assertEqual(read_streaming_response(response), content)
 
         # Invalid
         response = self.client.get("/share/123notaslug", follow=True)

src/documents/tests/utils.py

@@ -17,6 +17,7 @@ import pytest
 from django.apps import apps
 from django.db import connection
 from django.db.migrations.executor import MigrationExecutor
+from django.http import StreamingHttpResponse
 from django.test import TransactionTestCase
 from django.test import override_settings
 
@@ -150,6 +151,13 @@ def util_call_with_backoff(
     return succeeded, result
 
 
+def read_streaming_response(response: StreamingHttpResponse) -> bytes:
+    """Consume a StreamingHttpResponse/FileResponse and close it."""
+    content = b"".join(response.streaming_content)
+    response.close()
+    return content
+
+
 class DirectoriesMixin:
     """
     Creates and overrides settings for all folders and paths, then ensures