Fix: pass document chat queries as a QuerySet instead of a materialized list

In tracemalloc based profiling, not materializing the whole Document list
reduced memory to approximately 20% of the baseline, with a peak memory
that scaled with the library size.  Now, the lazt queryset is used and only
the needed pk value is actually contributing to memory

docs/usage.md

@@ -1084,8 +1084,7 @@ Paperless-ngx supports performing OCR on documents using remote services. At the
 [Microsoft's Azure "Document Intelligence" service](https://azure.microsoft.com/en-us/products/ai-services/ai-document-intelligence).
 This is of course a paid service (with a free tier) which requires an Azure account and subscription. Azure AI is not affiliated with
 Paperless-ngx in any way. When enabled, Paperless-ngx will automatically send appropriate documents to Azure for OCR processing, bypassing
-the local OCR engine. See the [configuration](configuration.md#PAPERLESS_REMOTE_OCR_ENGINE) options for more details. These
+the local OCR engine. See the [configuration](configuration.md#PAPERLESS_REMOTE_OCR_ENGINE) options for more details.
-settings can be supplied as environment variables or via **Application Configuration**.
 
 Additionally, when using a commercial service with this feature, consider both potential costs as well as any associated file size
 or page limitations (e.g. with a free tier).

src-ui/messages.xlf

@@ -1703,7 +1703,7 @@
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/suggestions-dropdown/suggestions-dropdown.component.html</context>
-          <context context-type="linenumber">34</context>
+          <context context-type="linenumber">28</context>
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.html</context>
@@ -3279,7 +3279,7 @@
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/suggestions-dropdown/suggestions-dropdown.component.html</context>
-          <context context-type="linenumber">46</context>
+          <context context-type="linenumber">40</context>
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.html</context>
@@ -7070,39 +7070,32 @@
           <context context-type="linenumber">143</context>
         </context-group>
       </trans-unit>
-      <trans-unit id="8336346011691074629" datatype="html">
-        <source>No suggestions</source>
-        <context-group purpose="location">
-          <context context-type="sourcefile">src/app/components/common/suggestions-dropdown/suggestions-dropdown.component.html</context>
-          <context context-type="linenumber">11,12</context>
-        </context-group>
-      </trans-unit>
       <trans-unit id="5320136382998259826" datatype="html">
         <source>Suggest</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/suggestions-dropdown/suggestions-dropdown.component.html</context>
-          <context context-type="linenumber">13,14</context>
+          <context context-type="linenumber">8,9</context>
         </context-group>
       </trans-unit>
       <trans-unit id="6934085657687954669" datatype="html">
         <source>Show suggestions</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/suggestions-dropdown/suggestions-dropdown.component.html</context>
-          <context context-type="linenumber">23,24</context>
+          <context context-type="linenumber">17,18</context>
         </context-group>
       </trans-unit>
       <trans-unit id="3834115140127576673" datatype="html">
         <source>No novel suggestions</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/suggestions-dropdown/suggestions-dropdown.component.html</context>
-          <context context-type="linenumber">30,31</context>
+          <context context-type="linenumber">24,25</context>
         </context-group>
       </trans-unit>
       <trans-unit id="4369111787961525769" datatype="html">
         <source>Document Types</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/suggestions-dropdown/suggestions-dropdown.component.html</context>
-          <context context-type="linenumber">40</context>
+          <context context-type="linenumber">34</context>
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/dashboard/widgets/statistics-widget/statistics-widget.component.html</context>

src-ui/package.json

@@ -66,5 +66,5 @@
     "ts-node": "~10.9.1",
     "typescript": "^6.0.3"
   },
-  "packageManager": "pnpm@11.15.1"
+  "packageManager": "pnpm@10.26.0"
 }

src-ui/pnpm-workspace.yaml

@@ -5,7 +5,6 @@ trustPolicy: no-downgrade
 trustPolicyExclude:
   - "chokidar@4.0.3"
   - "semver@6.3.1 || 5.7.2"
-blockExoticSubdeps: true
 allowBuilds:
   "@parcel/watcher": true
   canvas: true

src-ui/src/app/components/admin/config/config.component.html

@@ -14,48 +14,43 @@
                 <a ngbNavLink>{{category}}</a>
                 <ng-template ngbNavContent>
                     <div class="p-3">
-                        @for (section of getCategorySections(category); track section) {
+                        <div class="row row-cols-1 row-cols-md-2 row-cols-lg-3 g-2">
-                            @if (section) {
+                            @for (option of getCategoryOptions(category); track option.key) {
-                                <h5 class="mt-4 mb-3">{{section}}</h5>
+                                <div class="col">
-                            }
+                                    <div class="card bg-light">
-                            <div class="row row-cols-1 row-cols-md-2 row-cols-lg-3 g-2">
+                                        <div class="card-body">
-                                @for (option of getCategoryOptions(category, section); track option.key) {
+                                            <div class="card-title d-flex align-items-center">
-                                    <div class="col">
+                                                <h6 class="mb-0">
-                                        <div class="card bg-light">
+                                                  {{option.title}}
-                                            <div class="card-body">
+                                                </h6>
-                                                <div class="card-title d-flex align-items-center">
+                                                <a class="btn btn-sm btn-link" title="Read the documentation about this setting" i18n-title [href]="getDocsUrl(option.config_key)" target="_blank" referrerpolicy="no-referrer">
-                                                    <h6 class="mb-0">
+                                                    <i-bs name="info-circle"></i-bs>
-                                                      {{option.title}}
+                                                </a>
-                                                    </h6>
+                                                @if (isSet(option.key)) {
-                                                    <a class="btn btn-sm btn-link" title="Read the documentation about this setting" i18n-title [href]="getDocsUrl(option.config_key)" target="_blank" referrerpolicy="no-referrer">
+                                                  <button type="button" class="btn btn-sm btn-link text-danger ms-auto pe-0" title="Reset" i18n-title (click)="resetOption(option.key)">
-                                                        <i-bs name="info-circle"></i-bs>
+                                                      <i-bs class="me-1" name="x"></i-bs><ng-container i18n>Reset</ng-container>
-                                                    </a>
+                                                  </button>
-                                                    @if (isSet(option.key)) {
-                                                      <button type="button" class="btn btn-sm btn-link text-danger ms-auto pe-0" title="Reset" i18n-title (click)="resetOption(option.key)">
-                                                          <i-bs class="me-1" name="x"></i-bs><ng-container i18n>Reset</ng-container>
-                                                      </button>
-                                                    }
-                                                </div>
-                                                <div class="mb-n3">
-                                                    @switch (option.type) {
-                                                        @case (ConfigOptionType.Select) { <pngx-input-select [formControlName]="option.key" [error]="errors[option.key]" [items]="option.choices" [allowNull]="true"></pngx-input-select> }
-                                                        @case (ConfigOptionType.Number) { <pngx-input-number [formControlName]="option.key" [error]="errors[option.key]" [showAdd]="false"></pngx-input-number> }
-                                                        @case (ConfigOptionType.Boolean) { <pngx-input-switch [formControlName]="option.key" [error]="errors[option.key]" [showUnsetNote]="true" [horizontal]="true" title="Enable" i18n-title></pngx-input-switch> }
-                                                        @case (ConfigOptionType.String) { <pngx-input-text [formControlName]="option.key" [error]="errors[option.key]"></pngx-input-text> }
-                                                        @case (ConfigOptionType.JSON) { <pngx-input-text [formControlName]="option.key" [error]="errors[option.key]"></pngx-input-text> }
-                                                        @case (ConfigOptionType.File) { <pngx-input-file [formControlName]="option.key" (upload)="uploadFile($event, option.key)" [error]="errors[option.key]"></pngx-input-file> }
-                                                        @case (ConfigOptionType.Password) { <pngx-input-password [formControlName]="option.key" [error]="errors[option.key]"></pngx-input-password> }
-                                                    }
-                                                </div>
-                                                @if (option.note) {
-                                                    <div class="form-text fst-italic">{{option.note}}</div>
                                                 }
                                             </div>
+                                            <div class="mb-n3">
+                                                @switch (option.type) {
+                                                    @case (ConfigOptionType.Select) { <pngx-input-select [formControlName]="option.key" [error]="errors[option.key]" [items]="option.choices" [allowNull]="true"></pngx-input-select> }
+                                                    @case (ConfigOptionType.Number) { <pngx-input-number [formControlName]="option.key" [error]="errors[option.key]" [showAdd]="false"></pngx-input-number> }
+                                                    @case (ConfigOptionType.Boolean) { <pngx-input-switch [formControlName]="option.key" [error]="errors[option.key]" [showUnsetNote]="true" [horizontal]="true" title="Enable" i18n-title></pngx-input-switch> }
+                                                    @case (ConfigOptionType.String) { <pngx-input-text [formControlName]="option.key" [error]="errors[option.key]"></pngx-input-text> }
+                                                    @case (ConfigOptionType.JSON) { <pngx-input-text [formControlName]="option.key" [error]="errors[option.key]"></pngx-input-text> }
+                                                    @case (ConfigOptionType.File) { <pngx-input-file [formControlName]="option.key" (upload)="uploadFile($event, option.key)" [error]="errors[option.key]"></pngx-input-file> }
+                                                    @case (ConfigOptionType.Password) { <pngx-input-password [formControlName]="option.key" [error]="errors[option.key]"></pngx-input-password> }
+                                                }
+                                            </div>
+                                            @if (option.note) {
+                                                <div class="form-text fst-italic">{{option.note}}</div>
+                                            }
                                         </div>
                                     </div>
-                                }
+                                </div>
-                            </div>
+                            }
-                        }
+                        </div>
                     </div>
                 </ng-template>
             </li>

src-ui/src/app/components/admin/config/config.component.spec.ts

@@ -8,11 +8,7 @@ import { NgbModule } from '@ng-bootstrap/ng-bootstrap'
 import { NgSelectModule } from '@ng-select/ng-select'
 import { NgxBootstrapIconsModule, allIcons } from 'ngx-bootstrap-icons'
 import { of, throwError } from 'rxjs'
-import {
+import { OutputTypeConfig } from 'src/app/data/paperless-config'
-  ConfigCategory,
-  ConfigSection,
-  OutputTypeConfig,
-} from 'src/app/data/paperless-config'
 import { ConfigService } from 'src/app/services/config.service'
 import { SettingsService } from 'src/app/services/settings.service'
 import { ToastService } from 'src/app/services/toast.service'
@@ -162,23 +158,4 @@ describe('ConfigComponent', () => {
     component.resetOption('barcodes_enabled')
     expect(component.configForm.get('barcodes_enabled').value).toBeNull()
   })
-
-  it('should group options into sections within a category, or not', () => {
-    const sections = component.getCategorySections(ConfigCategory.OCR)
-    expect(sections).toEqual([null, ConfigSection.RemoteOCR])
-    expect(
-      component
-        .getCategoryOptions(ConfigCategory.OCR)
-        .map((option) => option.key)
-    ).toContain('output_type')
-    expect(
-      component
-        .getCategoryOptions(ConfigCategory.OCR, ConfigSection.RemoteOCR)
-        .map((option) => option.key)
-    ).toEqual([
-      'remote_ocr_engine',
-      'remote_ocr_api_key',
-      'remote_ocr_endpoint',
-    ])
-  })
 })

src-ui/src/app/components/admin/config/config.component.ts

@@ -74,20 +74,8 @@ export class ConfigComponent
     return Object.values(ConfigCategory)
   }
 
-  getCategorySections(category: string): string[] {
+  getCategoryOptions(category: string): ConfigOption[] {
-    return [
+    return PaperlessConfigOptions.filter((o) => o.category === category)
-      ...new Set(
-        PaperlessConfigOptions.filter((o) => o.category === category).map(
-          (o) => o.section ?? null // null means no section
-        )
-      ),
-    ]
-  }
-
-  getCategoryOptions(category: string, section: string = null): ConfigOption[] {
-    return PaperlessConfigOptions.filter(
-      (o) => o.category === category && (o.section ?? null) === section
-    )
   }
 
   initialConfig: PaperlessConfig

src-ui/src/app/components/common/suggestions-dropdown/suggestions-dropdown.component.html

@@ -2,16 +2,10 @@
   <button type="button" class="btn btn-sm btn-outline-primary" (click)="clickSuggest()" [disabled]="disabled() || loading() || (suggestions() && !aiEnabled())">
     @if (loading()) {
       <div class="spinner-border spinner-border-sm" role="status"></div>
-    } @else if (noSuggestions) {
-      <i-bs width="1.2em" height="1.2em" name="check-circle"></i-bs>
     } @else {
       <i-bs width="1.2em" height="1.2em" name="stars"></i-bs>
     }
-    @if (noSuggestions) {
+    <span class="d-none d-lg-inline ps-1" i18n>Suggest</span>
-      <span class="d-none d-lg-inline ps-1" i18n>No suggestions</span>
-    } @else {
-      <span class="d-none d-lg-inline ps-1" i18n>Suggest</span>
-    }
     @if (totalSuggestions > 0) {
       <span class="badge bg-primary ms-2">{{ totalSuggestions }}</span>
     }
@@ -25,7 +19,7 @@
 
       <div ngbDropdownMenu aria-labelledby="suggestionsDropdown" class="shadow suggestions-dropdown">
         <div class="list-group list-group-flush small pb-0">
-          @if (totalSuggestions === 0) {
+          @if (!suggestions()?.suggested_tags && !suggestions()?.suggested_document_types && !suggestions()?.suggested_correspondents) {
             <div class="list-group-item text-muted fst-italic">
               <small class="text-muted small fst-italic" i18n>No novel suggestions</small>
             </div>

src-ui/src/app/components/common/suggestions-dropdown/suggestions-dropdown.component.spec.ts

@@ -30,34 +30,6 @@ describe('SuggestionsDropdownComponent', () => {
     expect(component.totalSuggestions).toBe(4)
   })
 
-  it('should show when a completed request returned no suggestions', () => {
-    fixture.componentRef.setInput('suggestions', {
-      correspondents: [],
-      tags: [],
-      document_types: [],
-      storage_paths: [],
-      dates: [],
-    })
-    fixture.detectChanges()
-
-    expect(component.noSuggestions).toBeTruthy()
-    expect(fixture.nativeElement.textContent).toContain('No suggestions')
-  })
-
-  it('should not show the empty state before a request or with suggestions', () => {
-    expect(component.noSuggestions).toBeFalsy()
-
-    fixture.componentRef.setInput('suggestions', {
-      correspondents: [],
-      tags: [42],
-      document_types: [],
-      storage_paths: [],
-      dates: [],
-    })
-
-    expect(component.noSuggestions).toBeFalsy()
-  })
-
   it('should emit getSuggestions when clickSuggest is called and suggestions are null', () => {
     jest.spyOn(component.getSuggestions, 'emit')
     fixture.componentRef.setInput('suggestions', null)
@@ -87,6 +59,5 @@ describe('SuggestionsDropdownComponent', () => {
     })
     component.clickSuggest()
     expect(component.dropdown.open).toBeTruthy()
-    expect(fixture.nativeElement.textContent).toContain('No novel suggestions')
   })
 })

src-ui/src/app/components/common/suggestions-dropdown/suggestions-dropdown.component.ts

@@ -61,21 +61,4 @@ export class SuggestionsDropdownComponent {
         this.suggestions()?.suggested_document_types?.length || 0
     )
   }
-
-  get noSuggestions(): boolean {
-    const suggestions = this.suggestions()
-    return (
-      suggestions != null &&
-      !suggestions.title &&
-      !suggestions.tags?.length &&
-      !suggestions.suggested_tags?.length &&
-      !suggestions.correspondents?.length &&
-      !suggestions.suggested_correspondents?.length &&
-      !suggestions.document_types?.length &&
-      !suggestions.suggested_document_types?.length &&
-      !suggestions.storage_paths?.length &&
-      !suggestions.suggested_storage_paths?.length &&
-      !suggestions.dates?.length
-    )
-  }
 }

src-ui/src/app/data/paperless-config.ts

@@ -54,10 +54,6 @@ export const ConfigCategory = {
   AI: $localize`AI Settings`,
 }
 
-export const ConfigSection = {
-  RemoteOCR: $localize`Remote OCR`,
-}
-
 export const LLMEmbeddingBackendConfig = {
   OPENAI_LIKE: 'openai-like',
   HUGGINGFACE: 'huggingface',
@@ -69,10 +65,6 @@ export const LLMBackendConfig = {
   OLLAMA: 'ollama',
 }
 
-export const RemoteOCREngineConfig = {
-  AZURE_AI: 'azureai',
-}
-
 export interface ConfigOption {
   key: string
   title: string
@@ -80,7 +72,6 @@ export interface ConfigOption {
   choices?: Array<{ id: string; name: string }>
   config_key?: string
   category: string
-  section?: string
   note?: string
 }
 
@@ -190,33 +181,6 @@ export const PaperlessConfigOptions: ConfigOption[] = [
     config_key: 'PAPERLESS_OCR_USER_ARGS',
     category: ConfigCategory.OCR,
   },
-  {
-    key: 'remote_ocr_engine',
-    title: $localize`Remote OCR Engine`,
-    type: ConfigOptionType.Select,
-    choices: mapToItems(RemoteOCREngineConfig),
-    config_key: 'PAPERLESS_REMOTE_OCR_ENGINE',
-    category: ConfigCategory.OCR,
-    section: ConfigSection.RemoteOCR,
-    note: $localize`Enabling remote OCR sends documents to a third-party service for processing. Consider the privacy implications as well as potential costs before enabling.`,
-  },
-  {
-    key: 'remote_ocr_api_key',
-    title: $localize`Remote OCR API Key`,
-    type: ConfigOptionType.Password,
-    config_key: 'PAPERLESS_REMOTE_OCR_API_KEY',
-    category: ConfigCategory.OCR,
-    section: ConfigSection.RemoteOCR,
-  },
-  {
-    key: 'remote_ocr_endpoint',
-    title: $localize`Remote OCR Endpoint`,
-    type: ConfigOptionType.String,
-    config_key: 'PAPERLESS_REMOTE_OCR_ENDPOINT',
-    category: ConfigCategory.OCR,
-    section: ConfigSection.RemoteOCR,
-    note: $localize`Required when using the Azure AI engine.`,
-  },
   {
     key: 'app_logo',
     title: $localize`Application Logo`,
@@ -434,9 +398,6 @@ export interface PaperlessConfig extends ObjectWithId {
   barcode_enable_tag: boolean
   barcode_tag_mapping: object
   barcode_tag_split: boolean
-  remote_ocr_engine: string
-  remote_ocr_api_key: string
-  remote_ocr_endpoint: string
   ai_enabled: boolean
   llm_embedding_backend: string
   llm_embedding_model: string

src/documents/filters.py

@@ -1047,12 +1047,6 @@ class PermittedObjectsFilter(BaseFilterBackend):
     perm_codename: str | None = None
 
     def filter_queryset(self, request, queryset, view):
-        # Before the superuser and owner-only paths, neither of which consults
-        # permitted_object_ids. Scoped to authenticated users so anonymous
-        # access (AnonymousUser.is_active is False) keeps its existing
-        # unowned-only behaviour.
-        if request.user.is_authenticated and not request.user.is_active:
-            return queryset.none()
         if request.user.is_superuser:
             return queryset
         if not self.include_granted:

src/documents/permissions.py

@@ -54,15 +54,11 @@ class PaperlessObjectPermissions(DjangoObjectPermissions):
 
 class PaperlessAdminPermissions(BasePermission):
     def has_permission(self, request, view):
-        return request.user.is_active and request.user.is_staff
+        return request.user.is_staff
 
 
 def has_global_statistics_permission(user: User | None) -> bool:
-    if (
+    if user is None or not getattr(user, "is_authenticated", False):
-        user is None
-        or not getattr(user, "is_active", False)
-        or not getattr(user, "is_authenticated", False)
-    ):
         return False
 
     return getattr(user, "is_superuser", False) or user.has_perm(
@@ -71,11 +67,7 @@ def has_global_statistics_permission(user: User | None) -> bool:
 
 
 def has_system_status_permission(user: User | None) -> bool:
-    if (
+    if user is None or not getattr(user, "is_authenticated", False):
-        user is None
-        or not getattr(user, "is_active", False)
-        or not getattr(user, "is_authenticated", False)
-    ):
         return False
 
     return (
@@ -196,13 +188,6 @@ def permitted_object_ids(
     if user is None or not getattr(user, "is_authenticated", False):
         return base_qs.filter(owner__isnull=True).values_list("id", flat=True)
 
-    # Deactivated users get nothing, deactivated superusers included, so this
-    # has to come before the superuser shortcut. guardian's
-    # ObjectPermissionChecker denies inactive users, but get_objects_for_user
-    # (the pattern this replaces) does not, so it would not be inherited.
-    if not getattr(user, "is_active", False):
-        return base_qs.none().values_list("id", flat=True)
-
     if getattr(user, "is_superuser", False):
         return base_qs.values_list("id", flat=True)
 

src/documents/tests/test_api_app_config.py

@@ -72,9 +72,6 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
                 "barcode_enable_tag": None,
                 "barcode_tag_mapping": None,
                 "barcode_tag_split": None,
-                "remote_ocr_engine": None,
-                "remote_ocr_api_key": None,
-                "remote_ocr_endpoint": None,
                 "ai_enabled": False,
                 "llm_embedding_backend": None,
                 "llm_embedding_model": None,
@@ -873,49 +870,6 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
         config.refresh_from_db()
         self.assertEqual(config.llm_api_key, None)
 
-    def test_update_remote_ocr_api_key(self) -> None:
-        """
-        GIVEN:
-            - Existing config with remote_ocr_api_key specified
-        WHEN:
-            - API to update remote_ocr_api_key is called with all *s
-            - API to update remote_ocr_api_key is called with empty string
-        THEN:
-            - remote_ocr_api_key is unchanged
-            - remote_ocr_api_key is set to None
-        """
-        config = ApplicationConfiguration.objects.first()
-        assert config is not None
-        config.remote_ocr_api_key = "1234567890"
-        config.save()
-
-        # Test with all *
-        response = self.client.patch(
-            f"{self.ENDPOINT}1/",
-            json.dumps(
-                {
-                    "remote_ocr_api_key": "*" * 32,
-                },
-            ),
-            content_type="application/json",
-        )
-        self.assertEqual(response.status_code, status.HTTP_200_OK)
-        config.refresh_from_db()
-        self.assertEqual(config.remote_ocr_api_key, "1234567890")
-        # Test with empty string
-        response = self.client.patch(
-            f"{self.ENDPOINT}1/",
-            json.dumps(
-                {
-                    "remote_ocr_api_key": "",
-                },
-            ),
-            content_type="application/json",
-        )
-        self.assertEqual(response.status_code, status.HTTP_200_OK)
-        config.refresh_from_db()
-        self.assertEqual(config.remote_ocr_api_key, None)
-
     def test_enable_ai_index_triggers_update(self) -> None:
         """
         GIVEN:

src/documents/tests/test_permission_filtering_security.py

@@ -496,28 +496,6 @@ class TestPermittedObjectIdsGenericModels:
             expected_hidden=[strangers.pk],
         )
 
-    @pytest.mark.parametrize("is_superuser", [False, True])
-    def test_inactive_user_sees_nothing(self, model, factory, perm, is_superuser):
-        suffix = f"{model.__name__}_{is_superuser}"
-        user = User.objects.create_user(
-            username=f"inactive_{suffix}",
-            is_active=False,
-            is_superuser=is_superuser,
-        )
-        other = User.objects.create_user(username=f"other_{suffix}")
-        granted = factory(owner=other)
-        assign_perm(perm, user, granted)
-
-        assert_visible_document_ids(
-            permitted_object_ids(user, model, perm),
-            expected_visible=[],
-            expected_hidden=[
-                factory(owner=None).pk,
-                factory(owner=user).pk,
-                granted.pk,
-            ],
-        )
-
     def test_unowned_object_visible_to_everyone(self, model, factory, perm):
         user = User.objects.create_user(username=f"user_{model.__name__}")
         unowned = factory(owner=None)

src/documents/tests/test_permitted_objects_filter.py

@@ -68,44 +68,3 @@ class TestPermittedObjectsFilter:
         visible_ids = set(result.values_list("id", flat=True))
         assert visible_ids == {owned.pk}
         assert granted.pk not in visible_ids
-
-    @pytest.mark.parametrize(
-        ("username", "is_superuser"),
-        [("inactive", False), ("inactive_super", True)],
-    )
-    def test_inactive_user_sees_nothing(self, username: str, *, is_superuser: bool):
-        user = User.objects.create_user(
-            username=username,
-            is_active=False,
-            is_superuser=is_superuser,
-        )
-        TagFactory(owner=None)
-        TagFactory(owner=user)
-        granted = TagFactory(owner=User.objects.create_user(username=f"o_{username}"))
-        assign_perm("view_tag", user, granted)
-        request = APIRequestFactory().get("/")
-        request.user = user
-
-        result = PermittedObjectsFilter().filter_queryset(
-            request,
-            Tag.objects.all(),
-            _DummyView(),
-        )
-        assert result.count() == 0
-
-    def test_inactive_user_sees_nothing_with_include_granted_false(self):
-        user = User.objects.create_user(username="inactive_owner", is_active=False)
-        TagFactory(owner=user)
-        TagFactory(owner=None)
-        request = APIRequestFactory().get("/")
-        request.user = user
-
-        class _OwnerOnlyFilter(PermittedObjectsFilter):
-            include_granted = False
-
-        result = _OwnerOnlyFilter().filter_queryset(
-            request,
-            Tag.objects.all(),
-            _DummyView(),
-        )
-        assert result.count() == 0

src/documents/views.py

@@ -2267,7 +2267,7 @@ class ChatStreamingView(GenericAPIView[Any]):
             if not has_perms_owner_aware(request.user, "view_document", document):
                 return HttpResponseForbidden("Insufficient permissions")
 
-            documents = [document]
+            documents = Document.objects.filter(pk=document.pk)
         else:
             documents = Document.objects.filter(
                 id__in=permitted_document_ids(request.user),

src/locale/en_US/LC_MESSAGES/django.po

@@ -2,7 +2,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: paperless-ngx\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2026-08-10 02:25+0000\n"
+"POT-Creation-Date: 2026-08-07 20:00+0000\n"
 "PO-Revision-Date: 2022-02-17 04:17\n"
 "Last-Translator: \n"
 "Language-Team: English\n"
@@ -21,39 +21,39 @@ msgstr ""
 msgid "Documents"
 msgstr ""
 
-#: documents/filters.py:471
+#: documents/filters.py:472
 msgid "Value must be valid JSON."
 msgstr ""
 
-#: documents/filters.py:490
+#: documents/filters.py:491
 msgid "Invalid custom field query expression"
 msgstr ""
 
-#: documents/filters.py:500
+#: documents/filters.py:501
 msgid "Invalid expression list. Must be nonempty."
 msgstr ""
 
-#: documents/filters.py:521
+#: documents/filters.py:522
 msgid "Invalid logical operator {op!r}"
 msgstr ""
 
-#: documents/filters.py:535
+#: documents/filters.py:536
 msgid "Maximum number of query conditions exceeded."
 msgstr ""
 
-#: documents/filters.py:599
+#: documents/filters.py:600
 msgid "{name!r} is not a valid custom field."
 msgstr ""
 
-#: documents/filters.py:636
+#: documents/filters.py:637
 msgid "{data_type} does not support query expr {expr!r}."
 msgstr ""
 
-#: documents/filters.py:755 documents/models.py:136
+#: documents/filters.py:756 documents/models.py:136
 msgid "Maximum nesting depth exceeded."
 msgstr ""
 
-#: documents/filters.py:1079
+#: documents/filters.py:1098
 msgid "Custom field not found"
 msgstr ""
 
@@ -1352,7 +1352,7 @@ msgid "workflow runs"
 msgstr ""
 
 #: documents/serialisers.py:521 documents/serialisers.py:873
-#: documents/serialisers.py:2767 documents/views.py:299 documents/views.py:2555
+#: documents/serialisers.py:2767 documents/views.py:300 documents/views.py:2556
 #: paperless_mail/serialisers.py:155
 msgid "Insufficient permissions."
 msgstr ""
@@ -1393,7 +1393,7 @@ msgstr ""
 msgid "Duplicate document identifiers are not allowed."
 msgstr ""
 
-#: documents/serialisers.py:2853 documents/views.py:4509
+#: documents/serialisers.py:2853 documents/views.py:4510
 #, python-format
 msgid "Documents not found: %(ids)s"
 msgstr ""
@@ -1661,36 +1661,36 @@ msgstr ""
 msgid "Unable to parse URI {value}"
 msgstr ""
 
-#: documents/views.py:292 documents/views.py:2552
+#: documents/views.py:293 documents/views.py:2553
 msgid "Invalid more_like_id"
 msgstr ""
 
-#: documents/views.py:1566
+#: documents/views.py:1567
 msgid "Invalid AI configuration."
 msgstr ""
 
-#: documents/views.py:1575
+#: documents/views.py:1576
 msgid "AI backend request timed out."
 msgstr ""
 
-#: documents/views.py:2377 documents/views.py:2698
+#: documents/views.py:2378 documents/views.py:2699
 msgid "Specify only one of text, title_search, query, or more_like_id."
 msgstr ""
 
-#: documents/views.py:4522
+#: documents/views.py:4523
 #, python-format
 msgid "Insufficient permissions to share document %(id)s."
 msgstr ""
 
-#: documents/views.py:4568
+#: documents/views.py:4569
 msgid "Bundle is already being processed."
 msgstr ""
 
-#: documents/views.py:4629
+#: documents/views.py:4630
 msgid "The share link bundle is still being prepared. Please try again later."
 msgstr ""
 
-#: documents/views.py:4639
+#: documents/views.py:4640
 msgid "The share link bundle is unavailable."
 msgstr ""
 

src/paperless/auth.py

@@ -19,10 +19,7 @@ class AutoLoginMiddleware(MiddlewareMixin):
         if request.path.startswith("/api/token/") and request.method == "POST":
             return None
         try:
-            request.user = User.objects.get(
+            request.user = User.objects.get(username=settings.AUTO_LOGIN_USERNAME)
-                username=settings.AUTO_LOGIN_USERNAME,
-                is_active=True,
-            )
             auth.login(
                 request=request,
                 user=request.user,

src/paperless/config.py

@@ -185,30 +185,6 @@ class GeneralConfig(BaseConfig):
         self.app_logo = app_config.app_logo.url if app_config.app_logo else None
 
 
-@dataclasses.dataclass
-class RemoteOCRConfig(BaseConfig):
-    """
-    Settings for the remote (cloud) OCR parser
-    """
-
-    remote_ocr_engine: str | None = dataclasses.field(init=False)
-    remote_ocr_api_key: str | None = dataclasses.field(init=False)
-    remote_ocr_endpoint: str | None = dataclasses.field(init=False)
-
-    def __post_init__(self) -> None:
-        app_config = self._get_config_instance()
-
-        self.remote_ocr_engine = (
-            app_config.remote_ocr_engine or settings.REMOTE_OCR_ENGINE
-        )
-        self.remote_ocr_api_key = (
-            app_config.remote_ocr_api_key or settings.REMOTE_OCR_API_KEY
-        )
-        self.remote_ocr_endpoint = (
-            app_config.remote_ocr_endpoint or settings.REMOTE_OCR_ENDPOINT
-        )
-
-
 @dataclasses.dataclass
 class AIConfig(BaseConfig):
     """

src/paperless/migrations/0014_applicationconfiguration_remote_ocr_api_key_and_more.py

@@ -1,44 +0,0 @@
-# Generated by Django 5.2.16 on 2026-08-10 14:37
-
-from django.db import migrations
-from django.db import models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("paperless", "0013_applicationconfiguration_llm_request_timeout"),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name="applicationconfiguration",
-            name="remote_ocr_api_key",
-            field=models.CharField(
-                blank=True,
-                max_length=1024,
-                null=True,
-                verbose_name="Sets the remote OCR API key",
-            ),
-        ),
-        migrations.AddField(
-            model_name="applicationconfiguration",
-            name="remote_ocr_endpoint",
-            field=models.CharField(
-                blank=True,
-                max_length=256,
-                null=True,
-                verbose_name="Sets the remote OCR endpoint",
-            ),
-        ),
-        migrations.AddField(
-            model_name="applicationconfiguration",
-            name="remote_ocr_engine",
-            field=models.CharField(
-                blank=True,
-                choices=[("azureai", "Azure AI Document Intelligence")],
-                max_length=32,
-                null=True,
-                verbose_name="Sets the remote OCR engine",
-            ),
-        ),
-    ]

src/paperless/models.py

@@ -74,14 +74,6 @@ class ColorConvertChoices(models.TextChoices):
     CMYK = ("CMYK", _("CMYK"))
 
 
-class RemoteOCREngine(models.TextChoices):
-    """
-    Matches to PAPERLESS_REMOTE_OCR_ENGINE
-    """
-
-    AZURE_AI = ("azureai", _("Azure AI Document Intelligence"))
-
-
 class LLMEmbeddingBackend(models.TextChoices):
     OPENAI_LIKE = ("openai-like", _("OpenAI-compatible"))
     HUGGINGFACE = ("huggingface", _("Huggingface"))
@@ -294,35 +286,6 @@ class ApplicationConfiguration(AbstractSingletonModel):
         null=True,
     )
 
-    """
-    Settings for the remote OCR parser
-    """
-
-    # PAPERLESS_REMOTE_OCR_ENGINE
-    remote_ocr_engine = models.CharField(
-        verbose_name=_("Sets the remote OCR engine"),
-        blank=True,
-        null=True,
-        max_length=32,
-        choices=RemoteOCREngine.choices,
-    )
-
-    # PAPERLESS_REMOTE_OCR_API_KEY
-    remote_ocr_api_key = models.CharField(
-        verbose_name=_("Sets the remote OCR API key"),
-        blank=True,
-        null=True,
-        max_length=1024,
-    )
-
-    # PAPERLESS_REMOTE_OCR_ENDPOINT
-    remote_ocr_endpoint = models.CharField(
-        verbose_name=_("Sets the remote OCR endpoint"),
-        blank=True,
-        null=True,
-        max_length=256,
-    )
-
     """
     AI related settings
     """

src/paperless/parsers/remote.py

@@ -57,18 +57,6 @@ class RemoteEngineConfig:
         self.api_key = api_key
         self.endpoint = endpoint
 
-    @classmethod
-    def from_app_config(cls) -> Self:
-        """Build the config from the app config, falling back to the env."""
-        from paperless.config import RemoteOCRConfig
-
-        app_config = RemoteOCRConfig()
-        return cls(
-            engine=app_config.remote_ocr_engine,
-            api_key=app_config.remote_ocr_api_key,
-            endpoint=app_config.remote_ocr_endpoint,
-        )
-
     def engine_is_valid(self) -> bool:
         """Return True when the engine is known and fully configured."""
         return (
@@ -150,7 +138,11 @@ class RemoteDocumentParser:
             20 when the remote engine is configured and the MIME type is
             supported, otherwise None.
         """
-        config = RemoteEngineConfig.from_app_config()
+        config = RemoteEngineConfig(
+            engine=settings.REMOTE_OCR_ENGINE,
+            api_key=settings.REMOTE_OCR_API_KEY,
+            endpoint=settings.REMOTE_OCR_ENDPOINT,
+        )
         if not config.engine_is_valid():
             return None
         if mime_type not in _SUPPORTED_MIME_TYPES:
@@ -236,7 +228,11 @@ class RemoteDocumentParser:
             Ignored — the remote engine always returns a searchable PDF,
             which is stored as the archive copy regardless of this flag.
         """
-        config = RemoteEngineConfig.from_app_config()
+        config = RemoteEngineConfig(
+            engine=settings.REMOTE_OCR_ENGINE,
+            api_key=settings.REMOTE_OCR_API_KEY,
+            endpoint=settings.REMOTE_OCR_ENDPOINT,
+        )
 
         if not config.engine_is_valid():
             logger.warning(

src/paperless/serialisers.py

@@ -217,15 +217,7 @@ class ApplicationConfigurationSerializer(
     llm_api_key = ObfuscatedPasswordField(
         required=False,
         allow_null=True,
-        max_length=1024,
     )
-    remote_ocr_api_key = ObfuscatedPasswordField(
-        required=False,
-        allow_null=True,
-        max_length=1024,
-    )
-
-    OBFUSCATED_FIELDS = ("llm_api_key", "remote_ocr_api_key")
 
     def run_validation(self, data):
         # Empty strings treated as None to avoid unexpected behavior
@@ -237,13 +229,11 @@ class ApplicationConfigurationSerializer(
             data["language"] = None
         if "llm_output_language" in data and data["llm_output_language"] == "":
             data["llm_output_language"] = None
-        for field in self.OBFUSCATED_FIELDS:
+        if "llm_api_key" in data and data["llm_api_key"] is not None:
-            if field in data and data[field] is not None:
+            if data["llm_api_key"] == "":
-                if data[field] == "":
+                data["llm_api_key"] = None
-                    data[field] = None
+            elif len(data["llm_api_key"].replace("*", "")) == 0:
-                # Not a real value, don't overwrite the stored one
+                del data["llm_api_key"]
-                elif len(data[field].replace("*", "")) == 0:
-                    del data[field]
         return super().run_validation(data)
 
     def update(self, instance, validated_data):

src/paperless/tests/parsers/test_remote_parser.py

@@ -21,7 +21,6 @@ from unittest.mock import Mock
 import pytest
 
 from documents.parsers import ParseError
-from paperless.models import ApplicationConfiguration
 from paperless.parsers import ParserContext
 from paperless.parsers import ParserProtocol
 from paperless.parsers.remote import RemoteDocumentParser
@@ -34,10 +33,6 @@ if TYPE_CHECKING:
     from pytest_mock import MockerFixture
 
 
-# Remote ocr config from ApplicationConfiguration needs DB access
-pytestmark = pytest.mark.django_db
-
-
 # ---------------------------------------------------------------------------
 # Module-local fixtures
 # ---------------------------------------------------------------------------
@@ -232,18 +227,6 @@ class TestRemoteParserScore:
         score = RemoteDocumentParser.score("application/pdf", "doc.pdf")
         assert score is not None and score > 10
 
-    @pytest.mark.usefixtures("no_engine_settings")
-    def test_score_uses_app_config_when_env_unset(self) -> None:
-        """The app config alone is enough to activate the parser."""
-        config = ApplicationConfiguration.objects.first()
-        assert config is not None
-        config.remote_ocr_engine = "azureai"
-        config.remote_ocr_api_key = "app-config-key"
-        config.remote_ocr_endpoint = "https://config.cognitiveservices.azure.com"
-        config.save()
-
-        assert RemoteDocumentParser.score("application/pdf", "doc.pdf") == 20
-
 
 # ---------------------------------------------------------------------------
 # Properties

src/paperless/tests/parsers/test_tesseract_parser.py

@@ -1277,8 +1277,6 @@ class TestParserFileTypes:
 # ---------------------------------------------------------------------------
 
 
-# Remote ocr config from ApplicationConfiguration needs DB access
-@pytest.mark.django_db
 class TestRasterisedDocumentParserRegistry:
     def test_registered_in_defaults(self) -> None:
         from paperless.parsers.registry import ParserRegistry

src/paperless/tests/test_auth_middleware.py

@@ -1,53 +0,0 @@
-from django.contrib.auth.models import AnonymousUser
-from django.contrib.auth.models import User
-from django.test import RequestFactory
-from django.test import TestCase
-from django.test import override_settings
-
-from paperless.auth import AutoLoginMiddleware
-
-
-@override_settings(AUTO_LOGIN_USERNAME="autologin")
-class TestAutoLoginMiddleware(TestCase):
-    def setUp(self) -> None:
-        super().setUp()
-        self.factory = RequestFactory()
-        self.middleware = AutoLoginMiddleware(lambda request: None)
-
-    def _process(self, request):
-        # login() needs a session to write to
-        request.session = self.client.session
-        self.middleware.process_request(request)
-        return request
-
-    def test_active_user_is_logged_in(self) -> None:
-        """
-        GIVEN:
-            - AUTO_LOGIN_USERNAME names an active user
-        WHEN:
-            - A request is processed by the middleware
-        THEN:
-            - That user is attached to the request
-        """
-        user = User.objects.create_user(username="autologin")
-
-        request = self._process(self.factory.get("/"))
-
-        self.assertEqual(request.user, user)
-
-    def test_deactivated_user_is_not_logged_in(self) -> None:
-        """
-        GIVEN:
-            - AUTO_LOGIN_USERNAME names a user who has been deactivated
-        WHEN:
-            - A request is processed by the middleware
-        THEN:
-            - The request is left anonymous rather than authenticated as them
-        """
-        User.objects.create_user(username="autologin", is_active=False)
-
-        request = self.factory.get("/")
-        request.user = AnonymousUser()
-        self._process(request)
-
-        self.assertFalse(request.user.is_authenticated)

src/paperless/tests/test_remote_ocr_config.py

@@ -1,88 +0,0 @@
-"""Tests for RemoteOCRConfig precedence between app config and Django settings."""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-import pytest
-from django.test import override_settings
-
-from paperless.config import RemoteOCRConfig
-
-if TYPE_CHECKING:
-    from unittest.mock import MagicMock
-
-
-@pytest.fixture()
-def null_app_config(mocker) -> MagicMock:
-    """Mock ApplicationConfiguration with all fields None → falls back to Django settings."""
-    return mocker.MagicMock(
-        remote_ocr_engine=None,
-        remote_ocr_api_key=None,
-        remote_ocr_endpoint=None,
-    )
-
-
-@pytest.fixture()
-def make_remote_ocr_config(mocker):
-    def _make(app_config, **django_settings_overrides):
-        mocker.patch(
-            "paperless.config.BaseConfig._get_config_instance",
-            return_value=app_config,
-        )
-        with override_settings(**django_settings_overrides):
-            return RemoteOCRConfig()
-
-    return _make
-
-
-class TestRemoteOCRConfig:
-    def test_falls_back_to_settings(
-        self,
-        make_remote_ocr_config,
-        null_app_config,
-    ) -> None:
-        cfg = make_remote_ocr_config(
-            null_app_config,
-            REMOTE_OCR_ENGINE="azureai",
-            REMOTE_OCR_API_KEY="env-key",
-            REMOTE_OCR_ENDPOINT="https://env.cognitiveservices.azure.com",
-        )
-        assert cfg.remote_ocr_engine == "azureai"
-        assert cfg.remote_ocr_api_key == "env-key"
-        assert cfg.remote_ocr_endpoint == "https://env.cognitiveservices.azure.com"
-
-    def test_app_config_takes_precedence(
-        self,
-        make_remote_ocr_config,
-        mocker,
-    ) -> None:
-        app_config = mocker.MagicMock(
-            remote_ocr_engine="azureai",
-            remote_ocr_api_key="db-key",
-            remote_ocr_endpoint="https://db.cognitiveservices.azure.com",
-        )
-        cfg = make_remote_ocr_config(
-            app_config,
-            REMOTE_OCR_ENGINE=None,
-            REMOTE_OCR_API_KEY="env-key",
-            REMOTE_OCR_ENDPOINT="https://env.cognitiveservices.azure.com",
-        )
-        assert cfg.remote_ocr_engine == "azureai"
-        assert cfg.remote_ocr_api_key == "db-key"
-        assert cfg.remote_ocr_endpoint == "https://db.cognitiveservices.azure.com"
-
-    def test_unset_everywhere(
-        self,
-        make_remote_ocr_config,
-        null_app_config,
-    ) -> None:
-        cfg = make_remote_ocr_config(
-            null_app_config,
-            REMOTE_OCR_ENGINE=None,
-            REMOTE_OCR_API_KEY=None,
-            REMOTE_OCR_ENDPOINT=None,
-        )
-        assert cfg.remote_ocr_engine is None
-        assert cfg.remote_ocr_api_key is None
-        assert cfg.remote_ocr_endpoint is None

src/paperless_ai/chat.py

@@ -2,6 +2,8 @@ import json
 import logging
 import sys
 
+from django.db.models import QuerySet
+
 from documents.models import Document
 from paperless.config import AIConfig
 from paperless_ai.client import AIClient
@@ -82,10 +84,21 @@ def _build_document_reference(
 
 
 def _get_document_references(
-    documents: list[Document],
+    documents: QuerySet[Document],
     top_nodes: list,
 ) -> list[dict[str, int | str]]:
-    allowed_documents = {doc.pk: doc for doc in documents}
+    candidate_ids: set[int] = set()
+    for node in top_nodes:
+        try:
+            candidate_ids.add(int(node.metadata["document_id"]))
+        except (KeyError, TypeError, ValueError):  # pragma: no cover
+            continue
+
+    if not candidate_ids:
+        return []
+
+    allowed_documents = {doc.pk: doc for doc in documents.filter(pk__in=candidate_ids)}
+
     references: list[dict[str, int | str]] = []
     seen_document_ids: set[int] = set()
 
@@ -119,7 +132,7 @@ def _format_chat_metadata_trailer(references: list[dict[str, int | str]]) -> str
 
 def stream_chat_with_documents(
     query_str: str,
-    documents: list[Document],
+    documents: QuerySet[Document],
     output_language: str | None = None,
 ):
     try:
@@ -135,10 +148,10 @@ def stream_chat_with_documents(
 
 def _stream_chat_with_documents(
     query_str: str,
-    documents: list[Document],
+    documents: QuerySet[Document],
     output_language: str | None = None,
 ):
-    if not documents:
+    if not documents.exists():
         yield CHAT_NO_CONTENT_MESSAGE
         return
 
@@ -148,7 +161,9 @@ def _stream_chat_with_documents(
     from llama_index.core.retrievers import VectorIndexRetriever
 
     config = AIConfig()
-    filters = _document_id_filters(str(doc.pk) for doc in documents)
+    filters = _document_id_filters(
+        str(pk) for pk in documents.values_list("pk", flat=True)
+    )
 
     # Hold the shared read lock for the whole operation: the query engine
     # retrieves from the vector store again during synthesis, so the connection

src/paperless_ai/tests/test_chat.py

@@ -3,10 +3,12 @@ from unittest.mock import MagicMock
 from unittest.mock import patch
 
 import pytest
+from django.db.models.signals import post_init
 from llama_index.core import settings as llama_settings
 from llama_index.core.embeddings.mock_embed_model import MockEmbedding
 from llama_index.core.schema import TextNode
 
+from documents.models import Document
 from documents.tests.factories import DocumentFactory
 from paperless_ai import chat
 from paperless_ai import indexing
@@ -36,16 +38,6 @@ def patch_embed_nodes():
         yield mock_embed_nodes
 
 
-@pytest.fixture
-def mock_document():
-    doc = MagicMock()
-    doc.pk = 1
-    doc.title = "Test Document"
-    doc.filename = "test_file.pdf"
-    doc.content = "This is the document content."
-    return doc
-
-
 def assert_chat_output(
     output: list[str],
     *,
@@ -61,6 +53,13 @@ def assert_chat_output(
     }
 
 
+def _fake_documents_queryset(pks: list[int]) -> MagicMock:
+    qs = MagicMock()
+    qs.exists.return_value = bool(pks)
+    qs.values_list.return_value = pks
+    return qs
+
+
 @pytest.mark.parametrize(
     ("output_language", "expected_language_line"),
     [
@@ -107,9 +106,10 @@ def test_build_refine_prompt(
 
 @pytest.mark.django_db
 def test_stream_chat_with_one_document_retrieval(
-    mock_document,
     patch_embed_nodes,
 ) -> None:
+    document = DocumentFactory.create(title="Test Document", content="ignored")
+    documents = Document.objects.filter(pk=document.pk)
     with (
         patch("paperless_ai.chat.AIClient") as mock_client_cls,
         patch("paperless_ai.chat.load_or_build_index") as mock_load_index,
@@ -124,22 +124,19 @@ def test_stream_chat_with_one_document_retrieval(
         mock_client_cls.return_value = mock_client
         mock_client.llm = MagicMock()
 
-        mock_node = TextNode(
-            text="This is node content.",
-            metadata={"document_id": str(mock_document.pk), "title": "Test Document"},
-        )
         mock_index = MagicMock()
-        # Simulate get_nodes returning nodes (content exists)
+        mock_index.vector_store.get_nodes.return_value = [
-        mock_index.vector_store.get_nodes.return_value = [mock_node]
+            TextNode(
+                text="This is node content.",
+                metadata={"document_id": str(document.pk), "title": "Test Document"},
+            ),
+        ]
         mock_load_index.return_value = mock_index
 
         mock_retriever_instance = MagicMock()
         mock_retriever_instance.retrieve.return_value = [
             MagicMock(
-                metadata={
+                metadata={"document_id": str(document.pk), "title": "Test Document"},
-                    "document_id": str(mock_document.pk),
-                    "title": "Test Document",
-                },
             ),
         ]
 
@@ -153,7 +150,7 @@ def test_stream_chat_with_one_document_retrieval(
             "llama_index.core.retrievers.VectorIndexRetriever",
             return_value=mock_retriever_instance,
         ):
-            output = list(stream_chat_with_documents("What is this?", [mock_document]))
+            output = list(stream_chat_with_documents("What is this?", documents))
 
         mock_query_engine.query.assert_called_once_with("What is this?")
         synthesizer_kwargs = mock_get_response_synthesizer.call_args.kwargs
@@ -166,13 +163,16 @@ def test_stream_chat_with_one_document_retrieval(
             output,
             expected_chunks=["chunk1", "chunk2"],
             expected_references=[
-                {"id": mock_document.pk, "title": "Test Document"},
+                {"id": document.pk, "title": "Test Document"},
             ],
         )
 
 
 @pytest.mark.django_db
 def test_stream_chat_with_multiple_documents_retrieval(patch_embed_nodes) -> None:
+    doc1 = DocumentFactory.create(title="Document 1", content="ignored")
+    doc2 = DocumentFactory.create(title="Document 2", content="ignored")
+    documents = Document.objects.filter(pk__in=[doc1.pk, doc2.pk])
     with (
         patch("paperless_ai.chat.AIClient") as mock_client_cls,
         patch("paperless_ai.chat.load_or_build_index") as mock_load_index,
@@ -184,23 +184,23 @@ def test_stream_chat_with_multiple_documents_retrieval(patch_embed_nodes) -> Non
         mock_client_cls.return_value = mock_client
         mock_client.llm = MagicMock()
 
-        mock_node1 = TextNode(
-            text="Content for doc 1.",
-            metadata={"document_id": "1", "title": "Document 1"},
-        )
-        mock_node2 = TextNode(
-            text="Content for doc 2.",
-            metadata={"document_id": "2", "title": "Document 2"},
-        )
         mock_index = MagicMock()
-        # Simulate get_nodes returning nodes (content exists)
+        mock_index.vector_store.get_nodes.return_value = [
-        mock_index.vector_store.get_nodes.return_value = [mock_node1, mock_node2]
+            TextNode(
+                text="Content for doc 1.",
+                metadata={"document_id": str(doc1.pk), "title": "Document 1"},
+            ),
+            TextNode(
+                text="Content for doc 2.",
+                metadata={"document_id": str(doc2.pk), "title": "Document 2"},
+            ),
+        ]
         mock_load_index.return_value = mock_index
 
         mock_retriever_instance = MagicMock()
         mock_retriever_instance.retrieve.return_value = [
-            MagicMock(metadata={"document_id": "1", "title": "Document 1"}),
+            MagicMock(metadata={"document_id": str(doc1.pk), "title": "Document 1"}),
-            MagicMock(metadata={"document_id": "2", "title": "Document 2"}),
+            MagicMock(metadata={"document_id": str(doc2.pk), "title": "Document 2"}),
         ]
 
         mock_response_stream = MagicMock()
@@ -210,14 +210,11 @@ def test_stream_chat_with_multiple_documents_retrieval(patch_embed_nodes) -> Non
         mock_query_engine_cls.return_value = mock_query_engine
         mock_query_engine.query.return_value = mock_response_stream
 
-        doc1 = MagicMock(pk=1, title="Document 1", filename="doc1.pdf")
-        doc2 = MagicMock(pk=2, title="Document 2", filename="doc2.pdf")
-
         with patch(
             "llama_index.core.retrievers.VectorIndexRetriever",
             return_value=mock_retriever_instance,
         ):
-            output = list(stream_chat_with_documents("What's up?", [doc1, doc2]))
+            output = list(stream_chat_with_documents("What's up?", documents))
 
         mock_query_engine.query.assert_called_once_with("What's up?")
         patch_embed_nodes.assert_not_called()
@@ -225,15 +222,15 @@ def test_stream_chat_with_multiple_documents_retrieval(patch_embed_nodes) -> Non
             output,
             expected_chunks=["chunk1", "chunk2"],
             expected_references=[
-                {"id": 1, "title": "Document 1"},
+                {"id": doc1.pk, "title": "Document 1"},
-                {"id": 2, "title": "Document 2"},
+                {"id": doc2.pk, "title": "Document 2"},
             ],
         )
 
 
 def test_stream_chat_empty_document_list() -> None:
     with patch("paperless_ai.chat.load_or_build_index") as mock_load_index:
-        output = list(stream_chat_with_documents("Any info?", []))
+        output = list(stream_chat_with_documents("Any info?", Document.objects.none()))
         mock_load_index.assert_not_called()
         assert output == ["Sorry, I couldn't find any content to answer your question."]
 
@@ -253,7 +250,9 @@ def test_stream_chat_no_matching_nodes() -> None:
         mock_index.vector_store.get_nodes.return_value = []
         mock_load_index.return_value = mock_index
 
-        output = list(stream_chat_with_documents("Any info?", [MagicMock(pk=1)]))
+        output = list(
+            stream_chat_with_documents("Any info?", _fake_documents_queryset([1])),
+        )
 
         assert output == ["Sorry, I couldn't find any content to answer your question."]
 
@@ -282,7 +281,9 @@ def test_stream_chat_unexpected_failure_returns_generic_error(caplog) -> None:
             )
             mock_retriever_cls.return_value = mock_retriever
 
-            output = list(stream_chat_with_documents("Any info?", [MagicMock(pk=1)]))
+            output = list(
+                stream_chat_with_documents("Any info?", _fake_documents_queryset([1])),
+            )
 
         assert output == [CHAT_ERROR_MESSAGE]
         assert "Failed to stream document chat response" in caplog.text
@@ -298,7 +299,12 @@ class TestStreamChatRetrieval:
     ) -> None:
         doc = DocumentFactory.create(content="hello world")
         # Nothing indexed for this document yet.
-        out = list(chat.stream_chat_with_documents("question?", [doc]))
+        out = list(
+            chat.stream_chat_with_documents(
+                "question?",
+                Document.objects.filter(pk=doc.pk),
+            ),
+        )
         assert chat.CHAT_NO_CONTENT_MESSAGE in out
 
     def test_chat_filter_contains_only_requested_document_ids(
@@ -332,7 +338,12 @@ class TestStreamChatRetrieval:
             side_effect=capture_retriever,
         )
 
-        list(chat.stream_chat_with_documents("question?", [included]))
+        list(
+            chat.stream_chat_with_documents(
+                "question?",
+                Document.objects.filter(pk=included.pk),
+            ),
+        )
 
         assert captured_filters, "VectorIndexRetriever was never constructed"
         filt = captured_filters[0]
@@ -340,3 +351,47 @@ class TestStreamChatRetrieval:
         filter_values = filt.filters[0].value
         assert str(included.pk) in filter_values
         assert str(excluded.pk) not in filter_values
+
+    @pytest.mark.django_db
+    def test_get_document_references_only_queries_referenced_documents(
+        self,
+        django_assert_num_queries,
+    ) -> None:
+        """Building references must not hydrate every document the caller is
+        permitted to see -- only the (<= CHAT_RETRIEVER_TOP_K) documents that
+        the retriever actually returned nodes for.
+        """
+        referenced = DocumentFactory.create(title="Referenced Document")
+        # Many more documents are "accessible" but never referenced by a node.
+        DocumentFactory.create_batch(200)
+
+        documents = Document.objects.all()
+        top_nodes = [
+            MagicMock(
+                metadata={
+                    "document_id": str(referenced.pk),
+                    "title": "Referenced Document",
+                },
+            ),
+        ]
+
+        hydrated_count = 0
+
+        def _count_hydration(sender, instance, **kwargs):
+            nonlocal hydrated_count
+            hydrated_count += 1
+
+        post_init.connect(_count_hydration, sender=Document)
+        try:
+            # One query: `documents.filter(pk__in=candidate_ids)` for the single
+            # referenced id. No query should scale with the 200 unreferenced documents.
+            with django_assert_num_queries(1):
+                references = chat._get_document_references(documents, top_nodes)
+        finally:
+            post_init.disconnect(_count_hydration, sender=Document)
+
+        # The bug this guards against: the old code hydrated all 201 accessible
+        # documents via `{doc.pk: doc for doc in documents}` before filtering by
+        # top_nodes. Only the referenced document should ever be constructed.
+        assert hydrated_count == 1
+        assert references == [{"id": referenced.pk, "title": "Referenced Document"}]

src/paperless_mail/tests/test_api.py

@@ -757,30 +757,3 @@ class TestAPIProcessedMails(DirectoriesMixin, APITestCase):
             format="json",
         )
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-
-    def test_bulk_delete_processed_mails_rejects_mixed_batch_atomically(self) -> None:
-        """
-        GIVEN:
-            - A permitted processed mail and one the user may not delete
-        WHEN:
-            - API call bulk deletes both in a single request
-        THEN:
-            - The request is rejected and neither mail is deleted
-        """
-        user2 = User.objects.create_user(username="temp_admin2")
-        rule = MailRuleFactory()
-        # Created first so it sorts ahead of the forbidden mail, i.e. the
-        # permission check has to cover the whole batch before deleting rather
-        # than rejecting only once it reaches the forbidden one.
-        pm_owned = ProcessedMailFactory(rule=rule, owner=self.user)
-        pm_forbidden = ProcessedMailFactory(rule=rule, owner=user2)
-
-        response = self.client.post(
-            f"{self.ENDPOINT}bulk_delete/",
-            data={"mail_ids": [pm_owned.id, pm_forbidden.id]},
-            format="json",
-        )
-
-        self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
-        self.assertTrue(ProcessedMail.objects.filter(id=pm_owned.id).exists())
-        self.assertTrue(ProcessedMail.objects.filter(id=pm_forbidden.id).exists())

src/paperless_mail/views.py

@@ -27,7 +27,6 @@ from documents.filters import PermittedObjectsFilter
 from documents.models import PaperlessTask
 from documents.permissions import PaperlessObjectPermissions
 from documents.permissions import has_perms_owner_aware
-from documents.permissions import permitted_object_ids
 from documents.views import PassUserMixin
 from paperless.views import StandardPagination
 from paperless_mail.filters import ProcessedMailFilterSet
@@ -212,17 +211,10 @@ class ProcessedMailViewSet(PassUserMixin, ReadOnlyModelViewSet[ProcessedMail]):
         ):
             return HttpResponseBadRequest("mail_ids must be a list of integers")
         mails = ProcessedMail.objects.filter(id__in=mail_ids)
-        # Check every id up front so an unpermitted one rejects the whole
+        for mail in mails:
-        # request rather than deleting the mails ahead of it first.
+            if not has_perms_owner_aware(request.user, "delete_processedmail", mail):
-        if mails.exclude(
+                return HttpResponseForbidden("Insufficient permissions")
-            pk__in=permitted_object_ids(
+            mail.delete()
-                request.user,
-                ProcessedMail,
-                "delete_processedmail",
-            ),
-        ).exists():
-            return HttpResponseForbidden("Insufficient permissions")
-        mails.delete()
         return Response({"result": "OK", "deleted_mail_ids": mail_ids})
 
 

uv.lock

@@ -1298,16 +1298,16 @@ wheels = [
 
 [[package]]
 name = "fpdf2"
-version = "2.8.8"
+version = "2.8.7"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "defusedxml", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "fonttools", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "pillow", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/1e/bc/8fd4321aed40cadadddc8f311c65b6082346b252bca048f7b476d8f35d72/fpdf2-2.8.8.tar.gz", hash = "sha256:9e94e155e85e8053329a9a1fce8b566fd7a7c5bb79e98a1a3952d379b947c5b9", size = 374689, upload-time = "2026-08-09T23:32:45.334Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/27/f2/72feae0b2827ed38013e4307b14f95bf0b3d124adfef4d38a7d57533f7be/fpdf2-2.8.7.tar.gz", hash = "sha256:7060ccee5a9c7ab0a271fb765a36a23639f83ef8996c34e3d46af0a17ede57f9", size = 362351, upload-time = "2026-02-28T05:39:16.456Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/f5/be/af012eda9507494f28b99b077423806c43a11573eb6225dd46f19ae2d263/fpdf2-2.8.8-py3-none-any.whl", hash = "sha256:3557a478fc577a929c94aace9666aed4dcc432b5ab6764232e6a59f1ccd75f17", size = 337000, upload-time = "2026-08-09T23:32:43.728Z" },
+    { url = "https://files.pythonhosted.org/packages/66/0a/cf50ecffa1e3747ed9380a3adfc829259f1f86b3fdbd9e505af789003141/fpdf2-2.8.7-py3-none-any.whl", hash = "sha256:d391fc508a3ce02fc43a577c830cda4fe6f37646f2d143d489839940932fbc19", size = 327056, upload-time = "2026-02-28T05:39:14.619Z" },
 ]
 
 [[package]]
@@ -2927,8 +2927,8 @@ dependencies = [
     { name = "sqlite-vec", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "tantivy", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "tika-client", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
-    { name = "torch", version = "2.13.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "python_full_version < '3.15' and sys_platform == 'darwin'" },
+    { name = "torch", version = "2.13.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'darwin'" },
-    { name = "torch", version = "2.13.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(python_full_version >= '3.15' and sys_platform == 'darwin') or sys_platform == 'linux'" },
+    { name = "torch", version = "2.13.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'linux'" },
     { name = "watchfiles", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "whitenoise", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "zxing-cpp", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -4511,8 +4511,8 @@ dependencies = [
     { name = "numpy", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "scikit-learn", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "scipy", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
-    { name = "torch", version = "2.13.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "python_full_version < '3.15' and sys_platform == 'darwin'" },
+    { name = "torch", version = "2.13.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'darwin'" },
-    { name = "torch", version = "2.13.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(python_full_version >= '3.15' and sys_platform == 'darwin') or sys_platform == 'linux'" },
+    { name = "torch", version = "2.13.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'linux'" },
     { name = "tqdm", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "transformers", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -4957,17 +4957,18 @@ name = "torch"
 version = "2.13.0"
 source = { registry = "https://download.pytorch.org/whl/cpu" }
 resolution-markers = [
+    "python_full_version >= '3.15' and sys_platform == 'darwin'",
     "python_full_version >= '3.12' and python_full_version < '3.15' and sys_platform == 'darwin'",
     "python_full_version < '3.12' and sys_platform == 'darwin'",
 ]
 dependencies = [
-    { name = "filelock", marker = "python_full_version < '3.15' and sys_platform == 'darwin'" },
+    { name = "filelock", marker = "sys_platform == 'darwin'" },
-    { name = "fsspec", marker = "python_full_version < '3.15' and sys_platform == 'darwin'" },
+    { name = "fsspec", marker = "sys_platform == 'darwin'" },
-    { name = "jinja2", marker = "python_full_version < '3.15' and sys_platform == 'darwin'" },
+    { name = "jinja2", marker = "sys_platform == 'darwin'" },
-    { name = "networkx", marker = "python_full_version < '3.15' and sys_platform == 'darwin'" },
+    { name = "networkx", marker = "sys_platform == 'darwin'" },
-    { name = "setuptools", marker = "python_full_version < '3.15' and sys_platform == 'darwin'" },
+    { name = "setuptools", marker = "sys_platform == 'darwin'" },
-    { name = "sympy", marker = "python_full_version < '3.15' and sys_platform == 'darwin'" },
+    { name = "sympy", marker = "sys_platform == 'darwin'" },
-    { name = "typing-extensions", marker = "python_full_version < '3.15' and sys_platform == 'darwin'" },
+    { name = "typing-extensions", marker = "sys_platform == 'darwin'" },
 ]
 wheels = [
     { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.13.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:e76f9bcecc52b8ff711239a2f7547d5353df95878ab232f0773c1d95928b92f8", upload-time = "2026-07-08T12:26:13Z" },
@@ -4982,7 +4983,6 @@ name = "torch"
 version = "2.13.0+cpu"
 source = { registry = "https://download.pytorch.org/whl/cpu" }
 resolution-markers = [
-    "python_full_version >= '3.15' and sys_platform == 'darwin'",
     "python_full_version == '3.12.*' and platform_machine == 'x86_64' and sys_platform == 'linux'",
     "python_full_version == '3.12.*' and platform_machine == 'aarch64' and sys_platform == 'linux'",
     "python_full_version >= '3.15' and sys_platform == 'linux'",
@@ -4990,13 +4990,13 @@ resolution-markers = [
     "python_full_version < '3.12' and sys_platform == 'linux'",
 ]
 dependencies = [
-    { name = "filelock", marker = "(python_full_version >= '3.15' and sys_platform == 'darwin') or sys_platform == 'linux'" },
+    { name = "filelock", marker = "sys_platform == 'linux'" },
-    { name = "fsspec", marker = "(python_full_version >= '3.15' and sys_platform == 'darwin') or sys_platform == 'linux'" },
+    { name = "fsspec", marker = "sys_platform == 'linux'" },
-    { name = "jinja2", marker = "(python_full_version >= '3.15' and sys_platform == 'darwin') or sys_platform == 'linux'" },
+    { name = "jinja2", marker = "sys_platform == 'linux'" },
-    { name = "networkx", marker = "(python_full_version >= '3.15' and sys_platform == 'darwin') or sys_platform == 'linux'" },
+    { name = "networkx", marker = "sys_platform == 'linux'" },
-    { name = "setuptools", marker = "(python_full_version >= '3.15' and sys_platform == 'darwin') or sys_platform == 'linux'" },
+    { name = "setuptools", marker = "sys_platform == 'linux'" },
-    { name = "sympy", marker = "(python_full_version >= '3.15' and sys_platform == 'darwin') or sys_platform == 'linux'" },
+    { name = "sympy", marker = "sys_platform == 'linux'" },
-    { name = "typing-extensions", marker = "(python_full_version >= '3.15' and sys_platform == 'darwin') or sys_platform == 'linux'" },
+    { name = "typing-extensions", marker = "sys_platform == 'linux'" },
 ]
 wheels = [
     { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.13.0%2Bcpu-cp311-cp311-linux_s390x.whl", hash = "sha256:6e9817dbdf5ea76789babd46e457eac5bf14ff566cf85f8addbfdff2d56601ce", upload-time = "2026-07-08T19:27:52Z" },