Wording

* Fix: skip vector store document id filter for unrestricted chat users

ChatStreamingView built an IN filter from every permitted document id
for the "chat over all documents" case, which exceeds the vector
store's SQLite bound-parameter safety limit on installs with more
than ~32700 documents, silently returning no context. For a user who
can see every document (an active superuser), that filter never
narrows anything, so skip it and let the retriever search the whole
index instead.

* Minor improvements from a Claude review

* When a user is unrestricted chatting, still exclude trashed documents using a 'NOT IN' SQL statement.  Wire that up where we need it

* Update src/paperless_ai/chat.py

Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com>

docs/advanced_usage.md

@@ -138,7 +138,9 @@ for suggested generation and embedding models.
 With AI enabled, Paperless-ngx can suggest a title, tags, correspondent, document type,
 storage path and dates by sending the document to the LLM. This is **opt-in per request**
 and surfaces through the "Suggest" control on the document detail page, alongside the
-classic classifier-based suggestions — it does not disable them. Suggestion output
+classic classifier-based suggestions — it does not disable them. Suggestions are requested
+automatically when you open a document that carries an inbox tag unless "Automatically request
+suggestions for inbox documents" under Settings > Documents is disabled. Suggestion output
 language can be steered with
 [`PAPERLESS_AI_LLM_OUTPUT_LANGUAGE`](configuration.md#PAPERLESS_AI_LLM_OUTPUT_LANGUAGE)
 (otherwise it follows the user's UI language).

docs/usage.md

@@ -317,6 +317,8 @@ a "document already exists" message.
 
 Paperless-ngx can suggest tags, correspondents, document types and storage paths for documents based on the content of the document. This is done using a (non-LLM) machine learning model that is trained on the documents in your database. The suggestions are shown in the document detail page and can be accepted or rejected by the user.
 
+Suggestions are requested automatically when you open a document that still has an inbox tag. To only request them by pressing the "Suggest" button instead, turn off "Automatically request suggestions for inbox documents" under Settings > Documents.
+
 ## AI Features
 
 Paperless-ngx includes several features that use AI to enhance the document management experience. These features are optional and can be enabled or disabled in the settings. If you are using the AI features, you may want to also enable the "LLM index" feature, which supports Retrieval-Augmented Generation (RAG) designed to improve the quality of AI responses. The LLM index feature is not enabled by default and requires additional configuration.

src-ui/src/app/components/admin/settings/settings.component.html

@@ -237,6 +237,12 @@
               </div>
             </div>
 
+            <div class="row">
+              <div class="col">
+                <pngx-input-check i18n-title title="Automatically request suggestions for inbox documents" i18n-hint hint="If un-checked, suggestions must be requested via the Suggest button." formControlName="documentEditingAutoSuggest"></pngx-input-check>
+              </div>
+            </div>
+
             <div class="row">
               <div class="col">
                 <pngx-input-check i18n-title title="Show document thumbnail during loading" formControlName="documentEditingOverlayThumbnail"></pngx-input-check>

src-ui/src/app/components/admin/settings/settings.component.spec.ts

@@ -267,7 +267,7 @@ describe('SettingsComponent', () => {
     expect(toastErrorSpy).toHaveBeenCalled()
     expect(storeSpy).toHaveBeenCalled()
     expect(appearanceSettingsSpy).not.toHaveBeenCalled()
-    expect(setSpy).toHaveBeenCalledTimes(32)
+    expect(setSpy).toHaveBeenCalledTimes(33)
 
     // succeed
     storeSpy.mockReturnValueOnce(of(true))

src-ui/src/app/components/admin/settings/settings.component.ts

@@ -168,6 +168,7 @@ export class SettingsComponent
     pdfEditorDefaultEditMode: new FormControl(null),
     documentEditingRemoveInboxTags: new FormControl(null),
     documentEditingOverlayThumbnail: new FormControl(null),
+    documentEditingAutoSuggest: new FormControl(null),
     documentDetailsHiddenFields: new FormControl([]),
     searchDbOnly: new FormControl(null),
     searchLink: new FormControl(null),
@@ -368,6 +369,9 @@ export class SettingsComponent
       documentEditingOverlayThumbnail: this.settings.get(
         SETTINGS_KEYS.DOCUMENT_EDITING_OVERLAY_THUMBNAIL
       ),
+      documentEditingAutoSuggest: this.settings.get(
+        SETTINGS_KEYS.DOCUMENT_EDITING_AUTO_SUGGEST
+      ),
       documentDetailsHiddenFields: this.settings.get(
         SETTINGS_KEYS.DOCUMENT_DETAILS_HIDDEN_FIELDS
       ),
@@ -565,6 +569,10 @@ export class SettingsComponent
       SETTINGS_KEYS.DOCUMENT_EDITING_OVERLAY_THUMBNAIL,
       this.settingsForm.value.documentEditingOverlayThumbnail
     )
+    this.settings.set(
+      SETTINGS_KEYS.DOCUMENT_EDITING_AUTO_SUGGEST,
+      this.settingsForm.value.documentEditingAutoSuggest
+    )
     this.settings.set(
       SETTINGS_KEYS.DOCUMENT_DETAILS_HIDDEN_FIELDS,
       this.settingsForm.value.documentDetailsHiddenFields

src-ui/src/app/components/common/filterable-dropdown/toggleable-dropdown-button/toggleable-dropdown-button.component.scss

@@ -7,6 +7,8 @@
   padding-left: calc(calc(var(--depth) - 2) * 1rem);
   display: flex;
   align-items: center;
+    min-width: 0;
+    overflow-wrap: anywhere;
 
   .indicator {
     display: inline-block;
@@ -18,3 +20,7 @@
     margin-left: .5rem;
   }
 }
+
+  .badge {
+    flex-shrink: 0;
+  }

src-ui/src/app/components/document-detail/document-detail.component.spec.ts

@@ -1473,6 +1473,35 @@ describe('DocumentDetailComponent', () => {
     })
   })
 
+  it('should not automatically get suggestions if auto-suggest is disabled', () => {
+    settingsService.set(SETTINGS_KEYS.DOCUMENT_EDITING_AUTO_SUGGEST, false)
+    const suggestionsSpy = jest.spyOn(documentService, 'getSuggestions')
+    suggestionsSpy.mockReturnValue(of({ tags: [42] }))
+    initNormally()
+    expect(suggestionsSpy).not.toHaveBeenCalled()
+
+    // still available on demand
+    component.getSuggestions()
+    expect(suggestionsSpy).toHaveBeenCalled()
+  })
+
+  it('should not automatically get AI suggestions if auto-suggest is disabled', () => {
+    settingsService.set(SETTINGS_KEYS.DOCUMENT_EDITING_AUTO_SUGGEST, false)
+    const getSetting = settingsService.get.bind(settingsService)
+    jest
+      .spyOn(settingsService, 'get')
+      .mockImplementation((key) =>
+        key === SETTINGS_KEYS.AI_ENABLED ? true : getSetting(key)
+      )
+    const aiSuggestionsSpy = jest.spyOn(documentService, 'getAiSuggestions')
+    aiSuggestionsSpy.mockReturnValue(of({ tags: [42] }))
+    initNormally()
+    expect(aiSuggestionsSpy).not.toHaveBeenCalled()
+
+    component.getSuggestions()
+    expect(aiSuggestionsSpy).toHaveBeenCalled()
+  })
+
   it('should reset the suggestions loading state if the document changes mid-request', () => {
     const getSetting = settingsService.get.bind(settingsService)
     jest

src-ui/src/app/components/document-detail/document-detail.component.ts

@@ -237,6 +237,9 @@ export class DocumentDetailComponent
     this.settings.getSignal<boolean>(
       SETTINGS_KEYS.DOCUMENT_EDITING_OVERLAY_THUMBNAIL
     )
+  private readonly autoSuggestSetting = this.settings.getSignal<boolean>(
+    SETTINGS_KEYS.DOCUMENT_EDITING_AUTO_SUGGEST
+  )
   private readonly hiddenFieldsSetting = this.settings.getSignal<
     DocumentDetailFieldID[]
   >(SETTINGS_KEYS.DOCUMENT_DETAILS_HIDDEN_FIELDS)
@@ -357,6 +360,10 @@ export class DocumentDetailComponent
     return this.aiEnabledSetting()
   }
 
+  get autoSuggest(): boolean {
+    return this.autoSuggestSetting()
+  }
+
   get archiveContentRenderType(): ContentRenderType {
     const hasArchiveVersion =
       this.metadata()?.has_archive_version ??
@@ -904,6 +911,7 @@ export class DocumentDetailComponent
     this.updateFormForCustomFields()
     this.loadMetadataForSelectedVersion()
     if (
+      this.autoSuggest &&
       this.permissionsService.currentUserHasObjectPermissions(
         PermissionAction.Change,
         doc

src-ui/src/app/data/ui-settings.ts

@@ -84,6 +84,8 @@ export const SETTINGS_KEYS = {
     'general-settings:document-editing:remove-inbox-tags',
   DOCUMENT_EDITING_OVERLAY_THUMBNAIL:
     'general-settings:document-editing:overlay-thumbnail',
+  DOCUMENT_EDITING_AUTO_SUGGEST:
+    'general-settings:document-editing:auto-suggest',
   DOCUMENT_DETAILS_HIDDEN_FIELDS:
     'general-settings:document-details:hidden-fields',
   SEARCH_DB_ONLY: 'general-settings:search:db-only',
@@ -300,6 +302,11 @@ export const SETTINGS: UiSetting[] = [
     type: 'boolean',
     default: true,
   },
+  {
+    key: SETTINGS_KEYS.DOCUMENT_EDITING_AUTO_SUGGEST,
+    type: 'boolean',
+    default: true,
+  },
   {
     key: SETTINGS_KEYS.DOCUMENT_DETAILS_HIDDEN_FIELDS,
     type: 'array',

src/documents/management/commands/document_consumer.py

@@ -314,7 +314,7 @@ def _consume_file(
     consumption_dir: Path,
     *,
     subdirs_as_tags: bool,
-) -> None:
+) -> bool:
     """
     Queue a file for consumption.
 
@@ -322,15 +322,20 @@ def _consume_file(
         filepath: Path to the file to consume.
         consumption_dir: Base consumption directory.
         subdirs_as_tags: Whether to create tags from subdirectory names.
+
+    Returns:
+        True if the file was successfully handed to Celery, False otherwise.
+        Callers must not record the file as queued on failure, or the rescan
+        will never retry it.
     """
     # Verify file still exists and is accessible
     try:
         if not filepath.is_file():
             logger.debug(f"Not consuming {filepath}: not a file or doesn't exist")
-            return
+            return False
     except OSError as e:
         logger.warning(f"Not consuming {filepath}: {e}")
-        return
+        return False
 
     # Get tags from path if configured
     tag_ids: list[int] | None = None
@@ -355,6 +360,9 @@ def _consume_file(
         )
     except Exception:
         logger.exception(f"Error while queuing document {filepath}")
+        return False
+
+    return True
 
 
 class Command(BaseCommand):
@@ -492,12 +500,12 @@ class Command(BaseCommand):
             if not consumer_filter(Change.added, str(filepath)):
                 continue
 
-            _consume_file(
+            if _consume_file(
                 filepath=filepath,
                 consumption_dir=directory,
                 subdirs_as_tags=subdirs_as_tags,
-            )
-            queued.add(filepath.resolve())
+            ):
+                queued.add(filepath.resolve())
 
         return queued
 
@@ -651,14 +659,16 @@ class Command(BaseCommand):
 
                     # Check for stable files
                     for stable_path in tracker.get_stable_files():
-                        _consume_file(
+                        # Only remember files that were actually queued, so the
+                        # rescan does not re-queue them while the consume task
+                        # has yet to remove them from disk, but does retry a
+                        # failed publish instead of stranding it
+                        if _consume_file(
                             filepath=stable_path,
                             consumption_dir=directory,
                             subdirs_as_tags=subdirs_as_tags,
-                        )
-                        # Remember it so the rescan does not re-queue it while
-                        # the consume task has yet to remove it from disk
-                        queued.add(stable_path)
+                        ):
+                            queued.add(stable_path)
 
                     # Exit watch loop to reconfigure timeout
                     break

src/documents/signals/handlers.py

@@ -1003,7 +1003,7 @@ def run_workflows(
 
                         # kwargs so the PaperlessTask record can note the
                         # document, see _extract_input_data
-                        apply_ai_suggestions.delay(
+                        apply_ai_suggestions.delay_on_commit(
                             action_id=action.pk,
                             document_id=document.pk,
                         )

src/documents/tests/test_management_consumer.py

@@ -445,12 +445,13 @@ class TestConsumeFile:
         target = consumption_dir / "document.pdf"
         shutil.copy(sample_pdf, target)
 
-        _consume_file(
+        result = _consume_file(
             filepath=target,
             consumption_dir=consumption_dir,
             subdirs_as_tags=False,
         )
 
+        assert result is True
         mock_consume_file_delay.apply_async.assert_called_once()
         call_args = mock_consume_file_delay.apply_async.call_args
         consumable_doc = call_args.kwargs["kwargs"]["input_doc"]
@@ -464,11 +465,12 @@ class TestConsumeFile:
         mock_consume_file_delay: MagicMock,
     ) -> None:
         """Test _consume_file handles nonexistent files gracefully."""
-        _consume_file(
+        result = _consume_file(
             filepath=consumption_dir / "nonexistent.pdf",
             consumption_dir=consumption_dir,
             subdirs_as_tags=False,
         )
+        assert result is False
         mock_consume_file_delay.apply_async.assert_not_called()
 
     def test_consume_directory(
@@ -480,11 +482,12 @@ class TestConsumeFile:
         subdir = consumption_dir / "subdir"
         subdir.mkdir()
 
-        _consume_file(
+        result = _consume_file(
             filepath=subdir,
             consumption_dir=consumption_dir,
             subdirs_as_tags=False,
         )
+        assert result is False
         mock_consume_file_delay.apply_async.assert_not_called()
 
     def test_consume_with_permission_error(
@@ -499,13 +502,33 @@ class TestConsumeFile:
         shutil.copy(sample_pdf, target)
 
         mocker.patch.object(Path, "is_file", side_effect=PermissionError("denied"))
-        _consume_file(
+        result = _consume_file(
             filepath=target,
             consumption_dir=consumption_dir,
             subdirs_as_tags=False,
         )
+        assert result is False
         mock_consume_file_delay.apply_async.assert_not_called()
 
+    def test_consume_with_apply_async_failure(
+        self,
+        consumption_dir: Path,
+        sample_pdf: Path,
+        mock_consume_file_delay: MagicMock,
+    ) -> None:
+        """Test _consume_file reports failure when apply_async raises."""
+        target = consumption_dir / "document.pdf"
+        shutil.copy(sample_pdf, target)
+
+        mock_consume_file_delay.apply_async.side_effect = Exception("broker down")
+
+        result = _consume_file(
+            filepath=target,
+            consumption_dir=consumption_dir,
+            subdirs_as_tags=False,
+        )
+        assert result is False
+
     def test_consume_with_tags_error(
         self,
         consumption_dir: Path,
@@ -522,11 +545,12 @@ class TestConsumeFile:
             side_effect=DatabaseError("Something happened"),
         )
 
-        _consume_file(
+        result = _consume_file(
             filepath=target,
             consumption_dir=consumption_dir,
             subdirs_as_tags=True,
         )
+        assert result is True
         mock_consume_file_delay.apply_async.assert_called_once()
         call_args = mock_consume_file_delay.apply_async.call_args
         overrides = call_args.kwargs["kwargs"]["overrides"]
@@ -1249,6 +1273,52 @@ class TestProcessExistingFilesQueued:
         assert target.resolve() in queued
 
 
+@pytest.mark.management
+@pytest.mark.django_db
+class TestCommandRetryAfterQueueFailure:
+    """
+    Regression test for GH #13923.
+
+    A file whose ``apply_async`` publish fails (e.g. broker briefly down)
+    must not be marked as queued, so the periodic rescan retries it once
+    the broker recovers, instead of stranding it until the consumer
+    process is restarted.
+    """
+
+    def test_watch_loop_retries_failed_publish_on_rescan(
+        self,
+        consumption_dir: Path,
+        sample_pdf: Path,
+        mock_consume_file_delay: MagicMock,
+        start_consumer: Callable[..., ConsumerThread],
+    ) -> None:
+        """A publish failure from the watch loop is retried by the rescan."""
+        apply_async = mock_consume_file_delay.apply_async
+
+        def fail_first_call(*args: object, **kwargs: object) -> None:
+            if apply_async.call_count == 1:
+                raise Exception("broker down")
+
+        apply_async.side_effect = fail_first_call
+
+        thread = start_consumer(stability_delay=0.1, rescan_interval=0.3)
+
+        target = consumption_dir / "document.pdf"
+        shutil.copy(sample_pdf, target)
+
+        deadline = monotonic() + 5.0
+        while apply_async.call_count < 2 and monotonic() < deadline:
+            sleep(0.1)
+
+        if thread.exception:
+            raise thread.exception
+
+        assert apply_async.call_count >= 2, (
+            "Expected the failed publish to be retried by the rescan, "
+            f"but apply_async was only called {apply_async.call_count} time(s)"
+        )
+
+
 @pytest.mark.management
 @pytest.mark.django_db
 class TestCommandRescanRecovery:

src/documents/tests/test_workflows.py

@@ -5621,11 +5621,15 @@ class TestApplyAISuggestionsWorkflowAction(
         action = self.make_action()
         self.make_workflow(action, WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED)
 
-        with mock.patch("documents.tasks.apply_ai_suggestions.delay") as delay:
+        with (
+            mock.patch("documents.tasks.apply_ai_suggestions.delay") as delay,
+            self.captureOnCommitCallbacks(execute=True),
+        ):
             run_workflows(
                 WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
                 self.doc,
             )
+            delay.assert_not_called()
 
         delay.assert_called_once_with(action_id=action.pk, document_id=self.doc.pk)
 

src/locale/en_US/LC_MESSAGES/django.po

@@ -2,7 +2,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: paperless-ngx\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2026-09-01 19:54+0000\n"
+"POT-Creation-Date: 2026-09-02 18:09+0000\n"
 "PO-Revision-Date: 2022-02-17 04:17\n"
 "Last-Translator: \n"
 "Language-Team: English\n"
@@ -1628,7 +1628,7 @@ msgid "workflow runs"
 msgstr ""
 
 #: documents/serialisers.py:524 documents/serialisers.py:878
-#: documents/serialisers.py:2830 documents/views.py:313 documents/views.py:2619
+#: documents/serialisers.py:2830 documents/views.py:314 documents/views.py:2623
 #: paperless_mail/serialisers.py:156
 msgid "Insufficient permissions."
 msgstr ""
@@ -1669,7 +1669,7 @@ msgstr ""
 msgid "Duplicate document identifiers are not allowed."
 msgstr ""
 
-#: documents/serialisers.py:2916 documents/views.py:4620
+#: documents/serialisers.py:2916 documents/views.py:4624
 #, python-format
 msgid "Documents not found: %(ids)s"
 msgstr ""
@@ -1937,36 +1937,36 @@ msgstr ""
 msgid "Unable to parse URI {value}"
 msgstr ""
 
-#: documents/views.py:306 documents/views.py:2616
+#: documents/views.py:307 documents/views.py:2620
 msgid "Invalid more_like_id"
 msgstr ""
 
-#: documents/views.py:1590
+#: documents/views.py:1591
 msgid "Invalid AI configuration."
 msgstr ""
 
-#: documents/views.py:1601
+#: documents/views.py:1602
 msgid "AI backend request timed out."
 msgstr ""
 
-#: documents/views.py:2441 documents/views.py:2762
+#: documents/views.py:2445 documents/views.py:2766
 msgid "Specify only one of text, title_search, query, or more_like_id."
 msgstr ""
 
-#: documents/views.py:4633
+#: documents/views.py:4637
 #, python-format
 msgid "Insufficient permissions to share document %(id)s."
 msgstr ""
 
-#: documents/views.py:4679
+#: documents/views.py:4683
 msgid "Bundle is already being processed."
 msgstr ""
 
-#: documents/views.py:4743
+#: documents/views.py:4747
 msgid "The share link bundle is still being prepared. Please try again later."
 msgstr ""
 
-#: documents/views.py:4757
+#: documents/views.py:4761
 msgid "The share link bundle is unavailable."
 msgstr ""
 

src/paperless/network.py

@@ -129,7 +129,7 @@ def _rewrite_request_to_pinned_ip(
         method=request.method,
         url=new_url,
         headers=new_headers,
-        content=request.stream,
+        stream=request.stream,
         extensions=request.extensions,
     )
     rewritten_request.extensions["sni_hostname"] = hostname

src/paperless/settings/__init__.py

@@ -705,6 +705,12 @@ CELERY_BROKER_TRANSPORT_OPTIONS = {
 CELERY_TASK_TRACK_STARTED = True
 CELERY_TASK_TIME_LIMIT: Final[int] = get_int_from_env("PAPERLESS_WORKER_TIMEOUT", 1800)
 
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#std-setting-task_allow_error_cb_on_chord_header
+# Without this, a failing chord header never triggers the errback, so a mail
+# whose attachments all fail is never recorded and is re-fetched forever.
+# The errback runs once per failed header task, so it must be idempotent.
+CELERY_TASK_ALLOW_ERROR_CB_ON_CHORD_HEADER = True
+
 CELERY_CACHE_BACKEND = "default"
 
 # https://docs.celeryq.dev/en/stable/userguide/configuration.html#task-serializer

src/paperless_ai/chat.py

@@ -135,9 +135,7 @@ def _stream_chat_with_documents(
         # limit (_MAX_IN_VALUES) on large installs. Trashed documents stay
         # indexed until permanent deletion (delete_document_from_llm_index
         # hangs off post_delete, not trash), so must be excluded explicitly.
-        trashed_ids = Document.global_objects.filter(
-            deleted_at__isnull=False,
-        ).values_list("pk", flat=True)
+        trashed_ids = Document.deleted_objects.values_list("pk", flat=True)
         filters = exclude_document_ids_filter(str(pk) for pk in trashed_ids)
     else:
         filters = document_id_filters(

src/paperless_ai/client.py

@@ -131,11 +131,10 @@ class AIClient:
 
         from llama_index.core.llms import ChatMessage
 
-        user_msg = ChatMessage(role="user", content=prompt)
         if self.settings.llm_backend == LLMBackend.OLLAMA:
             with self._normalize_timeouts():
                 result = self.llm.chat(
-                    [user_msg],
+                    [ChatMessage(role="user", content=prompt)],
                     format=DocumentClassifierSchema.model_json_schema(),
                     think=False,
                 )
@@ -149,6 +148,11 @@ class AIClient:
         from llama_index.core.program.function_program import get_function_tool
 
         tool = get_function_tool(DocumentClassifierSchema)
+        user_msg = ChatMessage(
+            role="user",
+            content=f"{prompt}\n\n"
+            f"Answer by calling the {tool.metadata.name} tool. Do not write the answer as text.",
+        )
         with self._normalize_timeouts():
             result = self.llm.chat_with_tools(
                 tools=[tool],

src/paperless_ai/prompts/localization.j2

@@ -4,7 +4,7 @@ Rewrite only the "title", "tags", "document_types", and "storage_paths" fields i
 
 Do not translate correspondents or dates.
 Preserve proper nouns, organization names, product names, and exact official document names. Translate generic category words when a {{ language_name }} equivalent exists.
-Return the same JSON schema with all fields present.
+Keep every entry you were given in those four fields, in the same order, using the original wording where no translation applies.
 
 Suggestions:
 {{ suggestions_json }}

src/paperless_ai/tables.py

@@ -154,35 +154,6 @@ class DocumentMetaTable:
         }
 
 
-class PermittedIdsTable:
-    """Per-connection scratch space for an oversized IN-filter id list.
-
-    A literal ``IN (?,?,...)`` list binds one SQL parameter per id, capped by
-    SQLite's own SQLITE_MAX_VARIABLE_NUMBER (see _MAX_IN_VALUES in
-    vector_store.py). Loading the ids into a TEMP TABLE and filtering via a
-    subquery instead has no such limit. TEMP tables live in a
-    connection-private namespace -- never visible to another connection,
-    even under this identical name -- so this is safe under the vector
-    store's one-connection-per-request model without any extra locking or
-    per-call naming scheme.
-    """
-
-    TABLE_NAME = "permitted_document_ids"
-
-    @staticmethod
-    def load(conn: sqlite3.Connection, ids: Iterable[int]) -> None:
-        """Replace this connection's scratch table with ``ids``."""
-        conn.execute(f"DROP TABLE IF EXISTS temp.{PermittedIdsTable.TABLE_NAME}")
-        conn.execute(
-            f"CREATE TEMP TABLE {PermittedIdsTable.TABLE_NAME} "
-            "(id INTEGER PRIMARY KEY)",
-        )
-        conn.executemany(
-            f"INSERT INTO {PermittedIdsTable.TABLE_NAME} (id) VALUES (?)",
-            ((i,) for i in ids),
-        )
-
-
 class IndexMetaTable:
     """Typed accessors over index_meta's key/value rows -- replaces
     PaperlessSqliteVecVectorStore._meta_get_on/_meta_set_on, which returned

src/paperless_ai/tests/test_client.py

@@ -146,6 +146,8 @@ def test_run_llm_query_ollama_uses_structured_json(mock_ai_config, mock_ollama_l
         format=ANY,
         think=False,
     )
+    messages = mock_llm_instance.chat.call_args.args[0]
+    assert messages[0].content == "test_prompt"
 
 
 def test_run_llm_query_openai_uses_tools(mock_ai_config, mock_openai_llm):
@@ -183,6 +185,13 @@ def test_run_llm_query_openai_uses_tools(mock_ai_config, mock_openai_llm):
     assert result["title"] == "Test Title"
     assert result["tags"] == {"existing_ids": [1], "new_names": []}
     mock_llm_instance.chat_with_tools.assert_called_once()
+    kwargs = mock_llm_instance.chat_with_tools.call_args.kwargs
+    offered_tool_name = kwargs["tools"][0].metadata.name
+    assert kwargs["user_msg"].content == (
+        "test_prompt\n\n"
+        f"Answer by calling the {offered_tool_name} tool. "
+        "Do not write the answer as text."
+    )
 
 
 def test_run_llm_query_openai_timeout_raises_local_error(

src/paperless_ai/tests/test_tables.py

@@ -9,7 +9,6 @@ from paperless_ai.tables import DocumentChunksTable
 from paperless_ai.tables import DocumentMetaRow
 from paperless_ai.tables import DocumentMetaTable
 from paperless_ai.tables import IndexMetaTable
-from paperless_ai.tables import PermittedIdsTable
 
 
 @pytest.fixture
@@ -339,85 +338,3 @@ class TestIndexMetaTable:
         IndexMetaTable.increment_total_inserts(conn, 100)
         IndexMetaTable.reset_total_inserts(conn, 7)
         assert IndexMetaTable.get_total_inserts(conn) == 7
-
-
-class TestPermittedIdsTable:
-    def _loaded_ids(self, conn: sqlite3.Connection) -> list[int]:
-        return [
-            row["id"]
-            for row in conn.execute(
-                f"SELECT id FROM {PermittedIdsTable.TABLE_NAME} ORDER BY id",
-            )
-        ]
-
-    def test_load_then_read_back_all_ids(self, conn: sqlite3.Connection) -> None:
-        """
-        GIVEN:
-            - A bare sqlite3 connection
-        WHEN:
-            - load() is called with a set of ids
-        THEN:
-            - Every id is present in the TEMP TABLE, and only those ids
-        """
-        PermittedIdsTable.load(conn, [3, 1, 2])
-        assert self._loaded_ids(conn) == [1, 2, 3]
-
-    def test_load_replaces_previous_contents(self, conn: sqlite3.Connection) -> None:
-        """
-        GIVEN:
-            - A connection whose PermittedIdsTable already holds one id set
-        WHEN:
-            - load() is called again with a different id set
-        THEN:
-            - Only the new ids are present -- a connection reused across
-              multiple queries in one request never leaks a stale filter
-        """
-        PermittedIdsTable.load(conn, [1, 2, 3])
-        PermittedIdsTable.load(conn, [4, 5])
-        assert self._loaded_ids(conn) == [4, 5]
-
-    def test_load_is_connection_private(self) -> None:
-        """
-        GIVEN:
-            - Two separate connections
-        WHEN:
-            - Each loads PermittedIdsTable with a different id set, under
-              the identical TABLE_NAME
-        THEN:
-            - Each connection sees only its own ids -- TEMP TABLE is
-              connection-private, so concurrent requests never collide or
-              cross-contaminate despite sharing the same table name (the
-              vector store opens one connection per request; see
-              PaperlessSqliteVecVectorStore)
-        """
-        conn_a = sqlite3.connect(":memory:")
-        conn_a.row_factory = sqlite3.Row
-        conn_b = sqlite3.connect(":memory:")
-        conn_b.row_factory = sqlite3.Row
-        try:
-            PermittedIdsTable.load(conn_a, [1, 2, 3])
-            PermittedIdsTable.load(conn_b, [4, 5, 6])
-            assert self._loaded_ids(conn_a) == [1, 2, 3]
-            assert self._loaded_ids(conn_b) == [4, 5, 6]
-        finally:
-            conn_a.close()
-            conn_b.close()
-
-    def test_load_handles_more_ids_than_a_bound_parameter_list_could(
-        self,
-        conn: sqlite3.Connection,
-    ) -> None:
-        """
-        GIVEN:
-            - An id count over SQLite's own bound-parameter limit
-              (SQLITE_MAX_VARIABLE_NUMBER, 32766 by default) -- more than a
-              literal IN(?,?,...) list could ever bind in one statement
-        WHEN:
-            - load() is called with that many ids
-        THEN:
-            - Every id is loaded without error, since executemany() binds
-              one row at a time rather than one statement with N parameters
-        """
-        ids = list(range(40_000))
-        PermittedIdsTable.load(conn, ids)
-        assert self._loaded_ids(conn) == ids

src/paperless_ai/tests/test_vector_store.py

@@ -18,7 +18,6 @@ from paperless_ai.migrations import Migration
 from paperless_ai.migrations import m0001_v1_to_v2
 from paperless_ai.tables import DocumentChunksTable
 from paperless_ai.tables import DocumentMetaTable
-from paperless_ai.tables import PermittedIdsTable
 from paperless_ai.vector_store import _MAX_IN_VALUES
 from paperless_ai.vector_store import DB_FILENAME
 from paperless_ai.vector_store import DEFAULT_TABLE_NAME
@@ -281,23 +280,8 @@ class TestCrud:
 
 
 class TestBuildWhere:
-    @pytest.fixture
-    def conn(self) -> Generator[sqlite3.Connection, None, None]:
-        """A bare connection, sufficient for _build_where(): it only ever
-        touches the connection via PermittedIdsTable, which needs no vec0
-        extension loaded.
-        """
-        connection = sqlite3.connect(":memory:")
-        try:
-            yield connection
-        finally:
-            connection.close()
-
-    def test_ne_filter_translates_to_not_equal_clause(
-        self,
-        conn: sqlite3.Connection,
-    ) -> None:
-        where, params = _build_where(conn, _ne_filter(1))
+    def test_ne_filter_translates_to_not_equal_clause(self) -> None:
+        where, params = _build_where(_ne_filter(1))
         assert where == "(document_id != ?)"
         assert params == [1]
 
@@ -309,11 +293,8 @@ class TestBuildWhere:
             "b1",
         ]
 
-    def test_nin_filter_translates_to_not_in_clause(
-        self,
-        conn: sqlite3.Connection,
-    ) -> None:
-        where, params = _build_where(conn, _nin_filter([1, 2]))
+    def test_nin_filter_translates_to_not_in_clause(self) -> None:
+        where, params = _build_where(_nin_filter([1, 2]))
         assert where == "(document_id NOT IN (?,?))"
         assert params == [1, 2]
 
@@ -323,10 +304,7 @@ class TestBuildWhere:
             _query(store, [0.0] * DIM, top_k=5, filters=_nin_filter([1, 2])).ids,
         ) == ["c1"]
 
-    def test_empty_in_filter_excludes_everything(
-        self,
-        conn: sqlite3.Connection,
-    ) -> None:
+    def test_empty_in_filter_excludes_everything(self) -> None:
         """
         GIVEN:
             - An IN filter with an empty value list
@@ -336,14 +314,11 @@ class TestBuildWhere:
             - It excludes everything (the opposite of an empty NOT IN
               filter) -- an empty inclusion list must never widen results
         """
-        where, params = _build_where(conn, _in_filter([]))
+        where, params = _build_where(_in_filter([]))
         assert where == "(1 = 0)"
         assert params == []
 
-    def test_empty_nin_filter_excludes_nothing(
-        self,
-        conn: sqlite3.Connection,
-    ) -> None:
+    def test_empty_nin_filter_excludes_nothing(self) -> None:
         """
         GIVEN:
             - A NOT IN filter with an empty value list -- e.g. an
@@ -355,14 +330,11 @@ class TestBuildWhere:
               excludes everything) -- an empty exclusion list must never
               narrow results
         """
-        where, params = _build_where(conn, _nin_filter([]))
+        where, params = _build_where(_nin_filter([]))
         assert where == "(1 = 1)"
         assert params == []
 
-    def test_fails_closed_when_no_filter_is_translatable(
-        self,
-        conn: sqlite3.Connection,
-    ) -> None:
+    def test_fails_closed_when_no_filter_is_translatable(self) -> None:
         # A nested MetadataFilters is not a MetadataFilter, so it is skipped.
         # With no translatable clauses, the function must fail closed rather
         # than emit "()" (invalid SQL) and never widen document access.
@@ -375,88 +347,42 @@ class TestBuildWhere:
                 ),
             ],
         )
-        where, params = _build_where(conn, MetadataFilters(filters=[nested]))
+        where, params = _build_where(MetadataFilters(filters=[nested]))
         assert where == "1 = 0"
         assert params == []
 
     @pytest.mark.parametrize(
-        ("build_filter", "sql_op"),
-        [(_in_filter, "IN"), (_nin_filter, "NOT IN")],
+        "build_filter",
+        [_in_filter, _nin_filter],
         ids=["in", "nin"],
     )
-    def test_filter_over_max_values_uses_permitted_ids_table(
+    def test_fails_closed_when_filter_exceeds_max_values(
         self,
-        conn: sqlite3.Connection,
         build_filter: Callable[[list[str]], MetadataFilters],
-        sql_op: str,
+        caplog: pytest.LogCaptureFixture,
     ) -> None:
         """
         GIVEN:
             - An IN or NOT IN filter with more values than _MAX_IN_VALUES
-              (SQLite's own bound-parameter limit is 32766; this threshold
-              sits below that with headroom for the query's other bound
-              parameters)
+              (SQLite's own bound-parameter limit is 32766; this guard sits
+              below that with headroom for the query's other bound parameters)
         WHEN:
             - _build_where() translates it to SQL
         THEN:
-            - It builds a subquery against PermittedIdsTable's TEMP TABLE,
-              loaded with every id, instead of a literal list SQLite would
-              reject past its own limit -- true for NOT IN too (e.g. an
-              install with an enormous trash), not just IN
+            - It fails closed ("1 = 0", no params) instead of building a
+              clause SQLite would reject, and logs a warning -- this filter
+              scopes document access, so refusing to build it must never
+              widen the scope to "everything" by accident. Failing open on
+              a NOT IN would surface exactly the excluded rows
         """
-        ids = list(range(_MAX_IN_VALUES + 1))
-        oversized = build_filter([str(i) for i in ids])
+        oversized = build_filter([str(i) for i in range(_MAX_IN_VALUES + 1)])
 
-        where, params = _build_where(conn, oversized)
+        with caplog.at_level("WARNING"):
+            where, params = _build_where(oversized)
 
-        assert where == (
-            f"(document_id {sql_op} (SELECT id FROM {PermittedIdsTable.TABLE_NAME}))"
-        )
+        assert where == "(1 = 0)"
         assert params == []
-        loaded = [
-            row[0]
-            for row in conn.execute(
-                f"SELECT id FROM {PermittedIdsTable.TABLE_NAME} ORDER BY id",
-            )
-        ]
-        assert loaded == ids
-
-    @pytest.mark.parametrize(
-        ("build_filter", "expected_ids"),
-        [(_in_filter, ["b1", "c1"]), (_nin_filter, ["a1"])],
-        ids=["in", "nin"],
-    )
-    def test_query_and_get_nodes_scope_correctly_when_filter_exceeds_max_values(
-        self,
-        store: PaperlessSqliteVecVectorStore,
-        mocker: MockerFixture,
-        build_filter: Callable[[list[int]], MetadataFilters],
-        expected_ids: list[str],
-    ) -> None:
-        """
-        GIVEN:
-            - _MAX_IN_VALUES lowered so a small IN/NOT IN filter exceeds it
-        WHEN:
-            - query() and get_nodes() are called with that filter
-        THEN:
-            - Both still correctly scope results -- the PermittedIdsTable
-              temp-table path behaves identically to the literal
-              IN(...)/NOT IN(...) path it replaces above the threshold
-        """
-        mocker.patch("paperless_ai.vector_store._MAX_IN_VALUES", 1)
-        store.add(
-            [
-                make_node("a1", 1, seed=0.0),
-                make_node("b1", 2, seed=1.0),
-                make_node("c1", 3, seed=2.0),
-            ],
-        )
-
-        result = _query(store, [0.0] * DIM, top_k=10, filters=build_filter([2, 3]))
-        nodes = store.get_nodes(filters=build_filter([2, 3]))
-
-        assert sorted(result.ids) == expected_ids
-        assert sorted(n.node_id for n in nodes) == expected_ids
+        assert "document_id" in caplog.text
 
     def test_query_with_untranslatable_filter_returns_no_rows(
         self,

src/paperless_ai/vector_store.py

@@ -30,7 +30,6 @@ from paperless_ai.tables import DocumentChunksTable
 from paperless_ai.tables import DocumentMetaRow
 from paperless_ai.tables import DocumentMetaTable
 from paperless_ai.tables import IndexMetaTable
-from paperless_ai.tables import PermittedIdsTable
 
 logger = logging.getLogger("paperless_ai.vector_store")
 
@@ -76,12 +75,14 @@ class _Row(NamedTuple):
     embedding: bytes
 
 
-# _build_where(): the largest IN value list translated into a literal
-# IN (?,?,...) clause. SQLite's own hard limit (SQLITE_MAX_VARIABLE_NUMBER)
-# is 32766 by default; this leaves headroom below that for the query's other
-# bound parameters (the embedding blob, k, and any NE clause) and for the
-# limit itself to move. Above this threshold _build_where() switches to
-# PermittedIdsTable instead of failing closed -- see its docstring.
+# _build_where(): the largest IN value list translated into bound SQL
+# parameters. SQLite's own hard limit (SQLITE_MAX_VARIABLE_NUMBER) is 32766
+# by default; this leaves headroom below that for the query's other bound
+# parameters (the embedding blob, k, and any NE clause) and for the limit
+# itself to move. An IN filter this large should not happen in practice --
+# callers are expected to pass None (no filter) rather than every id when
+# the filter would not actually narrow anything -- so this is a guard
+# against a future regression, not a normal code path.
 _MAX_IN_VALUES = 32700
 
 
@@ -105,10 +106,7 @@ def _vec0_params(rows: list[_Row]) -> list[tuple[str, int, str, bytes]]:
     return [(r.chunk_id, r.document_id, r.node_content, r.embedding) for r in rows]
 
 
-def _build_where(
-    conn: sqlite3.Connection,
-    filters: MetadataFilters | None,
-) -> tuple[str, list[int]]:
+def _build_where(filters: MetadataFilters | None) -> tuple[str, list[int]]:
     """Translate the EQ / IN / NIN / NE filters we use into a parameterized
     SQL clause on vec0 metadata columns. Returns ("", []) when there is
     nothing to filter. document_id is vec0's only filterable column and is
@@ -116,10 +114,6 @@ def _build_where(
     still pass strings in places, e.g. indexing.py's MetadataFilter
     construction) don't have to be individually correct -- vec0 doesn't
     coerce types itself.
-
-    ``conn`` is only used for an IN/NOT IN filter over _MAX_IN_VALUES: it
-    loads the ids into PermittedIdsTable's TEMP TABLE on that connection
-    rather than binding them as SQL parameters.
     """
     if filters is None or not filters.filters:
         return "", []
@@ -142,15 +136,20 @@ def _build_where(
                 clauses.append("1 = 0" if is_in else "1 = 1")
                 continue
             if len(values) > _MAX_IN_VALUES:
-                # A literal list this large would exceed SQLite's own
-                # bound-parameter limit. Load the ids into a TEMP TABLE on
-                # this connection instead and filter via subquery, which has
-                # no such limit -- see PermittedIdsTable. Applies to NOT IN
-                # too (e.g. an install with an enormous trash), not just IN.
-                PermittedIdsTable.load(conn, values)
-                clauses.append(
-                    f"{f.key} {sql_op} (SELECT id FROM {PermittedIdsTable.TABLE_NAME})",
+                # Refuse rather than risk SQLite's own bound-parameter limit
+                # ("too many SQL variables"): a list this large must match no
+                # rows, never widen the scope to "everything" -- true for
+                # NOT IN too, where failing open would surface every
+                # excluded row.
+                logger.warning(
+                    "Refusing to build a %s filter on %r with %d values "
+                    "(over the %d-value safety limit); returning no rows.",
+                    sql_op,
+                    f.key,
+                    len(values),
+                    _MAX_IN_VALUES,
                 )
+                clauses.append("1 = 0")
                 continue
             placeholders = ",".join("?" for _ in values)
             clauses.append(f"{f.key} {sql_op} ({placeholders})")
@@ -489,7 +488,7 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore):
             )
         if not self.table_exists():
             return []
-        where, params = _build_where(self._conn, filters)
+        where, params = _build_where(filters)
         sql = "SELECT node_content, embedding FROM " + DEFAULT_TABLE_NAME
         if where:
             sql += " WHERE " + where
@@ -505,7 +504,7 @@ class PaperlessSqliteVecVectorStore(BasePydanticVectorStore):
         if query.query_embedding is None:  # pragma: no cover
             return VectorStoreQueryResult(nodes=[], similarities=[], ids=[])
         top_k = query.similarity_top_k if query.similarity_top_k is not None else 10
-        where, params = _build_where(self._conn, query.filters)
+        where, params = _build_where(query.filters)
         sql = (
             "SELECT id, node_content, embedding, distance FROM "
             + DEFAULT_TABLE_NAME

src/paperless_mail/mail.py

@@ -334,18 +334,24 @@ def error_callback(
     """
     A shared task that is called whenever something goes wrong during
     consumption of a file. See queue_consumption_tasks.
+
+    With CELERY_TASK_ALLOW_ERROR_CB_ON_CHORD_HEADER enabled this runs once per
+    failed header task, not once per chord, so it must be idempotent.
     """
     rule = MailRule.objects.get(pk=rule_id)
+    received = make_aware(message_date) if is_naive(message_date) else message_date
 
-    ProcessedMail.objects.create(
+    ProcessedMail.objects.get_or_create(
         rule=rule,
         folder=rule.folder,
         uid=message_uid,
         uid_validity=uid_validity,
-        subject=message_subject,
-        received=make_aware(message_date) if is_naive(message_date) else message_date,
-        status="FAILED",
-        error=traceback.format_exc(),
+        defaults={
+            "subject": message_subject,
+            "received": received,
+            "status": "FAILED",
+            "error": traceback.format_exc(),
+        },
     )
 
 

src/paperless_mail/tests/test_mail.py

@@ -36,6 +36,7 @@ from paperless_mail.mail import MailAccountHandler
 from paperless_mail.mail import MailError
 from paperless_mail.mail import TagMailAction
 from paperless_mail.mail import apply_mail_action
+from paperless_mail.mail import error_callback
 from paperless_mail.mail import get_mailbox
 from paperless_mail.models import MailAccount
 from paperless_mail.models import MailRule
@@ -2045,6 +2046,44 @@ class TestPostConsumeAction(TestCase):
         self.assertIn("Test Exception", processed_mail.error)
 
 
+@pytest.mark.django_db
+class TestErrorCallback:
+    def test_error_callback_is_idempotent_for_same_mail(self) -> None:
+        """
+        GIVEN:
+            - A mail rule and a mail that failed to be consumed
+        WHEN:
+            - error_callback is invoked more than once for the same mail, as
+              happens when task_allow_error_cb_on_chord_header fires the
+              errback once per failed header task in a chord
+        THEN:
+            - Only one ProcessedMail row is created for that mail
+        """
+        rule = MailRuleFactory()
+        message_uid = "12345"
+
+        for _ in range(2):
+            error_callback(
+                None,
+                Exception("Test Exception"),
+                None,
+                rule_id=rule.pk,
+                message_uid=message_uid,
+                message_subject="Test Subject",
+                message_date=timezone.make_aware(
+                    timezone.datetime(2023, 1, 1, 12, 0, 0),
+                ),
+            )
+
+        processed_mails = ProcessedMail.objects.filter(
+            rule=rule,
+            uid=message_uid,
+            folder=rule.folder,
+        )
+        assert processed_mails.count() == 1
+        assert processed_mails.get().status == "FAILED"
+
+
 class TestManagementCommand(TestCase):
     @mock.patch(
         "paperless_mail.management.commands.mail_fetcher.tasks.process_mail_accounts",