Always these new ones with xdist, try a better condition

Add WriteBatch.add_or_update_ids() and use it in bulk_update_documents
and trash restore, cutting index writes from ~8 queries per document
to a constant handful per batch

src-ui/messages.xlf

@@ -1048,7 +1048,7 @@
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/app-frame/global-search/global-search.component.ts</context>
-          <context context-type="linenumber">124</context>
+          <context context-type="linenumber">130</context>
         </context-group>
       </trans-unit>
       <trans-unit id="2818183879511244335" datatype="html">
@@ -3714,22 +3714,22 @@
         <source>Successfully updated object.</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/app-frame/global-search/global-search.component.ts</context>
-          <context context-type="linenumber">213</context>
+          <context context-type="linenumber">219</context>
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/app-frame/global-search/global-search.component.ts</context>
-          <context context-type="linenumber">251</context>
+          <context context-type="linenumber">257</context>
         </context-group>
       </trans-unit>
       <trans-unit id="1801333259018423190" datatype="html">
         <source>Error occurred saving object.</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/app-frame/global-search/global-search.component.ts</context>
-          <context context-type="linenumber">216</context>
+          <context context-type="linenumber">222</context>
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/app-frame/global-search/global-search.component.ts</context>
-          <context context-type="linenumber">254</context>
+          <context context-type="linenumber">260</context>
         </context-group>
       </trans-unit>
       <trans-unit id="8193912662253833654" datatype="html">

src-ui/src/app/components/app-frame/global-search/global-search.component.html

@@ -9,7 +9,7 @@
                     autocomplete="off"
                     spellcheck="false"
                     [ngModel]="query()"
-                    (ngModelChange)="queryDebounce.next($event)"
+                    (ngModelChange)="onQueryChange($event)"
                     (keydown)="searchInputKeyDown($event)"
                     ngbDropdownAnchor>
                 <div class="position-absolute top-50 end-0 translate-middle">

src-ui/src/app/components/app-frame/global-search/global-search.component.spec.ts

@@ -272,6 +272,19 @@ describe('GlobalSearchComponent', () => {
     expect(advancedSearchSpy).toHaveBeenCalled()
   })
 
+  it('should set query immediately and run full search on enter without waiting for debounce', () => {
+    jest.useFakeTimers()
+    const searchSpy = jest.spyOn(searchService, 'globalSearch')
+    searchSpy.mockReturnValue(of({} as any))
+    const fullSearchSpy = jest.spyOn(component, 'runFullSearch')
+    component.onQueryChange('test')
+    expect(component.query()).toBe('test')
+    component.searchInputKeyDown(new KeyboardEvent('keydown', { key: 'Enter' }))
+    expect(fullSearchSpy).toHaveBeenCalled()
+    expect(searchSpy).not.toHaveBeenCalled()
+    jest.useRealTimers()
+  })
+
   it('should search on query debounce', () => {
     jest.useFakeTimers()
     const query = 'test'

src-ui/src/app/components/app-frame/global-search/global-search.component.ts

@@ -119,6 +119,12 @@ export class GlobalSearchComponent implements OnInit {
       })
   }
 
+  public onQueryChange(text: string) {
+    // set immediately so Enter / the full search button work without waiting for the debounce
+    this.query.set(text)
+    this.queryDebounce.next(text)
+  }
+
   public ngOnInit() {
     this.hotkeyService
       .addShortcut({ keys: '/', description: $localize`Global search` })

src/documents/caching.py

@@ -45,16 +45,14 @@ CLASSIFIER_HASH_KEY: Final[str] = "classifier_hash"
 CLASSIFIER_MODIFIED_KEY: Final[str] = "classifier_modified"
 # Marker distinguishing LLM suggestions from classifier-generated ones (whose
 # FORMAT_VERSION lives in a much lower range - see DocumentClassifier). Bump
-# this whenever cached suggestions must not be reused, including changes to
-# their shape or interpretation, so a previous release's result cannot leak
-# incompatible or obsolete behavior into the new one:
+# this whenever the *shape* of the cached `suggestions` dict changes, so a
+# cache entry written by a previous release can never be read back by code
+# that expects a different shape:
 #   1000 - initial LLM suggestions cache (flat lists of resolved object ids
 #          per taxonomy field)
 #   1001 - suggestions reshaped to {"existing_ids": [...], "new_names":
 #          [...]} per taxonomy field (#13676)
-#   1002 - names are always generated and optional candidate mappings are
-#          validated separately, so candidate-anchored 1001 results are stale
-LLM_CACHE_CLASSIFIER_VERSION: Final[int] = 1002
+LLM_CACHE_CLASSIFIER_VERSION: Final[int] = 1001
 
 CACHE_1_MINUTE: Final[int] = 60
 CACHE_5_MINUTES: Final[int] = 5 * CACHE_1_MINUTE

src/documents/search/_backend.py

@@ -284,6 +284,46 @@ class WriteBatch:
             tantivy.Query.term_query(self._backend._schema, "id", doc_id),
         )
 
+    def add_or_update_ids(self, ids: Sequence[int]) -> None:
+        """
+        Add or update multiple documents in the batch by primary key.
+
+        Unlike calling ``add_or_update()`` once per document, this resolves
+        viewer permissions and effective (versioned) content in bulk against
+        the ids as a whole, instead of once per document -- see
+        ``_DocumentViewerStream`` and ``annotate_effective_content``. Use
+        this whenever more than one document is being written in the same
+        batch.
+
+        An id with no matching document (e.g. deleted between the caller
+        collecting ids and the batch running) is silently skipped, matching
+        ``add_or_update()``'s existing single-document deferred-task behavior
+        rather than erroring or leaving a stale index entry.
+
+        Args:
+            ids: Primary keys of Document instances to index
+        """
+        from documents.models import Document
+        from documents.versioning import annotate_effective_content
+
+        ids = list(ids)
+        if not ids:
+            return
+
+        queryset = annotate_effective_content(
+            Document.objects.filter(pk__in=ids)
+            .select_related("correspondent", "document_type", "storage_path", "owner")
+            .prefetch_related("tags", "notes__user", "custom_fields__field"),
+        )
+        for document, grant in _DocumentViewerStream(queryset, chunk_size=1000):
+            self.remove(document.pk)
+            doc = self._backend._build_tantivy_doc(
+                document,
+                viewer_ids=grant.viewer_ids,
+                viewer_group_ids=grant.viewer_group_ids,
+            )
+            self._writer.add_document(doc)
+
 
 class TantivyBackend:
     """

src/documents/tasks.py

@@ -312,7 +312,10 @@ def bulk_update_documents(document_ids) -> None:
     from documents.search import get_backend
 
     document_ids = list(document_ids)
-    # Annotated so indexing below doesn't query the versions of each document
+    # Annotated so the signal handlers below (e.g. matching) don't query the
+    # versions of each document. Indexing re-queries and re-annotates its own
+    # copy via add_or_update_ids() below, after these signals (and any
+    # workflow they trigger) have had a chance to mutate the documents.
     documents = annotate_effective_content(
         Document.objects.filter(id__in=document_ids),
     )
@@ -328,8 +331,7 @@ def bulk_update_documents(document_ids) -> None:
         post_save.send(Document, instance=doc, created=False)
 
     with get_backend().batch_update() as batch:
-        for doc in documents:
-            batch.add_or_update(doc)
+        batch.add_or_update_ids(document_ids)
 
     ai_config = AIConfig()
     if ai_config.llm_index_enabled:

src/documents/tests/search/test_backend.py

@@ -4,6 +4,8 @@ from pathlib import Path
 import pytest
 from django.contrib.auth.models import Group
 from django.contrib.auth.models import User
+from django.db import connection
+from django.test.utils import CaptureQueriesContext
 from guardian.shortcuts import assign_perm
 from pytest_mock import MockerFixture
 
@@ -102,6 +104,191 @@ class TestWriteBatch:
         assert len(backend.search_ids("indexable", user=None)) == 1
 
 
+class TestAddOrUpdateIds:
+    """Test WriteBatch.add_or_update_ids(), the bulk id-based upsert path.
+
+    Unlike add_or_update() called once per document, this resolves viewer
+    permissions and effective (versioned) content in bulk against the ids as
+    a whole, so it must produce identical indexed output to the per-document
+    path while issuing a constant number of queries regardless of batch size.
+    """
+
+    def test_missing_id_is_skipped_not_errored(
+        self,
+        backend: TantivyBackend,
+    ) -> None:
+        doc = Document.objects.create(
+            title="doc",
+            content="present",
+            checksum="EXIST1",
+            pk=1,
+        )
+        missing_pk = 999
+
+        with backend.batch_update() as batch:
+            batch.add_or_update_ids([doc.pk, missing_pk])
+
+        assert backend.search_ids("present", user=None) == [doc.pk]
+
+    def test_query_count_does_not_scale_with_batch_size(
+        self,
+        backend: TantivyBackend,
+    ) -> None:
+        """Each query count must stay far below N, not merely match between
+        two runs -- an exact-equality assertion between two measurements is
+        at the mercy of incidental process-level caches (e.g. Django's
+        ContentType.objects.get_for_model) warming on whichever run happens
+        first, which makes counts differ by a query for reasons unrelated to
+        batch size. A generous fixed bound sidesteps that: the old
+        per-document path issued roughly 8 queries per document, so 50
+        documents under a bound this low proves the fix regardless of cache
+        state.
+        """
+        max_queries_for_any_batch_size = 15
+
+        small_docs = [
+            Document.objects.create(
+                title="doc",
+                content=f"unique{i}",
+                checksum=f"SMALL{i}",
+                pk=i,
+            )
+            for i in range(1, 3)
+        ]
+        with CaptureQueriesContext(connection) as ctx_small:
+            with backend.batch_update() as batch:
+                batch.add_or_update_ids([d.pk for d in small_docs])
+        assert len(ctx_small.captured_queries) <= max_queries_for_any_batch_size
+
+        large_docs = [
+            Document.objects.create(
+                title="doc",
+                content=f"unique{i}",
+                checksum=f"LARGE{i}",
+                pk=i,
+            )
+            for i in range(100, 150)
+        ]
+        with CaptureQueriesContext(connection) as ctx_large:
+            with backend.batch_update() as batch:
+                batch.add_or_update_ids([d.pk for d in large_docs])
+        assert len(ctx_large.captured_queries) <= max_queries_for_any_batch_size
+
+        for doc in large_docs:
+            assert backend.search_ids(f"unique{doc.pk}", user=None) == [doc.pk]
+
+    def test_resolves_direct_user_grant_in_bulk(
+        self,
+        backend: TantivyBackend,
+    ) -> None:
+        owner = UserFactory()
+        user = UserFactory()
+        doc = Document.objects.create(
+            title="doc",
+            checksum="PERM1",
+            pk=1,
+            owner=owner,
+        )
+        assign_perm("view_document", user, doc)
+
+        with backend.batch_update() as batch:
+            batch.add_or_update_ids([doc.pk])
+
+        assert backend.search_ids("doc", user=user) == [doc.pk]
+        other = UserFactory()
+        assert backend.search_ids("doc", user=other) == []
+
+    def test_resolves_group_grant_in_bulk(self, backend: TantivyBackend) -> None:
+        owner = UserFactory()
+        group = Group.objects.create(name="reviewers")
+        user = UserFactory()
+        user.groups.add(group)
+        doc = Document.objects.create(
+            title="doc",
+            checksum="GPERM1",
+            pk=1,
+            owner=owner,
+        )
+        assign_perm("view_document", group, doc)
+
+        with backend.batch_update() as batch:
+            batch.add_or_update_ids([doc.pk])
+
+        assert backend.search_ids("doc", user=user) == [doc.pk]
+        other = UserFactory()
+        assert backend.search_ids("doc", user=other) == []
+
+    def test_indexes_notes_and_custom_fields(self, backend: TantivyBackend) -> None:
+        note_author = UserFactory(username="noter")
+        field = CustomField.objects.create(
+            name="Invoice Number",
+            data_type=CustomField.FieldDataType.STRING,
+        )
+        doc = Document.objects.create(title="doc", checksum="RICH1", pk=1)
+        Note.objects.create(document=doc, note="Reviewed", user=note_author)
+        CustomFieldInstance.objects.create(
+            document=doc,
+            field=field,
+            value_text="INV-42",
+        )
+
+        with backend.batch_update() as batch:
+            batch.add_or_update_ids([doc.pk])
+
+        assert backend.search_ids("notes.user:noter", user=None) == [doc.pk]
+        assert backend.search_ids("custom_fields.value:INV-42", user=None) == [
+            doc.pk,
+        ]
+
+    def test_uses_effective_content_for_versioned_documents(
+        self,
+        backend: TantivyBackend,
+    ) -> None:
+        root = Document.objects.create(
+            title="Statement",
+            content="stale text",
+            checksum="ROOT1",
+            pk=1,
+        )
+        Document.objects.create(
+            title="Statement",
+            content="latest version text",
+            checksum="VER1",
+            pk=2,
+            root_document=root,
+            version_index=1,
+        )
+
+        with backend.batch_update() as batch:
+            batch.add_or_update_ids([root.pk])
+
+        assert backend.search_ids("latest", user=None) == [root.pk]
+        assert backend.search_ids("stale", user=None) == []
+
+    def test_reindexes_documents_already_in_the_index(
+        self,
+        backend: TantivyBackend,
+    ) -> None:
+        """add_or_update_ids must upsert, matching add_or_update's behaviour."""
+        doc = Document.objects.create(
+            title="doc",
+            content="original",
+            checksum="UP1",
+            pk=1,
+        )
+        backend.add_or_update(doc)
+        assert backend.search_ids("original", user=None) == [doc.pk]
+
+        doc.content = "updated"
+        doc.save()
+
+        with backend.batch_update() as batch:
+            batch.add_or_update_ids([doc.pk])
+
+        assert backend.search_ids("original", user=None) == []
+        assert backend.search_ids("updated", user=None) == [doc.pk]
+
+
 class TestSearch:
     """Test search query parsing and matching via search_ids."""
 

src/documents/views.py

@@ -5444,8 +5444,7 @@ class TrashView(ListModelMixin, PassUserMixin):
                 from documents.search import get_backend
 
                 with get_backend().batch_update() as batch:
-                    for doc in restored:
-                        batch.add_or_update(doc)
+                    batch.add_or_update_ids([doc.pk for doc in restored])
         elif action == "empty":
             if doc_ids is None:
                 doc_ids = [doc.id for doc in docs]

src/paperless_ai/ai_classifier.py

@@ -67,6 +67,7 @@ def build_prompt_without_rag(
     document: Document,
     config: AIConfig,
     candidates: TaxonomyCandidates | None = None,
+    assigned: AssignedMetadata | None = None,
 ) -> str:
     filename = document.filename or ""
     content = truncate_content(
@@ -76,7 +77,9 @@ def build_prompt_without_rag(
     )
 
     taxonomy_block = (
-        format_taxonomy_for_prompt(candidates) if candidates is not None else ""
+        format_taxonomy_for_prompt(candidates, assigned)
+        if candidates is not None and assigned is not None
+        else ""
     )
     has_candidates = candidates is not None and any(candidates.values())
 
@@ -94,12 +97,14 @@ def build_prompt_with_rag(
     document: Document,
     config: AIConfig,
     candidates: TaxonomyCandidates | None = None,
+    assigned: AssignedMetadata | None = None,
     context: str = "",
 ) -> str:
     base_prompt = build_prompt_without_rag(
         document,
         config,
         candidates=candidates,
+        assigned=assigned,
     )
     truncated_context = truncate_content(
         context,
@@ -264,22 +269,6 @@ def _restrict_to_shown_candidates(
     )
 
 
-def _candidate_id_allowlist(
-    candidates: TaxonomyCandidates,
-) -> dict[str, set[int]]:
-    """Candidate IDs grouped by category for validating model mappings."""
-    return {
-        "tags": {candidate["id"] for candidate in candidates["tags"]},
-        "document_types": {
-            candidate["id"] for candidate in candidates["document_types"]
-        },
-        "correspondents": {
-            candidate["id"] for candidate in candidates["correspondents"]
-        },
-        "storage_paths": {candidate["id"] for candidate in candidates["storage_paths"]},
-    }
-
-
 def get_ai_document_classification(
     document: Document,
     user: User | None = None,
@@ -288,25 +277,28 @@ def get_ai_document_classification(
     ai_config = AIConfig()
 
     if ai_config.llm_embedding_backend:
-        candidates, _assigned, context = get_taxonomy_context(document, user)
+        candidates, assigned, context = get_taxonomy_context(document, user)
         prompt = build_prompt_with_rag(
             document,
             ai_config,
             candidates=candidates,
+            assigned=assigned,
             context=context,
         )
     else:
         candidates = empty_taxonomy_candidates()
-        prompt = build_prompt_without_rag(document, ai_config, candidates=candidates)
+        prompt = build_prompt_without_rag(
+            document,
+            ai_config,
+            candidates=candidates,
+            assigned=get_assigned_metadata(document, user),
+        )
 
     client = AIClient()
     # Hand the pooled DB connection back while the (slow) LLM query runs so it
     # is not pinned for the call's duration; see paperless_ai.db and #12976.
     with db_connection_released():
-        result = client.run_llm_query(
-            prompt,
-            allowed_candidate_ids=_candidate_id_allowlist(candidates),
-        )
+        result = client.run_llm_query(prompt)
         suggestions = _restrict_to_shown_candidates(
             parse_ai_response(result),
             candidates,

src/paperless_ai/base_model.py

@@ -48,107 +48,75 @@ class DocumentClassifierSchema(BaseModel):
         default_factory=list,
         max_length=MAX_NEW_NAMES,
         description=(
-            "All topic labels you would suggest from the document itself, e.g. "
-            "'Insurance', 'Car', 'Warranty'. Always include every suggested "
-            "name here, even when it matches an available tag."
-        ),
-    )
-    matched_tags: list[str] = Field(
-        default_factory=list,
-        max_length=MAX_NEW_NAMES,
-        description=(
-            "Names copied exactly from tags that mean the same thing as an "
-            "available tag. Align each name by position with tag_ids."
+            "Names of topic labels describing what this document is about, "
+            "e.g. 'Insurance', 'Car', 'Warranty'. When an available tag "
+            "represents the same label, use its ID in tag_ids instead."
         ),
     )
     tag_ids: list[int] = Field(
         default_factory=list,
         max_length=MAX_EXISTING_IDS,
         description=(
-            "Available tag IDs matching matched_tags, in the same order. "
-            "Only use IDs shown in the prompt."
+            "IDs of available tags that clearly apply to this document. Only "
+            "use IDs shown in the prompt; never invent one or choose a weak match."
         ),
     )
     correspondents: list[str] = Field(
         default_factory=list,
         max_length=MAX_NEW_NAMES,
         description=(
-            "All people, institutions or companies you would suggest as who "
-            "this document is from or was sent to, not every party merely "
-            "mentioned. Always include every suggested name here, even when it "
-            "matches an available correspondent."
-        ),
-    )
-    matched_correspondents: list[str] = Field(
-        default_factory=list,
-        max_length=MAX_NEW_NAMES,
-        description=(
-            "Names copied exactly from correspondents that identify the same "
-            "entity as an available correspondent. Align each name by position "
-            "with correspondent_ids."
+            "Names of people, institutions or companies this document is from "
+            "or was sent to, not every party merely mentioned. When an "
+            "available correspondent is the same entity, use its ID in "
+            "correspondent_ids instead."
         ),
     )
     correspondent_ids: list[int] = Field(
         default_factory=list,
         max_length=MAX_EXISTING_IDS,
         description=(
-            "Available correspondent IDs matching matched_correspondents, in "
-            "the same order. Only use IDs shown in the prompt."
+            "IDs of available correspondents that clearly apply to this "
+            "document. Only use IDs shown in the prompt; never invent one or "
+            "choose a weak match."
         ),
     )
     document_types: list[str] = Field(
         default_factory=list,
         max_length=MAX_NEW_NAMES,
         description=(
-            "All names describing what kind of document this is, e.g. 'Invoice', "
+            "Names describing what kind of document this is, e.g. 'Invoice', "
             "'Contract', 'Bank Statement', 'Letter'. Never use its subject or "
-            "sender as a document type. Always include every suggested name "
-            "here, even when it matches an available document type."
-        ),
-    )
-    matched_document_types: list[str] = Field(
-        default_factory=list,
-        max_length=MAX_NEW_NAMES,
-        description=(
-            "Names copied exactly from document_types that mean the same thing "
-            "as an available document type. Align each name by position with "
-            "document_type_ids."
+            "sender as a document type. When an available document type is the "
+            "same kind, use its ID in document_type_ids instead."
         ),
     )
     document_type_ids: list[int] = Field(
         default_factory=list,
         max_length=MAX_EXISTING_IDS,
         description=(
-            "Available document type IDs matching matched_document_types, in "
-            "the same order. Only use IDs shown in the prompt."
+            "IDs of available document types that clearly apply to this "
+            "document. Only use IDs shown in the prompt; never invent one or "
+            "choose a weak match."
         ),
     )
     storage_paths: list[str] = Field(
         default_factory=list,
         max_length=MAX_NEW_NAMES,
         description=(
-            "All folder-style filing locations you would suggest, e.g. "
+            "Names of folder-style filing locations, e.g. "
             "'Finance/Invoices'. Leave empty unless a filing location is "
             "clearly implied - never put tags, document types or "
-            "correspondents here. Always include every suggested name here, "
-            "even when it matches an available storage path."
-        ),
-    )
-    matched_storage_paths: list[str] = Field(
-        default_factory=list,
-        max_length=MAX_NEW_NAMES,
-        description=(
-            "Names copied exactly from storage_paths that mean the same filing "
-            "location as an available storage path. Align each name by position "
-            "with storage_path_ids."
+            "correspondents here. When an available storage path is the same "
+            "location, use its ID in storage_path_ids instead."
         ),
     )
     storage_path_ids: list[int] = Field(
         default_factory=list,
         max_length=MAX_EXISTING_IDS,
         description=(
-            "Available storage path IDs matching matched_storage_paths, in the "
-            "same order. Only use IDs shown in the prompt."
+            "IDs of available storage paths that clearly apply to this "
+            "document. Only use IDs shown in the prompt; never invent one or "
+            "choose a weak match."
         ),
     )
     dates: list[str] = Field(
@@ -164,16 +132,12 @@ class DocumentClassifierSchema(BaseModel):
     @field_validator(
         "title",
         "tags",
-        "matched_tags",
         "tag_ids",
         "correspondents",
-        "matched_correspondents",
         "correspondent_ids",
         "document_types",
-        "matched_document_types",
         "document_type_ids",
         "storage_paths",
-        "matched_storage_paths",
         "storage_path_ids",
         "dates",
         mode="before",
@@ -203,64 +167,25 @@ class ClassificationSuggestions(TypedDict):
 
 def model_to_classification_suggestions(
     model: DocumentClassifierSchema,
-    allowed_candidate_ids: dict[str, set[int]] | None = None,
 ) -> ClassificationSuggestions:
-    """Validate optional candidate mappings and convert to the internal shape.
-
-    A mapping is accepted only when its name is copied from the model's own
-    complete suggestion list and its ID was actually shown for that category.
-    Invalid or unpaired mappings leave the original name untouched.
-    """
-    allowed_candidate_ids = allowed_candidate_ids or {}
-
-    def _choice(
-        names: list[str],
-        matched_names: list[str],
-        ids: list[int],
-        category: str,
-    ) -> TaxonomyChoiceDict:
-        remaining_names = list(names)
-        existing_ids: list[int] = []
-        allowed_ids = allowed_candidate_ids.get(category, set())
-        for name, object_id in zip(matched_names, ids, strict=False):
-            if (
-                name not in remaining_names
-                or object_id not in allowed_ids
-                or object_id in existing_ids
-            ):
-                continue
-            remaining_names.remove(name)
-            existing_ids.append(object_id)
-        return TaxonomyChoiceDict(
-            existing_ids=existing_ids,
-            new_names=remaining_names,
-        )
-
+    """Convert the flat, model-friendly response to the internal shape."""
     return ClassificationSuggestions(
         title=model.title,
-        tags=_choice(
-            model.tags,
-            model.matched_tags,
-            model.tag_ids,
-            "tags",
+        tags=TaxonomyChoiceDict(
+            existing_ids=model.tag_ids,
+            new_names=model.tags,
         ),
-        correspondents=_choice(
-            model.correspondents,
-            model.matched_correspondents,
-            model.correspondent_ids,
-            "correspondents",
+        correspondents=TaxonomyChoiceDict(
+            existing_ids=model.correspondent_ids,
+            new_names=model.correspondents,
         ),
-        document_types=_choice(
-            model.document_types,
-            model.matched_document_types,
-            model.document_type_ids,
-            "document_types",
+        document_types=TaxonomyChoiceDict(
+            existing_ids=model.document_type_ids,
+            new_names=model.document_types,
         ),
-        storage_paths=_choice(
-            model.storage_paths,
-            model.matched_storage_paths,
-            model.storage_path_ids,
-            "storage_paths",
+        storage_paths=TaxonomyChoiceDict(
+            existing_ids=model.storage_path_ids,
+            new_names=model.storage_paths,
         ),
         dates=model.dates,
     )
@@ -273,16 +198,12 @@ def classification_suggestions_to_model(
     return DocumentClassifierSchema(
         title=suggestions["title"],
         tags=suggestions["tags"]["new_names"],
-        matched_tags=[],
-        tag_ids=[],
+        tag_ids=suggestions["tags"]["existing_ids"],
         correspondents=suggestions["correspondents"]["new_names"],
-        matched_correspondents=[],
-        correspondent_ids=[],
+        correspondent_ids=suggestions["correspondents"]["existing_ids"],
         document_types=suggestions["document_types"]["new_names"],
-        matched_document_types=[],
-        document_type_ids=[],
+        document_type_ids=suggestions["document_types"]["existing_ids"],
         storage_paths=suggestions["storage_paths"]["new_names"],
-        matched_storage_paths=[],
-        storage_path_ids=[],
+        storage_path_ids=suggestions["storage_paths"]["existing_ids"],
         dates=suggestions["dates"],
     )

src/paperless_ai/client.py

@@ -117,12 +117,7 @@ class AIClient:
         else:
             raise ValueError(f"Unsupported LLM backend: {self.settings.llm_backend}")
 
-    def run_llm_query(
-        self,
-        prompt: str,
-        *,
-        allowed_candidate_ids: dict[str, set[int]] | None = None,
-    ) -> ClassificationSuggestions:
+    def run_llm_query(self, prompt: str) -> ClassificationSuggestions:
         logger.debug(
             "Running LLM query against %s with model %s",
             self.settings.llm_backend,
@@ -141,10 +136,7 @@ class AIClient:
                 )
             logger.debug("LLM query result: %s", result)
             parsed = DocumentClassifierSchema(**json.loads(result.message.content))
-            return model_to_classification_suggestions(
-                parsed,
-                allowed_candidate_ids,
-            )
+            return model_to_classification_suggestions(parsed)
 
         from llama_index.core.program.function_program import get_function_tool
 
@@ -163,10 +155,7 @@ class AIClient:
             )
         logger.debug("LLM query result: %s", tool_calls)
         parsed = DocumentClassifierSchema(**tool_calls[0].tool_kwargs)
-        return model_to_classification_suggestions(
-            parsed,
-            allowed_candidate_ids,
-        )
+        return model_to_classification_suggestions(parsed)
 
     @contextmanager
     def _normalize_timeouts(self) -> Iterator[None]:

src/paperless_ai/prompts/classification.j2

@@ -13,7 +13,10 @@ Analyze the following document and fill in these fields:
 - dates: up to 3 relevant dates in YYYY-MM-DD format
 {% if has_candidates %}
 
-First produce the complete name suggestions from the document itself in tags, correspondents, document_types, and storage_paths. Always include every suggested name in those fields, even when an available candidate represents the same value. Then, as a separate reconciliation step, copy each name that means the same thing as an available candidate into the matching matched_* field and put that candidate's ID at the same position in the corresponding *_ids field. Candidates must not create, replace, or suppress suggestions. Do not match a candidate that is merely related.
+For tags, correspondents, document types, and storage paths: first decide whether there is a useful, well-supported suggestion. If an available candidate clearly represents that suggestion, put its id in the matching tag_ids, correspondent_ids, document_type_ids, or storage_path_ids field instead of repeating its name. If no candidate represents the suggestion, put its name in tags, correspondents, document_types, or storage_paths. Do not choose a weak candidate merely because it exists.
+{% else %}
+
+No candidates are shown for this document, so leave every field ending in "_ids" empty and put suggestions in the corresponding name fields.
 {% endif %}
 
 Filename:

src/paperless_ai/prompts/localization.j2

@@ -1,6 +1,6 @@
 You are localizing document classification suggestions for display in Paperless-ngx.
 
-Rewrite only the "title", "tags", "document_types", and "storage_paths" fields in {{ language_name }}.
+Rewrite only the "title", "tags", "document_types", and "storage_paths" fields in {{ language_name }}. Leave every field ending in "_ids" exactly as given - these are database identifiers, not text, and are not used from your response even if changed.
 
 Do not translate correspondents or dates.
 Preserve proper nouns, organization names, product names, and exact official document names. Translate generic category words when a {{ language_name }} equivalent exists.

src/paperless_ai/taxonomy.py

@@ -245,7 +245,7 @@ def _assigned_block(assigned: AssignedMetadata) -> str:
 
 def format_taxonomy_for_prompt(
     candidates: TaxonomyCandidates,
-    assigned: AssignedMetadata | None = None,
+    assigned: AssignedMetadata,
 ) -> str:
     """Render assigned metadata and ranked candidates as labelled prompt
     blocks. Candidate names are untrusted, user-controlled data, so they are
@@ -255,7 +255,7 @@ def format_taxonomy_for_prompt(
     is nothing to say (no assigned metadata and no candidates), so callers can
     treat the result the same as no hints at all.
     """
-    has_assigned = assigned is not None and any(
+    has_assigned = any(
         [
             assigned["tags"],
             assigned["document_type"],
@@ -271,11 +271,7 @@ def format_taxonomy_for_prompt(
 
     return render_prompt(
         TaxonomyBlockPromptContext(
-            assigned_block=(
-                _assigned_block(assigned)
-                if assigned is not None and has_assigned
-                else ""
-            ),
+            assigned_block=_assigned_block(assigned) if has_assigned else "",
             candidate_payload_json=(
                 json.dumps(candidate_payload, ensure_ascii=False)
                 if candidate_payload

src/paperless_ai/tests/test_ai_classifier.py

@@ -175,7 +175,6 @@ def test_get_ai_document_classification_failure(mock_run_llm_query, mock_documen
 @pytest.mark.django_db
 @patch("paperless_ai.client.AIClient.run_llm_query")
 @patch("paperless_ai.ai_classifier.build_prompt_with_rag")
-@patch("paperless_ai.ai_classifier.build_taxonomy_candidates")
 @patch("paperless_ai.ai_classifier.retrieve_similar_nodes")
 @override_settings(
     LLM_EMBEDDING_BACKEND="huggingface",
@@ -185,7 +184,6 @@ def test_get_ai_document_classification_failure(mock_run_llm_query, mock_documen
 )
 def test_use_rag_if_configured(
     mock_retrieve,
-    mock_build_candidates,
     mock_build_prompt_with_rag,
     mock_run_llm_query,
     mock_document,
@@ -197,29 +195,12 @@ def test_use_rag_if_configured(
         - get_ai_document_classification() is called
     THEN:
         - The RAG-augmented prompt builder is used
-        - Classification and candidate reconciliation happen in one LLM call
-        - Only candidate IDs from the permission-filtered candidate set are allowed
     """
     mock_retrieve.return_value = []
-    mock_build_candidates.return_value = TaxonomyCandidates(
-        tags=[TaxonomyCandidate(id=12, name="Contractor", weight=1.0)],
-        document_types=[],
-        correspondents=[],
-        storage_paths=[],
-    )
     mock_build_prompt_with_rag.return_value = "Prompt with RAG"
     mock_run_llm_query.return_value = NESTED_SUGGESTIONS
     get_ai_document_classification(mock_document)
     mock_build_prompt_with_rag.assert_called_once()
-    mock_run_llm_query.assert_called_once_with(
-        "Prompt with RAG",
-        allowed_candidate_ids={
-            "tags": {12},
-            "document_types": set(),
-            "correspondents": set(),
-            "storage_paths": set(),
-        },
-    )
 
 
 @pytest.mark.django_db
@@ -625,11 +606,10 @@ def test_build_prompt_without_rag_includes_taxonomy_block():
     GIVEN:
         - Non-empty taxonomy candidates
     WHEN:
-        - build_prompt_without_rag() is called with candidates
+        - build_prompt_without_rag() is called with candidates and assigned metadata
     THEN:
-        - The candidate and single-call reconciliation instructions appear
-        - Complete name suggestions remain mandatory
-        - Assigned metadata is not included
+        - The candidate's id and the flat name/ID instructions appear
+        - Candidates are presented as deduplication options, not requirements
     """
     document = DocumentFactory.create(content="Some content")
     config = AIConfig()
@@ -639,31 +619,40 @@ def test_build_prompt_without_rag_includes_taxonomy_block():
         "correspondents": [],
         "storage_paths": [],
     }
+    assigned = {
+        "tags": [],
+        "document_type": None,
+        "correspondent": None,
+        "storage_path": None,
+    }
+
     prompt = build_prompt_without_rag(
         document,
         config,
         candidates=candidates,
+        assigned=assigned,
     )
 
     assert '"id": 12' in prompt
-    assert "Always include every suggested name" in prompt
-    assert "matched_*" in prompt
-    assert "corresponding *_ids" in prompt
-    assert "Candidates must not create, replace, or suppress suggestions" in prompt
-    assert "already assigned" not in prompt
+    assert "tag_ids" in prompt
+    assert "correspondent_ids" in prompt
+    assert "not requirements" in prompt
+    assert "weak candidate" in prompt
 
 
 @pytest.mark.django_db
-def test_build_prompt_without_rag_identical_when_no_candidates():
+def test_build_prompt_without_rag_identical_when_no_hints():
     """
     GIVEN:
-        - Empty taxonomy candidates
+        - Empty taxonomy candidates and empty assigned metadata
     WHEN:
         - build_prompt_without_rag() is called with those empty values, and
-          separately with no candidates at all
+          separately with no candidates/assigned at all
     THEN:
         - Both prompts are identical
-        - Neither carries candidate reconciliation instructions
+        - Neither carries the "Available ..." candidate block or the
+          id-vs-name routing instruction
+        - Both still tell the model to leave every ID field empty
     """
     document = DocumentFactory.create(content="Some content")
     config = AIConfig()
@@ -673,37 +662,67 @@ def test_build_prompt_without_rag_identical_when_no_candidates():
         "correspondents": [],
         "storage_paths": [],
     }
+    empty_assigned = {
+        "tags": [],
+        "document_type": None,
+        "correspondent": None,
+        "storage_path": None,
+    }
+
     with_empty_hints = build_prompt_without_rag(
         document,
         config,
         candidates=empty_candidates,
+        assigned=empty_assigned,
     )
     with_no_hints = build_prompt_without_rag(document, config)
 
     assert with_empty_hints == with_no_hints
     assert "Available " not in with_no_hints
-    assert "matched_*" not in with_no_hints
+    assert "put its id in the matching" not in with_no_hints
+    assert 'leave every field ending in "_ids" empty' in with_no_hints
 
 
 @pytest.mark.django_db
-def test_build_prompt_without_rag_never_includes_assigned_metadata():
+def test_build_prompt_without_rag_tells_model_to_skip_ids_when_no_candidates():
     """
     GIVEN:
-        - A document with assigned taxonomy metadata
+        - Assigned metadata but empty taxonomy candidates
     WHEN:
-        - build_prompt_without_rag() is called
+        - build_prompt_without_rag() is called with candidates and assigned metadata
     THEN:
-        - Assigned metadata is absent so it cannot anchor classification
+        - The assigned-metadata block appears (taxonomy_block is non-empty)
+        - The prompt tells the model to leave every ID field empty
+
+    Staying silent about ID fields here is not enough: the response schema
+    advertises the field whatever the prompt says, and models fill it with
+    placeholder ids that resolve to real but unrelated objects (#13831).
     """
     document = DocumentFactory.create(content="Some content")
     config = AIConfig()
-    assigned_tag = TagFactory.create(name="Bloodwork")
-    document.tags.add(assigned_tag)
+    empty_candidates = {
+        "tags": [],
+        "document_types": [],
+        "correspondents": [],
+        "storage_paths": [],
+    }
+    assigned = {
+        "tags": ["Bloodwork"],
+        "document_type": None,
+        "correspondent": None,
+        "storage_path": None,
+    }
 
-    prompt = build_prompt_without_rag(document, config)
+    prompt = build_prompt_without_rag(
+        document,
+        config,
+        candidates=empty_candidates,
+        assigned=assigned,
+    )
 
-    assert "Bloodwork" not in prompt
-    assert "already assigned" not in prompt
+    assert "already assigned" in prompt
+    assert "No candidates are shown" in prompt
+    assert 'leave every field ending in "_ids" empty' in prompt
 
 
 @pytest.mark.django_db

src/paperless_ai/tests/test_base_model.py

@@ -1,6 +1,7 @@
 import json
 
 from paperless_ai.base_model import MAX_DATES
+from paperless_ai.base_model import MAX_EXISTING_IDS
 from paperless_ai.base_model import MAX_NEW_NAMES
 from paperless_ai.base_model import MAX_TITLE_LENGTH
 from paperless_ai.base_model import ClassificationSuggestions
@@ -18,7 +19,7 @@ def test_document_classifier_schema_declared_defaults():
     WHEN:
         - The schema is dumped to a dict via model_dump()
     THEN:
-        - Every optional name field, and dates, dump as empty lists
+        - Every name and ID field, and dates, dump as empty lists
 
     The model may omit optional fields, so the schema must provide the complete
     empty shape expected by the conversion and matching pipeline.
@@ -30,106 +31,57 @@ def test_document_classifier_schema_declared_defaults():
     assert dumped == {
         "title": "Test Title",
         "tags": [],
-        "matched_tags": [],
         "tag_ids": [],
         "correspondents": [],
-        "matched_correspondents": [],
         "correspondent_ids": [],
         "document_types": [],
-        "matched_document_types": [],
         "document_type_ids": [],
         "storage_paths": [],
-        "matched_storage_paths": [],
         "storage_path_ids": [],
         "dates": [],
     }
 
 
-def test_model_response_converts_names_to_internal_taxonomy_choices():
+def test_flat_model_response_converts_to_internal_taxonomy_choices():
     """
     GIVEN:
-        - A model response containing taxonomy names
+        - A flat model response with separate name and candidate-ID fields
     WHEN:
         - It is converted to Paperless' internal suggestion representation
     THEN:
-        - Names enter the internal taxonomy representation as new names
-        - Existing IDs remain empty for deterministic application-side matching
+        - Names and IDs are paired under their taxonomy category
     """
     parsed = DocumentClassifierSchema(
         title="Electricity Bill",
         tags=["Utilities", "Electricity"],
+        tag_ids=[12],
         correspondents=["Power Company"],
+        correspondent_ids=[23],
         document_types=["Utility Bill"],
+        document_type_ids=[34],
         storage_paths=["Finance/Utilities"],
+        storage_path_ids=[45],
     )
     suggestions = model_to_classification_suggestions(parsed)
 
     assert suggestions["tags"] == {
-        "existing_ids": [],
+        "existing_ids": [12],
         "new_names": ["Utilities", "Electricity"],
     }
     assert suggestions["correspondents"] == {
-        "existing_ids": [],
+        "existing_ids": [23],
         "new_names": ["Power Company"],
     }
     assert suggestions["document_types"] == {
-        "existing_ids": [],
+        "existing_ids": [34],
         "new_names": ["Utility Bill"],
     }
     assert suggestions["storage_paths"] == {
-        "existing_ids": [],
+        "existing_ids": [45],
         "new_names": ["Finance/Utilities"],
     }
 
 
-def test_valid_candidate_mappings_replace_only_the_matched_names():
-    parsed = DocumentClassifierSchema(
-        title="Electricity Bill",
-        tags=["Utilities", "Electricity"],
-        matched_tags=["Utilities"],
-        tag_ids=[12],
-        correspondents=["Power Company"],
-        matched_correspondents=["Power Company"],
-        correspondent_ids=[23],
-    )
-
-    suggestions = model_to_classification_suggestions(
-        parsed,
-        {
-            "tags": {12},
-            "correspondents": {23},
-        },
-    )
-
-    assert suggestions["tags"] == {
-        "existing_ids": [12],
-        "new_names": ["Electricity"],
-    }
-    assert suggestions["correspondents"] == {
-        "existing_ids": [23],
-        "new_names": [],
-    }
-
-
-def test_invalid_or_unpaired_candidate_mappings_do_not_remove_names():
-    parsed = DocumentClassifierSchema(
-        title="Electricity Bill",
-        tags=["Utilities", "Electricity", "Energy"],
-        matched_tags=["Invented", "Utilities", "Electricity"],
-        tag_ids=[12, 999],
-    )
-
-    suggestions = model_to_classification_suggestions(
-        parsed,
-        {"tags": {12}},
-    )
-
-    assert suggestions["tags"] == {
-        "existing_ids": [],
-        "new_names": ["Utilities", "Electricity", "Energy"],
-    }
-
-
 def test_document_classifier_schema_json_schema_is_self_contained():
     """
     GIVEN:
@@ -223,11 +175,13 @@ def test_over_long_response_is_truncated_rather_than_rejected():
     parsed = DocumentClassifierSchema(
         title="T" * (MAX_TITLE_LENGTH + 50),
         tags=["n"] * (MAX_NEW_NAMES + 20),
+        tag_ids=list(range(MAX_EXISTING_IDS + 20)),
         dates=[f"2016-{month:02d}-01" for month in range(1, 13)],
     )
 
     assert len(parsed.title) == MAX_TITLE_LENGTH
     assert len(parsed.dates) == MAX_DATES
+    assert len(parsed.tag_ids) == MAX_EXISTING_IDS
     assert len(parsed.tags) == MAX_NEW_NAMES
 
 
@@ -260,7 +214,7 @@ def test_model_conversion_matches_internal_typed_dict_keys():
         - The converted tags dict's keys exactly match TaxonomyChoiceDict's
           declared keys
     """
-    schema = DocumentClassifierSchema(title="T", tags=["Tag"])
+    schema = DocumentClassifierSchema(title="T", tags=["Tag"], tag_ids=[1])
     suggestions = model_to_classification_suggestions(schema)
 
     assert set(suggestions.keys()) == set(
@@ -271,7 +225,7 @@ def test_model_conversion_matches_internal_typed_dict_keys():
     )
 
 
-def test_internal_suggestions_convert_to_names_only_model():
+def test_internal_suggestions_round_trip_through_flat_model():
     suggestions = ClassificationSuggestions(
         title="Electricity Bill",
         tags=TaxonomyChoiceDict(existing_ids=[1], new_names=["Utilities"]),
@@ -292,22 +246,4 @@ def test_internal_suggestions_convert_to_names_only_model():
 
     model = classification_suggestions_to_model(suggestions)
 
-    converted = model_to_classification_suggestions(model)
-
-    assert converted == ClassificationSuggestions(
-        title="Electricity Bill",
-        tags=TaxonomyChoiceDict(existing_ids=[], new_names=["Utilities"]),
-        correspondents=TaxonomyChoiceDict(
-            existing_ids=[],
-            new_names=["Power Company"],
-        ),
-        document_types=TaxonomyChoiceDict(
-            existing_ids=[],
-            new_names=["Utility Bill"],
-        ),
-        storage_paths=TaxonomyChoiceDict(
-            existing_ids=[],
-            new_names=["Finance/Utilities"],
-        ),
-        dates=["2026-08-30"],
-    )
+    assert model_to_classification_suggestions(model) == suggestions

src/paperless_ai/tests/test_client.py

@@ -124,23 +124,22 @@ def test_run_llm_query_ollama_uses_structured_json(mock_ai_config, mock_ollama_l
         {
             "title": "Test Title",
             "tags": ["document"],
-            "matched_tags": ["document"],
             "tag_ids": [1],
             "correspondents": ["John Doe"],
+            "correspondent_ids": [],
             "document_types": ["report"],
+            "document_type_ids": [],
             "storage_paths": ["Reports"],
+            "storage_path_ids": [],
             "dates": ["2023-01-01"],
         },
     )
 
     client = AIClient()
-    result = client.run_llm_query(
-        "test_prompt",
-        allowed_candidate_ids={"tags": {1}},
-    )
+    result = client.run_llm_query("test_prompt")
 
     assert result["title"] == "Test Title"
-    assert result["tags"] == {"existing_ids": [1], "new_names": []}
+    assert result["tags"] == {"existing_ids": [1], "new_names": ["document"]}
     mock_llm_instance.chat.assert_called_once_with(
         [ANY],
         format=ANY,
@@ -162,11 +161,13 @@ def test_run_llm_query_openai_uses_tools(mock_ai_config, mock_openai_llm):
         tool_kwargs={
             "title": "Test Title",
             "tags": ["document"],
-            "matched_tags": ["document"],
             "tag_ids": [1],
             "correspondents": ["John Doe"],
+            "correspondent_ids": [],
             "document_types": ["report"],
+            "document_type_ids": [],
             "storage_paths": ["Reports"],
+            "storage_path_ids": [],
             "dates": ["2023-01-01"],
         },
     )
@@ -175,13 +176,10 @@ def test_run_llm_query_openai_uses_tools(mock_ai_config, mock_openai_llm):
     mock_llm_instance.get_tool_calls_from_response.return_value = [tool_selection]
 
     client = AIClient()
-    result = client.run_llm_query(
-        "test_prompt",
-        allowed_candidate_ids={"tags": {1}},
-    )
+    result = client.run_llm_query("test_prompt")
 
     assert result["title"] == "Test Title"
-    assert result["tags"] == {"existing_ids": [1], "new_names": []}
+    assert result["tags"] == {"existing_ids": [1], "new_names": ["document"]}
     mock_llm_instance.chat_with_tools.assert_called_once()