sort similar documents by weight, use unrestricted-user check consistently (existed before, but good to be consistent)

This brings users without an embedding backend configured to closer
parity with those who do.  Reuse the search backend to locate similar
documents and use them to provide the LLM with the better suggestion pool
to draw from

src-ui/src/app/components/app-frame/app-frame.component.scss

@@ -385,7 +385,7 @@ main {
   }
 
   ::ng-deep .navbar-official-logo {
-    will-change: filter; // prevent resizing the filter region on hover and nudging the logo a pixel
+    will-change: filter; // Safari repaints the whole navbar on filter change without this
     filter: drop-shadow(0 1px 2px rgba(var(--pngx-navbar-brand-shadow-rgb), .3));
     transition: filter .15s ease-in-out;
 
@@ -398,8 +398,6 @@ main {
     width: 1.65rem;
     height: 1.65rem;
     flex: 0 0 auto;
-    will-change: filter; // prevent resizing the filter region on hover and nudging the logo a pixel
-    filter: drop-shadow(0 2px 3px rgba(var(--pngx-navbar-brand-shadow-rgb), 0));
     transition: filter .15s ease-in-out;
   }
 
@@ -432,8 +430,6 @@ main {
     max-width: 5rem;
     flex: 0 0 auto;
     object-fit: contain;
-    will-change: filter; // prevent resizing the filter region on hover and nudging the logo a pixel
-    filter: drop-shadow(0 2px 3px rgba(var(--pngx-navbar-brand-shadow-rgb), 0));
     transition: filter .15s ease-in-out, transform .15s ease-in-out;
   }
 

src/paperless_ai/ai_classifier.py

@@ -1,17 +1,18 @@
+import json
 import logging
 
 from django.conf import settings
 from django.contrib.auth.models import User
 
 from documents.models import Document
-from documents.permissions import get_objects_for_user_owner_aware
+from documents.permissions import permitted_object_ids
+from documents.permissions import restrict_queryset_to_visible
+from documents.permissions import user_is_unrestricted
 from paperless.config import AIConfig
 from paperless_ai.base_model import ClassificationSuggestions
 from paperless_ai.base_model import TaxonomyChoiceDict
-from paperless_ai.base_model import classification_suggestions_to_model
 from paperless_ai.client import AIClient
 from paperless_ai.db import db_connection_released
-from paperless_ai.indexing import _node_document_ids
 from paperless_ai.indexing import retrieve_similar_nodes
 from paperless_ai.indexing import truncate_content
 from paperless_ai.prompts.context import ClassificationPromptContext
@@ -19,7 +20,9 @@ from paperless_ai.prompts.context import LocalizationPromptContext
 from paperless_ai.prompts.context import RagContextPromptContext
 from paperless_ai.prompts.render import render_prompt
 from paperless_ai.taxonomy import AssignedMetadata
+from paperless_ai.taxonomy import SimilarDocument
 from paperless_ai.taxonomy import TaxonomyCandidates
+from paperless_ai.taxonomy import _node_document_weights
 from paperless_ai.taxonomy import build_taxonomy_candidates
 from paperless_ai.taxonomy import empty_taxonomy_candidates
 from paperless_ai.taxonomy import format_taxonomy_for_prompt
@@ -39,6 +42,48 @@ logger = logging.getLogger("paperless_ai.rag_classifier")
 TAXONOMY_CANDIDATE_TOP_K = 15
 
 
+def _fulltext_similar_documents(
+    document: Document,
+    user: User | None,
+    top_k: int,
+) -> list[SimilarDocument]:
+    """Rank-based fallback when no embedding backend is configured. Uses
+    Tantivy's "More Like This" (term-overlap similarity) instead of vector
+    similarity - cruder, but far better than no candidates at all.
+    more_like_this_ids returns only a ranked ID list, no scores, so weight is
+    synthesized from rank (descending from top_k) rather than claiming a
+    similarity magnitude that doesn't exist. An unrestricted user (none, or an
+    active superuser - see user_is_unrestricted) is normalized to ``None``
+    before calling, since the backend's permission filter has no superuser
+    short-circuit of its own. Results are re-checked with
+    restrict_queryset_to_visible() since Tantivy's indexed permission fields
+    lag the DB via async reindexing.
+    """
+    from documents.search import get_backend
+
+    unrestricted = user_is_unrestricted(user)
+    search_user = None if unrestricted else user
+    backend = get_backend()
+    similar_ids = backend.more_like_this_ids(
+        document.pk,
+        user=search_user,
+        limit=top_k,
+    )
+    if not unrestricted:
+        allowed_ids = set(
+            restrict_queryset_to_visible(
+                Document.objects.filter(pk__in=similar_ids),
+                user,
+                "view_document",
+            ).values_list("pk", flat=True),
+        )
+        similar_ids = [doc_id for doc_id in similar_ids if doc_id in allowed_ids]
+    return [
+        SimilarDocument(document_id=doc_id, weight=float(top_k - rank))
+        for rank, doc_id in enumerate(similar_ids)
+    ]
+
+
 def get_language_name(language_code: str) -> str:
     normalized_language_code = language_code.lower()
     for code, name in settings.LANGUAGES:
@@ -124,16 +169,20 @@ def build_localization_prompt(
     suggestions: ClassificationSuggestions,
     output_language: str,
 ) -> str:
-    """Render internal suggestions in the same flat shape the model returns.
-    Only the name fields and title are used from the localized response; the
-    merge step always keeps the original ID fields.
+    """``suggestions`` is the full nested-shape result of parse_ai_response
+    (each taxonomy field a ``{"existing_ids": [...], "new_names": [...]}``
+    dict) - passed through as-is so the model receives and returns the exact
+    DocumentClassifierSchema shape run_llm_query() always parses against.
+    Only each field's new_names (never existing_ids, which are plain
+    resolved-object IDs, not text) and title get used from the response; see
+    get_ai_document_classification's merge step, which always keeps the
+    *original* existing_ids regardless of what the model echoes back here.
     """
     language_name = get_language_name(output_language)
-    model_suggestions = classification_suggestions_to_model(suggestions)
     return render_prompt(
         LocalizationPromptContext(
             language_name=language_name,
-            suggestions_json=model_suggestions.model_dump_json(),
+            suggestions_json=json.dumps(suggestions, ensure_ascii=False),
         ),
     )
 
@@ -143,44 +192,53 @@ def get_taxonomy_context(
     user: User | None = None,
     max_docs: int = 5,
 ) -> tuple[TaxonomyCandidates, AssignedMetadata, str]:
-    """One retrieval feeds both taxonomy candidates and RAG text context.
-    On any retrieval failure, degrades to empty candidates/context rather than
-    propagating the exception - a vector-store outage should not block
-    classification, only its RAG-assisted enrichment.
+    """One retrieval feeds both taxonomy candidates and RAG text context. Uses
+    vector similarity when an embedding backend is configured, otherwise
+    falls back to Tantivy full-text "More Like This" similarity - see
+    _fulltext_similar_documents. On any retrieval failure, degrades to empty
+    candidates/context rather than propagating the exception - neither a
+    vector-store outage nor a search-index issue should block classification,
+    only its context-assisted enrichment.
     """
     assigned = get_assigned_metadata(document, user)
+    ai_config = AIConfig()
     try:
-        # None means "no restriction" to retrieve_similar_nodes. A superuser
-        # (like no user at all) can see every document, so skip materializing
-        # every visible pk into a Python list and passing it through as an IN
-        # filter: for a large library that is a wasted quadratic scan in the
-        # vector store at best, and past ~32,763 documents a hard
-        # sqlite3.OperationalError (SQLite's bound-parameter limit) at worst.
-        # get_objects_for_user_owner_aware() would return every Document for a
-        # superuser anyway (guardian's own with_superuser shortcut), so this
-        # changes nothing about which documents are considered -- only how we
-        # get there.
-        visible_document_ids = (
-            None
-            if user is None or user.is_superuser
-            else list(
-                get_objects_for_user_owner_aware(
-                    user,
-                    "view_document",
-                    Document,
-                ).values_list("pk", flat=True),
+        if ai_config.llm_embedding_backend:
+            # None means "no restriction" to retrieve_similar_nodes. An
+            # unrestricted user (no user at all, or an active superuser -- see
+            # user_is_unrestricted) can see every document, so skip
+            # materializing every visible pk into a Python list and passing it
+            # through as an IN filter: for a large library that is a wasted
+            # quadratic scan in the vector store at best, and past ~32,763
+            # documents a hard sqlite3.OperationalError (SQLite's
+            # bound-parameter limit) at worst.
+            # permitted_object_ids() has its own superuser shortcut that would
+            # return every Document's id anyway, so this changes nothing about
+            # which documents are considered -- only how we get there.
+            visible_document_ids = (
+                None
+                if user_is_unrestricted(user)
+                else list(permitted_object_ids(user, Document, "view_document"))
+            )
+            nodes = retrieve_similar_nodes(
+                document,
+                top_k=TAXONOMY_CANDIDATE_TOP_K,
+                document_ids=visible_document_ids,
+            )
+            similar_documents = _node_document_weights(nodes)
+        else:
+            # See _fulltext_similar_documents: it applies its own permission
+            # filter via `user`, so no visible-document-id list is needed here.
+            similar_documents = _fulltext_similar_documents(
+                document,
+                user,
+                top_k=TAXONOMY_CANDIDATE_TOP_K,
             )
-        )
-        nodes = retrieve_similar_nodes(
-            document,
-            top_k=TAXONOMY_CANDIDATE_TOP_K,
-            document_ids=visible_document_ids,
-        )
 
-        candidates = build_taxonomy_candidates(nodes, user)
+        candidates = build_taxonomy_candidates(similar_documents, user)
 
-        # ``nodes`` are already ordered by descending vector similarity; don't lose it.
-        similar_document_ids = list(dict.fromkeys(_node_document_ids(nodes)))
+        # similar_documents is already ordered by descending weight; don't lose it.
+        similar_document_ids = [s["document_id"] for s in similar_documents]
         similar_documents_by_id = Document.objects.in_bulk(similar_document_ids)
         similar_docs = [
             similar_documents_by_id[document_id]
@@ -194,8 +252,8 @@ def get_taxonomy_context(
             context_blocks.append(f"TITLE: {title}\n{text}")
     except Exception:
         logger.exception(
-            "Failed to retrieve RAG neighbours for document %s; continuing "
-            "without taxonomy candidates or similar-document context.",
+            "Failed to retrieve similar-document context for document %s; "
+            "continuing without taxonomy candidates or similar-document context.",
             document.pk,
         )
         return empty_taxonomy_candidates(), assigned, ""
@@ -204,9 +262,11 @@ def get_taxonomy_context(
 
 
 def parse_ai_response(raw: dict) -> ClassificationSuggestions:
-    """``raw`` is AIClient.run_llm_query()'s validated internal-shape result.
-    This gives the rest of the module a named, typed boundary instead of
-    passing the client's bare dict straight through everywhere.
+    """``raw`` is AIClient.run_llm_query()'s return value - already a
+    DocumentClassifierSchema.model_dump(), so every key below is always
+    present with the right shape; this only exists to give the rest of the
+    module a named, typed boundary instead of passing the client's bare dict
+    straight through everywhere.
     """
 
     def _choice(value: dict | None) -> TaxonomyChoiceDict:
@@ -276,23 +336,14 @@ def get_ai_document_classification(
 ) -> ClassificationSuggestions:
     ai_config = AIConfig()
 
-    if ai_config.llm_embedding_backend:
-        candidates, assigned, context = get_taxonomy_context(document, user)
-        prompt = build_prompt_with_rag(
-            document,
-            ai_config,
-            candidates=candidates,
-            assigned=assigned,
-            context=context,
-        )
-    else:
-        candidates = empty_taxonomy_candidates()
-        prompt = build_prompt_without_rag(
-            document,
-            ai_config,
-            candidates=candidates,
-            assigned=get_assigned_metadata(document, user),
-        )
+    candidates, assigned, context = get_taxonomy_context(document, user)
+    prompt = build_prompt_with_rag(
+        document,
+        ai_config,
+        candidates=candidates,
+        assigned=assigned,
+        context=context,
+    )
 
     client = AIClient()
     # Hand the pooled DB connection back while the (slow) LLM query runs so it

src/paperless_ai/base_model.py

@@ -31,9 +31,38 @@ def _truncate_to_field_limit(value: Any, field: FieldInfo) -> Any:
     )
 
 
-# This model is serialized into the schema handed to the LLM, so its docstring
-# and field descriptions are instructions for the model. Keep implementation
-# details in code comments instead.
+# Docstrings and field descriptions on both models below are serialized into
+# the schema handed to the LLM, so write them for the model. Code comments
+# should go here only.
+class TaxonomyChoice(BaseModel):
+    """One field's suggestions: existing values to reuse, plus new ones to create."""
+
+    existing_ids: list[int] = Field(
+        default_factory=list,
+        max_length=MAX_EXISTING_IDS,
+        description=(
+            "IDs from the candidate list shown in the prompt that clearly "
+            "represent values you would suggest for this field. Never invent "
+            "an ID, select a weak match merely because it exists, or use an "
+            "ID when no candidates are shown."
+        ),
+    )
+    new_names: list[str] = Field(
+        default_factory=list,
+        max_length=MAX_NEW_NAMES,
+        description=(
+            "Names for clearly supported values that no shown candidate "
+            "represents. When a candidate represents the same value, use its "
+            "ID instead so an existing value is not duplicated under a new name."
+        ),
+    )
+
+    @field_validator("existing_ids", "new_names", mode="before")
+    @classmethod
+    def _truncate(cls, value: Any, info: ValidationInfo) -> Any:
+        return _truncate_to_field_limit(value, cls.model_fields[info.field_name])
+
+
 class DocumentClassifierSchema(BaseModel):
     """Classification suggestions for a single document."""
 
@@ -44,79 +73,36 @@ class DocumentClassifierSchema(BaseModel):
             f"{MAX_TITLE_LENGTH} characters."
         ),
     )
-    tags: list[str] = Field(
-        default_factory=list,
-        max_length=MAX_NEW_NAMES,
+    tags: TaxonomyChoice = Field(
+        default_factory=TaxonomyChoice,
         description=(
-            "Names of topic labels describing what this document is about, "
-            "e.g. 'Insurance', 'Car', 'Warranty'. When an available tag "
-            "represents the same label, use its ID in tag_ids instead."
+            "Topic labels describing what this document is about. A document "
+            "may have several, e.g. 'Insurance', 'Car', 'Warranty'."
         ),
     )
-    tag_ids: list[int] = Field(
-        default_factory=list,
-        max_length=MAX_EXISTING_IDS,
+    correspondents: TaxonomyChoice = Field(
+        default_factory=TaxonomyChoice,
         description=(
-            "IDs of available tags that clearly apply to this document. Only "
-            "use IDs shown in the prompt; never invent one or choose a weak match."
+            "The person, institution or company this document originates "
+            "from, or was sent to. Not every party merely mentioned in the "
+            "text, and not the subject of the document."
         ),
     )
-    correspondents: list[str] = Field(
-        default_factory=list,
-        max_length=MAX_NEW_NAMES,
+    document_types: TaxonomyChoice = Field(
+        default_factory=TaxonomyChoice,
         description=(
-            "Names of people, institutions or companies this document is from "
-            "or was sent to, not every party merely mentioned. When an "
-            "available correspondent is the same entity, use its ID in "
-            "correspondent_ids instead."
+            "What kind of document this is, e.g. 'Invoice', 'Contract', "
+            "'Bank Statement', 'Letter'. Never its subject matter and never "
+            "who sent it."
         ),
     )
-    correspondent_ids: list[int] = Field(
-        default_factory=list,
-        max_length=MAX_EXISTING_IDS,
+    storage_paths: TaxonomyChoice = Field(
+        default_factory=TaxonomyChoice,
         description=(
-            "IDs of available correspondents that clearly apply to this "
-            "document. Only use IDs shown in the prompt; never invent one or "
-            "choose a weak match."
-        ),
-    )
-    document_types: list[str] = Field(
-        default_factory=list,
-        max_length=MAX_NEW_NAMES,
-        description=(
-            "Names describing what kind of document this is, e.g. 'Invoice', "
-            "'Contract', 'Bank Statement', 'Letter'. Never use its subject or "
-            "sender as a document type. When an available document type is the "
-            "same kind, use its ID in document_type_ids instead."
-        ),
-    )
-    document_type_ids: list[int] = Field(
-        default_factory=list,
-        max_length=MAX_EXISTING_IDS,
-        description=(
-            "IDs of available document types that clearly apply to this "
-            "document. Only use IDs shown in the prompt; never invent one or "
-            "choose a weak match."
-        ),
-    )
-    storage_paths: list[str] = Field(
-        default_factory=list,
-        max_length=MAX_NEW_NAMES,
-        description=(
-            "Names of folder-style filing locations, e.g. "
+            "A folder-style filing location for this document, e.g. "
             "'Finance/Invoices'. Leave empty unless a filing location is "
             "clearly implied - never put tags, document types or "
-            "correspondents here. When an available storage path is the same "
-            "location, use its ID in storage_path_ids instead."
-        ),
-    )
-    storage_path_ids: list[int] = Field(
-        default_factory=list,
-        max_length=MAX_EXISTING_IDS,
-        description=(
-            "IDs of available storage paths that clearly apply to this "
-            "document. Only use IDs shown in the prompt; never invent one or "
-            "choose a weak match."
+            "correspondents here."
         ),
     )
     dates: list[str] = Field(
@@ -129,33 +115,41 @@ class DocumentClassifierSchema(BaseModel):
         ),
     )
 
-    @field_validator(
-        "title",
-        "tags",
-        "tag_ids",
-        "correspondents",
-        "correspondent_ids",
-        "document_types",
-        "document_type_ids",
-        "storage_paths",
-        "storage_path_ids",
-        "dates",
-        mode="before",
-    )
+    @field_validator("title", "dates", mode="before")
     @classmethod
     def _truncate(cls, value: Any, info: ValidationInfo) -> Any:
         return _truncate_to_field_limit(value, cls.model_fields[info.field_name])
 
+    @classmethod
+    def model_json_schema(cls, *args: Any, **kwargs: Any) -> dict[str, Any]:
+        """Inline TaxonomyChoice for backends that reject JSON Schema refs."""
+        schema = super().model_json_schema(*args, **kwargs)
+        taxonomy_choice = schema.pop("$defs")["TaxonomyChoice"]
+        for field in ("tags", "correspondents", "document_types", "storage_paths"):
+            # Pydantic emits a field's description as a sibling of its $ref;
+            # those keys must survive and win over the shared definition.
+            siblings = {
+                key: value
+                for key, value in schema["properties"][field].items()
+                if key != "$ref"
+            }
+            schema["properties"][field] = taxonomy_choice | siblings
+        return schema
+
 
 class TaxonomyChoiceDict(TypedDict):
-    """Internal representation of names and existing IDs for one taxonomy."""
+    """Plain-dict counterpart of TaxonomyChoice - what
+    TaxonomyChoice.model_dump() actually produces, typed for callers that
+    work with the dumped dict rather than the pydantic instance."""
 
     existing_ids: list[int]
     new_names: list[str]
 
 
 class ClassificationSuggestions(TypedDict):
-    """Internal shape used after the flat LLM response is validated."""
+    """Plain-dict counterpart of DocumentClassifierSchema.model_dump() -
+    the shape threaded through parse_ai_response, build_localization_prompt,
+    get_ai_document_classification, and the ai_suggestions view."""
 
     title: str
     tags: TaxonomyChoiceDict
@@ -163,47 +157,3 @@ class ClassificationSuggestions(TypedDict):
     document_types: TaxonomyChoiceDict
     storage_paths: TaxonomyChoiceDict
     dates: list[str]
-
-
-def model_to_classification_suggestions(
-    model: DocumentClassifierSchema,
-) -> ClassificationSuggestions:
-    """Convert the flat, model-friendly response to the internal shape."""
-    return ClassificationSuggestions(
-        title=model.title,
-        tags=TaxonomyChoiceDict(
-            existing_ids=model.tag_ids,
-            new_names=model.tags,
-        ),
-        correspondents=TaxonomyChoiceDict(
-            existing_ids=model.correspondent_ids,
-            new_names=model.correspondents,
-        ),
-        document_types=TaxonomyChoiceDict(
-            existing_ids=model.document_type_ids,
-            new_names=model.document_types,
-        ),
-        storage_paths=TaxonomyChoiceDict(
-            existing_ids=model.storage_path_ids,
-            new_names=model.storage_paths,
-        ),
-        dates=model.dates,
-    )
-
-
-def classification_suggestions_to_model(
-    suggestions: ClassificationSuggestions,
-) -> DocumentClassifierSchema:
-    """Convert internal suggestions to the flat shape used for localization."""
-    return DocumentClassifierSchema(
-        title=suggestions["title"],
-        tags=suggestions["tags"]["new_names"],
-        tag_ids=suggestions["tags"]["existing_ids"],
-        correspondents=suggestions["correspondents"]["new_names"],
-        correspondent_ids=suggestions["correspondents"]["existing_ids"],
-        document_types=suggestions["document_types"]["new_names"],
-        document_type_ids=suggestions["document_types"]["existing_ids"],
-        storage_paths=suggestions["storage_paths"]["new_names"],
-        storage_path_ids=suggestions["storage_paths"]["existing_ids"],
-        dates=suggestions["dates"],
-    )

src/paperless_ai/client.py

@@ -19,9 +19,7 @@ from paperless.network import PinnedHostHTTPTransport
 from paperless.network import create_pinned_async_httpx_client
 from paperless.network import create_pinned_httpx_client
 from paperless.network import validate_outbound_http_url
-from paperless_ai.base_model import ClassificationSuggestions
 from paperless_ai.base_model import DocumentClassifierSchema
-from paperless_ai.base_model import model_to_classification_suggestions
 from paperless_ai.exceptions import LLMTimeoutError
 
 logger = logging.getLogger("paperless_ai.client")
@@ -117,7 +115,7 @@ class AIClient:
         else:
             raise ValueError(f"Unsupported LLM backend: {self.settings.llm_backend}")
 
-    def run_llm_query(self, prompt: str) -> ClassificationSuggestions:
+    def run_llm_query(self, prompt: str) -> str:
         logger.debug(
             "Running LLM query against %s with model %s",
             self.settings.llm_backend,
@@ -136,7 +134,7 @@ class AIClient:
                 )
             logger.debug("LLM query result: %s", result)
             parsed = DocumentClassifierSchema(**json.loads(result.message.content))
-            return model_to_classification_suggestions(parsed)
+            return parsed.model_dump()
 
         from llama_index.core.program.function_program import get_function_tool
 
@@ -155,7 +153,7 @@ class AIClient:
             )
         logger.debug("LLM query result: %s", tool_calls)
         parsed = DocumentClassifierSchema(**tool_calls[0].tool_kwargs)
-        return model_to_classification_suggestions(parsed)
+        return parsed.model_dump()
 
     @contextmanager
     def _normalize_timeouts(self) -> Iterator[None]:

src/paperless_ai/prompts/classification.j2

@@ -13,10 +13,10 @@ Analyze the following document and fill in these fields:
 - dates: up to 3 relevant dates in YYYY-MM-DD format
 {% if has_candidates %}
 
-For tags, correspondents, document types, and storage paths: first decide whether there is a useful, well-supported suggestion. If an available candidate clearly represents that suggestion, put its id in the matching tag_ids, correspondent_ids, document_type_ids, or storage_path_ids field instead of repeating its name. If no candidate represents the suggestion, put its name in tags, correspondents, document_types, or storage_paths. Do not choose a weak candidate merely because it exists.
+For tags, correspondents, document types, and storage paths: first decide whether there is a useful, well-supported suggestion. If an available candidate clearly represents that suggestion, put its id in existing_ids instead of duplicating it in new_names. If no candidate represents the suggestion, put its name in new_names. Do not choose a weak candidate merely because it exists.
 {% else %}
 
-No candidates are shown for this document, so leave every field ending in "_ids" empty and put suggestions in the corresponding name fields.
+No candidates are shown for this document, so leave every existing_ids list empty and put each suggestion's name in new_names.
 {% endif %}
 
 Filename:

src/paperless_ai/prompts/localization.j2

@@ -1,6 +1,6 @@
 You are localizing document classification suggestions for display in Paperless-ngx.
 
-Rewrite only the "title", "tags", "document_types", and "storage_paths" fields in {{ language_name }}. Leave every field ending in "_ids" exactly as given - these are database identifiers, not text, and are not used from your response even if changed.
+Rewrite only the "title" field and each taxonomy field's "new_names" list in {{ language_name }}. Leave every "existing_ids" list exactly as given - these are database identifiers, not text, and are not used from your response even if changed.
 
 Do not translate correspondents or dates.
 Preserve proper nouns, organization names, product names, and exact official document names. Translate generic category words when a {{ language_name }} equivalent exists.

src/paperless_ai/taxonomy.py

@@ -33,6 +33,11 @@ class TaxonomyCandidate(TypedDict):
     weight: float
 
 
+class SimilarDocument(TypedDict):
+    document_id: int
+    weight: float
+
+
 class TaxonomyCandidates(TypedDict):
     tags: list[TaxonomyCandidate]
     document_types: list[TaxonomyCandidate]
@@ -105,10 +110,10 @@ def get_assigned_metadata(document: Document, user: User | None) -> AssignedMeta
     )
 
 
-def _node_document_weights(nodes: list["NodeWithScore"]) -> dict[int, float]:
-    """document_id -> that node's similarity score, summed if a document_id
-    appears more than once across the retrieved nodes (e.g. multiple chunks
-    of the same source document)."""
+def _node_document_weights(nodes: list["NodeWithScore"]) -> list[SimilarDocument]:
+    """Sum each node's similarity score into its document_id (a document can
+    appear via multiple chunks/nodes) and return one SimilarDocument per
+    distinct document_id."""
     weights: dict[int, float] = defaultdict(float)
     for node in nodes:
         document_id = node.metadata.get("document_id")
@@ -121,7 +126,14 @@ def _node_document_weights(nodes: list["NodeWithScore"]) -> dict[int, float]:
             weights[int(document_id)] += float(node.score or 0.0)
         except (TypeError, ValueError):  # pragma: no cover
             continue
-    return weights
+    return sorted(
+        (
+            SimilarDocument(document_id=document_id, weight=weight)
+            for document_id, weight in weights.items()
+        ),
+        key=lambda similar: similar["weight"],
+        reverse=True,
+    )
 
 
 def _visible_ranked_candidates(
@@ -157,21 +169,26 @@ def _visible_ranked_candidates(
 
 
 def build_taxonomy_candidates(
-    nodes: list["NodeWithScore"],
+    similar_documents: list[SimilarDocument],
     user: User | None,
 ) -> TaxonomyCandidates:
-    """Resolve each neighbour node's document_id to a live Document, read its
-    *current* tags/type/correspondent/storage_path via the ORM (never the
-    possibly-stale names cached in vector-index node metadata), weight each
-    distinct taxonomy object by aggregate neighbour similarity, permission-filter
+    """Resolve each similar document's id to a live Document, read its
+    *current* tags/type/correspondent/storage_path via the ORM (never any
+    possibly-stale names an adapter's source might have cached), weight each
+    distinct taxonomy object by aggregate similarity weight, permission-filter
     against what ``user`` can see, and return each category ranked by weight
-    and capped.
+    and capped. ``similar_documents`` may come from either the vector-RAG
+    adapter or the full-text fallback adapter - both produce this same shape.
     """
-
-    document_weights = _node_document_weights(nodes)
-    if not document_weights:
+    if not similar_documents:
         return empty_taxonomy_candidates()
 
+    # Both adapters guarantee at most one SimilarDocument per document_id, so
+    # this never silently drops a duplicate's weight.
+    document_weights: dict[int, float] = {
+        s["document_id"]: s["weight"] for s in similar_documents
+    }
+
     # Only .tags.all() needs prefetching (a reverse M2M, one extra query for
     # the whole batch). document_type/correspondent/storage_path are read
     # below via their *_id columns (neighbour.document_type_id, etc.), which

src/paperless_ai/tests/test_ai_classifier.py

@@ -1,4 +1,5 @@
 import datetime
+from collections.abc import Generator
 from types import SimpleNamespace
 from unittest.mock import MagicMock
 from unittest.mock import patch
@@ -8,10 +9,13 @@ import pytest_mock
 from django.test import override_settings
 
 from documents.models import Document
+from documents.search import TantivyBackend
 from documents.tests.factories import DocumentFactory
 from documents.tests.factories import TagFactory
 from documents.tests.factories import UserFactory
 from paperless.config import AIConfig
+from paperless_ai.ai_classifier import TAXONOMY_CANDIDATE_TOP_K
+from paperless_ai.ai_classifier import _fulltext_similar_documents
 from paperless_ai.ai_classifier import _restrict_to_shown_candidates
 from paperless_ai.ai_classifier import build_localization_prompt
 from paperless_ai.ai_classifier import build_prompt_with_rag
@@ -21,6 +25,7 @@ from paperless_ai.ai_classifier import get_language_name
 from paperless_ai.ai_classifier import get_taxonomy_context
 from paperless_ai.base_model import ClassificationSuggestions
 from paperless_ai.base_model import TaxonomyChoiceDict
+from paperless_ai.taxonomy import SimilarDocument
 from paperless_ai.taxonomy import TaxonomyCandidate
 from paperless_ai.taxonomy import TaxonomyCandidates
 from paperless_ai.taxonomy import empty_taxonomy_candidates
@@ -205,12 +210,10 @@ def test_use_rag_if_configured(
 
 @pytest.mark.django_db
 @patch("paperless_ai.client.AIClient.run_llm_query")
-@patch("paperless_ai.ai_classifier.build_prompt_without_rag")
-@patch("paperless_ai.ai_classifier.AIConfig")
+@patch("paperless_ai.ai_classifier.build_prompt_with_rag")
 @override_settings(LLM_BACKEND="ollama", LLM_MODEL="some_model")
-def test_use_without_rag_if_not_configured(
-    mock_ai_config,
-    mock_build_prompt_without_rag,
+def test_use_rag_prompt_even_without_embedding_backend(
+    mock_build_prompt_with_rag,
     mock_run_llm_query,
     mock_document,
 ):
@@ -220,13 +223,13 @@ def test_use_without_rag_if_not_configured(
     WHEN:
         - get_ai_document_classification() is called
     THEN:
-        - The non-RAG prompt builder is used
+        - The RAG-context prompt builder is still used (fed by the full-text
+          fallback's context/candidates instead of the vector store's)
     """
-    mock_ai_config.return_value.llm_embedding_backend = None
-    mock_build_prompt_without_rag.return_value = "Prompt without RAG"
+    mock_build_prompt_with_rag.return_value = "Prompt with RAG"
     mock_run_llm_query.return_value = NESTED_SUGGESTIONS
     get_ai_document_classification(mock_document)
-    mock_build_prompt_without_rag.assert_called_once()
+    mock_build_prompt_with_rag.assert_called_once()
 
 
 @pytest.mark.django_db
@@ -245,7 +248,7 @@ def test_prompt_with_without_rag(mock_document):
     THEN:
         - build_prompt_without_rag() has no similar-documents section
         - build_prompt_with_rag() includes the similar-documents context
-        - build_localization_prompt() asks to rewrite only names/title and
+        - build_localization_prompt() asks to rewrite only new_names/title and
           not to translate correspondents or dates
     """
     config = AIConfig()
@@ -264,7 +267,6 @@ def test_prompt_with_without_rag(mock_document):
     prompt = build_localization_prompt(NESTED_SUGGESTIONS, output_language="de-de")
     assert "Rewrite only the" in prompt
     assert "Do not translate correspondents or dates" in prompt
-    assert '"tag_ids":[]' in prompt
 
 
 def test_get_language_name_falls_back_to_language_code():
@@ -305,6 +307,7 @@ def test_build_localization_prompt_preserves_unicode_characters():
 
 
 @pytest.mark.django_db
+@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
 def test_get_taxonomy_context_assembles_rag_text_and_candidates():
     """
     GIVEN:
@@ -346,6 +349,7 @@ def test_get_taxonomy_context_assembles_rag_text_and_candidates():
 
 
 @pytest.mark.django_db
+@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
 def test_get_taxonomy_context_preserves_similarity_order_and_distinct_documents():
     """
     GIVEN:
@@ -416,6 +420,7 @@ def test_get_taxonomy_context_preserves_similarity_order_and_distinct_documents(
 
 
 @pytest.mark.django_db
+@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
 def test_get_taxonomy_context_no_similar_docs():
     """
     GIVEN:
@@ -439,6 +444,67 @@ def test_get_taxonomy_context_no_similar_docs():
     }
 
 
+@pytest.mark.django_db
+def test_get_taxonomy_context_uses_fulltext_fallback_when_no_embedding_backend(
+    mocker: pytest_mock.MockerFixture,
+) -> None:
+    """
+    GIVEN:
+        - No LLM embedding backend is configured (the default test settings)
+    WHEN:
+        - get_taxonomy_context() is called
+    THEN:
+        - _fulltext_similar_documents() is called with the document, the user
+          and TAXONOMY_CANDIDATE_TOP_K
+        - retrieve_similar_nodes() (the vector path) is never called
+    """
+    document = DocumentFactory.create(content="Some content")
+    mock_fulltext = mocker.patch(
+        "paperless_ai.ai_classifier._fulltext_similar_documents",
+        return_value=[],
+    )
+    mock_retrieve = mocker.patch("paperless_ai.ai_classifier.retrieve_similar_nodes")
+
+    get_taxonomy_context(document, user=None)
+
+    mock_fulltext.assert_called_once_with(
+        document,
+        None,
+        top_k=TAXONOMY_CANDIDATE_TOP_K,
+    )
+    mock_retrieve.assert_not_called()
+
+
+@pytest.mark.django_db
+@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
+def test_get_taxonomy_context_uses_vector_path_when_embedding_backend_configured(
+    mocker: pytest_mock.MockerFixture,
+) -> None:
+    """
+    GIVEN:
+        - An LLM embedding backend is configured
+    WHEN:
+        - get_taxonomy_context() is called
+    THEN:
+        - retrieve_similar_nodes() (the vector path) is called
+        - _fulltext_similar_documents() (the no-embedding-backend fallback)
+          is never called
+    """
+    document = DocumentFactory.create(content="Some content")
+    mock_retrieve = mocker.patch(
+        "paperless_ai.ai_classifier.retrieve_similar_nodes",
+        return_value=[],
+    )
+    mock_fulltext = mocker.patch(
+        "paperless_ai.ai_classifier._fulltext_similar_documents",
+    )
+
+    get_taxonomy_context(document, user=None)
+
+    mock_retrieve.assert_called_once()
+    mock_fulltext.assert_not_called()
+
+
 class TestGetTaxonomyContextVisibility:
     """get_taxonomy_context must not materialize every visible document id
     for a user who can already see the whole library: a superuser (like no
@@ -451,6 +517,7 @@ class TestGetTaxonomyContextVisibility:
     """
 
     @pytest.mark.django_db
+    @override_settings(LLM_EMBEDDING_BACKEND="huggingface")
     def test_skips_permission_lookup_for_superuser(
         self,
         mocker: pytest_mock.MockerFixture,
@@ -469,17 +536,18 @@ class TestGetTaxonomyContextVisibility:
             "paperless_ai.ai_classifier.retrieve_similar_nodes",
             return_value=[],
         )
-        mock_get_objects = mocker.patch(
-            "paperless_ai.ai_classifier.get_objects_for_user_owner_aware",
+        mock_permitted = mocker.patch(
+            "paperless_ai.ai_classifier.permitted_object_ids",
         )
         user = UserFactory.create(is_superuser=True)
 
         get_taxonomy_context(document, user)
 
-        mock_get_objects.assert_not_called()
+        mock_permitted.assert_not_called()
         assert mock_retrieve.call_args.kwargs["document_ids"] is None
 
     @pytest.mark.django_db
+    @override_settings(LLM_EMBEDDING_BACKEND="huggingface")
     def test_skips_permission_lookup_when_no_user(
         self,
         mocker: pytest_mock.MockerFixture,
@@ -498,16 +566,17 @@ class TestGetTaxonomyContextVisibility:
             "paperless_ai.ai_classifier.retrieve_similar_nodes",
             return_value=[],
         )
-        mock_get_objects = mocker.patch(
-            "paperless_ai.ai_classifier.get_objects_for_user_owner_aware",
+        mock_permitted = mocker.patch(
+            "paperless_ai.ai_classifier.permitted_object_ids",
         )
 
         get_taxonomy_context(document, None)
 
-        mock_get_objects.assert_not_called()
+        mock_permitted.assert_not_called()
         assert mock_retrieve.call_args.kwargs["document_ids"] is None
 
     @pytest.mark.django_db
+    @override_settings(LLM_EMBEDDING_BACKEND="huggingface")
     def test_restricts_to_visible_documents_for_non_superuser(
         self,
         mocker: pytest_mock.MockerFixture,
@@ -518,7 +587,7 @@ class TestGetTaxonomyContextVisibility:
         WHEN:
             - get_taxonomy_context() is called
         THEN:
-            - The user's visible document ids are looked up and passed to
+            - The user's permitted document ids are looked up and passed to
               retrieve_similar_nodes() as a restriction
         """
         document = DocumentFactory.create(content="Some content")
@@ -526,21 +595,186 @@ class TestGetTaxonomyContextVisibility:
             "paperless_ai.ai_classifier.retrieve_similar_nodes",
             return_value=[],
         )
-        mock_queryset = mocker.MagicMock()
-        mock_queryset.values_list.return_value = [1, 2, 3]
-        mock_get_objects = mocker.patch(
-            "paperless_ai.ai_classifier.get_objects_for_user_owner_aware",
-            return_value=mock_queryset,
+        mock_permitted = mocker.patch(
+            "paperless_ai.ai_classifier.permitted_object_ids",
+            return_value=[1, 2, 3],
         )
         user = UserFactory.create(is_superuser=False)
 
         get_taxonomy_context(document, user)
 
-        mock_get_objects.assert_called_once_with(user, "view_document", Document)
+        mock_permitted.assert_called_once_with(user, Document, "view_document")
         assert mock_retrieve.call_args.kwargs["document_ids"] == [1, 2, 3]
 
 
 @pytest.mark.django_db
+class TestFulltextSimilarDocuments:
+    """_fulltext_similar_documents is the no-embedding-backend fallback: it
+    asks the Tantivy full-text index for "More Like This" neighbours instead
+    of the vector store, and synthesizes a rank-based weight since Tantivy's
+    more_like_this_ids returns only an ordered id list, no scores.
+    """
+
+    @pytest.fixture
+    def fulltext_backend(
+        self,
+        mocker: pytest_mock.MockerFixture,
+    ) -> Generator[TantivyBackend, None, None]:
+        """An in-memory Tantivy backend, wired up as the module-level
+        singleton _fulltext_similar_documents resolves via get_backend()."""
+        backend = TantivyBackend(path=None)
+        backend.open()
+        mocker.patch("documents.search.get_backend", return_value=backend)
+        try:
+            yield backend
+        finally:
+            backend.close()
+
+    def test_ranks_by_rank_based_weight_descending(
+        self,
+        fulltext_backend: TantivyBackend,
+    ) -> None:
+        """
+        GIVEN:
+            - A source document and two similar documents indexed in Tantivy
+        WHEN:
+            - _fulltext_similar_documents() is called
+        THEN:
+            - Each result's weight reflects its rank (first result weighted
+              higher than the second), not a raw similarity score
+        """
+        source = DocumentFactory.create(content="quarterly financial report details")
+        first = DocumentFactory.create(content="quarterly financial report details")
+        second = DocumentFactory.create(content="financial report")
+        for doc in (source, first, second):
+            fulltext_backend.add_or_update(doc)
+
+        result = _fulltext_similar_documents(source, user=None, top_k=5)
+
+        assert len(result) == 2
+        weight_by_id = {s["document_id"]: s["weight"] for s in result}
+        assert weight_by_id[first.pk] > weight_by_id[second.pk]
+
+    def test_excludes_source_document(
+        self,
+        fulltext_backend: TantivyBackend,
+    ) -> None:
+        """
+        GIVEN:
+            - A source document indexed in Tantivy with no other documents
+        WHEN:
+            - _fulltext_similar_documents() is called
+        THEN:
+            - An empty list is returned - the source document is never its
+              own similar document
+        """
+        source = DocumentFactory.create(content="unique unrelated content")
+        fulltext_backend.add_or_update(source)
+
+        result = _fulltext_similar_documents(source, user=None, top_k=5)
+
+        assert result == []
+
+    def test_empty_index_returns_empty_list(
+        self,
+        fulltext_backend: TantivyBackend,
+    ) -> None:
+        """
+        GIVEN:
+            - A document that has never been indexed (fresh/empty Tantivy index)
+        WHEN:
+            - _fulltext_similar_documents() is called
+        THEN:
+            - An empty list is returned rather than raising
+        """
+        source = DocumentFactory.create(content="never indexed")
+
+        result = _fulltext_similar_documents(source, user=None, top_k=5)
+
+        assert result == []
+
+    def test_respects_top_k_limit(
+        self,
+        fulltext_backend: TantivyBackend,
+    ) -> None:
+        """
+        GIVEN:
+            - A source document and four similar documents indexed
+        WHEN:
+            - _fulltext_similar_documents() is called with top_k=2
+        THEN:
+            - At most 2 results are returned
+        """
+        source = DocumentFactory.create(content="shared overlapping keyword text")
+        fulltext_backend.add_or_update(source)
+        for _ in range(4):
+            fulltext_backend.add_or_update(
+                DocumentFactory.create(content="shared overlapping keyword text"),
+            )
+
+        result = _fulltext_similar_documents(source, user=None, top_k=2)
+
+        assert len(result) == 2
+
+    def test_result_shape_is_similar_document(
+        self,
+        fulltext_backend: TantivyBackend,
+    ) -> None:
+        """
+        GIVEN:
+            - A source document and one similar document indexed
+        WHEN:
+            - _fulltext_similar_documents() is called
+        THEN:
+            - Each result is a SimilarDocument (document_id + weight only)
+        """
+        source = DocumentFactory.create(content="shared content phrase")
+        other = DocumentFactory.create(content="shared content phrase")
+        fulltext_backend.add_or_update(source)
+        fulltext_backend.add_or_update(other)
+
+        result = _fulltext_similar_documents(source, user=None, top_k=5)
+
+        # rank 0 (the only/best result) with top_k=5 -> weight = top_k - rank = 5.0,
+        # per the "first result gets top_k, the last gets 1" formula.
+        assert result == [SimilarDocument(document_id=other.pk, weight=5.0)]
+
+    def test_superuser_sees_other_users_documents(
+        self,
+        fulltext_backend: TantivyBackend,
+    ) -> None:
+        """
+        GIVEN:
+            - A source document owned by one user and a similar document
+              owned by a different user, with no sharing between them
+        WHEN:
+            - _fulltext_similar_documents() is called with a superuser
+        THEN:
+            - The other user's document is still returned as a similar
+              document - a superuser must not be narrowed by the backend's
+              owner-based permission filter
+        """
+        owner = UserFactory.create()
+        other_owner = UserFactory.create()
+        superuser = UserFactory.create(is_superuser=True)
+        source = DocumentFactory.create(
+            content="shared content phrase",
+            owner=owner,
+        )
+        other = DocumentFactory.create(
+            content="shared content phrase",
+            owner=other_owner,
+        )
+        fulltext_backend.add_or_update(source)
+        fulltext_backend.add_or_update(other)
+
+        result = _fulltext_similar_documents(source, user=superuser, top_k=5)
+
+        assert [s["document_id"] for s in result] == [other.pk]
+
+
+@pytest.mark.django_db
+@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
 @patch("paperless_ai.ai_classifier.retrieve_similar_nodes")
 def test_get_taxonomy_context_retrieval_failure_degrades_to_no_hints(mock_retrieve):
     """
@@ -567,6 +801,7 @@ def test_get_taxonomy_context_retrieval_failure_degrades_to_no_hints(mock_retrie
 
 
 @pytest.mark.django_db
+@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
 @patch("paperless_ai.ai_classifier.build_taxonomy_candidates")
 @patch("paperless_ai.ai_classifier.retrieve_similar_nodes")
 def test_get_taxonomy_context_candidate_building_failure_degrades_to_no_hints(
@@ -608,7 +843,7 @@ def test_build_prompt_without_rag_includes_taxonomy_block():
     WHEN:
         - build_prompt_without_rag() is called with candidates and assigned metadata
     THEN:
-        - The candidate's id and the flat name/ID instructions appear
+        - The candidate's id and the existing_ids/new_names instructions appear
         - Candidates are presented as deduplication options, not requirements
     """
     document = DocumentFactory.create(content="Some content")
@@ -634,8 +869,8 @@ def test_build_prompt_without_rag_includes_taxonomy_block():
     )
 
     assert '"id": 12' in prompt
-    assert "tag_ids" in prompt
-    assert "correspondent_ids" in prompt
+    assert "existing_ids" in prompt
+    assert "new_names" in prompt
     assert "not requirements" in prompt
     assert "weak candidate" in prompt
 
@@ -652,7 +887,7 @@ def test_build_prompt_without_rag_identical_when_no_hints():
         - Both prompts are identical
         - Neither carries the "Available ..." candidate block or the
           id-vs-name routing instruction
-        - Both still tell the model to leave every ID field empty
+        - Both still tell the model to leave existing_ids empty
     """
     document = DocumentFactory.create(content="Some content")
     config = AIConfig()
@@ -679,8 +914,8 @@ def test_build_prompt_without_rag_identical_when_no_hints():
 
     assert with_empty_hints == with_no_hints
     assert "Available " not in with_no_hints
-    assert "put its id in the matching" not in with_no_hints
-    assert 'leave every field ending in "_ids" empty' in with_no_hints
+    assert "put its id in existing_ids" not in with_no_hints
+    assert "leave every existing_ids list empty" in with_no_hints
 
 
 @pytest.mark.django_db
@@ -692,9 +927,9 @@ def test_build_prompt_without_rag_tells_model_to_skip_ids_when_no_candidates():
         - build_prompt_without_rag() is called with candidates and assigned metadata
     THEN:
         - The assigned-metadata block appears (taxonomy_block is non-empty)
-        - The prompt tells the model to leave every ID field empty
+        - The prompt tells the model to leave existing_ids empty
 
-    Staying silent about ID fields here is not enough: the response schema
+    Staying silent about existing_ids here is not enough: the response schema
     advertises the field whatever the prompt says, and models fill it with
     placeholder ids that resolve to real but unrelated objects (#13831).
     """
@@ -722,7 +957,7 @@ def test_build_prompt_without_rag_tells_model_to_skip_ids_when_no_candidates():
 
     assert "already assigned" in prompt
     assert "No candidates are shown" in prompt
-    assert 'leave every field ending in "_ids" empty' in prompt
+    assert "leave every existing_ids list empty" in prompt
 
 
 @pytest.mark.django_db

src/paperless_ai/tests/test_base_model.py

@@ -6,9 +6,8 @@ from paperless_ai.base_model import MAX_NEW_NAMES
 from paperless_ai.base_model import MAX_TITLE_LENGTH
 from paperless_ai.base_model import ClassificationSuggestions
 from paperless_ai.base_model import DocumentClassifierSchema
+from paperless_ai.base_model import TaxonomyChoice
 from paperless_ai.base_model import TaxonomyChoiceDict
-from paperless_ai.base_model import classification_suggestions_to_model
-from paperless_ai.base_model import model_to_classification_suggestions
 
 
 def test_document_classifier_schema_declared_defaults():
@@ -19,67 +18,29 @@ def test_document_classifier_schema_declared_defaults():
     WHEN:
         - The schema is dumped to a dict via model_dump()
     THEN:
-        - Every name and ID field, and dates, dump as empty lists
+        - Every taxonomy field dumps as an empty existing_ids/new_names
+          dict, and dates dumps as an empty list
 
-    The model may omit optional fields, so the schema must provide the complete
-    empty shape expected by the conversion and matching pipeline.
+    This is the one project-owned fact worth pinning down here: which
+    defaults this schema declares for a partial LLM response (see
+    client.py's DocumentClassifierSchema(**json.loads(...)) call sites,
+    which construct from whatever subset of fields the backend actually
+    returned). It deliberately hardcodes the expected literal rather than
+    re-deriving it from TaxonomyChoice()/[] - pydantic's own
+    default_factory machinery is not this project's to re-test, and a
+    test that recomputes the expected value from the model under test
+    can't ever catch a wrong default.
     """
     schema = DocumentClassifierSchema(title="Test Title")
 
     dumped = schema.model_dump()
 
-    assert dumped == {
-        "title": "Test Title",
-        "tags": [],
-        "tag_ids": [],
-        "correspondents": [],
-        "correspondent_ids": [],
-        "document_types": [],
-        "document_type_ids": [],
-        "storage_paths": [],
-        "storage_path_ids": [],
-        "dates": [],
-    }
-
-
-def test_flat_model_response_converts_to_internal_taxonomy_choices():
-    """
-    GIVEN:
-        - A flat model response with separate name and candidate-ID fields
-    WHEN:
-        - It is converted to Paperless' internal suggestion representation
-    THEN:
-        - Names and IDs are paired under their taxonomy category
-    """
-    parsed = DocumentClassifierSchema(
-        title="Electricity Bill",
-        tags=["Utilities", "Electricity"],
-        tag_ids=[12],
-        correspondents=["Power Company"],
-        correspondent_ids=[23],
-        document_types=["Utility Bill"],
-        document_type_ids=[34],
-        storage_paths=["Finance/Utilities"],
-        storage_path_ids=[45],
-    )
-    suggestions = model_to_classification_suggestions(parsed)
-
-    assert suggestions["tags"] == {
-        "existing_ids": [12],
-        "new_names": ["Utilities", "Electricity"],
-    }
-    assert suggestions["correspondents"] == {
-        "existing_ids": [23],
-        "new_names": ["Power Company"],
-    }
-    assert suggestions["document_types"] == {
-        "existing_ids": [34],
-        "new_names": ["Utility Bill"],
-    }
-    assert suggestions["storage_paths"] == {
-        "existing_ids": [45],
-        "new_names": ["Finance/Utilities"],
-    }
+    empty_choice = {"existing_ids": [], "new_names": []}
+    assert dumped["tags"] == empty_choice
+    assert dumped["correspondents"] == empty_choice
+    assert dumped["document_types"] == empty_choice
+    assert dumped["storage_paths"] == empty_choice
+    assert dumped["dates"] == []
 
 
 def test_document_classifier_schema_json_schema_is_self_contained():
@@ -89,20 +50,23 @@ def test_document_classifier_schema_json_schema_is_self_contained():
     WHEN:
         - Its JSON schema is generated via model_json_schema()
     THEN:
-        - The schema contains no definitions, references, or nested objects
-        - Every response field is a scalar or flat array
+        - No $defs section and no $ref at any depth survives in the schema
+        - Each taxonomy property carries existing_ids/new_names inline
 
-    This keeps the function declaration compatible with backends that reject
-    JSON Schema references and with smaller models that struggle with nesting.
+    Regression guard: Google's function-declaration schema rejects the $ref
+    Pydantic normally emits for the nested TaxonomyChoice model.
     """
     schema = DocumentClassifierSchema.model_json_schema()
 
     assert "$defs" not in schema
     assert "$ref" not in json.dumps(schema)
-    assert all(
-        field_schema.get("type") != "object"
-        for field_schema in schema["properties"].values()
-    )
+    for field in ("tags", "correspondents", "document_types", "storage_paths"):
+        field_schema = schema["properties"][field]
+        assert "$ref" not in field_schema
+        assert set(field_schema["properties"].keys()) == {
+            "existing_ids",
+            "new_names",
+        }
 
 
 def test_every_field_describes_itself_to_the_model():
@@ -112,7 +76,8 @@ def test_every_field_describes_itself_to_the_model():
     WHEN:
         - Its JSON schema is generated via model_json_schema()
     THEN:
-        - Every property carries a non-empty description
+        - Every property, and every property of each inlined TaxonomyChoice,
+          carries a non-empty description
 
     In tool-calling mode the schema is most of what tells the model how to
     fill these fields; on field names alone, small models can bin tags and
@@ -121,14 +86,48 @@ def test_every_field_describes_itself_to_the_model():
     schema = DocumentClassifierSchema.model_json_schema()
 
     undescribed = [
-        name
-        for name, prop in schema["properties"].items()
+        f"{owner}.{name}"
+        for owner, definition in [
+            ("DocumentClassifierSchema", schema),
+            *(
+                (name, prop)
+                for name, prop in schema["properties"].items()
+                if prop.get("type") == "object"
+            ),
+        ]
+        for name, prop in definition.get("properties", {}).items()
         if not prop.get("description")
     ]
 
     assert undescribed == []
 
 
+def test_inlining_keeps_each_taxonomy_fields_own_description():
+    """
+    GIVEN:
+        - The DocumentClassifierSchema pydantic model
+    WHEN:
+        - Its JSON schema is generated via model_json_schema()
+    THEN:
+        - Each taxonomy field keeps its own description, not the shared one
+        - The inlined TaxonomyChoice properties survive underneath it
+
+    Pydantic emits a field's description as a sibling of its $ref, so
+    replacing the property outright collapses all four onto TaxonomyChoice's
+    docstring - which still passes a "has a description" check.
+    """
+    properties = DocumentClassifierSchema.model_json_schema()["properties"]
+
+    taxonomy_fields = ("tags", "correspondents", "document_types", "storage_paths")
+    descriptions = {
+        field: properties[field]["description"] for field in taxonomy_fields
+    }
+
+    assert len(set(descriptions.values())) == len(taxonomy_fields)
+    for field in taxonomy_fields:
+        assert properties[field]["properties"]["existing_ids"]["description"]
+
+
 def test_every_sequence_in_the_emitted_schema_is_bounded():
     """
     GIVEN:
@@ -136,13 +135,22 @@ def test_every_sequence_in_the_emitted_schema_is_bounded():
     WHEN:
         - Its JSON schema is generated via model_json_schema()
     THEN:
-        - Every array property in the schema carries a maxItems
+        - Every array property in the schema, including those on each
+          inlined TaxonomyChoice, carries a maxItems
     """
     schema = DocumentClassifierSchema.model_json_schema()
 
     unbounded = [
-        name
-        for name, prop in schema["properties"].items()
+        f"{owner}.{name}"
+        for owner, definition in [
+            ("DocumentClassifierSchema", schema),
+            *(
+                (name, prop)
+                for name, prop in schema["properties"].items()
+                if prop.get("type") == "object"
+            ),
+        ]
+        for name, prop in definition.get("properties", {}).items()
         if prop.get("type") == "array" and "maxItems" not in prop
     ]
 
@@ -174,15 +182,17 @@ def test_over_long_response_is_truncated_rather_than_rejected():
     """
     parsed = DocumentClassifierSchema(
         title="T" * (MAX_TITLE_LENGTH + 50),
-        tags=["n"] * (MAX_NEW_NAMES + 20),
-        tag_ids=list(range(MAX_EXISTING_IDS + 20)),
+        tags=TaxonomyChoice(
+            existing_ids=list(range(MAX_EXISTING_IDS + 20)),
+            new_names=["n"] * (MAX_NEW_NAMES + 20),
+        ),
         dates=[f"2016-{month:02d}-01" for month in range(1, 13)],
     )
 
     assert len(parsed.title) == MAX_TITLE_LENGTH
     assert len(parsed.dates) == MAX_DATES
-    assert len(parsed.tag_ids) == MAX_EXISTING_IDS
-    assert len(parsed.tags) == MAX_NEW_NAMES
+    assert len(parsed.tags.existing_ids) == MAX_EXISTING_IDS
+    assert len(parsed.tags.new_names) == MAX_NEW_NAMES
 
 
 def test_truncation_keeps_the_earliest_entries():
@@ -202,48 +212,24 @@ def test_truncation_keeps_the_earliest_entries():
     assert parsed.dates == ["2016-10-01", "2016-09-01", "2016-08-01"]
 
 
-def test_model_conversion_matches_internal_typed_dict_keys():
+def test_model_dump_matches_typed_dict_keys():
     """
     GIVEN:
         - A DocumentClassifierSchema instance
     WHEN:
-        - It is converted to ClassificationSuggestions
+        - It is dumped to a dict via model_dump()
     THEN:
-        - The converted dict's keys exactly match ClassificationSuggestions'
+        - The dumped dict's keys exactly match ClassificationSuggestions'
           declared keys
-        - The converted tags dict's keys exactly match TaxonomyChoiceDict's
+        - The dumped tags dict's keys exactly match TaxonomyChoiceDict's
           declared keys
     """
-    schema = DocumentClassifierSchema(title="T", tags=["Tag"], tag_ids=[1])
-    suggestions = model_to_classification_suggestions(schema)
+    # TaxonomyChoiceDict/ClassificationSuggestions are the static-typing
+    # counterparts of TaxonomyChoice/DocumentClassifierSchema - this pins
+    # down that .model_dump()'s actual runtime keys are exactly what the
+    # TypedDicts declare, so the two don't silently drift apart.
+    schema = DocumentClassifierSchema(title="T", tags=TaxonomyChoice(existing_ids=[1]))
+    dumped = schema.model_dump()
 
-    assert set(suggestions.keys()) == set(
-        ClassificationSuggestions.__annotations__.keys(),
-    )
-    assert set(suggestions["tags"].keys()) == set(
-        TaxonomyChoiceDict.__annotations__.keys(),
-    )
-
-
-def test_internal_suggestions_round_trip_through_flat_model():
-    suggestions = ClassificationSuggestions(
-        title="Electricity Bill",
-        tags=TaxonomyChoiceDict(existing_ids=[1], new_names=["Utilities"]),
-        correspondents=TaxonomyChoiceDict(
-            existing_ids=[2],
-            new_names=["Power Company"],
-        ),
-        document_types=TaxonomyChoiceDict(
-            existing_ids=[3],
-            new_names=["Utility Bill"],
-        ),
-        storage_paths=TaxonomyChoiceDict(
-            existing_ids=[4],
-            new_names=["Finance/Utilities"],
-        ),
-        dates=["2026-08-30"],
-    )
-
-    model = classification_suggestions_to_model(suggestions)
-
-    assert model_to_classification_suggestions(model) == suggestions
+    assert set(dumped.keys()) == set(ClassificationSuggestions.__annotations__.keys())
+    assert set(dumped["tags"].keys()) == set(TaxonomyChoiceDict.__annotations__.keys())

src/paperless_ai/tests/test_client.py

@@ -123,14 +123,10 @@ def test_run_llm_query_ollama_uses_structured_json(mock_ai_config, mock_ollama_l
     mock_llm_instance.chat.return_value.message.content = json.dumps(
         {
             "title": "Test Title",
-            "tags": ["document"],
-            "tag_ids": [1],
-            "correspondents": ["John Doe"],
-            "correspondent_ids": [],
-            "document_types": ["report"],
-            "document_type_ids": [],
-            "storage_paths": ["Reports"],
-            "storage_path_ids": [],
+            "tags": {"existing_ids": [1], "new_names": ["document"]},
+            "correspondents": {"existing_ids": [], "new_names": ["John Doe"]},
+            "document_types": {"existing_ids": [], "new_names": ["report"]},
+            "storage_paths": {"existing_ids": [], "new_names": ["Reports"]},
             "dates": ["2023-01-01"],
         },
     )
@@ -160,14 +156,10 @@ def test_run_llm_query_openai_uses_tools(mock_ai_config, mock_openai_llm):
         tool_name="DocumentClassifierSchema",
         tool_kwargs={
             "title": "Test Title",
-            "tags": ["document"],
-            "tag_ids": [1],
-            "correspondents": ["John Doe"],
-            "correspondent_ids": [],
-            "document_types": ["report"],
-            "document_type_ids": [],
-            "storage_paths": ["Reports"],
-            "storage_path_ids": [],
+            "tags": {"existing_ids": [1], "new_names": ["document"]},
+            "correspondents": {"existing_ids": [], "new_names": ["John Doe"]},
+            "document_types": {"existing_ids": [], "new_names": ["report"]},
+            "storage_paths": {"existing_ids": [], "new_names": ["Reports"]},
             "dates": ["2023-01-01"],
         },
     )

src/paperless_ai/tests/test_taxonomy.py

@@ -1,5 +1,4 @@
 import json
-from types import SimpleNamespace
 
 import pytest
 import pytest_mock
@@ -11,6 +10,7 @@ from documents.tests.factories import StoragePathFactory
 from documents.tests.factories import TagFactory
 from documents.tests.factories import UserFactory
 from paperless_ai.taxonomy import AssignedMetadata
+from paperless_ai.taxonomy import SimilarDocument
 from paperless_ai.taxonomy import TaxonomyCandidates
 from paperless_ai.taxonomy import build_taxonomy_candidates
 from paperless_ai.taxonomy import format_taxonomy_for_prompt
@@ -132,9 +132,8 @@ class TestGetAssignedMetadata:
         assert result["tags"] == ["Owned By Someone Else"]
 
 
-def make_node(document_id: int, score: float) -> SimpleNamespace:
-    """A stand-in for NodeWithScore: only ``.metadata``/``.score`` are read."""
-    return SimpleNamespace(metadata={"document_id": str(document_id)}, score=score)
+def make_similar(document_id: int, weight: float) -> SimilarDocument:
+    return SimilarDocument(document_id=document_id, weight=weight)
 
 
 @pytest.mark.django_db
@@ -170,9 +169,9 @@ class TestBuildTaxonomyCandidates:
         doc_a.tags.add(tag)
         doc_b = DocumentFactory.create()
         doc_b.tags.add(tag)
-        nodes = [make_node(doc_a.pk, 0.9), make_node(doc_b.pk, 0.4)]
+        similar_documents = [make_similar(doc_a.pk, 0.9), make_similar(doc_b.pk, 0.4)]
 
-        result = build_taxonomy_candidates(nodes, user=None)
+        result = build_taxonomy_candidates(similar_documents, user=None)
 
         assert len(result["tags"]) == 1
         assert result["tags"][0]["id"] == tag.pk
@@ -197,9 +196,9 @@ class TestBuildTaxonomyCandidates:
         document.tags.add(tag)
         tag.name = "New Name"
         tag.save()
-        nodes = [make_node(document.pk, 0.5)]
+        similar_documents = [make_similar(document.pk, 0.5)]
 
-        result = build_taxonomy_candidates(nodes, user=None)
+        result = build_taxonomy_candidates(similar_documents, user=None)
 
         assert result["tags"][0]["name"] == "New Name"
 
@@ -219,9 +218,9 @@ class TestBuildTaxonomyCandidates:
         document = DocumentFactory.create()
         document.tags.add(tag)
         tag.delete()
-        nodes = [make_node(document.pk, 0.5)]
+        similar_documents = [make_similar(document.pk, 0.5)]
 
-        result = build_taxonomy_candidates(nodes, user=None)
+        result = build_taxonomy_candidates(similar_documents, user=None)
 
         assert result["tags"] == []
 
@@ -240,9 +239,12 @@ class TestBuildTaxonomyCandidates:
         strong_doc.tags.add(strong_tag)
         weak_doc = DocumentFactory.create()
         weak_doc.tags.add(weak_tag)
-        nodes = [make_node(strong_doc.pk, 0.9), make_node(weak_doc.pk, 0.1)]
+        similar_documents = [
+            make_similar(strong_doc.pk, 0.9),
+            make_similar(weak_doc.pk, 0.1),
+        ]
 
-        result = build_taxonomy_candidates(nodes, user=None)
+        result = build_taxonomy_candidates(similar_documents, user=None)
 
         assert [c["name"] for c in result["tags"]] == ["Strong", "Weak"]
 
@@ -258,9 +260,9 @@ class TestBuildTaxonomyCandidates:
         document = DocumentFactory.create()
         for i in range(15):
             document.tags.add(TagFactory.create(name=f"Tag{i}"))
-        nodes = [make_node(document.pk, 0.5)]
+        similar_documents = [make_similar(document.pk, 0.5)]
 
-        result = build_taxonomy_candidates(nodes, user=None)
+        result = build_taxonomy_candidates(similar_documents, user=None)
 
         assert len(result["tags"]) == 10
 
@@ -274,12 +276,12 @@ class TestBuildTaxonomyCandidates:
             - Only 5 correspondents are returned
         """
         correspondents = CorrespondentFactory.create_batch(7)
-        nodes = [
-            make_node(DocumentFactory.create(correspondent=c).pk, 0.5)
+        similar_documents = [
+            make_similar(DocumentFactory.create(correspondent=c).pk, 0.5)
             for c in correspondents
         ]
 
-        result = build_taxonomy_candidates(nodes, user=None)
+        result = build_taxonomy_candidates(similar_documents, user=None)
 
         assert len(result["correspondents"]) == 5
 
@@ -294,9 +296,9 @@ class TestBuildTaxonomyCandidates:
         """
         document_type = DocumentTypeFactory.create(name="Invoice")
         document = DocumentFactory.create(document_type=document_type)
-        nodes = [make_node(document.pk, 0.5)]
+        similar_documents = [make_similar(document.pk, 0.5)]
 
-        result = build_taxonomy_candidates(nodes, user=None)
+        result = build_taxonomy_candidates(similar_documents, user=None)
 
         assert len(result["document_types"]) == 1
         assert result["document_types"][0]["id"] == document_type.pk
@@ -312,12 +314,12 @@ class TestBuildTaxonomyCandidates:
             - Only 5 document_types are returned
         """
         document_types = DocumentTypeFactory.create_batch(7)
-        nodes = [
-            make_node(DocumentFactory.create(document_type=dt).pk, 0.5)
+        similar_documents = [
+            make_similar(DocumentFactory.create(document_type=dt).pk, 0.5)
             for dt in document_types
         ]
 
-        result = build_taxonomy_candidates(nodes, user=None)
+        result = build_taxonomy_candidates(similar_documents, user=None)
 
         assert len(result["document_types"]) == 5
 
@@ -332,9 +334,9 @@ class TestBuildTaxonomyCandidates:
         """
         storage_path = StoragePathFactory.create(name="Invoices")
         document = DocumentFactory.create(storage_path=storage_path)
-        nodes = [make_node(document.pk, 0.5)]
+        similar_documents = [make_similar(document.pk, 0.5)]
 
-        result = build_taxonomy_candidates(nodes, user=None)
+        result = build_taxonomy_candidates(similar_documents, user=None)
 
         assert len(result["storage_paths"]) == 1
         assert result["storage_paths"][0]["id"] == storage_path.pk
@@ -350,12 +352,12 @@ class TestBuildTaxonomyCandidates:
             - Only 5 storage_paths are returned
         """
         storage_paths = StoragePathFactory.create_batch(7)
-        nodes = [
-            make_node(DocumentFactory.create(storage_path=sp).pk, 0.5)
+        similar_documents = [
+            make_similar(DocumentFactory.create(storage_path=sp).pk, 0.5)
             for sp in storage_paths
         ]
 
-        result = build_taxonomy_candidates(nodes, user=None)
+        result = build_taxonomy_candidates(similar_documents, user=None)
 
         assert len(result["storage_paths"]) == 5
 
@@ -375,14 +377,14 @@ class TestBuildTaxonomyCandidates:
         tag = TagFactory.create(name="Restricted")
         document = DocumentFactory.create()
         document.tags.add(tag)
-        nodes = [make_node(document.pk, 0.5)]
+        similar_documents = [make_similar(document.pk, 0.5)]
         user = UserFactory.create()
         mocker.patch(
             "documents.permissions.permitted_object_ids",
             return_value=[],  # user cannot see this tag
         )
 
-        result = build_taxonomy_candidates(nodes, user=user)
+        result = build_taxonomy_candidates(similar_documents, user=user)
 
         assert result["tags"] == []
 
@@ -412,10 +414,10 @@ class TestBuildTaxonomyCandidates:
         tag.save()
         document = DocumentFactory.create()
         document.tags.add(tag)
-        nodes = [make_node(document.pk, 0.5)]
+        similar_documents = [make_similar(document.pk, 0.5)]
         spy = mocker.patch("documents.permissions.permitted_object_ids")
 
-        result = build_taxonomy_candidates(nodes, user=None)
+        result = build_taxonomy_candidates(similar_documents, user=None)
 
         assert result["tags"][0]["name"] == "Owned"
         spy.assert_not_called()