diff --git a/src/paperless_ai/embedding.py b/src/paperless_ai/embedding.py index 88ea80293..206e2480a 100644 --- a/src/paperless_ai/embedding.py +++ b/src/paperless_ai/embedding.py @@ -112,15 +112,11 @@ def _normalize_llm_index_text(text: str) -> str: def build_llm_index_text(doc: Document) -> str: - # TODO: Filename, Storage Path, and Archive Serial Number are short structured - # values that could move to node.metadata (excluded from embeddings, visible to - # LLM via metadata prepend) — same pattern as title/tags/correspondent. Notes - # and Custom Fields should stay here: Notes can be long free text, Custom Fields - # are dynamic in count and best kept in the embedding. + # Short structured fields (filename, storage path, ASN, title, tags, ...) live + # in node.metadata: excluded from embeddings, shown to the LLM via metadata + # prepend. Notes and Custom Fields stay in the body: Notes can be long free + # text, Custom Fields are dynamic in count and best kept in the embedding. lines = [ - f"Filename: {doc.filename}", - f"Storage Path: {doc.storage_path.name if doc.storage_path else ''}", - f"Archive Serial Number: {doc.archive_serial_number or ''}", f"Notes: {','.join([str(c.note) for c in Note.objects.filter(document=doc)])}", ] diff --git a/src/paperless_ai/indexing.py b/src/paperless_ai/indexing.py index b9360774d..bd29273e0 100644 --- a/src/paperless_ai/indexing.py +++ b/src/paperless_ai/indexing.py @@ -130,6 +130,9 @@ def build_document_node( "document_type": document.document_type.name if document.document_type else None, + "filename": document.filename, + "storage_path": document.storage_path.name if document.storage_path else None, + "archive_serial_number": document.archive_serial_number, "created": document.created.isoformat() if document.created else None, "added": document.added.isoformat() if document.added else None, "modified": document.modified.isoformat(), diff --git a/src/paperless_ai/tests/test_ai_indexing.py b/src/paperless_ai/tests/test_ai_indexing.py index 9ae791613..d3ff256d1 100644 --- a/src/paperless_ai/tests/test_ai_indexing.py +++ b/src/paperless_ai/tests/test_ai_indexing.py @@ -33,6 +33,16 @@ def test_build_document_node(real_document: Document) -> None: nodes = indexing.build_document_node(real_document) assert len(nodes) > 0 assert nodes[0].metadata["document_id"] == str(real_document.id) + assert nodes[0].metadata["filename"] == real_document.filename + assert nodes[0].metadata["storage_path"] == ( + real_document.storage_path.name if real_document.storage_path else None + ) + assert ( + nodes[0].metadata["archive_serial_number"] + == real_document.archive_serial_number + ) + assert "filename" in nodes[0].excluded_embed_metadata_keys + assert "filename" not in nodes[0].excluded_llm_metadata_keys @pytest.mark.django_db diff --git a/src/paperless_ai/tests/test_embedding.py b/src/paperless_ai/tests/test_embedding.py index 251d3f90b..883b5172f 100644 --- a/src/paperless_ai/tests/test_embedding.py +++ b/src/paperless_ai/tests/test_embedding.py @@ -224,15 +224,17 @@ def test_build_llm_index_text(mock_document): result = build_llm_index_text(mock_document) - # Structured fields live in node.metadata for LLM context — not body text + # Structured fields live in node.metadata for LLM context -- not body text assert "Title: Test Title" not in result assert "Created: 2023-01-01" not in result assert "Tags: Tag1, Tag2" not in result assert "Document Type: Invoice" not in result assert "Correspondent: Test Correspondent" not in result + assert "Filename:" not in result + assert "Storage Path:" not in result + assert "Archive Serial Number:" not in result # Fields without a metadata equivalent stay in body text - assert "Filename: test_file.pdf" in result assert "Notes: Note1,Note2" in result assert "Content:\n\nThis is the document content." in result assert "Custom Field - Field1: Value1\nCustom Field - Field2: Value2" in result