Fix (beta): normalize long punctuation chunks to improve embedding (#12848)

This commit is contained in:
shamoon authored and GitHub committed 2026-05-26 09:32:38 -07:00
1 parent 0a6e0db186
commit da3e845b8b
2 files changed
+35 -1

No files matched your search

+25
View File
@@ -7,6 +7,7 @@ from django.conf import settings
from documents.models import Document
from paperless.models import LLMEmbeddingBackend
from paperless_ai.embedding import _normalize_llm_index_text
from paperless_ai.embedding import build_llm_index_text
from paperless_ai.embedding import get_embedding_dim
from paperless_ai.embedding import get_embedding_model
@@ -243,3 +244,27 @@ def test_build_llm_index_text(mock_document):
assert "Notes: Note1,Note2" in result
assert "Content:\n\nThis is the document content." in result
assert "Custom Field - Field1: Value1\nCustom Field - Field2: Value2" in result
def test_build_llm_index_text_normalizes_ocr_punctuation_runs(mock_document):
mock_document.content = (
"Introduction ................................................ 7\n"
"Hardware Limitation ________________________________________ 9\n"
"Keep short punctuation like INV-100 and ellipses..."
)
with patch("documents.models.Note.objects.filter", return_value=[]):
result = build_llm_index_text(mock_document)
assert "Introduction 7" in result
assert "Hardware Limitation 9" in result
assert "INV-100" in result
assert "ellipses..." in result
def test_normalize_llm_index_text_collapses_ocr_leaders_without_joining_lines():
assert _normalize_llm_index_text("A........B\nC____D----E") == "A B\nC D E"
def test_normalize_llm_index_text_collapses_non_breaking_spaces():
assert _normalize_llm_index_text("A\u00a0........\u00a0B") == "A B"