Fix: consolidate born-digital PDF detection between archive decision and OCR (#13409)

* Fix: unify born-digital PDF detection between archive decision and OCR

should_produce_archive() and RasterisedDocumentParser.parse() each
reimplemented the "does this PDF have real text" check independently,
using different normalization of pdftotext output. Raw pdftotext output
can be non-empty (whitespace/form-feed layout padding) even when there
is no real content, so the two checks could disagree: consumer.py
treated a tagged-but-textless PDF as born-digital and skipped the
archive, while the parser's own (stricter, normalized) check found no
text and ran OCR anyway, leaving the document with no archive despite
real OCR text (GH #13387).

Both call sites now share one predicate, pdf_born_digital_text() in
paperless/parsers/utils.py, so they can no longer drift apart.

* Fix: restore extract_text seam for born-digital detection in parse()

parse() had switched to calling pdf_born_digital_text() directly for
its initial text/born-digital check, bypassing the parser's own
extract_text instance method. That broke test mockability (tests patch
tesseract_parser.extract_text to control the born-digital decision)
and caused CI failures with mismatched OCR call counts and text.

Split pdf_born_digital_text() into is_born_digital_text(text, path,
log) - a pure decision function - and a thin pdf_born_digital_text()
wrapper for callers without text in hand (consumer.should_produce_archive).
parse() now extracts via self.extract_text(None, document_path) and
passes the result to is_born_digital_text(), restoring the seam with
no change to production behavior.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

* Cleanup: simplify born-digital detection, close #13387 test gap

Simplification pass over the born-digital detection consolidation:
- is_born_digital_text(): drop the has_text temp for an early return.
- consumer.py: standardize the archive-decision log lines on plain
  hyphens (was a mix of em-dash and hyphen) and hoist the duplicated
  text_length computation.
- Parametrize TestPdfBornDigitalText instead of four near-identical
  tests.

Code review follow-up: the existing tests only ever exercised
pdf_born_digital_text() through mocks, so the actual #13387 scenario
(a tagged PDF whose only "text" is layout padding) was never checked
against real pdftotext/pikepdf output - a regression in the
normalize-before-decide logic itself would have gone undetected.
Moved tagged_no_text_pdf_file from parsers/conftest.py up to the
shared paperless/tests/conftest.py (it was previously only visible to
tests under parsers/) and added a non-mocked regression test against
the real sample file.

Also fixed two stale comments in test_consumer.py referencing a
_extract_text_for_archive_check helper that no longer exists.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>

src/documents/consumer.py

@@ -57,9 +57,7 @@ from paperless.models import ArchiveFileGenerationChoices
 from paperless.parsers import ParserContext
 from paperless.parsers import ParserProtocol
 from paperless.parsers.registry import get_parser_registry
-from paperless.parsers.utils import PDF_TEXT_MIN_LENGTH
+from paperless.parsers.utils import pdf_born_digital_text
-from paperless.parsers.utils import extract_pdf_text
-from paperless.parsers.utils import is_tagged_pdf
 
 LOGGING_NAME: Final[str] = "paperless.consumer"
 
@@ -138,53 +136,45 @@ def should_produce_archive(
 
     # Must produce a PDF so the frontend can display the original format at all.
     if parser.requires_pdf_rendition:
-        _log.debug("Archive: yes — parser requires PDF rendition for frontend display")
+        _log.debug("Archive: yes - parser requires PDF rendition for frontend display")
         return True
 
     # Parser cannot produce an archive (e.g. TextDocumentParser).
     if not parser.can_produce_archive:
-        _log.debug("Archive: no — parser cannot produce archives")
+        _log.debug("Archive: no - parser cannot produce archives")
         return False
 
     generation = OcrConfig().archive_file_generation
 
     if generation == ArchiveFileGenerationChoices.ALWAYS:
-        _log.debug("Archive: yes — ARCHIVE_FILE_GENERATION=always")
+        _log.debug("Archive: yes - ARCHIVE_FILE_GENERATION=always")
         return True
     if generation == ArchiveFileGenerationChoices.NEVER:
-        _log.debug("Archive: no — ARCHIVE_FILE_GENERATION=never")
+        _log.debug("Archive: no - ARCHIVE_FILE_GENERATION=never")
         return False
 
     # auto: produce archives for scanned/image documents; skip for born-digital PDFs.
     if mime_type.startswith("image/"):
-        _log.debug("Archive: yes — image document, ARCHIVE_FILE_GENERATION=auto")
+        _log.debug("Archive: yes - image document, ARCHIVE_FILE_GENERATION=auto")
         return True
     if mime_type == "application/pdf":
-        text = extract_pdf_text(document_path)
+        text, born_digital = pdf_born_digital_text(document_path, log=_log)
-        has_text = text is not None and len(text) > 0
+        text_length = len(text) if text else 0
-        if has_text and is_tagged_pdf(document_path):
+        if born_digital:
             _log.debug(
-                "Archive: no — born-digital PDF (structure tags detected),"
+                "Archive: no - born-digital PDF (text_length=%d),"
                 " ARCHIVE_FILE_GENERATION=auto",
+                text_length,
             )
             return False
-        if text is None or len(text) <= PDF_TEXT_MIN_LENGTH:
-            _log.debug(
-                "Archive: yes — scanned PDF (text_length=%d ≤ %d),"
-                " ARCHIVE_FILE_GENERATION=auto",
-                len(text) if text else 0,
-                PDF_TEXT_MIN_LENGTH,
-            )
-            return True
         _log.debug(
-            "Archive: no — born-digital PDF (text_length=%d > %d),"
+            "Archive: yes - scanned/textless PDF (text_length=%d),"
             " ARCHIVE_FILE_GENERATION=auto",
-            len(text),
+            text_length,
-            PDF_TEXT_MIN_LENGTH,
         )
-        return False
+        return True
     _log.debug(
-        "Archive: no — MIME type %r not eligible for auto archive generation",
+        "Archive: no - MIME type %r not eligible for auto archive generation",
         mime_type,
     )
     return False

src/documents/tests/test_consumer.py

@@ -1329,7 +1329,7 @@ class PreConsumeTestCase(DirectoriesMixin, GetConsumerMixin, TestCase):
         with self.get_consumer(self.test_file) as c:
             c.run()
             # Verify no pre-consume script subprocess was invoked
-            # (run_subprocess may still be called by _extract_text_for_archive_check)
+            # (run_subprocess may still be called by pdf_born_digital_text via pdftotext)
             script_calls = [
                 call
                 for call in m.call_args_list
@@ -1354,7 +1354,7 @@ class PreConsumeTestCase(DirectoriesMixin, GetConsumerMixin, TestCase):
                     self.assertTrue(m.called)
 
                     # Find the call that invoked the pre-consume script
-                    # (run_subprocess may also be called by _extract_text_for_archive_check)
+                    # (run_subprocess may also be called by pdf_born_digital_text via pdftotext)
                     script_call = next(
                         call
                         for call in m.call_args_list

src/documents/tests/test_consumer_archive.py

@@ -134,60 +134,32 @@ class TestShouldProduceArchive:
         assert should_produce_archive(parser, mime, Path("/tmp/doc")) is expected
 
     @pytest.mark.parametrize(
-        ("extracted_text", "expected"),
+        ("born_digital", "expected"),
         [
-            pytest.param(
+            pytest.param(True, False, id="born-digital-skips-archive"),
-                "This is a born-digital PDF with lots of text content. " * 10,
+            pytest.param(False, True, id="not-born-digital-produces-archive"),
-                False,
-                id="born-digital-long-text-skips-archive",
-            ),
-            pytest.param(None, True, id="no-text-scanned-produces-archive"),
-            pytest.param("tiny", True, id="short-text-treated-as-scanned"),
         ],
     )
     def test_auto_pdf_archive_decision(
         self,
         mocker: MockerFixture,
         settings,
-        extracted_text: str | None,
+        born_digital: bool,  # noqa: FBT001
         expected: bool,  # noqa: FBT001
     ) -> None:
+        """Archive decision tracks pdf_born_digital_text()'s verdict exactly.
+
+        should_produce_archive() defers entirely to pdf_born_digital_text()
+        for the has-real-text decision, so both callers of that predicate
+        (this function and RasterisedDocumentParser.parse()) always agree.
+        """
         settings.ARCHIVE_FILE_GENERATION = "auto"
-        mocker.patch("documents.consumer.is_tagged_pdf", return_value=False)
+        mocker.patch(
-        mocker.patch("documents.consumer.extract_pdf_text", return_value=extracted_text)
+            "documents.consumer.pdf_born_digital_text",
+            return_value=("some text", born_digital),
+        )
         parser = _parser_instance(can_produce=True, requires_rendition=False)
         assert (
             should_produce_archive(parser, "application/pdf", Path("/tmp/doc.pdf"))
             is expected
         )
-
-    def test_tagged_pdf_skips_archive_in_auto_mode(
-        self,
-        mocker: MockerFixture,
-        settings,
-    ) -> None:
-        """Tagged PDFs (e.g. Word exports) with real text are treated as born-digital, even below PDF_TEXT_MIN_LENGTH."""
-        settings.ARCHIVE_FILE_GENERATION = "auto"
-        mocker.patch("documents.consumer.is_tagged_pdf", return_value=True)
-        mocker.patch("documents.consumer.extract_pdf_text", return_value="tiny")
-        parser = _parser_instance(can_produce=True, requires_rendition=False)
-        assert (
-            should_produce_archive(parser, "application/pdf", Path("/tmp/doc.pdf"))
-            is False
-        )
-
-    def test_tagged_pdf_without_text_produces_archive(
-        self,
-        mocker: MockerFixture,
-        settings,
-    ) -> None:
-        """A tagged PDF with no actual extractable text (e.g. some scanner firmware) is not
-        trusted as born-digital — the tag alone must not bypass OCR."""
-        settings.ARCHIVE_FILE_GENERATION = "auto"
-        mocker.patch("documents.consumer.is_tagged_pdf", return_value=True)
-        mocker.patch("documents.consumer.extract_pdf_text", return_value=None)
-        parser = _parser_instance(can_produce=True, requires_rendition=False)
-        assert (
-            should_produce_archive(parser, "application/pdf", Path("/tmp/doc.pdf"))
-            is True
-        )

src/paperless/parsers/tesseract.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 import importlib.resources
 import logging
 import os
-import re
 import shutil
 import tempfile
 from pathlib import Path
@@ -25,9 +24,9 @@ from paperless.config import OcrConfig
 from paperless.models import CleanChoices
 from paperless.models import ModeChoices
 from paperless.models import OutputTypeChoices
-from paperless.parsers.utils import PDF_TEXT_MIN_LENGTH
 from paperless.parsers.utils import extract_pdf_text
-from paperless.parsers.utils import is_tagged_pdf
+from paperless.parsers.utils import is_born_digital_text
+from paperless.parsers.utils import post_process_text
 from paperless.parsers.utils import read_file_handle_unicode_errors
 from paperless.version import __full_version_str__
 
@@ -510,10 +509,10 @@ class RasterisedDocumentParser:
 
         if mime_type == "application/pdf":
             text_original = self.extract_text(None, document_path)
-            has_text = text_original is not None and len(text_original) > 0
+            original_has_text = is_born_digital_text(
-            original_has_text = has_text and (
+                text_original,
-                is_tagged_pdf(document_path, log=self.log)
+                document_path,
-                or len(text_original) > PDF_TEXT_MIN_LENGTH
+                log=self.log,
             )
         else:
             text_original = None
@@ -658,17 +657,3 @@ class RasterisedDocumentParser:
                     f"No text was found in {document_path}, the content will be empty.",
                 )
                 self.text = ""
-
-
-def post_process_text(text: str | None) -> str | None:
-    if not text:
-        return None
-
-    collapsed_spaces = re.sub(r"([^\S\r\n]+)", " ", text)
-    no_leading_whitespace = re.sub(r"([\n\r]+)([^\S\n\r]+)", "\\1", collapsed_spaces)
-    no_trailing_whitespace = re.sub(r"([^\S\n\r]+)$", "", no_leading_whitespace)
-
-    # TODO: this needs a rework
-    # replace \0 prevents issues with saving to postgres.
-    # text may contain \0 when this character is present in PDF files.
-    return no_trailing_whitespace.strip().replace("\0", " ")

src/paperless/parsers/utils.py

@@ -111,6 +111,88 @@ def extract_pdf_text(
         return None
 
 
+def post_process_text(text: str | None) -> str | None:
+    """Normalize extracted PDF/OCR text: collapse whitespace, strip padding.
+
+    Returns ``None`` for ``None`` or whitespace-only input, so callers can
+    treat "no text" and "only layout padding" the same way.
+    """
+    if not text:
+        return None
+
+    collapsed_spaces = re.sub(r"([^\S\r\n]+)", " ", text)
+    no_leading_whitespace = re.sub(r"([\n\r]+)([^\S\n\r]+)", "\\1", collapsed_spaces)
+    no_trailing_whitespace = re.sub(r"([^\S\n\r]+)$", "", no_leading_whitespace)
+
+    # replace \0 prevents issues with saving to postgres.
+    # text may contain \0 when this character is present in PDF files.
+    result = no_trailing_whitespace.strip().replace("\0", " ")
+    return result or None
+
+
+def is_born_digital_text(
+    text: str | None,
+    path: Path,
+    log: logging.Logger | None = None,
+) -> bool:
+    """Decide whether already-extracted, normalized PDF text counts as born-digital.
+
+    This is the single source of truth for "does this PDF already have real
+    text", used both to decide whether to produce an archive file and to
+    decide whether OCR can be skipped. Both decisions must agree, or a
+    tagged-but-textless PDF can end up with no archive AND a forced OCR pass
+    (see GH #13387): raw ``pdftotext -layout`` output can be non-empty
+    (whitespace/form-feed padding) even when there is no real content, so
+    *text* must already be normalized via :func:`post_process_text`, not the
+    raw extraction.
+
+    Parameters
+    ----------
+    text:
+        The normalized extracted text (or ``None``) to evaluate.
+    path:
+        Absolute path to the PDF file, used for the tagged-PDF check.
+    log:
+        Logger for warnings.  Falls back to the module-level logger when omitted.
+
+    Returns
+    -------
+    bool
+        Whether the PDF counts as born-digital (has real text, and is either
+        tagged or exceeds ``PDF_TEXT_MIN_LENGTH``).
+    """
+    if not text:
+        return False
+    return is_tagged_pdf(path, log=log) or len(text) > PDF_TEXT_MIN_LENGTH
+
+
+def pdf_born_digital_text(
+    path: Path,
+    log: logging.Logger | None = None,
+) -> tuple[str | None, bool]:
+    """Extract a PDF's text and decide whether it should be treated as born-digital.
+
+    Convenience wrapper around :func:`is_born_digital_text` for callers that
+    don't already have the PDF's text extracted (e.g. the archive-generation
+    decision, which runs before any parser has touched the file).
+
+    Parameters
+    ----------
+    path:
+        Absolute path to the PDF file.
+    log:
+        Logger for warnings.  Falls back to the module-level logger when omitted.
+
+    Returns
+    -------
+    tuple[str | None, bool]
+        The normalized extracted text (or ``None``), and whether the PDF
+        counts as born-digital.
+    """
+    text = post_process_text(extract_pdf_text(path, log=log))
+    return text, is_born_digital_text(text, path, log=log)
+
+
 def read_file_handle_unicode_errors(
     filepath: Path,
     log: logging.Logger | None = None,

src/paperless/tests/conftest.py

@@ -36,6 +36,23 @@ def samples_dir() -> Path:
     return (Path(__file__).parent / "samples").resolve()
 
 
+@pytest.fixture(scope="session")
+def tagged_no_text_pdf_file(samples_dir: Path) -> Path:
+    """Path to a tagged PDF whose only "text" is pdftotext layout padding.
+
+    Reproduces GH #13387: ``/MarkInfo /Marked true`` is set, but the only
+    extractable content is a form-feed byte, not real text. Lives here
+    rather than in parsers/conftest.py so both parser tests and
+    paperless/tests/test_parser_utils.py can use it.
+
+    Returns
+    -------
+    Path
+        Absolute path to ``tesseract/tagged-but-no-text.pdf``.
+    """
+    return samples_dir / "tesseract" / "tagged-but-no-text.pdf"
+
+
 @pytest.fixture(autouse=True)
 def clean_registry() -> Generator[None, None, None]:
     """Reset the parser registry before and after every test.

src/paperless/tests/parsers/test_tesseract_parser.py

@@ -21,7 +21,7 @@ from documents.parsers import run_convert
 from paperless.models import ModeChoices
 from paperless.parsers import ParserProtocol
 from paperless.parsers.tesseract import RasterisedDocumentParser
-from paperless.parsers.tesseract import post_process_text
+from paperless.parsers.utils import is_tagged_pdf
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -151,36 +151,6 @@ class TestRasterisedDocumentParserLifecycle:
         assert tempdir is not None and not tempdir.exists()
 
 
-# ---------------------------------------------------------------------------
-# post_process_text
-# ---------------------------------------------------------------------------
-
-
-class TestPostProcessText:
-    @pytest.mark.parametrize(
-        ("source", "expected"),
-        [
-            pytest.param(
-                "simple     string",
-                "simple string",
-                id="collapse-spaces",
-            ),
-            pytest.param(
-                "simple    newline\n   testing string",
-                "simple newline\ntesting string",
-                id="preserve-newline",
-            ),
-            pytest.param(
-                "utf-8   строка с пробелами в конце  ",  # noqa: RUF001
-                "utf-8 строка с пробелами в конце",  # noqa: RUF001
-                id="utf8-trailing-spaces",
-            ),
-        ],
-    )
-    def test_post_process_text(self, source: str, expected: str) -> None:
-        assert post_process_text(source) == expected
-
-
 # ---------------------------------------------------------------------------
 # Page count
 # ---------------------------------------------------------------------------
@@ -910,25 +880,25 @@ class TestSkipArchive:
         self,
         mocker: MockerFixture,
         tesseract_parser: RasterisedDocumentParser,
-        tesseract_samples_dir: Path,
+        tagged_no_text_pdf_file: Path,
     ) -> None:
         """
         GIVEN:
-            - A PDF that reports itself as tagged (/MarkInfo /Marked true) but
+            - A real PDF that reports itself as tagged (/MarkInfo /Marked
-              has no actual extractable text (some scanner firmware produces
+              true) but whose only pdftotext output is layout padding (a
-              this — see GitHub issue #13349)
+              lone form-feed byte), not real text (see GitHub issue #13387,
+              originally reported against #13349's tagged-PDF handling)
             - Mode: auto, produce_archive=False
         WHEN:
             - Document is parsed
         THEN:
             - The tag alone is not trusted as "has text"; OCRmyPDF still runs
         """
+        assert is_tagged_pdf(tagged_no_text_pdf_file) is True
         tesseract_parser.settings.mode = ModeChoices.AUTO
-        mocker.patch("paperless.parsers.tesseract.is_tagged_pdf", return_value=True)
-        mocker.patch.object(tesseract_parser, "extract_text", return_value=None)
         mock_ocr = mocker.patch("ocrmypdf.ocr")
         tesseract_parser.parse(
-            tesseract_samples_dir / "multi-page-images.pdf",
+            tagged_no_text_pdf_file,
             "application/pdf",
             produce_archive=False,
         )

src/paperless/tests/test_parser_utils.py

@@ -4,10 +4,18 @@ from __future__ import annotations
 
 import codecs
 from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pytest
 
 from paperless.parsers.utils import is_tagged_pdf
+from paperless.parsers.utils import pdf_born_digital_text
+from paperless.parsers.utils import post_process_text
 from paperless.parsers.utils import read_file_handle_unicode_errors
 
+if TYPE_CHECKING:
+    from pytest_mock import MockerFixture
+
 SAMPLES = Path(__file__).parent / "samples" / "tesseract"
 
 
@@ -60,3 +68,105 @@ class TestIsTaggedPdf:
         bad = tmp_path / "bad.pdf"
         bad.write_bytes(b"not a pdf")
         assert is_tagged_pdf(bad) is False
+
+
+class TestPostProcessText:
+    @pytest.mark.parametrize(
+        ("source", "expected"),
+        [
+            pytest.param(
+                "simple     string",
+                "simple string",
+                id="collapse-spaces",
+            ),
+            pytest.param(
+                "simple    newline\n   testing string",
+                "simple newline\ntesting string",
+                id="preserve-newline",
+            ),
+            pytest.param(
+                "utf-8   строка с пробелами в конце  ",  # noqa: RUF001
+                "utf-8 строка с пробелами в конце",  # noqa: RUF001
+                id="utf8-trailing-spaces",
+            ),
+            pytest.param(None, None, id="none-input"),
+            pytest.param("", None, id="empty-string"),
+            pytest.param("   \n\x0c  \n ", None, id="whitespace-and-formfeed-only"),
+        ],
+    )
+    def test_post_process_text(
+        self,
+        source: str | None,
+        expected: str | None,
+    ) -> None:
+        assert post_process_text(source) == expected
+
+
+class TestPdfBornDigitalText:
+    """Regression coverage for GH #13387.
+
+    should_produce_archive() and RasterisedDocumentParser.parse() must agree
+    on whether a PDF has real text, so both go through this one function.
+    """
+
+    @pytest.mark.parametrize(
+        ("extracted", "tagged", "expected_text", "expected_born_digital"),
+        [
+            pytest.param("tiny", True, "tiny", True, id="tagged-with-real-text"),
+            pytest.param("tiny", False, "tiny", False, id="untagged-below-min-length"),
+            pytest.param(
+                "x" * 51,
+                False,
+                "x" * 51,
+                True,
+                id="untagged-above-min-length",
+            ),
+            pytest.param(None, True, None, False, id="tagged-but-no-text"),
+        ],
+    )
+    def test_born_digital_decision(
+        self,
+        mocker: MockerFixture,
+        tmp_path: Path,
+        extracted: str | None,
+        tagged: bool,  # noqa: FBT001
+        expected_text: str | None,
+        expected_born_digital: bool,  # noqa: FBT001
+    ) -> None:
+        """
+        GIVEN:
+            - A PDF whose pdftotext output and /MarkInfo tag status vary
+        WHEN:
+            - pdf_born_digital_text() is called
+        THEN:
+            - The normalized text and born-digital verdict match; the tag
+              alone never counts as "has text"
+        """
+        mocker.patch(
+            "paperless.parsers.utils.extract_pdf_text",
+            return_value=extracted,
+        )
+        mocker.patch("paperless.parsers.utils.is_tagged_pdf", return_value=tagged)
+        text, born_digital = pdf_born_digital_text(tmp_path / "doc.pdf")
+        assert text == expected_text
+        assert born_digital is expected_born_digital
+
+    def test_tagged_but_textless_pdf_is_not_born_digital(
+        self,
+        tagged_no_text_pdf_file: Path,
+    ) -> None:
+        """
+        GIVEN:
+            - A real PDF that is tagged (/MarkInfo /Marked true) but whose
+              only "text" is layout padding (a stray form-feed byte)
+        WHEN:
+            - pdf_born_digital_text() is called with no mocking
+        THEN:
+            - The normalized text is None and the PDF is not treated as
+              born-digital. The raw, unnormalized pdftotext output is
+              non-empty for this file, which is exactly what caused the
+              archive decision to disagree with the OCR decision in #13387.
+        """
+        text, born_digital = pdf_born_digital_text(tagged_no_text_pdf_file)
+        assert text is None
+        assert born_digital is False

src/paperless_ai/tables.py

@@ -1,229 +0,0 @@
-"""Thin gateways over the plain relational side tables that sit alongside the
-vec0 table. Each method takes the sqlite3.Connection to operate on
-explicitly, rather than owning one -- the store swaps connections during
-compact()/migration, and migrations always work across two connections
-(src_conn, dst_conn) at once.
-
-PRECONDITION: Callers must set conn.row_factory = sqlite3.Row before passing a
-connection to any of these gateways' read methods. The read methods across all
-three classes (DocumentChunksTable.chunk_ids_for_document, IndexMetaTable._get,
-DocumentMetaTable.all_modified_times, DocumentMetaTable.copy_all) use
-row["column_name"] dictionary-style indexing, which requires sqlite3.Row as the
-row factory -- without it, sqlite3.Row is not set, rows are returned as plain
-tuples, and tuple indices must be integers, raising TypeError.
-"""
-
-import sqlite3
-from collections.abc import Iterable
-from typing import NamedTuple
-
-
-class ChunkRow(NamedTuple):
-    chunk_id: str
-    document_id: int
-
-
-class DocumentMetaRow(NamedTuple):
-    document_id: int
-    modified: str
-
-
-class DocumentChunksTable:
-    """chunk_id -> document_id, indexed by document_id. Gives O(1)
-    per-document chunk lookup that vec0's own document_id metadata column
-    cannot (see PaperlessSqliteVecVectorStore._delete_chunks_by_document_id).
-    """
-
-    @staticmethod
-    def create(conn: sqlite3.Connection) -> None:
-        conn.execute(
-            "CREATE TABLE IF NOT EXISTS document_chunks "
-            "(chunk_id TEXT PRIMARY KEY, document_id INTEGER NOT NULL)",
-        )
-        conn.execute(
-            "CREATE INDEX IF NOT EXISTS idx_document_chunks_document_id "
-            "ON document_chunks (document_id)",
-        )
-
-    @staticmethod
-    def insert_many(conn: sqlite3.Connection, rows: Iterable[ChunkRow]) -> None:
-        """rows must already be batch-bounded by the caller (e.g. vec0's own
-        fetchmany() loop) -- this never reads, so it can't itself introduce
-        an unbounded scan, but a whole-table iterable defeats the point."""
-        conn.executemany(
-            "INSERT INTO document_chunks (chunk_id, document_id) VALUES (?, ?)",
-            rows,
-        )
-
-    @staticmethod
-    def chunk_ids_for_document(
-        conn: sqlite3.Connection,
-        document_id: int,
-    ) -> list[str]:
-        return [
-            row["chunk_id"]
-            for row in conn.execute(
-                "SELECT chunk_id FROM document_chunks WHERE document_id = ?",
-                (document_id,),
-            ).fetchall()
-        ]
-
-    @staticmethod
-    def delete_for_document(conn: sqlite3.Connection, document_id: int) -> None:
-        conn.execute(
-            "DELETE FROM document_chunks WHERE document_id = ?",
-            (document_id,),
-        )
-
-    @staticmethod
-    def delete_all(conn: sqlite3.Connection) -> None:
-        conn.execute("DELETE FROM document_chunks")
-
-    @staticmethod
-    def count(conn: sqlite3.Connection) -> int:
-        """Cheap stand-in for vec0's own row count -- see compact()."""
-        return conn.execute("SELECT count(*) FROM document_chunks").fetchone()[0]
-
-
-class DocumentMetaTable:
-    """document_id -> modified, one row per document. Lives outside vec0
-    because vec0 only inlines TEXT metadata up to 12 bytes and `modified`
-    (an ISO timestamp) is always longer.
-    """
-
-    @staticmethod
-    def create(conn: sqlite3.Connection) -> None:
-        conn.execute(
-            "CREATE TABLE IF NOT EXISTS document_meta "
-            "(document_id INTEGER PRIMARY KEY, modified TEXT NOT NULL)",
-        )
-
-    @staticmethod
-    def upsert_many(
-        conn: sqlite3.Connection,
-        rows: Iterable[DocumentMetaRow],
-    ) -> None:
-        conn.executemany(
-            "INSERT INTO document_meta (document_id, modified) VALUES (?, ?) "
-            "ON CONFLICT(document_id) DO UPDATE SET modified = excluded.modified",
-            rows,
-        )
-
-    @staticmethod
-    def delete_for_document(conn: sqlite3.Connection, document_id: int) -> None:
-        conn.execute(
-            "DELETE FROM document_meta WHERE document_id = ?",
-            (document_id,),
-        )
-
-    @staticmethod
-    def delete_all(conn: sqlite3.Connection) -> None:
-        conn.execute("DELETE FROM document_meta")
-
-    @staticmethod
-    def copy_all(
-        src_conn: sqlite3.Connection,
-        dst_conn: sqlite3.Connection,
-        batch_size: int,
-    ) -> None:
-        """Stream document_meta from src_conn into dst_conn in bounded
-        batches. The *only* sanctioned way to move this table across
-        connections (compact()/migrations) -- an unbounded fetchall here
-        would defeat the same OOM-avoidance the vec0 row copy already relies
-        on. batch_size has no default: forces the call site to think about
-        it (pass COMPACT_BATCH_SIZE)."""
-        cursor = src_conn.execute(
-            "SELECT document_id, modified FROM document_meta",
-        )
-        while batch := cursor.fetchmany(batch_size):
-            DocumentMetaTable.upsert_many(
-                dst_conn,
-                (DocumentMetaRow(r["document_id"], r["modified"]) for r in batch),
-            )
-
-    @staticmethod
-    def all_modified_times(conn: sqlite3.Connection) -> dict[str, str]:
-        """Full document_id -> modified map, for get_modified_times()'s
-        public API only. One unbounded read by design (existing behavior).
-        Never use this for cross-connection copying; see copy_all()."""
-        return {
-            str(row["document_id"]): str(row["modified"] or "")
-            for row in conn.execute(
-                "SELECT document_id, modified FROM document_meta",
-            )
-        }
-
-
-class IndexMetaTable:
-    """Typed accessors over index_meta's key/value rows -- replaces
-    PaperlessSqliteVecVectorStore._meta_get_on/_meta_set_on, which returned
-    untyped str | None regardless of whether the key held an int (dim,
-    schema_version, total_inserts) or a string (embed_model).
-    """
-
-    @staticmethod
-    def create(conn: sqlite3.Connection) -> None:
-        conn.execute(
-            "CREATE TABLE IF NOT EXISTS index_meta (key TEXT PRIMARY KEY, value TEXT)",
-        )
-
-    @staticmethod
-    def _get(conn: sqlite3.Connection, key: str) -> str | None:
-        row = conn.execute(
-            "SELECT value FROM index_meta WHERE key = ?",
-            (key,),
-        ).fetchone()
-        return row["value"] if row else None
-
-    @staticmethod
-    def _set(conn: sqlite3.Connection, key: str, value: str) -> None:
-        conn.execute(
-            "INSERT INTO index_meta (key, value) VALUES (?, ?) "
-            "ON CONFLICT(key) DO UPDATE SET value = excluded.value",
-            (key, value),
-        )
-
-    @staticmethod
-    def get_dim(conn: sqlite3.Connection) -> int | None:
-        value = IndexMetaTable._get(conn, "dim")
-        return int(value) if value is not None else None
-
-    @staticmethod
-    def set_dim(conn: sqlite3.Connection, dim: int) -> None:
-        IndexMetaTable._set(conn, "dim", str(dim))
-
-    @staticmethod
-    def get_embed_model(conn: sqlite3.Connection) -> str | None:
-        return IndexMetaTable._get(conn, "embed_model")
-
-    @staticmethod
-    def set_embed_model(conn: sqlite3.Connection, name: str) -> None:
-        IndexMetaTable._set(conn, "embed_model", name)
-
-    @staticmethod
-    def get_schema_version(conn: sqlite3.Connection) -> int | None:
-        value = IndexMetaTable._get(conn, "schema_version")
-        return int(value) if value is not None else None
-
-    @staticmethod
-    def set_schema_version(conn: sqlite3.Connection, version: int) -> None:
-        IndexMetaTable._set(conn, "schema_version", str(version))
-
-    @staticmethod
-    def get_total_inserts(conn: sqlite3.Connection) -> int:
-        value = IndexMetaTable._get(conn, "total_inserts")
-        return int(value) if value is not None else 0
-
-    @staticmethod
-    def increment_total_inserts(conn: sqlite3.Connection, count: int) -> None:
-        current = IndexMetaTable.get_total_inserts(conn)
-        IndexMetaTable._set(conn, "total_inserts", str(current + count))
-
-    @staticmethod
-    def reset_total_inserts(conn: sqlite3.Connection, count: int) -> None:
-        """Set total_inserts to an absolute value -- distinct from
-        increment_total_inserts(): used by compact()'s rebuild and by
-        m0001_v1_to_v2 after copying live rows into a fresh file, where
-        total_inserts must become exactly the live row count, not add to
-        whatever the source file's counter held."""
-        IndexMetaTable._set(conn, "total_inserts", str(count))

src/paperless_ai/tests/test_tables.py

@@ -1,306 +0,0 @@
-import sqlite3
-from collections.abc import Generator
-
-import pytest
-
-from paperless_ai.tables import ChunkRow
-from paperless_ai.tables import DocumentChunksTable
-from paperless_ai.tables import DocumentMetaRow
-from paperless_ai.tables import DocumentMetaTable
-from paperless_ai.tables import IndexMetaTable
-
-
-@pytest.fixture
-def conn() -> Generator[sqlite3.Connection, None, None]:
-    connection = sqlite3.connect(":memory:")
-    connection.row_factory = sqlite3.Row
-    try:
-        yield connection
-    finally:
-        connection.close()
-
-
-class TestDocumentChunksTable:
-    def test_create_is_idempotent(self, conn: sqlite3.Connection) -> None:
-        """
-        GIVEN:
-            - A bare sqlite3 connection
-        WHEN:
-            - create() is called, a row is inserted, then create() is called again
-        THEN:
-            - No error is raised and the row survives uncorrupted
-        """
-        DocumentChunksTable.create(conn)
-        DocumentChunksTable.insert_many(conn, [ChunkRow("c1", 1)])
-        DocumentChunksTable.create(conn)
-        assert DocumentChunksTable.chunk_ids_for_document(conn, 1) == ["c1"]
-
-    def test_insert_many_then_lookup_by_document_id(
-        self,
-        conn: sqlite3.Connection,
-    ) -> None:
-        """
-        GIVEN:
-            - An empty document_chunks table
-        WHEN:
-            - Two chunks for document 1 and one for document 2 are inserted
-        THEN:
-            - chunk_ids_for_document returns exactly the matching chunk ids
-        """
-        DocumentChunksTable.create(conn)
-        DocumentChunksTable.insert_many(
-            conn,
-            [ChunkRow("c1", 1), ChunkRow("c2", 1), ChunkRow("c3", 2)],
-        )
-        assert sorted(DocumentChunksTable.chunk_ids_for_document(conn, 1)) == [
-            "c1",
-            "c2",
-        ]
-        assert DocumentChunksTable.chunk_ids_for_document(conn, 2) == ["c3"]
-        assert DocumentChunksTable.chunk_ids_for_document(conn, 999) == []
-
-    def test_delete_for_document_removes_only_that_document(
-        self,
-        conn: sqlite3.Connection,
-    ) -> None:
-        """
-        GIVEN:
-            - Chunks for two different documents
-        WHEN:
-            - delete_for_document() is called for one of them
-        THEN:
-            - Only that document's chunks are removed
-        """
-        DocumentChunksTable.create(conn)
-        DocumentChunksTable.insert_many(
-            conn,
-            [ChunkRow("c1", 1), ChunkRow("c2", 2)],
-        )
-        DocumentChunksTable.delete_for_document(conn, 1)
-        assert DocumentChunksTable.chunk_ids_for_document(conn, 1) == []
-        assert DocumentChunksTable.chunk_ids_for_document(conn, 2) == ["c2"]
-
-    def test_delete_all_clears_every_row(self, conn: sqlite3.Connection) -> None:
-        """
-        GIVEN:
-            - Chunks for multiple documents
-        WHEN:
-            - delete_all() is called
-        THEN:
-            - count() returns 0
-        """
-        DocumentChunksTable.create(conn)
-        DocumentChunksTable.insert_many(
-            conn,
-            [ChunkRow("c1", 1), ChunkRow("c2", 2)],
-        )
-        DocumentChunksTable.delete_all(conn)
-        assert DocumentChunksTable.count(conn) == 0
-
-    def test_count_reflects_live_rows(self, conn: sqlite3.Connection) -> None:
-        """
-        GIVEN:
-            - An empty document_chunks table
-        WHEN:
-            - Rows are inserted then one document's rows are deleted
-        THEN:
-            - count() reflects the remaining row count
-        """
-        DocumentChunksTable.create(conn)
-        DocumentChunksTable.insert_many(
-            conn,
-            [ChunkRow("c1", 1), ChunkRow("c2", 1), ChunkRow("c3", 2)],
-        )
-        assert DocumentChunksTable.count(conn) == 3
-        DocumentChunksTable.delete_for_document(conn, 1)
-        assert DocumentChunksTable.count(conn) == 1
-
-
-class TestDocumentMetaTable:
-    def test_upsert_many_then_all_modified_times(
-        self,
-        conn: sqlite3.Connection,
-    ) -> None:
-        """
-        GIVEN:
-            - An empty document_meta table
-        WHEN:
-            - Two documents' modified timestamps are upserted
-        THEN:
-            - all_modified_times() returns both, keyed by str(document_id)
-        """
-        DocumentMetaTable.create(conn)
-        DocumentMetaTable.upsert_many(
-            conn,
-            [
-                DocumentMetaRow(1, "2026-01-01T00:00:00"),
-                DocumentMetaRow(2, "2026-02-02T00:00:00"),
-            ],
-        )
-        assert DocumentMetaTable.all_modified_times(conn) == {
-            "1": "2026-01-01T00:00:00",
-            "2": "2026-02-02T00:00:00",
-        }
-
-    def test_upsert_many_overwrites_existing_value(
-        self,
-        conn: sqlite3.Connection,
-    ) -> None:
-        """
-        GIVEN:
-            - A document_meta row for document 1
-        WHEN:
-            - upsert_many() is called again with a new modified value for
-              the same document_id
-        THEN:
-            - The stored value is replaced, not duplicated
-        """
-        DocumentMetaTable.create(conn)
-        DocumentMetaTable.upsert_many(conn, [DocumentMetaRow(1, "old")])
-        DocumentMetaTable.upsert_many(conn, [DocumentMetaRow(1, "new")])
-        assert DocumentMetaTable.all_modified_times(conn) == {"1": "new"}
-
-    def test_delete_for_document_removes_only_that_row(
-        self,
-        conn: sqlite3.Connection,
-    ) -> None:
-        """
-        GIVEN:
-            - document_meta rows for two documents
-        WHEN:
-            - delete_for_document() is called for one of them
-        THEN:
-            - Only that document's row is removed
-        """
-        DocumentMetaTable.create(conn)
-        DocumentMetaTable.upsert_many(
-            conn,
-            [DocumentMetaRow(1, "a"), DocumentMetaRow(2, "b")],
-        )
-        DocumentMetaTable.delete_for_document(conn, 1)
-        assert DocumentMetaTable.all_modified_times(conn) == {"2": "b"}
-
-    def test_delete_all_clears_every_row(self, conn: sqlite3.Connection) -> None:
-        """
-        GIVEN:
-            - document_meta rows for multiple documents
-        WHEN:
-            - delete_all() is called
-        THEN:
-            - all_modified_times() returns an empty dict
-        """
-        DocumentMetaTable.create(conn)
-        DocumentMetaTable.upsert_many(
-            conn,
-            [DocumentMetaRow(1, "a"), DocumentMetaRow(2, "b")],
-        )
-        DocumentMetaTable.delete_all(conn)
-        assert DocumentMetaTable.all_modified_times(conn) == {}
-
-    def test_copy_all_streams_every_row_to_destination(
-        self,
-        conn: sqlite3.Connection,
-    ) -> None:
-        """
-        GIVEN:
-            - A source connection with document_meta rows for 5 documents
-            - A separate, empty destination connection
-        WHEN:
-            - copy_all() is called with a batch size smaller than the row
-              count, forcing multiple fetchmany() cycles
-        THEN:
-            - Every row is present on the destination connection
-        """
-        DocumentMetaTable.create(conn)
-        DocumentMetaTable.upsert_many(
-            conn,
-            [DocumentMetaRow(i, f"modified-{i}") for i in range(5)],
-        )
-        dst_conn = sqlite3.connect(":memory:")
-        dst_conn.row_factory = sqlite3.Row
-        try:
-            DocumentMetaTable.create(dst_conn)
-            DocumentMetaTable.copy_all(conn, dst_conn, batch_size=2)
-            assert DocumentMetaTable.all_modified_times(dst_conn) == {
-                str(i): f"modified-{i}" for i in range(5)
-            }
-        finally:
-            dst_conn.close()
-
-
-class TestIndexMetaTable:
-    @pytest.mark.parametrize(
-        ("setter_name", "getter_name", "value"),
-        [
-            ("set_dim", "get_dim", 384),
-            ("set_embed_model", "get_embed_model", "model-a"),
-            ("set_schema_version", "get_schema_version", 2),
-        ],
-    )
-    def test_typed_accessor_roundtrip(
-        self,
-        conn: sqlite3.Connection,
-        setter_name: str,
-        getter_name: str,
-        value: int | str,
-    ) -> None:
-        """
-        GIVEN:
-            - An empty index_meta table
-        WHEN:
-            - A typed accessor's setter is called then the getter is read back
-        THEN:
-            - The same value is returned, correctly typed (int or str)
-        """
-        IndexMetaTable.create(conn)
-        getter = getattr(IndexMetaTable, getter_name)
-        setter = getattr(IndexMetaTable, setter_name)
-        assert getter(conn) is None
-        setter(conn, value)
-        assert getter(conn) == value
-
-    def test_total_inserts_starts_at_zero(self, conn: sqlite3.Connection) -> None:
-        """
-        GIVEN:
-            - An empty index_meta table
-        WHEN:
-            - get_total_inserts() is read before anything is set
-        THEN:
-            - 0 is returned
-        """
-        IndexMetaTable.create(conn)
-        assert IndexMetaTable.get_total_inserts(conn) == 0
-
-    def test_increment_total_inserts_accumulates(
-        self,
-        conn: sqlite3.Connection,
-    ) -> None:
-        """
-        GIVEN:
-            - An empty index_meta table
-        WHEN:
-            - increment_total_inserts() is called twice
-        THEN:
-            - get_total_inserts() returns the running sum
-        """
-        IndexMetaTable.create(conn)
-        IndexMetaTable.increment_total_inserts(conn, 5)
-        IndexMetaTable.increment_total_inserts(conn, 3)
-        assert IndexMetaTable.get_total_inserts(conn) == 8
-
-    def test_reset_total_inserts_sets_absolute_value(
-        self,
-        conn: sqlite3.Connection,
-    ) -> None:
-        """
-        GIVEN:
-            - A total_inserts counter already at a high value
-        WHEN:
-            - reset_total_inserts() is called with a lower value
-        THEN:
-            - get_total_inserts() returns exactly that value, not a sum
-        """
-        IndexMetaTable.create(conn)
-        IndexMetaTable.increment_total_inserts(conn, 100)
-        IndexMetaTable.reset_total_inserts(conn, 7)
-        assert IndexMetaTable.get_total_inserts(conn) == 7