diff --git a/src/documents/search/_backend.py b/src/documents/search/_backend.py index 447bc6bbe..c04ccbfd4 100644 --- a/src/documents/search/_backend.py +++ b/src/documents/search/_backend.py @@ -196,52 +196,49 @@ class WriteBatch: return self._raw_writer def __enter__(self) -> Self: - if self._backend._path is not None: - lock_path = self._backend._path / ".tantivy.lock" - self._lock = filelock.FileLock(str(lock_path)) - for attempt in range(_LOCK_RETRY_ATTEMPTS): - try: - self._lock.acquire(timeout=self._lock_timeout) - break - except filelock.Timeout: - if attempt == _LOCK_RETRY_ATTEMPTS - 1: - raise SearchIndexLockError( - f"Could not acquire index lock after {_LOCK_RETRY_ATTEMPTS} " - f"attempts (timeout={self._lock_timeout}s each)", - ) - sleep_s = random.uniform( - 0, - min(_LOCK_BACKOFF_CAP, _LOCK_BACKOFF_BASE * (2**attempt)), + lock_path = self._backend._path / ".tantivy.lock" + self._lock = filelock.FileLock(str(lock_path)) + for attempt in range(_LOCK_RETRY_ATTEMPTS): + try: + self._lock.acquire(timeout=self._lock_timeout) + break + except filelock.Timeout: + if attempt == _LOCK_RETRY_ATTEMPTS - 1: + raise SearchIndexLockError( + f"Could not acquire index lock after {_LOCK_RETRY_ATTEMPTS} " + f"attempts (timeout={self._lock_timeout}s each)", ) - logger.debug( - "Index lock contention; retrying in %.2fs (attempt %d/%d)", - sleep_s, - attempt + 1, - _LOCK_RETRY_ATTEMPTS, - ) - time.sleep(sleep_s) + sleep_s = random.uniform( + 0, + min(_LOCK_BACKOFF_CAP, _LOCK_BACKOFF_BASE * (2**attempt)), + ) + logger.debug( + "Index lock contention; retrying in %.2fs (attempt %d/%d)", + sleep_s, + attempt + 1, + _LOCK_RETRY_ATTEMPTS, + ) + time.sleep(sleep_s) - # Open a fresh Index (and thus a fresh Tantivy ManagedDirectory) - # for the write, rather than reusing the process-local cached - # index. ManagedDirectory loads its GC bookkeeping (.managed.json) - # once, at construction, and never re-reads it; paperless runs - # several long-lived processes (Granian workers, Celery workers) - # that take turns writing under the file lock above. A cached, - # long-lived writer index would carry a stale managed-files view - # and, on commit, overwrite .managed.json with that stale view - - # permanently losing track of segment files other processes - # registered in the meantime, so they can never be garbage - # collected. Reopening fresh here always picks up the current - # on-disk state. The long-lived self._backend._index is used for - # reads only and is reloaded (not reopened) after commit below. - write_index = tantivy.Index( - build_schema(), - path=str(self._backend._path), - ) - register_tokenizers(write_index, settings.SEARCH_LANGUAGE) - self._raw_writer = write_index.writer() - else: - self._raw_writer = self._backend._index.writer() + # Open a fresh Index (and thus a fresh Tantivy ManagedDirectory) + # for the write, rather than reusing the process-local cached + # index. ManagedDirectory loads its GC bookkeeping (.managed.json) + # once, at construction, and never re-reads it; paperless runs + # several long-lived processes (Granian workers, Celery workers) + # that take turns writing under the file lock above. A cached, + # long-lived writer index would carry a stale managed-files view + # and, on commit, overwrite .managed.json with that stale view - + # permanently losing track of segment files other processes + # registered in the meantime, so they can never be garbage + # collected. Reopening fresh here always picks up the current + # on-disk state. The long-lived self._backend._index is used for + # reads only and is reloaded (not reopened) after commit below. + write_index = tantivy.Index( + build_schema(), + path=str(self._backend._path), + ) + register_tokenizers(write_index, settings.SEARCH_LANGUAGE) + self._raw_writer = write_index.writer() return self def __exit__(self, exc_type, exc_val, exc_tb): @@ -372,9 +369,8 @@ class TantivyBackend: Tantivy search backend with explicit lifecycle management. Provides full-text search capabilities using the Tantivy search engine. - Supports in-memory indexes (for testing) and persistent on-disk indexes - (for production use). Handles document indexing, search queries, autocompletion, - and "more like this" functionality. + Keeps a persistent on-disk index. Handles document indexing, search queries, + autocompletion, and "more like this" functionality. The backend manages its own connection lifecycle and can be reset when the underlying index directory changes (e.g., during test isolation). @@ -408,9 +404,7 @@ class TantivyBackend: }, ) - def __init__(self, path: Path | None = None): - # path=None → in-memory index (for tests) - # path=some_dir → on-disk index (for production) + def __init__(self, path: Path): self._path = path self._raw_index: tantivy.Index | None = None self._raw_schema: tantivy.Schema | None = None @@ -429,16 +423,13 @@ class TantivyBackend: """ Open or rebuild the index as needed. - For disk-based indexes, checks if rebuilding is needed due to schema - version or language changes. Registers custom tokenizers after opening. + Checks if rebuilding is needed due to schema version or language + changes. Registers custom tokenizers after opening. Safe to call multiple times - subsequent calls are no-ops. """ if self._raw_index is not None: return # pragma: no cover - if self._path is not None: - self._raw_index = open_or_rebuild_index(self._path) - else: - self._raw_index = tantivy.Index(build_schema()) + self._raw_index = open_or_rebuild_index(self._path) register_tokenizers(self._raw_index, settings.SEARCH_LANGUAGE) self._raw_schema = self._raw_index.schema @@ -1102,13 +1093,9 @@ class TantivyBackend: writer's threads). Larger values buffer more docs in RAM before flushing a segment, deferring merge work; they do not avoid it. """ - # Create new index (on-disk or in-memory) - if self._path is not None: - wipe_index(self._path) - new_index = tantivy.Index(build_schema(), path=str(self._path)) - _write_sentinels(self._path) - else: - new_index = tantivy.Index(build_schema()) + wipe_index(self._path) + new_index = tantivy.Index(build_schema(), path=str(self._path)) + _write_sentinels(self._path) register_tokenizers(new_index, settings.SEARCH_LANGUAGE) # Point instance at the new index so _build_tantivy_doc uses it diff --git a/src/documents/tests/search/conftest.py b/src/documents/tests/search/conftest.py index ce805cdd4..abeb99d76 100644 --- a/src/documents/tests/search/conftest.py +++ b/src/documents/tests/search/conftest.py @@ -19,6 +19,7 @@ if TYPE_CHECKING: from pytest_django.fixtures import Settings from documents.models import Document + from paperless_testing.dirs import PaperlessDirs @pytest.fixture @@ -30,8 +31,8 @@ def index_dir(tmp_path: Path, settings: Settings) -> Path: @pytest.fixture -def backend() -> Generator[TantivyBackend, None, None]: - b = TantivyBackend() # path=None → in-memory index +def backend(paperless_dirs: PaperlessDirs) -> Generator[TantivyBackend, None, None]: + b = TantivyBackend(path=paperless_dirs.index_dir) b.open() try: yield b diff --git a/src/paperless_ai/tests/test_ai_classifier.py b/src/paperless_ai/tests/test_ai_classifier.py index cbdd1e7f1..eeb8eabf7 100644 --- a/src/paperless_ai/tests/test_ai_classifier.py +++ b/src/paperless_ai/tests/test_ai_classifier.py @@ -1,6 +1,7 @@ import datetime from collections.abc import Generator from types import SimpleNamespace +from typing import TYPE_CHECKING from unittest.mock import MagicMock from unittest.mock import patch @@ -28,6 +29,9 @@ from paperless_testing.factories import TagFactory from paperless_testing.factories import UserFactory from paperless_testing.permissions import grant_object +if TYPE_CHECKING: + from paperless_testing.dirs import PaperlessDirs + @pytest.fixture def mock_document(): @@ -630,10 +634,11 @@ class TestFulltextSimilarDocuments: def fulltext_backend( self, mocker: pytest_mock.MockerFixture, + paperless_dirs: "PaperlessDirs", ) -> Generator[TantivyBackend, None, None]: - """An in-memory Tantivy backend, wired up as the module-level + """An on-disk Tantivy backend, wired up as the module-level singleton _fulltext_similar_documents resolves via get_backend().""" - backend = TantivyBackend(path=None) + backend = TantivyBackend(path=paperless_dirs.index_dir) backend.open() mocker.patch("documents.search.get_backend", return_value=backend) try: