Feature: Replace Whoosh with tantivy search backend (#12471)

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
Co-authored-by: Antoine Mérino <3023499+Merinorus@users.noreply.github.com>

src/paperless/settings/__init__.py

@@ -21,6 +21,7 @@ from paperless.settings.custom import parse_hosting_settings
 from paperless.settings.custom import parse_ignore_dates
 from paperless.settings.custom import parse_redis_url
 from paperless.settings.parsers import get_bool_from_env
+from paperless.settings.parsers import get_choice_from_env
 from paperless.settings.parsers import get_float_from_env
 from paperless.settings.parsers import get_int_from_env
 from paperless.settings.parsers import get_list_from_env
@@ -85,6 +86,11 @@ EMPTY_TRASH_DIR = (
 # threads.
 MEDIA_LOCK = MEDIA_ROOT / "media.lock"
 INDEX_DIR = DATA_DIR / "index"
+
+ADVANCED_FUZZY_SEARCH_THRESHOLD: float | None = get_float_from_env(
+    "PAPERLESS_ADVANCED_FUZZY_SEARCH_THRESHOLD",
+)
+
 MODEL_FILE = get_path_from_env(
     "PAPERLESS_MODEL_FILE",
     DATA_DIR / "classification_model.pickle",
@@ -1033,10 +1039,55 @@ def _get_nltk_language_setting(ocr_lang: str) -> str | None:
     return iso_code_to_nltk.get(ocr_lang)
 
 
+def _get_search_language_setting(ocr_lang: str) -> str | None:
+    """
+    Determine the Tantivy stemmer language.
+
+    If PAPERLESS_SEARCH_LANGUAGE is explicitly set, it is validated against
+    the languages supported by Tantivy's built-in stemmer and returned as-is.
+    Otherwise the primary Tesseract language code from PAPERLESS_OCR_LANGUAGE
+    is mapped to the corresponding ISO 639-1 code understood by Tantivy.
+    Returns None when unset and the OCR language has no Tantivy stemmer.
+    """
+    explicit = os.environ.get("PAPERLESS_SEARCH_LANGUAGE")
+    if explicit is not None:
+        # Lazy import avoids any app-loading order concerns; _tokenizer has no
+        # Django dependencies so this is safe.
+        from documents.search._tokenizer import SUPPORTED_LANGUAGES
+
+        return get_choice_from_env("PAPERLESS_SEARCH_LANGUAGE", SUPPORTED_LANGUAGES)
+
+    # Infer from the primary Tesseract language code (ISO 639-2/T → ISO 639-1)
+    primary = ocr_lang.split("+", maxsplit=1)[0].lower()
+    _ocr_to_search: dict[str, str] = {
+        "ara": "ar",
+        "dan": "da",
+        "nld": "nl",
+        "eng": "en",
+        "fin": "fi",
+        "fra": "fr",
+        "deu": "de",
+        "ell": "el",
+        "hun": "hu",
+        "ita": "it",
+        "nor": "no",
+        "por": "pt",
+        "ron": "ro",
+        "rus": "ru",
+        "spa": "es",
+        "swe": "sv",
+        "tam": "ta",
+        "tur": "tr",
+    }
+    return _ocr_to_search.get(primary)
+
+
 NLTK_ENABLED: Final[bool] = get_bool_from_env("PAPERLESS_ENABLE_NLTK", "yes")
 
 NLTK_LANGUAGE: str | None = _get_nltk_language_setting(OCR_LANGUAGE)
 
+SEARCH_LANGUAGE: str | None = _get_search_language_setting(OCR_LANGUAGE)
+
 ###############################################################################
 # Email Preprocessors                                                         #
 ###############################################################################

src/paperless/settings/parsers.py

@@ -260,7 +260,7 @@ def get_list_from_env(
 
 def get_choice_from_env(
     env_key: str,
-    choices: set[str],
+    choices: set[str] | frozenset[str],
     default: str | None = None,
 ) -> str:
     """

src/paperless/tests/parsers/test_tesseract_custom_settings.py

@@ -14,6 +14,11 @@ from paperless.parsers.tesseract import RasterisedDocumentParser
 
 
 class TestParserSettingsFromDb(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
+    @classmethod
+    def setUpTestData(cls) -> None:
+        super().setUpTestData()
+        ApplicationConfiguration.objects.get_or_create()
+
     @staticmethod
     def get_params():
         """

src/paperless/tests/settings/test_settings.py

@@ -2,6 +2,9 @@ import os
 from unittest import TestCase
 from unittest import mock
 
+import pytest
+
+from paperless.settings import _get_search_language_setting
 from paperless.settings import _parse_paperless_url
 from paperless.settings import default_threads_per_worker
 
@@ -32,6 +35,48 @@ class TestThreadCalculation(TestCase):
                 self.assertLessEqual(default_workers * default_threads, i)
 
 
+@pytest.mark.parametrize(
+    ("env_value", "expected"),
+    [
+        ("en", "en"),
+        ("de", "de"),
+        ("fr", "fr"),
+        ("swedish", "swedish"),
+    ],
+)
+def test_get_search_language_setting_explicit_valid(
+    monkeypatch: pytest.MonkeyPatch,
+    env_value: str,
+    expected: str,
+) -> None:
+    """
+    GIVEN:
+        - PAPERLESS_SEARCH_LANGUAGE is set to a valid Tantivy stemmer language
+    WHEN:
+        - _get_search_language_setting is called
+    THEN:
+        - The explicit value is returned regardless of the OCR language
+    """
+    monkeypatch.setenv("PAPERLESS_SEARCH_LANGUAGE", env_value)
+    assert _get_search_language_setting("deu") == expected
+
+
+def test_get_search_language_setting_explicit_invalid(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """
+    GIVEN:
+        - PAPERLESS_SEARCH_LANGUAGE is set to an unsupported language code
+    WHEN:
+        - _get_search_language_setting is called
+    THEN:
+        - ValueError is raised
+    """
+    monkeypatch.setenv("PAPERLESS_SEARCH_LANGUAGE", "klingon")
+    with pytest.raises(ValueError, match="klingon"):
+        _get_search_language_setting("eng")
+
+
 class TestPaperlessURLSettings(TestCase):
     def test_paperless_url(self) -> None:
         """

src/paperless/views.py

@@ -36,7 +36,6 @@ from rest_framework.permissions import IsAuthenticated
 from rest_framework.response import Response
 from rest_framework.viewsets import ModelViewSet
 
-from documents.index import DelayedQuery
 from documents.permissions import PaperlessObjectPermissions
 from documents.tasks import llmindex_index
 from paperless.filters import GroupFilterSet
@@ -83,20 +82,12 @@ class StandardPagination(PageNumberPagination):
         )
 
     def get_all_result_ids(self):
+        from documents.search import TantivyRelevanceList
+
         query = self.page.paginator.object_list
-        if isinstance(query, DelayedQuery):
-            try:
-                ids = [
-                    query.searcher.ixreader.stored_fields(
-                        doc_num,
-                    )["id"]
-                    for doc_num in query.saved_results.get(0).results.docs()
-                ]
-            except Exception:
-                pass
-        else:
-            ids = self.page.paginator.object_list.values_list("pk", flat=True)
-        return ids
+        if isinstance(query, TantivyRelevanceList):
+            return [h["id"] for h in query._hits]
+        return self.page.paginator.object_list.values_list("pk", flat=True)
 
     def get_paginated_response_schema(self, schema):
         response_schema = super().get_paginated_response_schema(schema)