Performance: Preprocess classifier text with Tantivy instead of NLTK (#14127)

* Preprocesses classifier content with Tantivy instead of NLTK

Tokenizing and stemming now happen in one Rust call instead of NLTK's
Python tokenizer and per word stemming, which also removes the Redis
backed stem cache from every preprocessing call. The output matches the
NLTK pipeline closely; tokens containing digits are now stemmed, and the
English stop words follow Snowball's list.

Stemming and stop word removal apply whenever the OCR language is one of
the supported classifier languages, so PAPERLESS_ENABLE_NLTK and
PAPERLESS_NLTK_DIR are removed.

* Copies packages instead of hardlinking them in backend CI, some NLTK thing

* Adds a normalization to NFC to better fit what Tantivy expects
This commit is contained in:
Trenton H authored and GitHub committed 2026-09-16 07:35:26 -07:00
1 parent 989f138556
commit 530059c5c0
15 files changed
+441 -378

No files matched your search

+21 -34
View File
@@ -73,8 +73,6 @@ SHARE_LINK_BUNDLE_DIR = MEDIA_ROOT / "documents" / "share_link_bundles"
DATA_DIR = get_path_from_env("PAPERLESS_DATA_DIR", BASE_DIR.parent / "data")
NLTK_DIR = get_path_from_env("PAPERLESS_NLTK_DIR", "/usr/share/nltk_data")
# Check deprecated setting first
EMPTY_TRASH_DIR = (
get_path_from_env("PAPERLESS_TRASH_DIR", os.getenv("PAPERLESS_EMPTY_TRASH_DIR"))
@@ -1067,39 +1065,30 @@ APP_LOGO = os.getenv("PAPERLESS_APP_LOGO", None)
###############################################################################
def _get_nltk_language_setting(ocr_lang: str) -> str | None:
CLASSIFIER_LANGUAGES: Final[dict[str, str]] = {
"dan": "danish",
"nld": "dutch",
"eng": "english",
"fin": "finnish",
"fra": "french",
"deu": "german",
"ita": "italian",
"nor": "norwegian",
"por": "portuguese",
"rus": "russian",
"spa": "spanish",
"swe": "swedish",
}
def _get_classifier_language_setting(ocr_lang: str) -> str | None:
"""
Maps an ISO-639-1 language code supported by Tesseract into
an optional NLTK language name. This is the set of common supported
languages for all the NLTK data used.
Maps the primary Tesseract language to the classifier's stemming
language, or None if unsupported.
Assumption: The primary language is first
NLTK Languages:
- https://www.nltk.org/api/nltk.stem.snowball.html#nltk.stem.snowball.SnowballStemmer
- https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/tokenizers/punkt.zip
- https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/corpora/stopwords.zip
The common intersection between all languages in those 3 is handled here
"""
ocr_lang = ocr_lang.split("+", maxsplit=1)[0]
iso_code_to_nltk = {
"dan": "danish",
"nld": "dutch",
"eng": "english",
"fin": "finnish",
"fra": "french",
"deu": "german",
"ita": "italian",
"nor": "norwegian",
"por": "portuguese",
"rus": "russian",
"spa": "spanish",
"swe": "swedish",
}
return iso_code_to_nltk.get(ocr_lang)
return CLASSIFIER_LANGUAGES.get(ocr_lang.split("+", maxsplit=1)[0])
def _get_search_language_setting(ocr_lang: str) -> str | None:
@@ -1145,9 +1134,7 @@ def _get_search_language_setting(ocr_lang: str) -> str | None:
return _ocr_to_search.get(primary)
NLTK_ENABLED: Final[bool] = get_bool_from_env("PAPERLESS_ENABLE_NLTK", "yes")
NLTK_LANGUAGE: str | None = _get_nltk_language_setting(OCR_LANGUAGE)
CLASSIFIER_LANGUAGE: str | None = _get_classifier_language_setting(OCR_LANGUAGE)
SEARCH_LANGUAGE: str | None = _get_search_language_setting(OCR_LANGUAGE)
@@ -6,6 +6,7 @@ import pytest
from django.core.exceptions import ImproperlyConfigured
from paperless.settings import _get_allauth_trusted_proxy_count
from paperless.settings import _get_classifier_language_setting
from paperless.settings import _get_search_language_setting
from paperless.settings import _parse_paperless_url
from paperless.settings import default_threads_per_worker
@@ -37,6 +38,45 @@ class TestThreadCalculation(TestCase):
self.assertLessEqual(default_workers * default_threads, i)
class TestClassifierLanguageSetting:
@pytest.mark.parametrize(
("ocr_language", "expected"),
[
pytest.param("dan", "danish", id="danish"),
pytest.param("nld", "dutch", id="dutch"),
pytest.param("eng", "english", id="english"),
pytest.param("fin", "finnish", id="finnish"),
pytest.param("fra", "french", id="french"),
pytest.param("deu", "german", id="german"),
pytest.param("ita", "italian", id="italian"),
pytest.param("nor", "norwegian", id="norwegian"),
pytest.param("por", "portuguese", id="portuguese"),
pytest.param("rus", "russian", id="russian"),
pytest.param("spa", "spanish", id="spanish"),
pytest.param("swe", "swedish", id="swedish"),
pytest.param("eng+deu", "english", id="primary-english"),
pytest.param("deu+eng", "german", id="primary-german"),
pytest.param("ell", None, id="greek-unsupported"),
pytest.param("chi_sim", None, id="chinese-unsupported"),
],
)
def test_maps_primary_ocr_language(
self,
ocr_language: str,
expected: str | None,
) -> None:
"""
GIVEN:
- An OCR language setting, possibly listing several languages
WHEN:
- The classifier language is determined
THEN:
- The first OCR language maps to its classifier language, or None
if unsupported
"""
assert _get_classifier_language_setting(ocr_language) == expected
def test_allauth_trusted_proxy_count_defaults_to_trusted_proxies(
monkeypatch: pytest.MonkeyPatch,
) -> None: