mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-11-22 05:14:22 +00:00
Performance: Preprocess classifier text with Tantivy instead of NLTK (#14127)
* Preprocesses classifier content with Tantivy instead of NLTK Tokenizing and stemming now happen in one Rust call instead of NLTK's Python tokenizer and per word stemming, which also removes the Redis backed stem cache from every preprocessing call. The output matches the NLTK pipeline closely; tokens containing digits are now stemmed, and the English stop words follow Snowball's list. Stemming and stop word removal apply whenever the OCR language is one of the supported classifier languages, so PAPERLESS_ENABLE_NLTK and PAPERLESS_NLTK_DIR are removed. * Copies packages instead of hardlinking them in backend CI, some NLTK thing * Adds a normalization to NFC to better fit what Tantivy expects
This commit is contained in:
1 parent
989f138556
commit
530059c5c0
15 files changed
+441
-378
No files matched your search
@@ -73,8 +73,6 @@ SHARE_LINK_BUNDLE_DIR = MEDIA_ROOT / "documents" / "share_link_bundles"
|
||||
|
||||
DATA_DIR = get_path_from_env("PAPERLESS_DATA_DIR", BASE_DIR.parent / "data")
|
||||
|
||||
NLTK_DIR = get_path_from_env("PAPERLESS_NLTK_DIR", "/usr/share/nltk_data")
|
||||
|
||||
# Check deprecated setting first
|
||||
EMPTY_TRASH_DIR = (
|
||||
get_path_from_env("PAPERLESS_TRASH_DIR", os.getenv("PAPERLESS_EMPTY_TRASH_DIR"))
|
||||
@@ -1067,39 +1065,30 @@ APP_LOGO = os.getenv("PAPERLESS_APP_LOGO", None)
|
||||
###############################################################################
|
||||
|
||||
|
||||
def _get_nltk_language_setting(ocr_lang: str) -> str | None:
|
||||
CLASSIFIER_LANGUAGES: Final[dict[str, str]] = {
|
||||
"dan": "danish",
|
||||
"nld": "dutch",
|
||||
"eng": "english",
|
||||
"fin": "finnish",
|
||||
"fra": "french",
|
||||
"deu": "german",
|
||||
"ita": "italian",
|
||||
"nor": "norwegian",
|
||||
"por": "portuguese",
|
||||
"rus": "russian",
|
||||
"spa": "spanish",
|
||||
"swe": "swedish",
|
||||
}
|
||||
|
||||
|
||||
def _get_classifier_language_setting(ocr_lang: str) -> str | None:
|
||||
"""
|
||||
Maps an ISO-639-1 language code supported by Tesseract into
|
||||
an optional NLTK language name. This is the set of common supported
|
||||
languages for all the NLTK data used.
|
||||
Maps the primary Tesseract language to the classifier's stemming
|
||||
language, or None if unsupported.
|
||||
|
||||
Assumption: The primary language is first
|
||||
|
||||
NLTK Languages:
|
||||
- https://www.nltk.org/api/nltk.stem.snowball.html#nltk.stem.snowball.SnowballStemmer
|
||||
- https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/tokenizers/punkt.zip
|
||||
- https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/corpora/stopwords.zip
|
||||
|
||||
The common intersection between all languages in those 3 is handled here
|
||||
|
||||
"""
|
||||
ocr_lang = ocr_lang.split("+", maxsplit=1)[0]
|
||||
iso_code_to_nltk = {
|
||||
"dan": "danish",
|
||||
"nld": "dutch",
|
||||
"eng": "english",
|
||||
"fin": "finnish",
|
||||
"fra": "french",
|
||||
"deu": "german",
|
||||
"ita": "italian",
|
||||
"nor": "norwegian",
|
||||
"por": "portuguese",
|
||||
"rus": "russian",
|
||||
"spa": "spanish",
|
||||
"swe": "swedish",
|
||||
}
|
||||
|
||||
return iso_code_to_nltk.get(ocr_lang)
|
||||
return CLASSIFIER_LANGUAGES.get(ocr_lang.split("+", maxsplit=1)[0])
|
||||
|
||||
|
||||
def _get_search_language_setting(ocr_lang: str) -> str | None:
|
||||
@@ -1145,9 +1134,7 @@ def _get_search_language_setting(ocr_lang: str) -> str | None:
|
||||
return _ocr_to_search.get(primary)
|
||||
|
||||
|
||||
NLTK_ENABLED: Final[bool] = get_bool_from_env("PAPERLESS_ENABLE_NLTK", "yes")
|
||||
|
||||
NLTK_LANGUAGE: str | None = _get_nltk_language_setting(OCR_LANGUAGE)
|
||||
CLASSIFIER_LANGUAGE: str | None = _get_classifier_language_setting(OCR_LANGUAGE)
|
||||
|
||||
SEARCH_LANGUAGE: str | None = _get_search_language_setting(OCR_LANGUAGE)
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import pytest
|
||||
from django.core.exceptions import ImproperlyConfigured
|
||||
|
||||
from paperless.settings import _get_allauth_trusted_proxy_count
|
||||
from paperless.settings import _get_classifier_language_setting
|
||||
from paperless.settings import _get_search_language_setting
|
||||
from paperless.settings import _parse_paperless_url
|
||||
from paperless.settings import default_threads_per_worker
|
||||
@@ -37,6 +38,45 @@ class TestThreadCalculation(TestCase):
|
||||
self.assertLessEqual(default_workers * default_threads, i)
|
||||
|
||||
|
||||
class TestClassifierLanguageSetting:
|
||||
@pytest.mark.parametrize(
|
||||
("ocr_language", "expected"),
|
||||
[
|
||||
pytest.param("dan", "danish", id="danish"),
|
||||
pytest.param("nld", "dutch", id="dutch"),
|
||||
pytest.param("eng", "english", id="english"),
|
||||
pytest.param("fin", "finnish", id="finnish"),
|
||||
pytest.param("fra", "french", id="french"),
|
||||
pytest.param("deu", "german", id="german"),
|
||||
pytest.param("ita", "italian", id="italian"),
|
||||
pytest.param("nor", "norwegian", id="norwegian"),
|
||||
pytest.param("por", "portuguese", id="portuguese"),
|
||||
pytest.param("rus", "russian", id="russian"),
|
||||
pytest.param("spa", "spanish", id="spanish"),
|
||||
pytest.param("swe", "swedish", id="swedish"),
|
||||
pytest.param("eng+deu", "english", id="primary-english"),
|
||||
pytest.param("deu+eng", "german", id="primary-german"),
|
||||
pytest.param("ell", None, id="greek-unsupported"),
|
||||
pytest.param("chi_sim", None, id="chinese-unsupported"),
|
||||
],
|
||||
)
|
||||
def test_maps_primary_ocr_language(
|
||||
self,
|
||||
ocr_language: str,
|
||||
expected: str | None,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An OCR language setting, possibly listing several languages
|
||||
WHEN:
|
||||
- The classifier language is determined
|
||||
THEN:
|
||||
- The first OCR language maps to its classifier language, or None
|
||||
if unsupported
|
||||
"""
|
||||
assert _get_classifier_language_setting(ocr_language) == expected
|
||||
|
||||
|
||||
def test_allauth_trusted_proxy_count_defaults_to_trusted_proxies(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
|
||||
Reference in new issue
Block a user