mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-11-22 13:24:22 +00:00
Performance: Preprocess classifier text with Tantivy instead of NLTK (#14127)
* Preprocesses classifier content with Tantivy instead of NLTK Tokenizing and stemming now happen in one Rust call instead of NLTK's Python tokenizer and per word stemming, which also removes the Redis backed stem cache from every preprocessing call. The output matches the NLTK pipeline closely; tokens containing digits are now stemmed, and the English stop words follow Snowball's list. Stemming and stop word removal apply whenever the OCR language is one of the supported classifier languages, so PAPERLESS_ENABLE_NLTK and PAPERLESS_NLTK_DIR are removed. * Copies packages instead of hardlinking them in backend CI, some NLTK thing * Adds a normalization to NFC to better fit what Tantivy expects
This commit is contained in:
1 parent
989f138556
commit
530059c5c0
15 files changed
+441
-378
No files matched your search
@@ -73,8 +73,6 @@ SHARE_LINK_BUNDLE_DIR = MEDIA_ROOT / "documents" / "share_link_bundles"
|
||||
|
||||
DATA_DIR = get_path_from_env("PAPERLESS_DATA_DIR", BASE_DIR.parent / "data")
|
||||
|
||||
NLTK_DIR = get_path_from_env("PAPERLESS_NLTK_DIR", "/usr/share/nltk_data")
|
||||
|
||||
# Check deprecated setting first
|
||||
EMPTY_TRASH_DIR = (
|
||||
get_path_from_env("PAPERLESS_TRASH_DIR", os.getenv("PAPERLESS_EMPTY_TRASH_DIR"))
|
||||
@@ -1067,39 +1065,30 @@ APP_LOGO = os.getenv("PAPERLESS_APP_LOGO", None)
|
||||
###############################################################################
|
||||
|
||||
|
||||
def _get_nltk_language_setting(ocr_lang: str) -> str | None:
|
||||
CLASSIFIER_LANGUAGES: Final[dict[str, str]] = {
|
||||
"dan": "danish",
|
||||
"nld": "dutch",
|
||||
"eng": "english",
|
||||
"fin": "finnish",
|
||||
"fra": "french",
|
||||
"deu": "german",
|
||||
"ita": "italian",
|
||||
"nor": "norwegian",
|
||||
"por": "portuguese",
|
||||
"rus": "russian",
|
||||
"spa": "spanish",
|
||||
"swe": "swedish",
|
||||
}
|
||||
|
||||
|
||||
def _get_classifier_language_setting(ocr_lang: str) -> str | None:
|
||||
"""
|
||||
Maps an ISO-639-1 language code supported by Tesseract into
|
||||
an optional NLTK language name. This is the set of common supported
|
||||
languages for all the NLTK data used.
|
||||
Maps the primary Tesseract language to the classifier's stemming
|
||||
language, or None if unsupported.
|
||||
|
||||
Assumption: The primary language is first
|
||||
|
||||
NLTK Languages:
|
||||
- https://www.nltk.org/api/nltk.stem.snowball.html#nltk.stem.snowball.SnowballStemmer
|
||||
- https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/tokenizers/punkt.zip
|
||||
- https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/corpora/stopwords.zip
|
||||
|
||||
The common intersection between all languages in those 3 is handled here
|
||||
|
||||
"""
|
||||
ocr_lang = ocr_lang.split("+", maxsplit=1)[0]
|
||||
iso_code_to_nltk = {
|
||||
"dan": "danish",
|
||||
"nld": "dutch",
|
||||
"eng": "english",
|
||||
"fin": "finnish",
|
||||
"fra": "french",
|
||||
"deu": "german",
|
||||
"ita": "italian",
|
||||
"nor": "norwegian",
|
||||
"por": "portuguese",
|
||||
"rus": "russian",
|
||||
"spa": "spanish",
|
||||
"swe": "swedish",
|
||||
}
|
||||
|
||||
return iso_code_to_nltk.get(ocr_lang)
|
||||
return CLASSIFIER_LANGUAGES.get(ocr_lang.split("+", maxsplit=1)[0])
|
||||
|
||||
|
||||
def _get_search_language_setting(ocr_lang: str) -> str | None:
|
||||
@@ -1145,9 +1134,7 @@ def _get_search_language_setting(ocr_lang: str) -> str | None:
|
||||
return _ocr_to_search.get(primary)
|
||||
|
||||
|
||||
NLTK_ENABLED: Final[bool] = get_bool_from_env("PAPERLESS_ENABLE_NLTK", "yes")
|
||||
|
||||
NLTK_LANGUAGE: str | None = _get_nltk_language_setting(OCR_LANGUAGE)
|
||||
CLASSIFIER_LANGUAGE: str | None = _get_classifier_language_setting(OCR_LANGUAGE)
|
||||
|
||||
SEARCH_LANGUAGE: str | None = _get_search_language_setting(OCR_LANGUAGE)
|
||||
|
||||
|
||||
Reference in new issue
Block a user