mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-09-24 18:30:31 +00:00
Performance: Preprocess classifier text with Tantivy instead of NLTK (#14127)
* Preprocesses classifier content with Tantivy instead of NLTK Tokenizing and stemming now happen in one Rust call instead of NLTK's Python tokenizer and per word stemming, which also removes the Redis backed stem cache from every preprocessing call. The output matches the NLTK pipeline closely; tokens containing digits are now stemmed, and the English stop words follow Snowball's list. Stemming and stop word removal apply whenever the OCR language is one of the supported classifier languages, so PAPERLESS_ENABLE_NLTK and PAPERLESS_NLTK_DIR are removed. * Copies packages instead of hardlinking them in backend CI, some NLTK thing * Adds a normalization to NFC to better fit what Tantivy expects
This commit is contained in:
Trenton H
@@ -1 +1 @@
|
||||
sampl textual document content includ mani charact possibl check classifi vector hey 00 test0707 content exampl document creat 2025 06 25 digit 0123456789 punctuat english text quick brown fox jump lazi dog english stop word accent latin diacrit àâäæçéèêëîïôœùûüÿñ arab لقد قام المترجم بعمل جيد greek αλφα βήτα γάμμα δέλτα ωμέγα cyril привет как дела добро пожаловать chines simplifi 你好 世界 今天的天气很好 chines tradit 歡迎來到世界 今天天氣很好 japanes kanji hiragana katakana 東京へ行きます カタカナ ひらがな 漢字 korean hangul 안녕하세요 오늘 날씨 어때요 arab مرحب ا كيف حالك hebrew שלום מה שלומך emoji symbol µ math ₀ x² dx π 3 14159 e ρ ε₀ currenc 1 date format 25 06 2025 june 25 2025 2025年6月25日 quot french bonjour ça va quot german guten tag wie geht newlin test r n r tab ttest tspace 192 33601010101 end document
|
||||
sampl textual document content includ mani charact possibl check classifi vector hey 00 test0707 content exampl document creat 2025 06 25 digit 0123456789 punctuat english text quick brown fox jump lazi dog english stop word accent latin diacrit àâäæçéèêëîïôœùûüÿñ arab لقد قام المترجم بعمل جيد greek αλφα βήτα γάμμα δέλτα ωμέγα cyril привет как дела добро пожаловать chines simplifi 你好 世界 今天的天气很好 chines tradit 歡迎來到世界 今天天氣很好 japanes kanji hiragana katakana 東京へ行きます カタカナ ひらがな 漢字 korean hangul 안녕하세요 오늘 날씨 어때요 arab مرحبًا كيف حالك hebrew שלום מה שלומך emoji ️ symbol µ math x dx π 3 14159 e ρ ε currenc 1 date format 25 06 2025 june 25 2025 2025年6月25日 quot french bonjour ça va quot german guten tag wie geht newlin test r n r tab ttest tspace 192 33601010101 end document
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
from documents.caching import StoredLRUCache
|
||||
from paperless.signed_pickle import HMAC_SIZE
|
||||
from paperless.signed_pickle import signed_pickle_dumps
|
||||
from paperless.signed_pickle import signed_pickle_loads
|
||||
|
||||
|
||||
def test_lru_cache_entries() -> None:
|
||||
CACHE_TTL = 1
|
||||
# LRU cache with a capacity of 2 elements
|
||||
cache = StoredLRUCache("test_lru_cache_key", 2, backend_ttl=CACHE_TTL)
|
||||
cache.set(1, 1)
|
||||
cache.set(2, 2)
|
||||
assert cache.get(2) == 2
|
||||
assert cache.get(1) == 1
|
||||
|
||||
# The oldest entry (2) should be removed
|
||||
cache.set(3, 3)
|
||||
assert cache.get(3) == 3
|
||||
assert not cache.get(2)
|
||||
assert cache.get(1) == 1
|
||||
|
||||
# Save the cache, restore it and check it overwrites the current cache in memory
|
||||
cache.save()
|
||||
cache.set(4, 4)
|
||||
assert not cache.get(3)
|
||||
cache.load()
|
||||
assert not cache.get(4)
|
||||
assert cache.get(3) == 3
|
||||
assert cache.get(1) == 1
|
||||
|
||||
|
||||
def test_stored_lru_cache_key_ttl(mocker) -> None:
|
||||
mock_backend = mocker.Mock()
|
||||
cache = StoredLRUCache("test_key", backend=mock_backend, backend_ttl=321)
|
||||
|
||||
# Simulate storing values
|
||||
cache.set("x", "X")
|
||||
cache.set("y", "Y")
|
||||
cache.save()
|
||||
|
||||
# Assert backend.set was called with pickled data, key and TTL
|
||||
mock_backend.set.assert_called_once()
|
||||
key, data, timeout = mock_backend.set.call_args[0]
|
||||
assert key == "test_key"
|
||||
assert timeout == 321
|
||||
assert signed_pickle_loads(data) == {"x": "X", "y": "Y"}
|
||||
|
||||
|
||||
def test_stored_lru_cache_rejects_tampered_data(mocker) -> None:
|
||||
serialized_data = bytearray(signed_pickle_dumps({"x": "X"}))
|
||||
serialized_data[HMAC_SIZE] ^= 0xFF
|
||||
mock_backend = mocker.Mock()
|
||||
mock_backend.get.return_value = bytes(serialized_data)
|
||||
cache = StoredLRUCache("test_key", backend=mock_backend)
|
||||
|
||||
cache.load()
|
||||
|
||||
assert cache.get("x") is None
|
||||
@@ -20,6 +20,7 @@ from documents.classifier import ClassifierModelCorruptError
|
||||
from documents.classifier import DocumentClassifier
|
||||
from documents.classifier import IncompatibleClassifierVersionError
|
||||
from documents.classifier import _predict_with_threshold
|
||||
from documents.classifier import _text_analyzer
|
||||
from documents.classifier import load_classifier
|
||||
from documents.models import Correspondent
|
||||
from documents.models import Document
|
||||
@@ -30,11 +31,12 @@ from documents.models import Tag
|
||||
from documents.tests.factories import DocumentFactory
|
||||
from documents.tests.factories import TagFactory
|
||||
from documents.tests.utils import DirectoriesMixin
|
||||
from paperless.settings import CLASSIFIER_LANGUAGES
|
||||
from paperless.signed_pickle import HMAC_SIZE
|
||||
from paperless.signed_pickle import signed_pickle_dumps
|
||||
|
||||
|
||||
def dummy_preprocess(content: str, **kwargs):
|
||||
def dummy_preprocess(content: str) -> str:
|
||||
"""
|
||||
Simpler, faster pre-processing for testing purposes
|
||||
"""
|
||||
@@ -1043,68 +1045,69 @@ def test_classifier_match_threshold_default() -> None:
|
||||
assert settings.CLASSIFIER_MATCH_THRESHOLD == 0.6
|
||||
|
||||
|
||||
def test_preprocess_content() -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Advanced text processing is enabled (default)
|
||||
WHEN:
|
||||
- Classifier preprocesses a document's content
|
||||
THEN:
|
||||
- Processed content matches the expected output (stemmed words)
|
||||
"""
|
||||
with (Path(__file__).parent / "samples" / "content.txt").open("r") as f:
|
||||
content = f.read()
|
||||
with (Path(__file__).parent / "samples" / "preprocessed_content_advanced.txt").open(
|
||||
"r",
|
||||
) as f:
|
||||
expected_preprocess_content = f.read().rstrip()
|
||||
classifier = DocumentClassifier()
|
||||
result = classifier.preprocess_content(content)
|
||||
assert result == expected_preprocess_content
|
||||
class TestPreprocessContent:
|
||||
@pytest.fixture
|
||||
def samples(self) -> Path:
|
||||
return Path(__file__).parent / "samples"
|
||||
|
||||
@pytest.fixture
|
||||
def content(self, samples: Path) -> str:
|
||||
return (samples / "content.txt").read_text()
|
||||
|
||||
def test_preprocess_content_nltk_disabled() -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Advanced text processing is disabled
|
||||
WHEN:
|
||||
- Classifier preprocesses a document's content
|
||||
THEN:
|
||||
- Processed content matches the expected output (unstemmed words)
|
||||
"""
|
||||
with (Path(__file__).parent / "samples" / "content.txt").open("r") as f:
|
||||
content = f.read()
|
||||
with (Path(__file__).parent / "samples" / "preprocessed_content.txt").open(
|
||||
"r",
|
||||
) as f:
|
||||
expected_preprocess_content = f.read().rstrip()
|
||||
classifier = DocumentClassifier()
|
||||
with mock.patch("documents.classifier.ADVANCED_TEXT_PROCESSING_ENABLED", new=False):
|
||||
result = classifier.preprocess_content(content)
|
||||
assert result == expected_preprocess_content
|
||||
def test_supported_language(
|
||||
self,
|
||||
settings: Settings,
|
||||
samples: Path,
|
||||
content: str,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- The classifier language is English, the default
|
||||
WHEN:
|
||||
- Document content is preprocessed
|
||||
THEN:
|
||||
- Stop words are removed and the remaining words are stemmed
|
||||
"""
|
||||
settings.CLASSIFIER_LANGUAGE = "english"
|
||||
expected = (samples / "preprocessed_content_advanced.txt").read_text()
|
||||
|
||||
assert DocumentClassifier().preprocess_content(content) == expected.rstrip()
|
||||
|
||||
def test_preprocess_content_nltk_load_fail(mocker) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- NLTK stop words fail to load
|
||||
WHEN:
|
||||
- Classifier preprocesses a document's content
|
||||
THEN:
|
||||
- Processed content matches the expected output (unstemmed words)
|
||||
"""
|
||||
_module = mocker.MagicMock(name="nltk_corpus_mock")
|
||||
_module.stopwords.words.side_effect = AttributeError()
|
||||
mocker.patch.dict("sys.modules", {"nltk.corpus": _module})
|
||||
classifier = DocumentClassifier()
|
||||
with (Path(__file__).parent / "samples" / "content.txt").open("r") as f:
|
||||
content = f.read()
|
||||
with (Path(__file__).parent / "samples" / "preprocessed_content.txt").open(
|
||||
"r",
|
||||
) as f:
|
||||
expected_preprocess_content = f.read().rstrip()
|
||||
result = classifier.preprocess_content(content)
|
||||
assert result == expected_preprocess_content
|
||||
def test_unsupported_language(
|
||||
self,
|
||||
settings: Settings,
|
||||
samples: Path,
|
||||
content: str,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- No classifier language (the OCR language has no stemming support)
|
||||
WHEN:
|
||||
- Document content is preprocessed
|
||||
THEN:
|
||||
- The content is only lowercased and split into words
|
||||
"""
|
||||
settings.CLASSIFIER_LANGUAGE = None
|
||||
expected = (samples / "preprocessed_content.txt").read_text()
|
||||
|
||||
assert DocumentClassifier().preprocess_content(content) == expected.rstrip()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"language",
|
||||
[pytest.param("english", id="supported"), pytest.param(None, id="unsupported")],
|
||||
)
|
||||
def test_empty_content(self, settings: Settings, language: str | None) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Empty document content
|
||||
WHEN:
|
||||
- The content is preprocessed
|
||||
THEN:
|
||||
- The result is empty
|
||||
"""
|
||||
settings.CLASSIFIER_LANGUAGE = language
|
||||
|
||||
assert DocumentClassifier().preprocess_content("") == ""
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@@ -1237,3 +1240,53 @@ class TestClassifierTrainContent:
|
||||
first.content,
|
||||
"",
|
||||
]
|
||||
|
||||
|
||||
class TestTextAnalyzer:
|
||||
@pytest.mark.parametrize(
|
||||
"language",
|
||||
[
|
||||
pytest.param(language, id=language)
|
||||
for language in sorted(set(CLASSIFIER_LANGUAGES.values()))
|
||||
],
|
||||
)
|
||||
def test_builds_for_every_classifier_language(self, language: str) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A language the classifier supports
|
||||
WHEN:
|
||||
- Text is analyzed with its classifier language
|
||||
THEN:
|
||||
- Tokens are produced
|
||||
"""
|
||||
assert _text_analyzer(language).analyze("Paperless invoice 2026")
|
||||
|
||||
def test_english_removes_snowball_stop_words(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- English text with a contraction and stop words missing from
|
||||
Tantivy's own English list
|
||||
WHEN:
|
||||
- The text is analyzed
|
||||
THEN:
|
||||
- All stop words are removed, including the contraction
|
||||
- The remaining words are stemmed
|
||||
"""
|
||||
tokens = _text_analyzer("english").analyze(
|
||||
"They were about to pay the invoices, don't worry",
|
||||
)
|
||||
|
||||
assert tokens == ["pay", "invoic", "worri"]
|
||||
|
||||
def test_keeps_underscores_within_tokens(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Text with a word joined by an underscore
|
||||
WHEN:
|
||||
- The text is analyzed
|
||||
THEN:
|
||||
- The word stays one token
|
||||
"""
|
||||
tokens = _text_analyzer("english").analyze("tax_id")
|
||||
|
||||
assert tokens == ["tax_id"]
|
||||
|
||||
Reference in New Issue
Block a user