mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-09-21 17:08:33 +00:00
Performance: Preprocess classifier text with Tantivy instead of NLTK (#14127)
* Preprocesses classifier content with Tantivy instead of NLTK Tokenizing and stemming now happen in one Rust call instead of NLTK's Python tokenizer and per word stemming, which also removes the Redis backed stem cache from every preprocessing call. The output matches the NLTK pipeline closely; tokens containing digits are now stemmed, and the English stop words follow Snowball's list. Stemming and stop word removal apply whenever the OCR language is one of the supported classifier languages, so PAPERLESS_ENABLE_NLTK and PAPERLESS_NLTK_DIR are removed. * Copies packages instead of hardlinking them in backend CI, some NLTK thing * Adds a normalization to NFC to better fit what Tantivy expects
This commit is contained in:
Trenton H
@@ -81,7 +81,6 @@ updates:
|
||||
# Data, NLP, and Search
|
||||
data-nlp-search:
|
||||
patterns:
|
||||
- "nltk"
|
||||
- "scikit-learn"
|
||||
- "langdetect"
|
||||
- "rapidfuzz"
|
||||
|
||||
@@ -12,7 +12,9 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
env:
|
||||
DEFAULT_UV_VERSION: "0.12.x"
|
||||
NLTK_DATA: "/usr/share/nltk_data"
|
||||
# Match the Docker image: nltk refuses to read hardlinked data files, such as
|
||||
# the copy bundled with llama-index when uv links packages from its cache
|
||||
UV_LINK_MODE: copy
|
||||
permissions: {}
|
||||
jobs:
|
||||
changes:
|
||||
@@ -125,12 +127,8 @@ jobs:
|
||||
- name: List installed Python dependencies
|
||||
run: |
|
||||
uv pip list
|
||||
- name: Install NLTK data
|
||||
run: |
|
||||
uv run python -m nltk.downloader punkt punkt_tab snowball_data stopwords -d "${NLTK_DATA}"
|
||||
- name: Run tests
|
||||
env:
|
||||
NLTK_DATA: ${{ env.NLTK_DATA }}
|
||||
PAPERLESS_CI_TEST: 1
|
||||
PYTHON_VERSION: ${{ steps.setup-python.outputs.python-version }}
|
||||
run: |
|
||||
|
||||
@@ -199,10 +199,6 @@ RUN set -eux \
|
||||
--index https://download.pytorch.org/whl/cpu \
|
||||
--index-strategy unsafe-best-match \
|
||||
--requirements requirements.txt \
|
||||
&& echo "Installing NLTK data" \
|
||||
&& python3 -W ignore::RuntimeWarning -m nltk.downloader -d "/usr/share/nltk_data" snowball_data \
|
||||
&& python3 -W ignore::RuntimeWarning -m nltk.downloader -d "/usr/share/nltk_data" stopwords \
|
||||
&& python3 -W ignore::RuntimeWarning -m nltk.downloader -d "/usr/share/nltk_data" punkt_tab \
|
||||
&& echo "Cleaning up image" \
|
||||
&& apt-get --yes purge ${BUILD_PACKAGES} \
|
||||
&& apt-get --yes autoremove --purge \
|
||||
|
||||
+10
-17
@@ -413,18 +413,12 @@ details.
|
||||
|
||||
Defaults to `PAPERLESS_DATA_DIR/log/`.
|
||||
|
||||
#### [`PAPERLESS_NLTK_DIR=<path>`](#PAPERLESS_NLTK_DIR) {#PAPERLESS_NLTK_DIR}
|
||||
#### ~~[`PAPERLESS_NLTK_DIR`](#PAPERLESS_NLTK_DIR)~~ {#PAPERLESS_NLTK_DIR}
|
||||
|
||||
: This is where paperless will search for the data required for NLTK
|
||||
processing, if you are using it. If you are using the Docker image,
|
||||
this should not be changed, as the data is included in the image
|
||||
already.
|
||||
!!! failure "Removed in v3.2"
|
||||
|
||||
Previously, the location defaulted to `PAPERLESS_DATA_DIR/nltk`.
|
||||
Unless you are using this in a bare metal install or other setup,
|
||||
this folder is no longer needed and can be removed manually.
|
||||
|
||||
Defaults to `/usr/share/nltk_data`
|
||||
Removed and ignored. Any previously downloaded NLTK data folder can be
|
||||
deleted.
|
||||
|
||||
#### [`PAPERLESS_MODEL_FILE=<path>`](#PAPERLESS_MODEL_FILE) {#PAPERLESS_MODEL_FILE}
|
||||
|
||||
@@ -1190,15 +1184,14 @@ for details on how to set it.
|
||||
|
||||
Defaults to UTC.
|
||||
|
||||
#### [`PAPERLESS_ENABLE_NLTK=<bool>`](#PAPERLESS_ENABLE_NLTK) {#PAPERLESS_ENABLE_NLTK}
|
||||
#### ~~[`PAPERLESS_ENABLE_NLTK`](#PAPERLESS_ENABLE_NLTK)~~ {#PAPERLESS_ENABLE_NLTK}
|
||||
|
||||
: Enables or disables the advanced natural language processing
|
||||
used during automatic classification. If disabled, paperless will
|
||||
still perform some basic text pre-processing before matching.
|
||||
!!! failure "Removed in v3.2"
|
||||
|
||||
: See also `PAPERLESS_NLTK_DIR`.
|
||||
|
||||
Defaults to true, enabling the feature.
|
||||
Removed and ignored. Automatic classification always removes stop words
|
||||
and stems words when the primary OCR language is Danish, Dutch, English,
|
||||
Finnish, French, German, Italian, Norwegian, Portuguese, Russian, Spanish
|
||||
or Swedish. Other languages are only lowercased and split into words.
|
||||
|
||||
#### [`PAPERLESS_CLASSIFIER_MATCH_THRESHOLD=<float>`](#PAPERLESS_CLASSIFIER_MATCH_THRESHOLD) {#PAPERLESS_CLASSIFIER_MATCH_THRESHOLD}
|
||||
|
||||
|
||||
@@ -430,11 +430,6 @@ to a positive number to enable polling and disable native filesystem notificatio
|
||||
This will reduce the size of generated PDF documents. You'll most likely need to compile this yourself, because this
|
||||
software has been patented until around 2017 and binary packages are not available for most distributions.
|
||||
|
||||
**Optional: download the NLTK data**
|
||||
If using the NLTK machine-learning processing (see [`PAPERLESS_ENABLE_NLTK`](configuration.md#PAPERLESS_ENABLE_NLTK) for details),
|
||||
download the NLTK data for the Snowball Stemmer, Stopwords and Punkt tokenizer to `/usr/share/nltk_data`. Refer to the [NLTK
|
||||
instructions](https://www.nltk.org/data.html) for details on how to download the data.
|
||||
|
||||
#### After installation
|
||||
|
||||
Your Paperless-ngx instance should now be accessible at `http://localhost:8000` (or similar, depending on your configuration).
|
||||
@@ -650,9 +645,6 @@ hardware, but a few settings can improve performance:
|
||||
`PAPERLESS_OCR_CLEAN=none`. This will speed up OCR times and use
|
||||
less memory at the expense of slightly worse OCR results.
|
||||
- If using Docker, consider setting [`PAPERLESS_WEBSERVER_WORKERS`](configuration.md#PAPERLESS_WEBSERVER_WORKERS) to 1. This will save some memory.
|
||||
- Consider setting [`PAPERLESS_ENABLE_NLTK`](configuration.md#PAPERLESS_ENABLE_NLTK) to false, to disable the
|
||||
more advanced language processing, which can take more memory and
|
||||
processing time.
|
||||
|
||||
For details, refer to [configuration](configuration.md).
|
||||
|
||||
|
||||
@@ -55,7 +55,6 @@ dependencies = [
|
||||
"llama-index-embeddings-openai-like>=0.2.2",
|
||||
"llama-index-llms-ollama>=0.9.1",
|
||||
"llama-index-llms-openai-like>=0.7.1",
|
||||
"nltk~=3.10.0",
|
||||
"ocrmypdf>=17.7,<17.12",
|
||||
"openai>=2.48",
|
||||
"pathvalidate~=3.3.1",
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
"""
|
||||
English stop words from the Snowball project, kept as published at
|
||||
https://snowballstem.org/algorithms/english/stop.txt, under this license:
|
||||
|
||||
Copyright (c) 2001, Dr Martin Porter
|
||||
Copyright (c) 2004,2005, Richard Boulton
|
||||
Copyright (c) 2013, Yoshiki Shibukawa
|
||||
Copyright (c) 2006-2025, Olly Betts
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions
|
||||
are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
3. Neither the name of the Snowball project nor the names of its contributors
|
||||
may be used to endorse or promote products derived from this software
|
||||
without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
|
||||
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
|
||||
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
"""
|
||||
|
||||
from typing import Final
|
||||
|
||||
ENGLISH: Final[tuple[str, ...]] = (
|
||||
"i",
|
||||
"me",
|
||||
"my",
|
||||
"myself",
|
||||
"we",
|
||||
"our",
|
||||
"ours",
|
||||
"ourselves",
|
||||
"you",
|
||||
"your",
|
||||
"yours",
|
||||
"yourself",
|
||||
"yourselves",
|
||||
"he",
|
||||
"him",
|
||||
"his",
|
||||
"himself",
|
||||
"she",
|
||||
"her",
|
||||
"hers",
|
||||
"herself",
|
||||
"it",
|
||||
"its",
|
||||
"itself",
|
||||
"they",
|
||||
"them",
|
||||
"their",
|
||||
"theirs",
|
||||
"themselves",
|
||||
"what",
|
||||
"which",
|
||||
"who",
|
||||
"whom",
|
||||
"this",
|
||||
"that",
|
||||
"these",
|
||||
"those",
|
||||
"am",
|
||||
"is",
|
||||
"are",
|
||||
"was",
|
||||
"were",
|
||||
"be",
|
||||
"been",
|
||||
"being",
|
||||
"have",
|
||||
"has",
|
||||
"had",
|
||||
"having",
|
||||
"do",
|
||||
"does",
|
||||
"did",
|
||||
"doing",
|
||||
"would",
|
||||
"should",
|
||||
"could",
|
||||
"ought",
|
||||
"i'm",
|
||||
"you're",
|
||||
"he's",
|
||||
"she's",
|
||||
"it's",
|
||||
"we're",
|
||||
"they're",
|
||||
"i've",
|
||||
"you've",
|
||||
"we've",
|
||||
"they've",
|
||||
"i'd",
|
||||
"you'd",
|
||||
"he'd",
|
||||
"she'd",
|
||||
"we'd",
|
||||
"they'd",
|
||||
"i'll",
|
||||
"you'll",
|
||||
"he'll",
|
||||
"she'll",
|
||||
"we'll",
|
||||
"they'll",
|
||||
"isn't",
|
||||
"aren't",
|
||||
"wasn't",
|
||||
"weren't",
|
||||
"hasn't",
|
||||
"haven't",
|
||||
"hadn't",
|
||||
"doesn't",
|
||||
"don't",
|
||||
"didn't",
|
||||
"won't",
|
||||
"wouldn't",
|
||||
"shan't",
|
||||
"shouldn't",
|
||||
"can't",
|
||||
"cannot",
|
||||
"couldn't",
|
||||
"mustn't",
|
||||
"let's",
|
||||
"that's",
|
||||
"who's",
|
||||
"what's",
|
||||
"here's",
|
||||
"there's",
|
||||
"when's",
|
||||
"where's",
|
||||
"why's",
|
||||
"how's",
|
||||
"a",
|
||||
"an",
|
||||
"the",
|
||||
"and",
|
||||
"but",
|
||||
"if",
|
||||
"or",
|
||||
"because",
|
||||
"as",
|
||||
"until",
|
||||
"while",
|
||||
"of",
|
||||
"at",
|
||||
"by",
|
||||
"for",
|
||||
"with",
|
||||
"about",
|
||||
"against",
|
||||
"between",
|
||||
"into",
|
||||
"through",
|
||||
"during",
|
||||
"before",
|
||||
"after",
|
||||
"above",
|
||||
"below",
|
||||
"to",
|
||||
"from",
|
||||
"up",
|
||||
"down",
|
||||
"in",
|
||||
"out",
|
||||
"on",
|
||||
"off",
|
||||
"over",
|
||||
"under",
|
||||
"again",
|
||||
"further",
|
||||
"then",
|
||||
"once",
|
||||
"here",
|
||||
"there",
|
||||
"when",
|
||||
"where",
|
||||
"why",
|
||||
"how",
|
||||
"all",
|
||||
"any",
|
||||
"both",
|
||||
"each",
|
||||
"few",
|
||||
"more",
|
||||
"most",
|
||||
"other",
|
||||
"some",
|
||||
"such",
|
||||
"no",
|
||||
"nor",
|
||||
"not",
|
||||
"only",
|
||||
"own",
|
||||
"same",
|
||||
"so",
|
||||
"than",
|
||||
"too",
|
||||
"very",
|
||||
)
|
||||
@@ -2,27 +2,17 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import pickle
|
||||
import uuid
|
||||
from binascii import hexlify
|
||||
from collections import OrderedDict
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import Any
|
||||
from typing import Final
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.cache import cache
|
||||
from django.core.cache import caches
|
||||
|
||||
from documents.models import Document
|
||||
from paperless.signed_pickle import SignedPickleError
|
||||
from paperless.signed_pickle import signed_pickle_dumps
|
||||
from paperless.signed_pickle import signed_pickle_loads
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from django.core.cache.backends.base import BaseCache
|
||||
|
||||
from documents.classifier import DocumentClassifier
|
||||
|
||||
logger = logging.getLogger("paperless.caching")
|
||||
@@ -65,82 +55,6 @@ CACHE_50_MINUTES: Final[int] = 50 * CACHE_1_MINUTE
|
||||
# Deliberately longer than any entry it names
|
||||
LLM_CACHE_GENERATION_TIMEOUT: Final[int] = 2 * CACHE_50_MINUTES
|
||||
|
||||
read_cache = caches["read-cache"]
|
||||
|
||||
|
||||
class LRUCache:
|
||||
def __init__(self, capacity: int = 128):
|
||||
self._data = OrderedDict()
|
||||
self.capacity = capacity
|
||||
|
||||
def get(self, key, default=None) -> Any | None:
|
||||
if key in self._data:
|
||||
self._data.move_to_end(key)
|
||||
return self._data[key]
|
||||
return default
|
||||
|
||||
def set(self, key, value) -> None:
|
||||
self._data[key] = value
|
||||
self._data.move_to_end(key)
|
||||
while len(self._data) > self.capacity:
|
||||
self._data.popitem(last=False)
|
||||
|
||||
|
||||
class StoredLRUCache(LRUCache):
|
||||
"""
|
||||
LRU cache that can persist its entire contents as a single entry in a backend cache.
|
||||
|
||||
Useful for sharing a cache across multiple workers or processes.
|
||||
|
||||
Workflow:
|
||||
1. Load the cache state from the backend using `load()`.
|
||||
2. Use `get()` and `set()` locally as usual.
|
||||
3. Persist changes back to the backend using `save()`.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
backend_key: str,
|
||||
capacity: int = 128,
|
||||
backend: BaseCache = read_cache,
|
||||
backend_ttl=settings.CACHALOT_TIMEOUT,
|
||||
):
|
||||
if backend_key is None:
|
||||
raise ValueError("backend_key is mandatory")
|
||||
super().__init__(capacity)
|
||||
self._backend_key = backend_key
|
||||
self._backend = backend
|
||||
self.backend_ttl = backend_ttl
|
||||
|
||||
def load(self) -> None:
|
||||
"""
|
||||
Load the whole cache content from backend storage.
|
||||
|
||||
If no valid cached data exists in the backend, the local cache is cleared.
|
||||
"""
|
||||
serialized_data = self._backend.get(self._backend_key)
|
||||
try:
|
||||
self._data = (
|
||||
signed_pickle_loads(serialized_data)
|
||||
if serialized_data
|
||||
else OrderedDict()
|
||||
)
|
||||
except (SignedPickleError, pickle.PickleError):
|
||||
logger.warning(
|
||||
"Cache exists in backend but could not be read (possibly invalid format)",
|
||||
)
|
||||
|
||||
def save(self) -> None:
|
||||
"""Save the entire local cache to the backend as a serialized object.
|
||||
|
||||
The backend entry will expire after the configured TTL.
|
||||
"""
|
||||
self._backend.set(
|
||||
self._backend_key,
|
||||
signed_pickle_dumps(self._data),
|
||||
self.backend_ttl,
|
||||
)
|
||||
|
||||
|
||||
def get_suggestion_cache_key(document_id: int) -> str:
|
||||
"""
|
||||
|
||||
+41
-102
@@ -1,9 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import hmac
|
||||
import logging
|
||||
import pickle
|
||||
import re
|
||||
import unicodedata
|
||||
import warnings
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
@@ -17,6 +19,7 @@ if TYPE_CHECKING:
|
||||
from typing import BinaryIO
|
||||
from typing import Self
|
||||
|
||||
import tantivy
|
||||
from numpy import ndarray
|
||||
from sklearn.neural_network import MLPClassifier
|
||||
|
||||
@@ -25,12 +28,12 @@ from django.core.cache import cache
|
||||
from django.core.cache import caches
|
||||
from django.db.models import Prefetch
|
||||
|
||||
from documents._snowball_stopwords import ENGLISH as ENGLISH_STOP_WORDS
|
||||
from documents.caching import CACHE_5_MINUTES
|
||||
from documents.caching import CACHE_50_MINUTES
|
||||
from documents.caching import CLASSIFIER_HASH_KEY
|
||||
from documents.caching import CLASSIFIER_MODIFIED_KEY
|
||||
from documents.caching import CLASSIFIER_VERSION_KEY
|
||||
from documents.caching import StoredLRUCache
|
||||
from documents.models import Document
|
||||
from documents.models import MatchingModel
|
||||
from documents.models import Tag
|
||||
@@ -61,14 +64,9 @@ def _predict_with_threshold(classifier, X, threshold: float) -> int | None:
|
||||
return best_class
|
||||
|
||||
|
||||
ADVANCED_TEXT_PROCESSING_ENABLED = (
|
||||
settings.NLTK_LANGUAGE is not None and settings.NLTK_ENABLED
|
||||
)
|
||||
|
||||
read_cache = caches["read-cache"]
|
||||
|
||||
|
||||
RE_DIGIT = re.compile(r"\d")
|
||||
RE_WORD = re.compile(r"\b[\w]+\b") # words that may contain digits
|
||||
|
||||
# Documents whose content is fetched per query while training
|
||||
@@ -118,6 +116,33 @@ class _SignedFileWriter:
|
||||
self._temp.unlink(missing_ok=True)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _text_analyzer(language: str) -> tantivy.TextAnalyzer:
|
||||
"""
|
||||
Builds the cached analyzer for a language: word tokens, lowercase, stop words, stemmer.
|
||||
|
||||
Long tokens are kept and accents are not folded to ASCII, since stemmers
|
||||
for languages such as French and German rely on accents.
|
||||
"""
|
||||
import tantivy
|
||||
|
||||
if language == "english":
|
||||
# Tantivy's builtin English list is much shorter than Snowball's.
|
||||
# Split contractions ("don't") on word characters, as content is, so they match
|
||||
stop_words = tantivy.Filter.custom_stopword(
|
||||
sorted({t for word in ENGLISH_STOP_WORDS for t in RE_WORD.findall(word)}),
|
||||
)
|
||||
else:
|
||||
stop_words = tantivy.Filter.stopword(language)
|
||||
return (
|
||||
tantivy.TextAnalyzerBuilder(tantivy.Tokenizer.regex(r"\w+"))
|
||||
.filter(tantivy.Filter.lowercase())
|
||||
.filter(stop_words)
|
||||
.filter(tantivy.Filter.stemmer(language))
|
||||
.build()
|
||||
)
|
||||
|
||||
|
||||
class IncompatibleClassifierVersionError(Exception):
|
||||
def __init__(self, message: str, *args: object) -> None:
|
||||
self.message: str = message
|
||||
@@ -177,6 +202,7 @@ class DocumentClassifier:
|
||||
# v10 - HMAC-signed model file
|
||||
# v11 - Use sample_weight for balanced training; predict_proba with threshold;
|
||||
# drop training-only MLP state before saving
|
||||
# Tantivy text preprocessing
|
||||
FORMAT_VERSION = 11
|
||||
|
||||
HMAC_SIZE = 32 # SHA-256 digest length
|
||||
@@ -194,16 +220,6 @@ class DocumentClassifier:
|
||||
self.correspondent_classifier = None
|
||||
self.document_type_classifier = None
|
||||
self.storage_path_classifier = None
|
||||
self._stemmer = None
|
||||
# 10,000 elements roughly use 200 to 500 KB per worker,
|
||||
# and also in the shared Redis cache,
|
||||
# Keep this cache small to minimize lookup and I/O latency.
|
||||
if ADVANCED_TEXT_PROCESSING_ENABLED:
|
||||
self._stem_cache = StoredLRUCache(
|
||||
f"stem_cache_v{self.FORMAT_VERSION}",
|
||||
capacity=10000,
|
||||
)
|
||||
self._stop_words = None
|
||||
|
||||
def _update_data_vectorizer_hash(self) -> None:
|
||||
self.data_vectorizer_hash = sha256(
|
||||
@@ -454,7 +470,6 @@ class DocumentClassifier:
|
||||
doc = docs.get(pk)
|
||||
yield self.preprocess_content(
|
||||
doc.content if doc is not None else "",
|
||||
shared_cache=False,
|
||||
)
|
||||
|
||||
self.data_vectorizer = CountVectorizer(
|
||||
@@ -564,99 +579,23 @@ class DocumentClassifier:
|
||||
|
||||
return True
|
||||
|
||||
def _init_advanced_text_processing(self):
|
||||
if self._stop_words is None or self._stemmer is None:
|
||||
import nltk
|
||||
from nltk.corpus import stopwords
|
||||
from nltk.stem import SnowballStemmer
|
||||
|
||||
# Not really hacky, since it isn't private and is documented, but
|
||||
# set the search path for NLTK data to the single location it should be in
|
||||
nltk.data.path = [settings.NLTK_DIR]
|
||||
try:
|
||||
# Preload the corpus early, to force the lazy loader to transform
|
||||
stopwords.ensure_loaded()
|
||||
|
||||
# Do some one time setup
|
||||
# Sometimes, somehow, there's multiple threads loading the corpus
|
||||
# and it's not thread safe, raising an AttributeError
|
||||
self._stemmer = SnowballStemmer(settings.NLTK_LANGUAGE)
|
||||
self._stop_words = frozenset(stopwords.words(settings.NLTK_LANGUAGE))
|
||||
except AttributeError:
|
||||
logger.debug("Could not initialize NLTK for advanced text processing.")
|
||||
return False
|
||||
return True
|
||||
|
||||
def stem_and_skip_stop_words(self, words: list[str], *, shared_cache=True):
|
||||
"""
|
||||
Reduce a list of words to their stem. Stop words are converted to empty strings.
|
||||
:param words: the list of words to stem
|
||||
"""
|
||||
|
||||
def _stem_and_skip_stop_word(word: str):
|
||||
"""
|
||||
Reduce a given word to its stem. If it's a stop word, return an empty string.
|
||||
E.g. "amazement", "amaze" and "amazed" all return "amaz".
|
||||
"""
|
||||
cached = self._stem_cache.get(word)
|
||||
if cached is not None:
|
||||
return cached
|
||||
elif word in self._stop_words:
|
||||
return ""
|
||||
# Assumption: words that contain numbers are never stemmed
|
||||
elif RE_DIGIT.search(word):
|
||||
return word
|
||||
else:
|
||||
result = self._stemmer.stem(word)
|
||||
self._stem_cache.set(word, result)
|
||||
return result
|
||||
|
||||
if shared_cache:
|
||||
self._stem_cache.load()
|
||||
|
||||
# Stem the words and skip stop words
|
||||
result = " ".join(
|
||||
filter(None, (_stem_and_skip_stop_word(w) for w in words)),
|
||||
)
|
||||
if shared_cache:
|
||||
self._stem_cache.save()
|
||||
return result
|
||||
|
||||
def preprocess_content(
|
||||
self,
|
||||
content: str,
|
||||
*,
|
||||
shared_cache=True,
|
||||
) -> str:
|
||||
def preprocess_content(self, content: str) -> str:
|
||||
"""
|
||||
Process the contents of a document, distilling it down into
|
||||
words which are meaningful to the content.
|
||||
|
||||
A stemmer cache is shared across workers with the parameter "shared_cache".
|
||||
This is unnecessary when training the classifier.
|
||||
"""
|
||||
|
||||
# Lower case the document, reduce space,
|
||||
# and keep only letters and digits.
|
||||
content = " ".join(match.group().lower() for match in RE_WORD.finditer(content))
|
||||
|
||||
if ADVANCED_TEXT_PROCESSING_ENABLED:
|
||||
from nltk.tokenize import word_tokenize
|
||||
|
||||
if not self._init_advanced_text_processing():
|
||||
return content
|
||||
# Tokenize
|
||||
# This splits the content into tokens, roughly words
|
||||
words = word_tokenize(content, language=settings.NLTK_LANGUAGE)
|
||||
# Stem the words and skip stop words
|
||||
content = self.stem_and_skip_stop_words(words, shared_cache=shared_cache)
|
||||
|
||||
return content
|
||||
language = settings.CLASSIFIER_LANGUAGE
|
||||
content = unicodedata.normalize("NFC", content)
|
||||
if language is None:
|
||||
return " ".join(
|
||||
match.group().lower() for match in RE_WORD.finditer(content)
|
||||
)
|
||||
return " ".join(_text_analyzer(language).analyze(content))
|
||||
|
||||
def _get_vectorizer_cache_key(self, content: str):
|
||||
hash = sha256(content.encode())
|
||||
hash.update(
|
||||
f"|{self.FORMAT_VERSION}|{settings.NLTK_LANGUAGE}|{settings.NLTK_ENABLED}|{self.data_vectorizer_hash}".encode(),
|
||||
f"|{self.FORMAT_VERSION}|{settings.CLASSIFIER_LANGUAGE}|{self.data_vectorizer_hash}".encode(),
|
||||
)
|
||||
return f"vectorized_content_{hash.hexdigest()}"
|
||||
|
||||
|
||||
@@ -1 +1 @@
|
||||
sampl textual document content includ mani charact possibl check classifi vector hey 00 test0707 content exampl document creat 2025 06 25 digit 0123456789 punctuat english text quick brown fox jump lazi dog english stop word accent latin diacrit àâäæçéèêëîïôœùûüÿñ arab لقد قام المترجم بعمل جيد greek αλφα βήτα γάμμα δέλτα ωμέγα cyril привет как дела добро пожаловать chines simplifi 你好 世界 今天的天气很好 chines tradit 歡迎來到世界 今天天氣很好 japanes kanji hiragana katakana 東京へ行きます カタカナ ひらがな 漢字 korean hangul 안녕하세요 오늘 날씨 어때요 arab مرحب ا كيف حالك hebrew שלום מה שלומך emoji symbol µ math ₀ x² dx π 3 14159 e ρ ε₀ currenc 1 date format 25 06 2025 june 25 2025 2025年6月25日 quot french bonjour ça va quot german guten tag wie geht newlin test r n r tab ttest tspace 192 33601010101 end document
|
||||
sampl textual document content includ mani charact possibl check classifi vector hey 00 test0707 content exampl document creat 2025 06 25 digit 0123456789 punctuat english text quick brown fox jump lazi dog english stop word accent latin diacrit àâäæçéèêëîïôœùûüÿñ arab لقد قام المترجم بعمل جيد greek αλφα βήτα γάμμα δέλτα ωμέγα cyril привет как дела добро пожаловать chines simplifi 你好 世界 今天的天气很好 chines tradit 歡迎來到世界 今天天氣很好 japanes kanji hiragana katakana 東京へ行きます カタカナ ひらがな 漢字 korean hangul 안녕하세요 오늘 날씨 어때요 arab مرحبًا كيف حالك hebrew שלום מה שלומך emoji ️ symbol µ math x dx π 3 14159 e ρ ε currenc 1 date format 25 06 2025 june 25 2025 2025年6月25日 quot french bonjour ça va quot german guten tag wie geht newlin test r n r tab ttest tspace 192 33601010101 end document
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
from documents.caching import StoredLRUCache
|
||||
from paperless.signed_pickle import HMAC_SIZE
|
||||
from paperless.signed_pickle import signed_pickle_dumps
|
||||
from paperless.signed_pickle import signed_pickle_loads
|
||||
|
||||
|
||||
def test_lru_cache_entries() -> None:
|
||||
CACHE_TTL = 1
|
||||
# LRU cache with a capacity of 2 elements
|
||||
cache = StoredLRUCache("test_lru_cache_key", 2, backend_ttl=CACHE_TTL)
|
||||
cache.set(1, 1)
|
||||
cache.set(2, 2)
|
||||
assert cache.get(2) == 2
|
||||
assert cache.get(1) == 1
|
||||
|
||||
# The oldest entry (2) should be removed
|
||||
cache.set(3, 3)
|
||||
assert cache.get(3) == 3
|
||||
assert not cache.get(2)
|
||||
assert cache.get(1) == 1
|
||||
|
||||
# Save the cache, restore it and check it overwrites the current cache in memory
|
||||
cache.save()
|
||||
cache.set(4, 4)
|
||||
assert not cache.get(3)
|
||||
cache.load()
|
||||
assert not cache.get(4)
|
||||
assert cache.get(3) == 3
|
||||
assert cache.get(1) == 1
|
||||
|
||||
|
||||
def test_stored_lru_cache_key_ttl(mocker) -> None:
|
||||
mock_backend = mocker.Mock()
|
||||
cache = StoredLRUCache("test_key", backend=mock_backend, backend_ttl=321)
|
||||
|
||||
# Simulate storing values
|
||||
cache.set("x", "X")
|
||||
cache.set("y", "Y")
|
||||
cache.save()
|
||||
|
||||
# Assert backend.set was called with pickled data, key and TTL
|
||||
mock_backend.set.assert_called_once()
|
||||
key, data, timeout = mock_backend.set.call_args[0]
|
||||
assert key == "test_key"
|
||||
assert timeout == 321
|
||||
assert signed_pickle_loads(data) == {"x": "X", "y": "Y"}
|
||||
|
||||
|
||||
def test_stored_lru_cache_rejects_tampered_data(mocker) -> None:
|
||||
serialized_data = bytearray(signed_pickle_dumps({"x": "X"}))
|
||||
serialized_data[HMAC_SIZE] ^= 0xFF
|
||||
mock_backend = mocker.Mock()
|
||||
mock_backend.get.return_value = bytes(serialized_data)
|
||||
cache = StoredLRUCache("test_key", backend=mock_backend)
|
||||
|
||||
cache.load()
|
||||
|
||||
assert cache.get("x") is None
|
||||
@@ -20,6 +20,7 @@ from documents.classifier import ClassifierModelCorruptError
|
||||
from documents.classifier import DocumentClassifier
|
||||
from documents.classifier import IncompatibleClassifierVersionError
|
||||
from documents.classifier import _predict_with_threshold
|
||||
from documents.classifier import _text_analyzer
|
||||
from documents.classifier import load_classifier
|
||||
from documents.models import Correspondent
|
||||
from documents.models import Document
|
||||
@@ -30,11 +31,12 @@ from documents.models import Tag
|
||||
from documents.tests.factories import DocumentFactory
|
||||
from documents.tests.factories import TagFactory
|
||||
from documents.tests.utils import DirectoriesMixin
|
||||
from paperless.settings import CLASSIFIER_LANGUAGES
|
||||
from paperless.signed_pickle import HMAC_SIZE
|
||||
from paperless.signed_pickle import signed_pickle_dumps
|
||||
|
||||
|
||||
def dummy_preprocess(content: str, **kwargs):
|
||||
def dummy_preprocess(content: str) -> str:
|
||||
"""
|
||||
Simpler, faster pre-processing for testing purposes
|
||||
"""
|
||||
@@ -1043,68 +1045,69 @@ def test_classifier_match_threshold_default() -> None:
|
||||
assert settings.CLASSIFIER_MATCH_THRESHOLD == 0.6
|
||||
|
||||
|
||||
def test_preprocess_content() -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Advanced text processing is enabled (default)
|
||||
WHEN:
|
||||
- Classifier preprocesses a document's content
|
||||
THEN:
|
||||
- Processed content matches the expected output (stemmed words)
|
||||
"""
|
||||
with (Path(__file__).parent / "samples" / "content.txt").open("r") as f:
|
||||
content = f.read()
|
||||
with (Path(__file__).parent / "samples" / "preprocessed_content_advanced.txt").open(
|
||||
"r",
|
||||
) as f:
|
||||
expected_preprocess_content = f.read().rstrip()
|
||||
classifier = DocumentClassifier()
|
||||
result = classifier.preprocess_content(content)
|
||||
assert result == expected_preprocess_content
|
||||
class TestPreprocessContent:
|
||||
@pytest.fixture
|
||||
def samples(self) -> Path:
|
||||
return Path(__file__).parent / "samples"
|
||||
|
||||
@pytest.fixture
|
||||
def content(self, samples: Path) -> str:
|
||||
return (samples / "content.txt").read_text()
|
||||
|
||||
def test_preprocess_content_nltk_disabled() -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Advanced text processing is disabled
|
||||
WHEN:
|
||||
- Classifier preprocesses a document's content
|
||||
THEN:
|
||||
- Processed content matches the expected output (unstemmed words)
|
||||
"""
|
||||
with (Path(__file__).parent / "samples" / "content.txt").open("r") as f:
|
||||
content = f.read()
|
||||
with (Path(__file__).parent / "samples" / "preprocessed_content.txt").open(
|
||||
"r",
|
||||
) as f:
|
||||
expected_preprocess_content = f.read().rstrip()
|
||||
classifier = DocumentClassifier()
|
||||
with mock.patch("documents.classifier.ADVANCED_TEXT_PROCESSING_ENABLED", new=False):
|
||||
result = classifier.preprocess_content(content)
|
||||
assert result == expected_preprocess_content
|
||||
def test_supported_language(
|
||||
self,
|
||||
settings: Settings,
|
||||
samples: Path,
|
||||
content: str,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- The classifier language is English, the default
|
||||
WHEN:
|
||||
- Document content is preprocessed
|
||||
THEN:
|
||||
- Stop words are removed and the remaining words are stemmed
|
||||
"""
|
||||
settings.CLASSIFIER_LANGUAGE = "english"
|
||||
expected = (samples / "preprocessed_content_advanced.txt").read_text()
|
||||
|
||||
assert DocumentClassifier().preprocess_content(content) == expected.rstrip()
|
||||
|
||||
def test_preprocess_content_nltk_load_fail(mocker) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- NLTK stop words fail to load
|
||||
WHEN:
|
||||
- Classifier preprocesses a document's content
|
||||
THEN:
|
||||
- Processed content matches the expected output (unstemmed words)
|
||||
"""
|
||||
_module = mocker.MagicMock(name="nltk_corpus_mock")
|
||||
_module.stopwords.words.side_effect = AttributeError()
|
||||
mocker.patch.dict("sys.modules", {"nltk.corpus": _module})
|
||||
classifier = DocumentClassifier()
|
||||
with (Path(__file__).parent / "samples" / "content.txt").open("r") as f:
|
||||
content = f.read()
|
||||
with (Path(__file__).parent / "samples" / "preprocessed_content.txt").open(
|
||||
"r",
|
||||
) as f:
|
||||
expected_preprocess_content = f.read().rstrip()
|
||||
result = classifier.preprocess_content(content)
|
||||
assert result == expected_preprocess_content
|
||||
def test_unsupported_language(
|
||||
self,
|
||||
settings: Settings,
|
||||
samples: Path,
|
||||
content: str,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- No classifier language (the OCR language has no stemming support)
|
||||
WHEN:
|
||||
- Document content is preprocessed
|
||||
THEN:
|
||||
- The content is only lowercased and split into words
|
||||
"""
|
||||
settings.CLASSIFIER_LANGUAGE = None
|
||||
expected = (samples / "preprocessed_content.txt").read_text()
|
||||
|
||||
assert DocumentClassifier().preprocess_content(content) == expected.rstrip()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"language",
|
||||
[pytest.param("english", id="supported"), pytest.param(None, id="unsupported")],
|
||||
)
|
||||
def test_empty_content(self, settings: Settings, language: str | None) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Empty document content
|
||||
WHEN:
|
||||
- The content is preprocessed
|
||||
THEN:
|
||||
- The result is empty
|
||||
"""
|
||||
settings.CLASSIFIER_LANGUAGE = language
|
||||
|
||||
assert DocumentClassifier().preprocess_content("") == ""
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@@ -1237,3 +1240,53 @@ class TestClassifierTrainContent:
|
||||
first.content,
|
||||
"",
|
||||
]
|
||||
|
||||
|
||||
class TestTextAnalyzer:
|
||||
@pytest.mark.parametrize(
|
||||
"language",
|
||||
[
|
||||
pytest.param(language, id=language)
|
||||
for language in sorted(set(CLASSIFIER_LANGUAGES.values()))
|
||||
],
|
||||
)
|
||||
def test_builds_for_every_classifier_language(self, language: str) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A language the classifier supports
|
||||
WHEN:
|
||||
- Text is analyzed with its classifier language
|
||||
THEN:
|
||||
- Tokens are produced
|
||||
"""
|
||||
assert _text_analyzer(language).analyze("Paperless invoice 2026")
|
||||
|
||||
def test_english_removes_snowball_stop_words(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- English text with a contraction and stop words missing from
|
||||
Tantivy's own English list
|
||||
WHEN:
|
||||
- The text is analyzed
|
||||
THEN:
|
||||
- All stop words are removed, including the contraction
|
||||
- The remaining words are stemmed
|
||||
"""
|
||||
tokens = _text_analyzer("english").analyze(
|
||||
"They were about to pay the invoices, don't worry",
|
||||
)
|
||||
|
||||
assert tokens == ["pay", "invoic", "worri"]
|
||||
|
||||
def test_keeps_underscores_within_tokens(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Text with a word joined by an underscore
|
||||
WHEN:
|
||||
- The text is analyzed
|
||||
THEN:
|
||||
- The word stays one token
|
||||
"""
|
||||
tokens = _text_analyzer("english").analyze("tax_id")
|
||||
|
||||
assert tokens == ["tax_id"]
|
||||
|
||||
@@ -73,8 +73,6 @@ SHARE_LINK_BUNDLE_DIR = MEDIA_ROOT / "documents" / "share_link_bundles"
|
||||
|
||||
DATA_DIR = get_path_from_env("PAPERLESS_DATA_DIR", BASE_DIR.parent / "data")
|
||||
|
||||
NLTK_DIR = get_path_from_env("PAPERLESS_NLTK_DIR", "/usr/share/nltk_data")
|
||||
|
||||
# Check deprecated setting first
|
||||
EMPTY_TRASH_DIR = (
|
||||
get_path_from_env("PAPERLESS_TRASH_DIR", os.getenv("PAPERLESS_EMPTY_TRASH_DIR"))
|
||||
@@ -1067,39 +1065,30 @@ APP_LOGO = os.getenv("PAPERLESS_APP_LOGO", None)
|
||||
###############################################################################
|
||||
|
||||
|
||||
def _get_nltk_language_setting(ocr_lang: str) -> str | None:
|
||||
CLASSIFIER_LANGUAGES: Final[dict[str, str]] = {
|
||||
"dan": "danish",
|
||||
"nld": "dutch",
|
||||
"eng": "english",
|
||||
"fin": "finnish",
|
||||
"fra": "french",
|
||||
"deu": "german",
|
||||
"ita": "italian",
|
||||
"nor": "norwegian",
|
||||
"por": "portuguese",
|
||||
"rus": "russian",
|
||||
"spa": "spanish",
|
||||
"swe": "swedish",
|
||||
}
|
||||
|
||||
|
||||
def _get_classifier_language_setting(ocr_lang: str) -> str | None:
|
||||
"""
|
||||
Maps an ISO-639-1 language code supported by Tesseract into
|
||||
an optional NLTK language name. This is the set of common supported
|
||||
languages for all the NLTK data used.
|
||||
Maps the primary Tesseract language to the classifier's stemming
|
||||
language, or None if unsupported.
|
||||
|
||||
Assumption: The primary language is first
|
||||
|
||||
NLTK Languages:
|
||||
- https://www.nltk.org/api/nltk.stem.snowball.html#nltk.stem.snowball.SnowballStemmer
|
||||
- https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/tokenizers/punkt.zip
|
||||
- https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/corpora/stopwords.zip
|
||||
|
||||
The common intersection between all languages in those 3 is handled here
|
||||
|
||||
"""
|
||||
ocr_lang = ocr_lang.split("+", maxsplit=1)[0]
|
||||
iso_code_to_nltk = {
|
||||
"dan": "danish",
|
||||
"nld": "dutch",
|
||||
"eng": "english",
|
||||
"fin": "finnish",
|
||||
"fra": "french",
|
||||
"deu": "german",
|
||||
"ita": "italian",
|
||||
"nor": "norwegian",
|
||||
"por": "portuguese",
|
||||
"rus": "russian",
|
||||
"spa": "spanish",
|
||||
"swe": "swedish",
|
||||
}
|
||||
|
||||
return iso_code_to_nltk.get(ocr_lang)
|
||||
return CLASSIFIER_LANGUAGES.get(ocr_lang.split("+", maxsplit=1)[0])
|
||||
|
||||
|
||||
def _get_search_language_setting(ocr_lang: str) -> str | None:
|
||||
@@ -1145,9 +1134,7 @@ def _get_search_language_setting(ocr_lang: str) -> str | None:
|
||||
return _ocr_to_search.get(primary)
|
||||
|
||||
|
||||
NLTK_ENABLED: Final[bool] = get_bool_from_env("PAPERLESS_ENABLE_NLTK", "yes")
|
||||
|
||||
NLTK_LANGUAGE: str | None = _get_nltk_language_setting(OCR_LANGUAGE)
|
||||
CLASSIFIER_LANGUAGE: str | None = _get_classifier_language_setting(OCR_LANGUAGE)
|
||||
|
||||
SEARCH_LANGUAGE: str | None = _get_search_language_setting(OCR_LANGUAGE)
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import pytest
|
||||
from django.core.exceptions import ImproperlyConfigured
|
||||
|
||||
from paperless.settings import _get_allauth_trusted_proxy_count
|
||||
from paperless.settings import _get_classifier_language_setting
|
||||
from paperless.settings import _get_search_language_setting
|
||||
from paperless.settings import _parse_paperless_url
|
||||
from paperless.settings import default_threads_per_worker
|
||||
@@ -37,6 +38,45 @@ class TestThreadCalculation(TestCase):
|
||||
self.assertLessEqual(default_workers * default_threads, i)
|
||||
|
||||
|
||||
class TestClassifierLanguageSetting:
|
||||
@pytest.mark.parametrize(
|
||||
("ocr_language", "expected"),
|
||||
[
|
||||
pytest.param("dan", "danish", id="danish"),
|
||||
pytest.param("nld", "dutch", id="dutch"),
|
||||
pytest.param("eng", "english", id="english"),
|
||||
pytest.param("fin", "finnish", id="finnish"),
|
||||
pytest.param("fra", "french", id="french"),
|
||||
pytest.param("deu", "german", id="german"),
|
||||
pytest.param("ita", "italian", id="italian"),
|
||||
pytest.param("nor", "norwegian", id="norwegian"),
|
||||
pytest.param("por", "portuguese", id="portuguese"),
|
||||
pytest.param("rus", "russian", id="russian"),
|
||||
pytest.param("spa", "spanish", id="spanish"),
|
||||
pytest.param("swe", "swedish", id="swedish"),
|
||||
pytest.param("eng+deu", "english", id="primary-english"),
|
||||
pytest.param("deu+eng", "german", id="primary-german"),
|
||||
pytest.param("ell", None, id="greek-unsupported"),
|
||||
pytest.param("chi_sim", None, id="chinese-unsupported"),
|
||||
],
|
||||
)
|
||||
def test_maps_primary_ocr_language(
|
||||
self,
|
||||
ocr_language: str,
|
||||
expected: str | None,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- An OCR language setting, possibly listing several languages
|
||||
WHEN:
|
||||
- The classifier language is determined
|
||||
THEN:
|
||||
- The first OCR language maps to its classifier language, or None
|
||||
if unsupported
|
||||
"""
|
||||
assert _get_classifier_language_setting(ocr_language) == expected
|
||||
|
||||
|
||||
def test_allauth_trusted_proxy_count_defaults_to_trusted_proxies(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
|
||||
@@ -2909,7 +2909,6 @@ dependencies = [
|
||||
{ name = "llama-index-embeddings-openai-like" },
|
||||
{ name = "llama-index-llms-ollama" },
|
||||
{ name = "llama-index-llms-openai-like" },
|
||||
{ name = "nltk" },
|
||||
{ name = "ocrmypdf" },
|
||||
{ name = "openai" },
|
||||
{ name = "pathvalidate" },
|
||||
@@ -3062,7 +3061,6 @@ requires-dist = [
|
||||
{ name = "llama-index-llms-ollama", specifier = ">=0.9.1" },
|
||||
{ name = "llama-index-llms-openai-like", specifier = ">=0.7.1" },
|
||||
{ name = "mysqlclient", marker = "extra == 'mariadb'", specifier = "~=2.2.7" },
|
||||
{ name = "nltk", specifier = "~=3.10.0" },
|
||||
{ name = "ocrmypdf", specifier = ">=17.7,<17.12" },
|
||||
{ name = "openai", specifier = ">=2.48" },
|
||||
{ name = "pathvalidate", specifier = "~=3.3.1" },
|
||||
|
||||
Reference in New Issue
Block a user