amorfo77/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2026-09-21 17:08:33 +00:00
Code Issues Projects Releases Wiki Activity
Files
684b3c9885fd44ef05873a6c8225f9cb02318478
paperless-ngx/pyproject.toml
T
Trenton H 530059c5c0 Performance: Preprocess classifier text with Tantivy instead of NLTK (#14127) 
* Preprocesses classifier content with Tantivy instead of NLTK

Tokenizing and stemming now happen in one Rust call instead of NLTK's
Python tokenizer and per word stemming, which also removes the Redis
backed stem cache from every preprocessing call. The output matches the
NLTK pipeline closely; tokens containing digits are now stemmed, and the
English stop words follow Snowball's list.

Stemming and stop word removal apply whenever the OCR language is one of
the supported classifier languages, so PAPERLESS_ENABLE_NLTK and
PAPERLESS_NLTK_DIR are removed.

* Copies packages instead of hardlinking them in backend CI, some NLTK thing

* Adds a normalization to NFC to better fit what Tantivy expects
2026-09-16 07:35:26 -07:00

351 lines
10 KiB
TOML
Raw Blame History

[project]
name = "paperless-ngx"
version = "3.1.3"
description = "A community-supported supercharged document management system: scan, index and archive all your physical documents"
readme = "README.md"
requires-python = ">=3.11"
classifiers = [
"Programming Language :: Python :: 3 :: Only",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Programming Language :: Python :: 3.14",
]
# TODO: Move certain things to groups and then utilize that further
# This will allow testing to not install a webserver, mysql, etc
dependencies = [
"azure-ai-documentintelligence>=1.0.2",
"babel>=2.17",
"bleach~=6.4.0",
"celery[redis]~=5.6.2",
"channels~=4.2",
"channels-redis~=4.2",
"concurrent-log-handler~=0.9.25",
"dateparser~=1.4",
# WARNING: django does not use semver.
# Only patch versions are guaranteed to not introduce breaking changes.
"django~=5.2.13",
"django-allauth[mfa,socialaccount]~=65.16.0",
"django-auditlog~=3.4.1",
"django-cachalot~=2.9.0",
"django-compression-middleware~=0.5.0",
"django-cors-headers~=4.9.0",
"django-extensions~=4.1",
"django-filter~=25.1",
"django-guardian>=3.3.3,<3.5",
"django-multiselectfield~=1.0.1",
"django-rich~=2.2.0",
"django-soft-delete~=1.0.18",
"django-treenode>=0.24",
"djangorestframework~=3.16",
"drf-spectacular~=0.30",
"drf-spectacular-sidecar>=2026.7.1,<2026.9",
"drf-writable-nested~=0.7.1",
"filelock~=3.32.0",
"flower>=2.0.1,<2.2",
"gotenberg-client[httpx]~=1.0",
"httpx-oauth~=0.17",
"ijson>=3.5.1",
"imap-tools>=1.14,<1.16",
"jinja2~=3.1.6",
"langdetect~=1.0.9",
"llama-index-core>=0.14.23",
"llama-index-embeddings-huggingface>=0.6.1",
"llama-index-embeddings-ollama>=0.9",
"llama-index-embeddings-openai-like>=0.2.2",
"llama-index-llms-ollama>=0.9.1",
"llama-index-llms-openai-like>=0.7.1",
"ocrmypdf>=17.7,<17.12",
"openai>=2.48",
"pathvalidate~=3.3.1",
"pdf2image~=1.17.0",
"python-dateutil~=2.9.0",
"python-dotenv~=1.2.1",
"python-gnupg~=0.5.4",
"python-ipware~=3.0.0",
"python-magic~=0.4.27",
"rapidfuzz~=3.14.5",
"redis[hiredis]~=6.4.0",
"regex>=2026.7.19",
"scikit-learn~=1.9.0",
"sentence-transformers>=5.6.1",
"setproctitle~=1.3.4",
"sqlite-vec==0.1.9",
"tantivy~=0.26.0",
"tika-client[httpx]~=1.0",
"torch~=2.13.0",
"watchfiles>=1.2",
"whitenoise~=6.11",
"whoosh-compat[tantivy]==0.2",
"zxing-cpp~=3.1.0",
]
[project.optional-dependencies]
mariadb = [
"mysqlclient~=2.2.7",
]
postgres = [
"psycopg[c,pool]==3.3.4",
# Direct dependency for proper resolution of the pre-built wheels
"psycopg-c==3.3.4",
"psycopg-pool==3.3.1",
]
webserver = [
"granian[uvloop]>=2.7,<2.9",
]
[dependency-groups]
dev = [
{ include-group = "docs" },
{ include-group = "lint" },
{ include-group = "testing" },
]
docs = [
"zensical>=0.0.51",
]
lint = [
"prek>=0.4.11,<0.6",
"ruff~=0.16.1",
]
testing = [
"daphne",
"factory-boy~=3.3.1",
"faker>=40.36,<40.38",
"imagehash",
"pytest~=9.1.1",
"pytest-cov~=7.1.0",
"pytest-django>=4.12,<4.15",
"pytest-env~=1.7.0",
"pytest-httpx",
"pytest-mock~=3.15.1",
# "pytest-randomly~=4.0.1",
"pytest-rerunfailures~=16.4",
"pytest-sugar",
"pytest-xdist~=3.8.0",
"time-machine>=2.13",
]
typing = [
"celery-types",
"django-filter-stubs",
"django-stubs[compatible-mypy]",
"djangorestframework-stubs[compatible-mypy]",
"lxml-stubs",
"microsoft-python-type-stubs @ git+https://github.com/microsoft/python-type-stubs.git",
"mypy",
"mypy-baseline",
"pyrefly",
"types-bleach",
"types-channels",
"types-colorama",
"types-dateparser",
"types-markdown",
"types-pygments",
"types-python-dateutil",
"types-pytz",
"types-redis",
"types-regex",
"types-setuptools"
]
[tool.uv]
required-version = ">=0.9.0"
environments = [
"sys_platform == 'darwin'",
"sys_platform == 'linux'",
]
package = false
[tool.uv.sources]
# Markers are chosen to select these almost exclusively when building the Docker image
psycopg-c = [
{ url = "https://github.com/paperless-ngx/builder/releases/download/psycopg-trixie-3.3.4/psycopg_c-3.3.4-cp312-cp312-linux_x86_64.whl", marker = "sys_platform == 'linux' and platform_machine == 'x86_64' and python_version == '3.12'" },
{ url = "https://github.com/paperless-ngx/builder/releases/download/psycopg-trixie-3.3.4/psycopg_c-3.3.4-cp312-cp312-linux_aarch64.whl", marker = "sys_platform == 'linux' and platform_machine == 'aarch64' and python_version == '3.12'" },
{ url = "https://github.com/paperless-ngx/builder/releases/download/psycopg-trixie-3.3.4/psycopg_c-3.3.4-cp314-cp314-linux_x86_64.whl", marker = "sys_platform == 'linux' and platform_machine == 'x86_64' and python_version == '3.14'" },
{ url = "https://github.com/paperless-ngx/builder/releases/download/psycopg-trixie-3.3.4/psycopg_c-3.3.4-cp314-cp314-linux_aarch64.whl", marker = "sys_platform == 'linux' and platform_machine == 'aarch64' and python_version == '3.14'" },
]
torch = [
{ index = "pytorch-cpu" },
]
[[tool.uv.index]]
name = "pytorch-cpu"
url = "https://download.pytorch.org/whl/cpu"
explicit = true
[tool.ruff]
target-version = "py311"
line-length = 88
src = [
"src",
]
respect-gitignore = true
# https://docs.astral.sh/ruff/settings/
fix = true
show-fixes = true
output-format = "grouped"
[tool.ruff.format]
line-ending = "lf"
[tool.ruff.lint]
# https://docs.astral.sh/ruff/rules/
select = [ "E4", "E7", "E9", "F" ]
extend-select = [
"COM", # https://docs.astral.sh/ruff/rules/#flake8-commas-com
"DJ", # https://docs.astral.sh/ruff/rules/#flake8-django-dj
"EXE", # https://docs.astral.sh/ruff/rules/#flake8-executable-exe
"FBT", # https://docs.astral.sh/ruff/rules/#flake8-boolean-trap-fbt
"FLY", # https://docs.astral.sh/ruff/rules/#flynt-fly
"G201", # https://docs.astral.sh/ruff/rules/#flake8-logging-format-g
"I", # https://docs.astral.sh/ruff/rules/#isort-i
"ICN", # https://docs.astral.sh/ruff/rules/#flake8-import-conventions-icn
"INP", # https://docs.astral.sh/ruff/rules/#flake8-no-pep420-inp
"ISC", # https://docs.astral.sh/ruff/rules/#flake8-implicit-str-concat-isc
"PIE", # https://docs.astral.sh/ruff/rules/#flake8-pie-pie
"PLC", # https://docs.astral.sh/ruff/rules/#pylint-pl
"PLE", # https://docs.astral.sh/ruff/rules/#pylint-pl
"PTH", # https://docs.astral.sh/ruff/rules/#flake8-use-pathlib-pth
"Q", # https://docs.astral.sh/ruff/rules/#flake8-quotes-q
"RSE", # https://docs.astral.sh/ruff/rules/#flake8-raise-rse
"RUF", # https://docs.astral.sh/ruff/rules/#ruff-specific-rules-ruf
"SIM", # https://docs.astral.sh/ruff/rules/#flake8-simplify-sim
"T20", # https://docs.astral.sh/ruff/rules/#flake8-print-t20
"TC", # https://docs.astral.sh/ruff/rules/#flake8-type-checking-tc
"TID", # https://docs.astral.sh/ruff/rules/#flake8-tidy-imports-tid
"UP", # https://docs.astral.sh/ruff/rules/#pyupgrade-up
"W", # https://docs.astral.sh/ruff/rules/#pycodestyle-e-w
]
ignore = [
"DJ001",
"PLC0415",
"RUF012",
"SIM105",
]
# Migrations
per-file-ignores."*/migrations/*.py" = [
"E501",
"SIM",
"T201",
]
# Testing
per-file-ignores."*/tests/*.py" = [
"E501",
"SIM117",
]
per-file-ignores.".github/scripts/*.py" = [
"E501",
"INP001",
"SIM117",
]
# Docker specific
per-file-ignores."docker/rootfs/usr/local/bin/wait-for-redis.py" = [
"INP001",
"T201",
]
per-file-ignores."docker/wait-for-redis.py" = [
"INP001",
"T201",
]
per-file-ignores."src/documents/models.py" = [
"SIM115",
]
isort.force-single-line = true
[tool.codespell]
ignore-words-list = "criterias,afterall,valeu,ureue,equest,ure,assertIn,Oktober,commitish,NIN,nin"
skip = """\
src-ui/src/locale/*,src-ui/pnpm-lock.yaml,src-ui/e2e/*,src/paperless_mail/tests/samples/*,src/paperless/tests/samples\
/mail/*,src/documents/tests/samples/*,*.po,*.json\
"""
write-changes = true
[tool.pyproject-fmt]
table_format = "long"
[tool.mypy]
mypy_path = "src"
disallow_any_generics = true
disallow_untyped_defs = true
disallow_incomplete_defs = true
check_untyped_defs = true
warn_redundant_casts = true
warn_unused_ignores = true
plugins = [
"mypy_django_plugin.main",
"mypy_drf_plugin.main",
]
[tool.pyrefly]
baseline = ".pyrefly-baseline.json"
python-platform = "linux"
search-path = [ "src" ]
[tool.django-stubs]
django_settings_module = "paperless.settings"
[tool.pytest]
addopts = [
"--pythonwarnings=all",
"--cov",
"--cov-report=html",
"--cov-report=xml",
"--numprocesses=auto",
"--maxprocesses=16",
"--dist=loadscope",
"--durations=50",
"--durations-min=0.5",
"--junitxml=junit.xml",
"-o",
"junit_family=legacy",
]
DJANGO_SETTINGS_MODULE = "paperless.settings"
markers = [
"live: Integration tests requiring external services (Gotenberg, Tika, nginx, etc)",
"nginx: Tests that make HTTP requests to the local nginx service",
"gotenberg: Tests requiring Gotenberg service",
"tika: Tests requiring Tika service",
"greenmail: Tests requiring Greenmail service",
"date_parsing: Tests which cover date parsing from content or filename",
"management: Tests which cover management commands/functionality",
"search: Tests for the Tantivy search backend",
"api: Tests for REST API endpoints",
]
minversion = "9.0"
norecursedirs = [ "src/locale/", ".venv/", "src-ui/" ]
pythonpath = [ "src" ]
strict_config = true
strict_markers = true
strict_parametrization_ids = true
strict_xfail = true
testpaths = [
"src/documents/tests/",
"src/paperless/tests/",
"src/paperless_mail/tests/",
"src/paperless_ai/tests",
]
[tool.pytest_env]
PAPERLESS_SECRET_KEY = "test-secret-key-do-not-use-in-production"
PAPERLESS_DISABLE_DBHANDLER = "true"
PAPERLESS_CACHE_BACKEND = "django.core.cache.backends.locmem.LocMemCache"
PAPERLESS_CHANNELS_BACKEND = "channels.layers.InMemoryChannelLayer"
# I don't think anything hits this, but just in case, basically infinite
PAPERLESS_TOKEN_THROTTLE_RATE = "1000/min"
[tool.coverage.run]
source = [
"src/",
]
omit = [
"*/tests/*",
"manage.py",
"paperless/wsgi.py",
"paperless/auth.py",
]
[tool.coverage.report]
exclude_also = [
"if settings.AUDIT_LOG_ENABLED:",
"if AUDIT_LOG_ENABLED:",
"if TYPE_CHECKING:",
]
[tool.mypy-baseline]
baseline_path = ".mypy-baseline.txt"
sort_baseline = true
ignore_categories = [ "note" ]
Reference in New Issue View Git Blame Copy Permalink