From e7c7978d6738e469673adc753056568ceaa70056 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Thu, 2 Apr 2026 20:24:28 -0700 Subject: [PATCH 01/11] Enhancement: allow opt-in blocking internal mail hosts (#12502) --- docs/configuration.md | 8 ++++++++ src/paperless/settings/__init__.py | 4 ++++ src/paperless_mail/mail.py | 9 +++++++++ src/paperless_mail/tests/test_mail.py | 20 ++++++++++++++++++++ src/paperless_mail/views.py | 22 +++++++++++----------- 5 files changed, 52 insertions(+), 11 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index fa0d32c51..7156b3553 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1440,6 +1440,14 @@ ports. ## Incoming Mail {#incoming_mail} +#### [`PAPERLESS_EMAIL_ALLOW_INTERNAL_HOSTS=`](#PAPERLESS_EMAIL_ALLOW_INTERNAL_HOSTS) {#PAPERLESS_EMAIL_ALLOW_INTERNAL_HOSTS} + +: If set to false, incoming mail account connections are blocked when the +configured IMAP hostname resolves to a non-public address (for example, +localhost, link-local, or RFC1918 private ranges). + + Defaults to true, which allows internal hosts. + ### Email OAuth {#email_oauth} #### [`PAPERLESS_OAUTH_CALLBACK_BASE_URL=`](#PAPERLESS_OAUTH_CALLBACK_BASE_URL) {#PAPERLESS_OAUTH_CALLBACK_BASE_URL} diff --git a/src/paperless/settings/__init__.py b/src/paperless/settings/__init__.py index a76c6ce75..b8a99e9fb 100644 --- a/src/paperless/settings/__init__.py +++ b/src/paperless/settings/__init__.py @@ -501,6 +501,10 @@ SESSION_COOKIE_NAME = f"{COOKIE_PREFIX}sessionid" LANGUAGE_COOKIE_NAME = f"{COOKIE_PREFIX}django_language" EMAIL_CERTIFICATE_FILE = get_path_from_env("PAPERLESS_EMAIL_CERTIFICATE_LOCATION") +EMAIL_ALLOW_INTERNAL_HOSTS = get_bool_from_env( + "PAPERLESS_EMAIL_ALLOW_INTERNAL_HOSTS", + "true", +) ############################################################################### diff --git a/src/paperless_mail/mail.py b/src/paperless_mail/mail.py index 56eaefaad..ecc785b1a 100644 --- a/src/paperless_mail/mail.py +++ b/src/paperless_mail/mail.py @@ -39,6 +39,8 @@ from documents.loggers import LoggingMixin from documents.models import Correspondent from documents.parsers import is_mime_type_supported from documents.tasks import consume_file +from paperless.network import is_public_ip +from paperless.network import resolve_hostname_ips from paperless_mail.models import MailAccount from paperless_mail.models import MailRule from paperless_mail.models import ProcessedMail @@ -412,6 +414,13 @@ def get_mailbox(server, port, security) -> MailBox: """ Returns the correct MailBox instance for the given configuration. """ + if not settings.EMAIL_ALLOW_INTERNAL_HOSTS: + for ip_str in resolve_hostname_ips(server): + if not is_public_ip(ip_str): + raise MailError( + f"Connection blocked: {server} resolves to a non-public address", + ) + ssl_context = ssl.create_default_context() if settings.EMAIL_CERTIFICATE_FILE is not None: # pragma: no cover ssl_context.load_verify_locations(cafile=settings.EMAIL_CERTIFICATE_FILE) diff --git a/src/paperless_mail/tests/test_mail.py b/src/paperless_mail/tests/test_mail.py index 72ee5331a..80a718a46 100644 --- a/src/paperless_mail/tests/test_mail.py +++ b/src/paperless_mail/tests/test_mail.py @@ -13,6 +13,7 @@ from django.contrib.auth.models import User from django.core.management import call_command from django.db import DatabaseError from django.test import TestCase +from django.test import override_settings from django.utils import timezone from imap_tools import NOT from imap_tools import EmailAddress @@ -1846,6 +1847,25 @@ class TestMailAccountTestView(APITestCase): self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) self.assertEqual(response.content.decode(), "Unable to connect to server") + @override_settings(EMAIL_ALLOW_INTERNAL_HOSTS=False) + @mock.patch("paperless_mail.mail.resolve_hostname_ips", return_value=["127.0.0.1"]) + def test_mail_account_test_view_blocks_internal_host_when_disabled( + self, + _mock_resolve_hostname_ips, + ) -> None: + data = { + "imap_server": "internal.example", + "imap_port": 993, + "imap_security": MailAccount.ImapSecurity.SSL, + "username": "admin", + "password": "secret", + "account_type": MailAccount.MailAccountType.IMAP, + "is_token": False, + } + response = self.client.post(self.url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(response.content.decode(), "Unable to connect to server") + @mock.patch( "paperless_mail.oauth.PaperlessMailOAuth2Manager.refresh_account_oauth_token", ) diff --git a/src/paperless_mail/views.py b/src/paperless_mail/views.py index 2593797f3..9e3850cfc 100644 --- a/src/paperless_mail/views.py +++ b/src/paperless_mail/views.py @@ -120,12 +120,12 @@ class MailAccountViewSet(ModelViewSet, PassUserMixin): serializer.validated_data["expiration"] = existing_account.expiration account = MailAccount(**serializer.validated_data) - with get_mailbox( - account.imap_server, - account.imap_port, - account.imap_security, - ) as M: - try: + try: + with get_mailbox( + account.imap_server, + account.imap_port, + account.imap_security, + ) as M: if ( existing_account is not None and account.is_token @@ -145,11 +145,11 @@ class MailAccountViewSet(ModelViewSet, PassUserMixin): mailbox_login(M, account) return Response({"success": True}) - except MailError: - logger.error( - "Mail account connectivity test failed", - ) - return HttpResponseBadRequest("Unable to connect to server") + except MailError: + logger.error( + "Mail account connectivity test failed", + ) + return HttpResponseBadRequest("Unable to connect to server") @action(methods=["post"], detail=True) def process(self, request, pk=None): From 2703c12f1add4f50da196ca2478ed857b6b61dce Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 3 Apr 2026 03:25:57 +0000 Subject: [PATCH 02/11] Auto translate strings --- src/locale/en_US/LC_MESSAGES/django.po | 76 +++++++++++++------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/src/locale/en_US/LC_MESSAGES/django.po b/src/locale/en_US/LC_MESSAGES/django.po index eba121b5b..57ade319a 100644 --- a/src/locale/en_US/LC_MESSAGES/django.po +++ b/src/locale/en_US/LC_MESSAGES/django.po @@ -2,7 +2,7 @@ msgid "" msgstr "" "Project-Id-Version: paperless-ngx\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2026-04-02 22:35+0000\n" +"POT-Creation-Date: 2026-04-03 03:25+0000\n" "PO-Revision-Date: 2022-02-17 04:17\n" "Last-Translator: \n" "Language-Team: English\n" @@ -1866,151 +1866,151 @@ msgstr "" msgid "paperless application settings" msgstr "" -#: paperless/settings/__init__.py:528 +#: paperless/settings/__init__.py:532 msgid "English (US)" msgstr "" -#: paperless/settings/__init__.py:529 +#: paperless/settings/__init__.py:533 msgid "Arabic" msgstr "" -#: paperless/settings/__init__.py:530 +#: paperless/settings/__init__.py:534 msgid "Afrikaans" msgstr "" -#: paperless/settings/__init__.py:531 +#: paperless/settings/__init__.py:535 msgid "Belarusian" msgstr "" -#: paperless/settings/__init__.py:532 +#: paperless/settings/__init__.py:536 msgid "Bulgarian" msgstr "" -#: paperless/settings/__init__.py:533 +#: paperless/settings/__init__.py:537 msgid "Catalan" msgstr "" -#: paperless/settings/__init__.py:534 +#: paperless/settings/__init__.py:538 msgid "Czech" msgstr "" -#: paperless/settings/__init__.py:535 +#: paperless/settings/__init__.py:539 msgid "Danish" msgstr "" -#: paperless/settings/__init__.py:536 +#: paperless/settings/__init__.py:540 msgid "German" msgstr "" -#: paperless/settings/__init__.py:537 +#: paperless/settings/__init__.py:541 msgid "Greek" msgstr "" -#: paperless/settings/__init__.py:538 +#: paperless/settings/__init__.py:542 msgid "English (GB)" msgstr "" -#: paperless/settings/__init__.py:539 +#: paperless/settings/__init__.py:543 msgid "Spanish" msgstr "" -#: paperless/settings/__init__.py:540 +#: paperless/settings/__init__.py:544 msgid "Persian" msgstr "" -#: paperless/settings/__init__.py:541 +#: paperless/settings/__init__.py:545 msgid "Finnish" msgstr "" -#: paperless/settings/__init__.py:542 +#: paperless/settings/__init__.py:546 msgid "French" msgstr "" -#: paperless/settings/__init__.py:543 +#: paperless/settings/__init__.py:547 msgid "Hungarian" msgstr "" -#: paperless/settings/__init__.py:544 +#: paperless/settings/__init__.py:548 msgid "Indonesian" msgstr "" -#: paperless/settings/__init__.py:545 +#: paperless/settings/__init__.py:549 msgid "Italian" msgstr "" -#: paperless/settings/__init__.py:546 +#: paperless/settings/__init__.py:550 msgid "Japanese" msgstr "" -#: paperless/settings/__init__.py:547 +#: paperless/settings/__init__.py:551 msgid "Korean" msgstr "" -#: paperless/settings/__init__.py:548 +#: paperless/settings/__init__.py:552 msgid "Luxembourgish" msgstr "" -#: paperless/settings/__init__.py:549 +#: paperless/settings/__init__.py:553 msgid "Norwegian" msgstr "" -#: paperless/settings/__init__.py:550 +#: paperless/settings/__init__.py:554 msgid "Dutch" msgstr "" -#: paperless/settings/__init__.py:551 +#: paperless/settings/__init__.py:555 msgid "Polish" msgstr "" -#: paperless/settings/__init__.py:552 +#: paperless/settings/__init__.py:556 msgid "Portuguese (Brazil)" msgstr "" -#: paperless/settings/__init__.py:553 +#: paperless/settings/__init__.py:557 msgid "Portuguese" msgstr "" -#: paperless/settings/__init__.py:554 +#: paperless/settings/__init__.py:558 msgid "Romanian" msgstr "" -#: paperless/settings/__init__.py:555 +#: paperless/settings/__init__.py:559 msgid "Russian" msgstr "" -#: paperless/settings/__init__.py:556 +#: paperless/settings/__init__.py:560 msgid "Slovak" msgstr "" -#: paperless/settings/__init__.py:557 +#: paperless/settings/__init__.py:561 msgid "Slovenian" msgstr "" -#: paperless/settings/__init__.py:558 +#: paperless/settings/__init__.py:562 msgid "Serbian" msgstr "" -#: paperless/settings/__init__.py:559 +#: paperless/settings/__init__.py:563 msgid "Swedish" msgstr "" -#: paperless/settings/__init__.py:560 +#: paperless/settings/__init__.py:564 msgid "Turkish" msgstr "" -#: paperless/settings/__init__.py:561 +#: paperless/settings/__init__.py:565 msgid "Ukrainian" msgstr "" -#: paperless/settings/__init__.py:562 +#: paperless/settings/__init__.py:566 msgid "Vietnamese" msgstr "" -#: paperless/settings/__init__.py:563 +#: paperless/settings/__init__.py:567 msgid "Chinese Simplified" msgstr "" -#: paperless/settings/__init__.py:564 +#: paperless/settings/__init__.py:568 msgid "Chinese Traditional" msgstr "" From d365f199627108a3d227e1585298121e8f6a8915 Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Thu, 2 Apr 2026 20:49:54 -0700 Subject: [PATCH 03/11] Security: Registers a custom serializer which signs the task payload (#12504) --- src/paperless/celery.py | 48 +++++++++++++++++++++ src/paperless/settings/__init__.py | 6 ++- src/paperless/tests/test_celery.py | 69 ++++++++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 src/paperless/tests/test_celery.py diff --git a/src/paperless/celery.py b/src/paperless/celery.py index d937b3ada..3797c840c 100644 --- a/src/paperless/celery.py +++ b/src/paperless/celery.py @@ -1,11 +1,59 @@ +import hmac import os +import pickle +from hashlib import sha256 from celery import Celery from celery.signals import worker_process_init +from kombu.serialization import register # Set the default Django settings module for the 'celery' program. os.environ.setdefault("DJANGO_SETTINGS_MODULE", "paperless.settings") +# --------------------------------------------------------------------------- +# Signed-pickle serializer: pickle with HMAC-SHA256 integrity verification. +# +# Protects against malicious pickle injection via an exposed Redis broker. +# Messages are signed on the producer side and verified before deserialization +# on the worker side using Django's SECRET_KEY. +# --------------------------------------------------------------------------- + +HMAC_SIZE = 32 # SHA-256 digest length + + +def _get_signing_key() -> bytes: + from django.conf import settings + + return settings.SECRET_KEY.encode() + + +def signed_pickle_dumps(obj: object) -> bytes: + data = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL) + signature = hmac.new(_get_signing_key(), data, sha256).digest() + return signature + data + + +def signed_pickle_loads(payload: bytes) -> object: + if len(payload) < HMAC_SIZE: + msg = "Signed-pickle payload too short" + raise ValueError(msg) + signature = payload[:HMAC_SIZE] + data = payload[HMAC_SIZE:] + expected = hmac.new(_get_signing_key(), data, sha256).digest() + if not hmac.compare_digest(signature, expected): + msg = "Signed-pickle HMAC verification failed — message may have been tampered with" + raise ValueError(msg) + return pickle.loads(data) + + +register( + "signed-pickle", + signed_pickle_dumps, + signed_pickle_loads, + content_type="application/x-signed-pickle", + content_encoding="binary", +) + app = Celery("paperless") # Using a string here means the worker doesn't have to serialize diff --git a/src/paperless/settings/__init__.py b/src/paperless/settings/__init__.py index b8a99e9fb..964295020 100644 --- a/src/paperless/settings/__init__.py +++ b/src/paperless/settings/__init__.py @@ -675,9 +675,11 @@ CELERY_RESULT_BACKEND = "django-db" CELERY_CACHE_BACKEND = "default" # https://docs.celeryq.dev/en/stable/userguide/configuration.html#task-serializer -CELERY_TASK_SERIALIZER = "pickle" +# Uses HMAC-signed pickle to prevent RCE via malicious messages on an exposed Redis broker. +# The signed-pickle serializer is registered in paperless/celery.py. +CELERY_TASK_SERIALIZER = "signed-pickle" # https://docs.celeryq.dev/en/stable/userguide/configuration.html#std-setting-accept_content -CELERY_ACCEPT_CONTENT = ["application/json", "application/x-python-serialize"] +CELERY_ACCEPT_CONTENT = ["application/json", "application/x-signed-pickle"] # https://docs.celeryq.dev/en/stable/userguide/configuration.html#beat-schedule CELERY_BEAT_SCHEDULE = parse_beat_schedule() diff --git a/src/paperless/tests/test_celery.py b/src/paperless/tests/test_celery.py new file mode 100644 index 000000000..0c0e51272 --- /dev/null +++ b/src/paperless/tests/test_celery.py @@ -0,0 +1,69 @@ +import hmac +import pickle +from hashlib import sha256 + +import pytest +from django.test import override_settings + +from paperless.celery import HMAC_SIZE +from paperless.celery import signed_pickle_dumps +from paperless.celery import signed_pickle_loads + + +class TestSignedPickleSerializer: + def test_roundtrip_simple_types(self): + """Signed pickle can round-trip basic JSON-like types.""" + for obj in [42, "hello", [1, 2, 3], {"key": "value"}, None, True]: + assert signed_pickle_loads(signed_pickle_dumps(obj)) == obj + + def test_roundtrip_complex_types(self): + """Signed pickle can round-trip types that JSON cannot.""" + from pathlib import Path + + obj = {"path": Path("/tmp/test"), "data": {1, 2, 3}} + result = signed_pickle_loads(signed_pickle_dumps(obj)) + assert result["path"] == Path("/tmp/test") + assert result["data"] == {1, 2, 3} + + def test_tampered_data_rejected(self): + """Flipping a byte in the data portion causes HMAC failure.""" + payload = signed_pickle_dumps({"task": "test"}) + tampered = bytearray(payload) + tampered[-1] ^= 0xFF + with pytest.raises(ValueError, match="HMAC verification failed"): + signed_pickle_loads(bytes(tampered)) + + def test_tampered_signature_rejected(self): + """Flipping a byte in the signature portion causes HMAC failure.""" + payload = signed_pickle_dumps({"task": "test"}) + tampered = bytearray(payload) + tampered[0] ^= 0xFF + with pytest.raises(ValueError, match="HMAC verification failed"): + signed_pickle_loads(bytes(tampered)) + + def test_truncated_payload_rejected(self): + """A payload shorter than HMAC_SIZE is rejected.""" + with pytest.raises(ValueError, match="too short"): + signed_pickle_loads(b"\x00" * (HMAC_SIZE - 1)) + + def test_empty_payload_rejected(self): + with pytest.raises(ValueError, match="too short"): + signed_pickle_loads(b"") + + @override_settings(SECRET_KEY="different-secret-key") + def test_wrong_secret_key_rejected(self): + """A message signed with one key cannot be loaded with another.""" + original_key = b"test-secret-key-do-not-use-in-production" + obj = {"task": "test"} + data = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL) + signature = hmac.new(original_key, data, sha256).digest() + payload = signature + data + with pytest.raises(ValueError, match="HMAC verification failed"): + signed_pickle_loads(payload) + + def test_forged_pickle_rejected(self): + """A raw pickle payload (no signature) is rejected.""" + raw_pickle = pickle.dumps({"task": "test"}) + # Raw pickle won't have a valid HMAC prefix + with pytest.raises(ValueError, match="HMAC verification failed"): + signed_pickle_loads(b"\x00" * HMAC_SIZE + raw_pickle) From 8c539bd862da3a13089c36e0bb91f9d950532e5a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 Apr 2026 13:25:17 +0000 Subject: [PATCH 04/11] Chore(deps): Bump the utilities-patch group across 1 directory with 5 updates (#12499) Bumps the utilities-patch group with 5 updates in the / directory: | Package | From | To | | --- | --- | --- | | [llama-index-core](https://github.com/run-llama/llama_index) | `0.14.16` | `0.14.19` | | [nltk](https://github.com/nltk/nltk) | `3.9.3` | `3.9.4` | | [zensical](https://github.com/zensical/zensical) | `0.0.26` | `0.0.29` | | [prek](https://github.com/j178/prek) | `0.3.5` | `0.3.8` | | [ruff](https://github.com/astral-sh/ruff) | `0.15.5` | `0.15.7` | Updates `llama-index-core` from 0.14.16 to 0.14.19 - [Release notes](https://github.com/run-llama/llama_index/releases) - [Changelog](https://github.com/run-llama/llama_index/blob/main/CHANGELOG.md) - [Commits](https://github.com/run-llama/llama_index/compare/v0.14.16...v0.14.19) Updates `nltk` from 3.9.3 to 3.9.4 - [Changelog](https://github.com/nltk/nltk/blob/develop/ChangeLog) - [Commits](https://github.com/nltk/nltk/compare/3.9.3...3.9.4) Updates `zensical` from 0.0.26 to 0.0.29 - [Release notes](https://github.com/zensical/zensical/releases) - [Commits](https://github.com/zensical/zensical/compare/v0.0.26...v0.0.29) Updates `prek` from 0.3.5 to 0.3.8 - [Release notes](https://github.com/j178/prek/releases) - [Changelog](https://github.com/j178/prek/blob/master/CHANGELOG.md) - [Commits](https://github.com/j178/prek/compare/v0.3.5...v0.3.8) Updates `ruff` from 0.15.5 to 0.15.7 - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.15.5...0.15.7) --- updated-dependencies: - dependency-name: llama-index-core dependency-version: 0.14.19 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: utilities-patch - dependency-name: nltk dependency-version: 3.9.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: utilities-patch - dependency-name: zensical dependency-version: 0.0.29 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: utilities-patch - dependency-name: prek dependency-version: 0.3.8 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: utilities-patch - dependency-name: ruff dependency-version: 0.15.7 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: utilities-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- uv.lock | 98 ++++++++++++++++++++++++++++----------------------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/uv.lock b/uv.lock index feffefce5..4cfd329f4 100644 --- a/uv.lock +++ b/uv.lock @@ -2139,7 +2139,7 @@ wheels = [ [[package]] name = "llama-index-core" -version = "0.14.16" +version = "0.14.19" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -2171,9 +2171,9 @@ dependencies = [ { name = "typing-inspect", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "wrapt", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/13/cb/1d7383f9f4520bb1d921c34f18c147b4b270007135212cedfa240edcd4c3/llama_index_core-0.14.16.tar.gz", hash = "sha256:cf2b7e4b798cb5ebad19c935174c200595c7ecff84a83793540cc27b03636a52", size = 11599715, upload-time = "2026-03-10T19:19:52.476Z" } +sdist = { url = "https://files.pythonhosted.org/packages/09/eb/a661cc2f70177f59cfe7bfcdb7a4e9352fb073ab46927068151bf2905fbb/llama_index_core-0.14.19.tar.gz", hash = "sha256:7b17f321f0d965495402890991b2bfde49d4197bc46ca5970300cc7b9c2df6a2", size = 11599592, upload-time = "2026-03-25T20:58:25.751Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b4/f5/a33839bae0bd07e4030969bdba1ac90665e359ae88c56c296991ae16b8a8/llama_index_core-0.14.16-py3-none-any.whl", hash = "sha256:0cc273ebc44d51ad636217661a25f9cd02fb2d0440641430f105da3ae9f43a6b", size = 11944927, upload-time = "2026-03-10T19:19:48.043Z" }, + { url = "https://files.pythonhosted.org/packages/1f/b6/6c2678b8597903503b804fe831a203d299bcbcc07bdf35789a484e67f7c0/llama_index_core-0.14.19-py3-none-any.whl", hash = "sha256:807352f16a300f9980d0110cfdaa81d07e201384965e9f7d940c8ead80d463ed", size = 11945679, upload-time = "2026-03-25T20:58:28.265Z" }, ] [[package]] @@ -2691,7 +2691,7 @@ wheels = [ [[package]] name = "nltk" -version = "3.9.3" +version = "3.9.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -2699,9 +2699,9 @@ dependencies = [ { name = "regex", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "tqdm", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e1/8f/915e1c12df07c70ed779d18ab83d065718a926e70d3ea33eb0cd66ffb7c0/nltk-3.9.3.tar.gz", hash = "sha256:cb5945d6424a98d694c2b9a0264519fab4363711065a46aa0ae7a2195b92e71f", size = 2923673, upload-time = "2026-02-24T12:05:53.833Z" } +sdist = { url = "https://files.pythonhosted.org/packages/74/a1/b3b4adf15585a5bc4c357adde150c01ebeeb642173ded4d871e89468767c/nltk-3.9.4.tar.gz", hash = "sha256:ed03bc098a40481310320808b2db712d95d13ca65b27372f8a403949c8b523d0", size = 2946864, upload-time = "2026-03-24T06:13:40.641Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/7e/9af5a710a1236e4772de8dfcc6af942a561327bb9f42b5b4a24d0cf100fd/nltk-3.9.3-py3-none-any.whl", hash = "sha256:60b3db6e9995b3dd976b1f0fa7dec22069b2677e759c28eb69b62ddd44870522", size = 1525385, upload-time = "2026-02-24T12:05:46.54Z" }, + { url = "https://files.pythonhosted.org/packages/9d/91/04e965f8e717ba0ab4bdca5c112deeab11c9e750d94c4d4602f050295d39/nltk-3.9.4-py3-none-any.whl", hash = "sha256:f2fa301c3a12718ce4a0e9305c5675299da5ad9e26068218b69d692fda84828f", size = 1552087, upload-time = "2026-03-24T06:13:38.47Z" }, ] [[package]] @@ -3346,23 +3346,23 @@ wheels = [ [[package]] name = "prek" -version = "0.3.5" +version = "0.3.8" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/46/d6/277e002e56eeab3a9d48f1ca4cc067d249d6326fc1783b770d70ad5ae2be/prek-0.3.5.tar.gz", hash = "sha256:ca40b6685a4192256bc807f32237af94bf9b8799c0d708b98735738250685642", size = 374806, upload-time = "2026-03-09T10:35:18.842Z" } +sdist = { url = "https://files.pythonhosted.org/packages/62/ee/03e8180e3fda9de25b6480bd15cc2bde40d573868d50648b0e527b35562f/prek-0.3.8.tar.gz", hash = "sha256:434a214256516f187a3ab15f869d950243be66b94ad47987ee4281b69643a2d9", size = 400224, upload-time = "2026-03-23T08:23:35.981Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8f/a9/16dd8d3a50362ebccffe58518af1f1f571c96f0695d7fcd8bbd386585f58/prek-0.3.5-py3-none-linux_armv6l.whl", hash = "sha256:44b3e12791805804f286d103682b42a84e0f98a2687faa37045e9d3375d3d73d", size = 5105604, upload-time = "2026-03-09T10:35:00.332Z" }, - { url = "https://files.pythonhosted.org/packages/e4/74/bc6036f5bf03860cda66ab040b32737e54802b71a81ec381839deb25df9e/prek-0.3.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:e3cb451cc51ac068974557491beb4c7d2d41dfde29ed559c1694c8ce23bf53e8", size = 5506155, upload-time = "2026-03-09T10:35:17.64Z" }, - { url = "https://files.pythonhosted.org/packages/02/d9/a3745c2a10509c63b6a118ada766614dd705efefd08f275804d5c807aa4a/prek-0.3.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:ad8f5f0d8da53dc94d00b76979af312b3dacccc9dcbc6417756c5dca3633c052", size = 5100383, upload-time = "2026-03-09T10:35:13.302Z" }, - { url = "https://files.pythonhosted.org/packages/43/8e/de965fc515d39309a332789cd3778161f7bc80cde15070bedf17f9f8cb93/prek-0.3.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:4511e15d34072851ac88e4b2006868fbe13655059ad941d7a0ff9ee17138fd9f", size = 5334913, upload-time = "2026-03-09T10:35:14.813Z" }, - { url = "https://files.pythonhosted.org/packages/3f/8c/44f07e8940256059cfd82520e3cbe0764ab06ddb4aa43148465db00b39ad/prek-0.3.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fcc0b63b8337e2046f51267facaac63ba755bc14aad53991840a5eccba3e5c28", size = 5033825, upload-time = "2026-03-09T10:35:06.976Z" }, - { url = "https://files.pythonhosted.org/packages/94/85/3ff0f96881ff2360c212d310ff23c3cf5a15b223d34fcfa8cdcef203be69/prek-0.3.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5fc0d78c3896a674aeb8247a83bbda7efec85274dbdfbc978ceff8d37e4ed20", size = 5438586, upload-time = "2026-03-09T10:34:58.779Z" }, - { url = "https://files.pythonhosted.org/packages/79/a5/c6d08d31293400fcb5d427f8e7e6bacfc959988e868ad3a9d97b4d87c4b7/prek-0.3.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:64cad21cb9072d985179495b77b312f6b81e7b45357d0c68dc1de66e0408eabc", size = 6359714, upload-time = "2026-03-09T10:34:57.454Z" }, - { url = "https://files.pythonhosted.org/packages/ba/18/321dcff9ece8065d42c8c1c7a53a23b45d2b4330aa70993be75dc5f2822f/prek-0.3.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45ee84199bb48e013bdfde0c84352c17a44cc42d5792681b86d94e9474aab6f8", size = 5717632, upload-time = "2026-03-09T10:35:08.634Z" }, - { url = "https://files.pythonhosted.org/packages/a3/7f/1288226aa381d0cea403157f4e6b64b356e1a745f2441c31dd9d8a1d63da/prek-0.3.5-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:f43275e5d564e18e52133129ebeb5cb071af7ce4a547766c7f025aa0955dfbb6", size = 5339040, upload-time = "2026-03-09T10:35:03.665Z" }, - { url = "https://files.pythonhosted.org/packages/22/94/cfec83df9c2b8e7ed1608087bcf9538a6a77b4c2e7365123e9e0a3162cd1/prek-0.3.5-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:abcee520d31522bcbad9311f21326b447694cd5edba33618c25fd023fc9865ec", size = 5162586, upload-time = "2026-03-09T10:35:11.564Z" }, - { url = "https://files.pythonhosted.org/packages/13/b7/741d62132f37a5f7cc0fad1168bd31f20dea9628f482f077f569547e0436/prek-0.3.5-py3-none-musllinux_1_1_armv7l.whl", hash = "sha256:499c56a94a155790c75a973d351a33f8065579d9094c93f6d451ada5d1e469be", size = 5002933, upload-time = "2026-03-09T10:35:16.347Z" }, - { url = "https://files.pythonhosted.org/packages/6f/83/630a5671df6550fcfa67c54955e8a8174eb9b4d97ac38fb05a362029245b/prek-0.3.5-py3-none-musllinux_1_1_i686.whl", hash = "sha256:de1065b59f194624adc9dea269d4ff6b50e98a1b5bb662374a9adaa496b3c1eb", size = 5304934, upload-time = "2026-03-09T10:35:09.975Z" }, - { url = "https://files.pythonhosted.org/packages/de/79/67a7afd0c0b6c436630b7dba6e586a42d21d5d6e5778fbd9eba7bbd3dd26/prek-0.3.5-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:a1c4869e45ee341735d07179da3a79fa2afb5959cef8b3c8a71906eb52dc6933", size = 5829914, upload-time = "2026-03-09T10:35:05.39Z" }, + { url = "https://files.pythonhosted.org/packages/00/84/40d2ddf362d12c4cd4a25a8c89a862edf87cdfbf1422aa41aac8e315d409/prek-0.3.8-py3-none-linux_armv6l.whl", hash = "sha256:6fb646ada60658fa6dd7771b2e0fb097f005151be222f869dada3eb26d79ed33", size = 5226646, upload-time = "2026-03-23T08:23:18.306Z" }, + { url = "https://files.pythonhosted.org/packages/e1/52/7308a033fa43b7e8e188797bd2b3b017c0f0adda70fa7af575b1f43ea888/prek-0.3.8-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:f3d7fdadb15efc19c09953c7a33cf2061a70f367d1e1957358d3ad5cc49d0616", size = 5620104, upload-time = "2026-03-23T08:23:40.053Z" }, + { url = "https://files.pythonhosted.org/packages/ff/b1/f106ac000a91511a9cd80169868daf2f5b693480ef5232cec5517a38a512/prek-0.3.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:72728c3295e79ca443f8c1ec037d2a5b914ec73a358f69cf1bc1964511876bf8", size = 5199867, upload-time = "2026-03-23T08:23:38.066Z" }, + { url = "https://files.pythonhosted.org/packages/b3/e9/970713f4b019f69de9844e1bab37b8ddb67558e410916f4eb5869a696165/prek-0.3.8-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:48efc28f2f53b5b8087efca9daaed91572d62df97d5f24a1c7a087fecb5017de", size = 5441801, upload-time = "2026-03-23T08:23:32.617Z" }, + { url = "https://files.pythonhosted.org/packages/12/a4/7ef44032b181753e19452ec3b09abb3a32607cf6b0a0508f0604becaaf2b/prek-0.3.8-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f6ca9d63bacbc448a5c18e955c78d3ac5176c3a17c3baacdd949b1a623e08a36", size = 5155107, upload-time = "2026-03-23T08:23:31.021Z" }, + { url = "https://files.pythonhosted.org/packages/bd/77/4d9c8985dbba84149760785dfe07093ea1e29d710257dfb7c89615e2234c/prek-0.3.8-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1000f7029696b4fe712fb1fefd4c55b9c4de72b65509c8e50296370a06f9dc3f", size = 5566541, upload-time = "2026-03-23T08:23:45.694Z" }, + { url = "https://files.pythonhosted.org/packages/1a/1a/81e6769ac1f7f8346d09ce2ab0b47cf06466acd9ff72e87e5d1f0d98cd32/prek-0.3.8-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6ff0bed0e2c1286522987d982168a86cbbd0d069d840506a46c9fda983515517", size = 6552991, upload-time = "2026-03-23T08:23:21.958Z" }, + { url = "https://files.pythonhosted.org/packages/6f/fa/ce2df0dd2dc75a9437a52463239d0782998943d7b04e191fb89b83016c34/prek-0.3.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4fb087ac0ffda3ac65bbbae9a38326a7fd27ee007bb4a94323ce1eb539d8bbec", size = 5832972, upload-time = "2026-03-23T08:23:20.258Z" }, + { url = "https://files.pythonhosted.org/packages/18/6b/9d4269df9073216d296244595a21c253b6475dfc9076c0bd2906be7a436c/prek-0.3.8-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:2e1e5e206ff7b31bd079cce525daddc96cd6bc544d20dc128921ad92f7a4c85d", size = 5448371, upload-time = "2026-03-23T08:23:41.835Z" }, + { url = "https://files.pythonhosted.org/packages/60/1d/1e4d8a78abefa5b9d086e5a9f1638a74b5e540eec8a648d9946707701f29/prek-0.3.8-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:dcea3fe23832a4481bccb7c45f55650cb233be7c805602e788bb7dba60f2d861", size = 5270546, upload-time = "2026-03-23T08:23:24.231Z" }, + { url = "https://files.pythonhosted.org/packages/77/07/34f36551a6319ae36e272bea63a42f59d41d2d47ab0d5fb00eb7b4e88e87/prek-0.3.8-py3-none-musllinux_1_1_armv7l.whl", hash = "sha256:4d25e647e9682f6818ab5c31e7a4b842993c14782a6ffcd128d22b784e0d677f", size = 5124032, upload-time = "2026-03-23T08:23:26.368Z" }, + { url = "https://files.pythonhosted.org/packages/e3/01/6d544009bb655e709993411796af77339f439526db4f3b3509c583ad8eb9/prek-0.3.8-py3-none-musllinux_1_1_i686.whl", hash = "sha256:de528b82935e33074815acff3c7c86026754d1212136295bc88fe9c43b4231d5", size = 5432245, upload-time = "2026-03-23T08:23:47.877Z" }, + { url = "https://files.pythonhosted.org/packages/54/96/1237ee269e9bfa283ffadbcba1f401f48a47aed2b2563eb1002740d6079d/prek-0.3.8-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:6d660f1c25a126e6d9f682fe61449441226514f412a4469f5d71f8f8cad56db2", size = 5950550, upload-time = "2026-03-23T08:23:43.8Z" }, ] [[package]] @@ -4326,24 +4326,24 @@ wheels = [ [[package]] name = "ruff" -version = "0.15.5" +version = "0.15.8" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/77/9b/840e0039e65fcf12758adf684d2289024d6140cde9268cc59887dc55189c/ruff-0.15.5.tar.gz", hash = "sha256:7c3601d3b6d76dce18c5c824fc8d06f4eef33d6df0c21ec7799510cde0f159a2", size = 4574214, upload-time = "2026-03-05T20:06:34.946Z" } +sdist = { url = "https://files.pythonhosted.org/packages/14/b0/73cf7550861e2b4824950b8b52eebdcc5adc792a00c514406556c5b80817/ruff-0.15.8.tar.gz", hash = "sha256:995f11f63597ee362130d1d5a327a87cb6f3f5eae3094c620bcc632329a4d26e", size = 4610921, upload-time = "2026-03-26T18:39:38.675Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/47/20/5369c3ce21588c708bcbe517a8fbe1a8dfdb5dfd5137e14790b1da71612c/ruff-0.15.5-py3-none-linux_armv6l.whl", hash = "sha256:4ae44c42281f42e3b06b988e442d344a5b9b72450ff3c892e30d11b29a96a57c", size = 10478185, upload-time = "2026-03-05T20:06:29.093Z" }, - { url = "https://files.pythonhosted.org/packages/44/ed/e81dd668547da281e5dce710cf0bc60193f8d3d43833e8241d006720e42b/ruff-0.15.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6edd3792d408ebcf61adabc01822da687579a1a023f297618ac27a5b51ef0080", size = 10859201, upload-time = "2026-03-05T20:06:32.632Z" }, - { url = "https://files.pythonhosted.org/packages/c4/8f/533075f00aaf19b07c5cd6aa6e5d89424b06b3b3f4583bfa9c640a079059/ruff-0.15.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:89f463f7c8205a9f8dea9d658d59eff49db05f88f89cc3047fb1a02d9f344010", size = 10184752, upload-time = "2026-03-05T20:06:40.312Z" }, - { url = "https://files.pythonhosted.org/packages/66/0e/ba49e2c3fa0395b3152bad634c7432f7edfc509c133b8f4529053ff024fb/ruff-0.15.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba786a8295c6574c1116704cf0b9e6563de3432ac888d8f83685654fe528fd65", size = 10534857, upload-time = "2026-03-05T20:06:19.581Z" }, - { url = "https://files.pythonhosted.org/packages/59/71/39234440f27a226475a0659561adb0d784b4d247dfe7f43ffc12dd02e288/ruff-0.15.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fd4b801e57955fe9f02b31d20375ab3a5c4415f2e5105b79fb94cf2642c91440", size = 10309120, upload-time = "2026-03-05T20:06:00.435Z" }, - { url = "https://files.pythonhosted.org/packages/f5/87/4140aa86a93df032156982b726f4952aaec4a883bb98cb6ef73c347da253/ruff-0.15.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:391f7c73388f3d8c11b794dbbc2959a5b5afe66642c142a6effa90b45f6f5204", size = 11047428, upload-time = "2026-03-05T20:05:51.867Z" }, - { url = "https://files.pythonhosted.org/packages/5a/f7/4953e7e3287676f78fbe85e3a0ca414c5ca81237b7575bdadc00229ac240/ruff-0.15.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8dc18f30302e379fe1e998548b0f5e9f4dff907f52f73ad6da419ea9c19d66c8", size = 11914251, upload-time = "2026-03-05T20:06:22.887Z" }, - { url = "https://files.pythonhosted.org/packages/77/46/0f7c865c10cf896ccf5a939c3e84e1cfaeed608ff5249584799a74d33835/ruff-0.15.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1cc6e7f90087e2d27f98dc34ed1b3ab7c8f0d273cc5431415454e22c0bd2a681", size = 11333801, upload-time = "2026-03-05T20:05:57.168Z" }, - { url = "https://files.pythonhosted.org/packages/d3/01/a10fe54b653061585e655f5286c2662ebddb68831ed3eaebfb0eb08c0a16/ruff-0.15.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1cb7169f53c1ddb06e71a9aebd7e98fc0fea936b39afb36d8e86d36ecc2636a", size = 11206821, upload-time = "2026-03-05T20:06:03.441Z" }, - { url = "https://files.pythonhosted.org/packages/7a/0d/2132ceaf20c5e8699aa83da2706ecb5c5dcdf78b453f77edca7fb70f8a93/ruff-0.15.5-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:9b037924500a31ee17389b5c8c4d88874cc6ea8e42f12e9c61a3d754ff72f1ca", size = 11133326, upload-time = "2026-03-05T20:06:25.655Z" }, - { url = "https://files.pythonhosted.org/packages/72/cb/2e5259a7eb2a0f87c08c0fe5bf5825a1e4b90883a52685524596bfc93072/ruff-0.15.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:65bb414e5b4eadd95a8c1e4804f6772bbe8995889f203a01f77ddf2d790929dd", size = 10510820, upload-time = "2026-03-05T20:06:37.79Z" }, - { url = "https://files.pythonhosted.org/packages/ff/20/b67ce78f9e6c59ffbdb5b4503d0090e749b5f2d31b599b554698a80d861c/ruff-0.15.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:d20aa469ae3b57033519c559e9bc9cd9e782842e39be05b50e852c7c981fa01d", size = 10302395, upload-time = "2026-03-05T20:05:54.504Z" }, - { url = "https://files.pythonhosted.org/packages/5f/e5/719f1acccd31b720d477751558ed74e9c88134adcc377e5e886af89d3072/ruff-0.15.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:15388dd28c9161cdb8eda68993533acc870aa4e646a0a277aa166de9ad5a8752", size = 10754069, upload-time = "2026-03-05T20:06:06.422Z" }, - { url = "https://files.pythonhosted.org/packages/c3/9c/d1db14469e32d98f3ca27079dbd30b7b44dbb5317d06ab36718dee3baf03/ruff-0.15.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:b30da330cbd03bed0c21420b6b953158f60c74c54c5f4c1dabbdf3a57bf355d2", size = 11304315, upload-time = "2026-03-05T20:06:10.867Z" }, + { url = "https://files.pythonhosted.org/packages/4a/92/c445b0cd6da6e7ae51e954939cb69f97e008dbe750cfca89b8cedc081be7/ruff-0.15.8-py3-none-linux_armv6l.whl", hash = "sha256:cbe05adeba76d58162762d6b239c9056f1a15a55bd4b346cfd21e26cd6ad7bc7", size = 10527394, upload-time = "2026-03-26T18:39:41.566Z" }, + { url = "https://files.pythonhosted.org/packages/eb/92/f1c662784d149ad1414cae450b082cf736430c12ca78367f20f5ed569d65/ruff-0.15.8-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:d3e3d0b6ba8dca1b7ef9ab80a28e840a20070c4b62e56d675c24f366ef330570", size = 10905693, upload-time = "2026-03-26T18:39:30.364Z" }, + { url = "https://files.pythonhosted.org/packages/ca/f2/7a631a8af6d88bcef997eb1bf87cc3da158294c57044aafd3e17030613de/ruff-0.15.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:6ee3ae5c65a42f273f126686353f2e08ff29927b7b7e203b711514370d500de3", size = 10323044, upload-time = "2026-03-26T18:39:33.37Z" }, + { url = "https://files.pythonhosted.org/packages/67/18/1bf38e20914a05e72ef3b9569b1d5c70a7ef26cd188d69e9ca8ef588d5bf/ruff-0.15.8-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdce027ada77baa448077ccc6ebb2fa9c3c62fd110d8659d601cf2f475858d94", size = 10629135, upload-time = "2026-03-26T18:39:44.142Z" }, + { url = "https://files.pythonhosted.org/packages/d2/e9/138c150ff9af60556121623d41aba18b7b57d95ac032e177b6a53789d279/ruff-0.15.8-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:12e617fc01a95e5821648a6df341d80456bd627bfab8a829f7cfc26a14a4b4a3", size = 10348041, upload-time = "2026-03-26T18:39:52.178Z" }, + { url = "https://files.pythonhosted.org/packages/02/f1/5bfb9298d9c323f842c5ddeb85f1f10ef51516ac7a34ba446c9347d898df/ruff-0.15.8-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:432701303b26416d22ba696c39f2c6f12499b89093b61360abc34bcc9bf07762", size = 11121987, upload-time = "2026-03-26T18:39:55.195Z" }, + { url = "https://files.pythonhosted.org/packages/10/11/6da2e538704e753c04e8d86b1fc55712fdbdcc266af1a1ece7a51fff0d10/ruff-0.15.8-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d910ae974b7a06a33a057cb87d2a10792a3b2b3b35e33d2699fdf63ec8f6b17a", size = 11951057, upload-time = "2026-03-26T18:39:19.18Z" }, + { url = "https://files.pythonhosted.org/packages/83/f0/c9208c5fd5101bf87002fed774ff25a96eea313d305f1e5d5744698dc314/ruff-0.15.8-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2033f963c43949d51e6fdccd3946633c6b37c484f5f98c3035f49c27395a8ab8", size = 11464613, upload-time = "2026-03-26T18:40:06.301Z" }, + { url = "https://files.pythonhosted.org/packages/f8/22/d7f2fabdba4fae9f3b570e5605d5eb4500dcb7b770d3217dca4428484b17/ruff-0.15.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f29b989a55572fb885b77464cf24af05500806ab4edf9a0fd8977f9759d85b1", size = 11257557, upload-time = "2026-03-26T18:39:57.972Z" }, + { url = "https://files.pythonhosted.org/packages/71/8c/382a9620038cf6906446b23ce8632ab8c0811b8f9d3e764f58bedd0c9a6f/ruff-0.15.8-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:ac51d486bf457cdc985a412fb1801b2dfd1bd8838372fc55de64b1510eff4bec", size = 11169440, upload-time = "2026-03-26T18:39:22.205Z" }, + { url = "https://files.pythonhosted.org/packages/4d/0d/0994c802a7eaaf99380085e4e40c845f8e32a562e20a38ec06174b52ef24/ruff-0.15.8-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c9861eb959edab053c10ad62c278835ee69ca527b6dcd72b47d5c1e5648964f6", size = 10605963, upload-time = "2026-03-26T18:39:46.682Z" }, + { url = "https://files.pythonhosted.org/packages/19/aa/d624b86f5b0aad7cef6bbf9cd47a6a02dfdc4f72c92a337d724e39c9d14b/ruff-0.15.8-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8d9a5b8ea13f26ae90838afc33f91b547e61b794865374f114f349e9036835fb", size = 10357484, upload-time = "2026-03-26T18:39:49.176Z" }, + { url = "https://files.pythonhosted.org/packages/35/c3/e0b7835d23001f7d999f3895c6b569927c4d39912286897f625736e1fd04/ruff-0.15.8-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c2a33a529fb3cbc23a7124b5c6ff121e4d6228029cba374777bd7649cc8598b8", size = 10830426, upload-time = "2026-03-26T18:40:03.702Z" }, + { url = "https://files.pythonhosted.org/packages/f0/51/ab20b322f637b369383adc341d761eaaa0f0203d6b9a7421cd6e783d81b9/ruff-0.15.8-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:75e5cd06b1cf3f47a3996cfc999226b19aa92e7cce682dcd62f80d7035f98f49", size = 11345125, upload-time = "2026-03-26T18:39:27.799Z" }, ] [[package]] @@ -5710,7 +5710,7 @@ wheels = [ [[package]] name = "zensical" -version = "0.0.26" +version = "0.0.29" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -5720,18 +5720,18 @@ dependencies = [ { name = "pymdown-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "pyyaml", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d5/1f/0a0b1ce8e0553a9dabaedc736d0f34b11fc33d71ff46bce44d674996d41f/zensical-0.0.26.tar.gz", hash = "sha256:f4d9c8403df25fbb3d6dd9577122dc2f23c73a2d16ab778bb7d40370dd71e987", size = 3841473, upload-time = "2026-03-11T09:51:38.838Z" } +sdist = { url = "https://files.pythonhosted.org/packages/78/bd/5786ab618a60bd7469ab243a7fd2c9eecb0790c85c784abb8b97edb77a54/zensical-0.0.29.tar.gz", hash = "sha256:0d6282be7cb551e12d5806badf5e94c54a5e2f2cf07057a3e36d1eaf97c33ada", size = 3842641, upload-time = "2026-03-24T13:37:27.587Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/41/58/fa3d9538ff1ea8cf4a193edbf47254f374fa7983fcfa876bb4336d72c53a/zensical-0.0.26-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:7823b25afe7d36099253aa59d643abaac940f80fd015d4a37954210c87d3da56", size = 12263607, upload-time = "2026-03-11T09:50:49.202Z" }, - { url = "https://files.pythonhosted.org/packages/5f/6e/44a3b21bd3569b9cad203364d73a956768d28a879e4c2be91bd889f74d2c/zensical-0.0.26-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c0254814382cdd3769bc7689180d09bf41de8879871dd736dc52d5f141e8ada7", size = 12144562, upload-time = "2026-03-11T09:50:53.685Z" }, - { url = "https://files.pythonhosted.org/packages/07/ae/31b9885745b3e7ef23a3ae7f175b879807288d11b3fb7e2d3c119c916258/zensical-0.0.26-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c8e601b2bbd239e564b04cf235eefb9777e7dfc7e1857b8871d6cdcfb577aa0", size = 12506728, upload-time = "2026-03-11T09:50:57.775Z" }, - { url = "https://files.pythonhosted.org/packages/bd/93/f5291e2c47076474f181f6eef35ef0428117d3f192da4358c0511e2ce09e/zensical-0.0.26-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2dc43c7e6c25d9724fc0450f0273ca4e5e2506eeb7f89f52f1405a592896ca3b", size = 12454975, upload-time = "2026-03-11T09:51:01.514Z" }, - { url = "https://files.pythonhosted.org/packages/aa/2e/61cac4f2ebad31dab768eb02753ffde9e56d4d34b8f876b949bf516fbd50/zensical-0.0.26-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:24ed236d1254cc474c19227eaa3670a1ccf921af53134ec5542b05853bdcd59c", size = 12791930, upload-time = "2026-03-11T09:51:05.162Z" }, - { url = "https://files.pythonhosted.org/packages/02/86/51995d1ed2dd6ad8a1a70bcdf3c5eb16b50e62ea70e638d454a6b9061c4d/zensical-0.0.26-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1110147710d1dd025d932c4a7eada836bdf079c91b70fb0ae5b202e14b094617", size = 12548166, upload-time = "2026-03-11T09:51:09.218Z" }, - { url = "https://files.pythonhosted.org/packages/3d/93/decbafdbfc77170cbc3851464632390846e9aaf45e743c8dd5a24d5673e9/zensical-0.0.26-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7d21596a785428cdebc20859bd94a05334abe14ad24f1bb9cd80d19219e3c220", size = 12682103, upload-time = "2026-03-11T09:51:12.68Z" }, - { url = "https://files.pythonhosted.org/packages/fb/e2/391d2d08dde621177da069a796a886b549fefb15734aeeb6e696af99b662/zensical-0.0.26-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:680a3c7bb71499b4da784d6072e44b3d7b8c0df3ce9bbd9974e24bd8058c2736", size = 12724219, upload-time = "2026-03-11T09:51:17.32Z" }, - { url = "https://files.pythonhosted.org/packages/80/2a/21b40c5c40a67da8a841f278d61dbd8d5e035e489de6fe1cef5f4e211b4f/zensical-0.0.26-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:e3294a79f98218b6fc2219232e166aa0932ae4dad58f6c8dbc0dbe0ecbff9c25", size = 12862117, upload-time = "2026-03-11T09:51:22.161Z" }, - { url = "https://files.pythonhosted.org/packages/51/76/e1910d6d75d207654c867b8efbda6822dedda9fed3601bf4a864a1f4fe26/zensical-0.0.26-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:630229587df1fb47be184a4a69d0772ce59a44cd2c481ae9f7e8852fffaff11e", size = 12815714, upload-time = "2026-03-11T09:51:26.24Z" }, + { url = "https://files.pythonhosted.org/packages/4b/9c/8b681daa024abca9763017bec09ecee8008e110cae1254217c8dd22cc339/zensical-0.0.29-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:20ae0709ea14fce25ab33d0a82acdaf454a7a2e232a9ee20c019942205174476", size = 12311399, upload-time = "2026-03-24T13:36:53.809Z" }, + { url = "https://files.pythonhosted.org/packages/81/ae/4ebb4d8bb2ef0164d473698b92f11caf431fc436e1625524acd5641102ca/zensical-0.0.29-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:599af3ba66fcd0146d7019f3493ed3c316051fae6c4d5599bc59f3a8f4b8a6f0", size = 12191845, upload-time = "2026-03-24T13:36:56.909Z" }, + { url = "https://files.pythonhosted.org/packages/d5/35/67f89db06571a52283b3ecbe3bcf32fd3115ca50436b3ae177a948b83ea7/zensical-0.0.29-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eea7e48a00a71c0586e875079b5f83a070c33a147e52ad4383e4b63ab524332b", size = 12554105, upload-time = "2026-03-24T13:36:59.945Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f6/ac79e5d9c18b28557c9ff1c7c23d695fbdd82645d69bfe02292f46d935e7/zensical-0.0.29-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:59a57db35542e98d2896b833de07d199320f8ada3b4e7ddccb7fe892292d8b74", size = 12498643, upload-time = "2026-03-24T13:37:02.376Z" }, + { url = "https://files.pythonhosted.org/packages/b1/70/5c22a96a69e0e91e569c26236918bb9bab1170f59b29ad04105ead64f199/zensical-0.0.29-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d42c2b2a96a80cf64c98ba7242f59ef95109914bd4c9499d7ebc12544663852c", size = 12854531, upload-time = "2026-03-24T13:37:04.962Z" }, + { url = "https://files.pythonhosted.org/packages/79/25/e32237a8fcb0ceae1ef8e192e7f8db53b38f1e48f1c7cdbacd0a7b713892/zensical-0.0.29-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b2fca39c5f6b1782c77cf6591cf346357cabee85ebdb956c5ddc0fd5169f3d9", size = 12596828, upload-time = "2026-03-24T13:37:07.817Z" }, + { url = "https://files.pythonhosted.org/packages/ff/74/89ac909cbb258903ea53802c184e4986c17ce0ba79b1c7f77b7e78a2dce3/zensical-0.0.29-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:dfc23a74ef672aa51088c080286319da1dc0b989cd5051e9e5e6d7d4abbc2fc1", size = 12732059, upload-time = "2026-03-24T13:37:11.651Z" }, + { url = "https://files.pythonhosted.org/packages/8c/31/2429de6a9328eed4acc7e9a3789f160294a15115be15f9870a0d02649302/zensical-0.0.29-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:c9336d4e4b232e3c9a70e30258e916dd7e60c0a2a08c8690065e60350c302028", size = 12768542, upload-time = "2026-03-24T13:37:14.39Z" }, + { url = "https://files.pythonhosted.org/packages/10/8a/55588b2a1dcbe86dad0404506c9ba367a06c663b1ff47147c84d26f7510e/zensical-0.0.29-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:30661148f0681199f3b598cbeb1d54f5cba773e54ae840bac639250d85907b84", size = 12917991, upload-time = "2026-03-24T13:37:16.795Z" }, + { url = "https://files.pythonhosted.org/packages/ec/5d/653901f0d3a3ca72daebc62746a148797f4e422cc3a2b66a4e6718e4398f/zensical-0.0.29-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6a566ac1fd4bfac5d711a7bd1ae06666712127c2718daa5083c7bf3f107e8578", size = 12868392, upload-time = "2026-03-24T13:37:19.42Z" }, ] [[package]] From eb758862c94ea6344da4b940e23ee30d21aff829 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 Apr 2026 07:22:04 -0700 Subject: [PATCH 05/11] Chore(deps): Bump the document-processing group with 3 updates (#12489) Bumps the document-processing group with 3 updates: [gotenberg-client](https://github.com/stumpylog/gotenberg-client), [ocrmypdf](https://github.com/ocrmypdf/OCRmyPDF) and [tika-client](https://github.com/stumpylog/tika-client). Updates `gotenberg-client` from 0.13.1 to 0.14.0 - [Release notes](https://github.com/stumpylog/gotenberg-client/releases) - [Changelog](https://github.com/stumpylog/gotenberg-client/blob/main/CHANGELOG.md) - [Commits](https://github.com/stumpylog/gotenberg-client/compare/0.13.1...0.14.0) Updates `ocrmypdf` from 17.3.0 to 17.4.0 - [Release notes](https://github.com/ocrmypdf/OCRmyPDF/releases) - [Commits](https://github.com/ocrmypdf/OCRmyPDF/compare/v17.3.0...v17.4.0) Updates `tika-client` from 0.10.0 to 0.11.0 - [Release notes](https://github.com/stumpylog/tika-client/releases) - [Changelog](https://github.com/stumpylog/tika-client/blob/main/CHANGELOG.md) - [Commits](https://github.com/stumpylog/tika-client/compare/0.10.0...0.11.0) --- updated-dependencies: - dependency-name: gotenberg-client dependency-version: 0.14.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: document-processing - dependency-name: ocrmypdf dependency-version: 17.4.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: document-processing - dependency-name: tika-client dependency-version: 0.11.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: document-processing ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 6 +++--- uv.lock | 24 ++++++++++++------------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7bb160956..30be1ae11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ dependencies = [ "faiss-cpu>=1.10", "filelock~=3.25.2", "flower~=2.0.1", - "gotenberg-client~=0.13.1", + "gotenberg-client~=0.14.0", "httpx-oauth~=0.16", "ijson>=3.2", "imap-tools~=1.11.0", @@ -59,7 +59,7 @@ dependencies = [ "llama-index-llms-openai>=0.6.13", "llama-index-vector-stores-faiss>=0.5.2", "nltk~=3.9.1", - "ocrmypdf~=17.3.0", + "ocrmypdf~=17.4.0", "openai>=1.76", "pathvalidate~=3.3.1", "pdf2image~=1.17.0", @@ -75,7 +75,7 @@ dependencies = [ "sentence-transformers>=4.1", "setproctitle~=1.3.4", "tantivy>=0.25.1", - "tika-client~=0.10.0", + "tika-client~=0.11.0", "torch~=2.10.0", "watchfiles>=1.1.1", "whitenoise~=6.11", diff --git a/uv.lock b/uv.lock index 4cfd329f4..8b72cbcc3 100644 --- a/uv.lock +++ b/uv.lock @@ -1434,14 +1434,14 @@ wheels = [ [[package]] name = "gotenberg-client" -version = "0.13.1" +version = "0.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx", extra = ["http2"], marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e4/6c/aaadd6657ca42fbd148b1c00604b98c1ead5a22552f4e5365ce5f0632430/gotenberg_client-0.13.1.tar.gz", hash = "sha256:cdd6bbb535cd739b87446cd1b4f6347ed7f9af6a0d4b19baf7c064b75528ee54", size = 1211143, upload-time = "2025-12-04T20:45:24.151Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/34/8e3be3a6a1b654d2a3bfa3e5d201183aeff6d50c42199ac0b8ed912c01c5/gotenberg_client-0.14.0.tar.gz", hash = "sha256:a853700c6b01c3372871264c4eb9ae3375addafbcbbfd3341e411f4217a8088c", size = 1214438, upload-time = "2026-03-11T17:23:11.122Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/79/f6/7a6e6785295332d2538f729ae19516cef712273a5ab8b90d015f08e37a45/gotenberg_client-0.13.1-py3-none-any.whl", hash = "sha256:613f7083a5e8a81699dd8d715c97e5806a424ac48920aad25d7c11b600cdfaf3", size = 51058, upload-time = "2025-12-04T20:45:22.603Z" }, + { url = "https://files.pythonhosted.org/packages/55/1a/67ff4cca162ae4195bd6f1a107779898b6f2977cc33ae7e05a5178a395fa/gotenberg_client-0.14.0-py3-none-any.whl", hash = "sha256:868f1be46d1ed0f327ca3efeb1888b4fe35641c35bfa39684d23a59365703156", size = 50977, upload-time = "2026-03-11T17:23:09.397Z" }, ] [[package]] @@ -2775,7 +2775,7 @@ wheels = [ [[package]] name = "ocrmypdf" -version = "17.3.0" +version = "17.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "deprecation", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -2792,9 +2792,9 @@ dependencies = [ { name = "rich", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "uharfbuzz", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fa/fe/60bdc79529be1ad8b151d426ed2020d5ac90328c54e9ba92bd808e1535c1/ocrmypdf-17.3.0.tar.gz", hash = "sha256:4022f13aad3f405e330056a07aa8bd63714b48b414693831b56e2cf2c325f52d", size = 7378015, upload-time = "2026-02-21T09:30:07.207Z" } +sdist = { url = "https://files.pythonhosted.org/packages/88/b9/01f5cbd062f680af8a3f8f883f8e71de8be7979c3256f509661c1e2e2065/ocrmypdf-17.4.0.tar.gz", hash = "sha256:4bbc53249f3981599565f670c5de774d6440832eede87c515e6608880fa02a34", size = 7378592, upload-time = "2026-03-21T19:06:50.966Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/b1/b7ae057a1bcb1495067ee3c4d48c1ce5fc66addd9492307c5a0ff799a7f2/ocrmypdf-17.3.0-py3-none-any.whl", hash = "sha256:c8882e7864954d3db6bcee49cc9f261b65bff66b7e5925eb68a1c281f41cad23", size = 488130, upload-time = "2026-02-21T09:30:05.236Z" }, + { url = "https://files.pythonhosted.org/packages/ff/b7/71ae51a11669e63f1fea76153db4475079ea8d7b60802faf080f25b4a262/ocrmypdf-17.4.0-py3-none-any.whl", hash = "sha256:b93fd4d736a71a241e44d1b48e1305b9bf4581cd2ae9a96fcd89f4db1051dd87", size = 488312, upload-time = "2026-03-21T19:06:48.456Z" }, ] [[package]] @@ -3022,7 +3022,7 @@ requires-dist = [ { name = "faiss-cpu", specifier = ">=1.10" }, { name = "filelock", specifier = "~=3.25.2" }, { name = "flower", specifier = "~=2.0.1" }, - { name = "gotenberg-client", specifier = "~=0.13.1" }, + { name = "gotenberg-client", specifier = "~=0.14.0" }, { name = "granian", extras = ["uvloop"], marker = "extra == 'webserver'", specifier = "~=2.7.0" }, { name = "httpx-oauth", specifier = "~=0.16" }, { name = "ijson", specifier = ">=3.2" }, @@ -3037,7 +3037,7 @@ requires-dist = [ { name = "llama-index-vector-stores-faiss", specifier = ">=0.5.2" }, { name = "mysqlclient", marker = "extra == 'mariadb'", specifier = "~=2.2.7" }, { name = "nltk", specifier = "~=3.9.1" }, - { name = "ocrmypdf", specifier = "~=17.3.0" }, + { name = "ocrmypdf", specifier = "~=17.4.0" }, { name = "openai", specifier = ">=1.76" }, { name = "pathvalidate", specifier = "~=3.3.1" }, { name = "pdf2image", specifier = "~=1.17.0" }, @@ -3058,7 +3058,7 @@ requires-dist = [ { name = "sentence-transformers", specifier = ">=4.1" }, { name = "setproctitle", specifier = "~=1.3.4" }, { name = "tantivy", specifier = ">=0.25.1" }, - { name = "tika-client", specifier = "~=0.10.0" }, + { name = "tika-client", specifier = "~=0.11.0" }, { name = "torch", specifier = "~=2.10.0", index = "https://download.pytorch.org/whl/cpu" }, { name = "watchfiles", specifier = ">=1.1.1" }, { name = "whitenoise", specifier = "~=6.11" }, @@ -4716,15 +4716,15 @@ wheels = [ [[package]] name = "tika-client" -version = "0.10.0" +version = "0.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "httpx", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/21/be/65bfc47e4689ecd5ead20cf47dc0084fd767b7e71e8cfabf5fddc42aae3c/tika_client-0.10.0.tar.gz", hash = "sha256:3101e8b2482ae4cb7f87be13ada970ff691bdc3404d94cd52f5e57a09c99370c", size = 2178257, upload-time = "2025-08-04T17:47:30.414Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4d/d9/01f2049240dacf67c9be61d9c59e72b6827a862e8fd87e77e458e0a3b797/tika_client-0.11.0.tar.gz", hash = "sha256:c741caaca08bbd715a8db3fe6f0430a54d075fef3d59a441e8b8d810f58de4f0", size = 2178828, upload-time = "2026-03-11T16:50:25.865Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/31/002e0fa5bca67d6a19da8c294273486f6c46cbcc83d6879719a38a181461/tika_client-0.10.0-py3-none-any.whl", hash = "sha256:f5486cc884e4522575662aa295bda761bf9f101ac8d92840155b58ab8b96f6e2", size = 18237, upload-time = "2025-08-04T17:47:28.966Z" }, + { url = "https://files.pythonhosted.org/packages/53/04/5a433d621ec559d1d216d200eea43b0ac63435beb5dd52bbc75f4aaef465/tika_client-0.11.0-py3-none-any.whl", hash = "sha256:461903ccbe705d84dd3e4a1ca83e04174776d4b06dc57b902f9281633a3836e6", size = 18470, upload-time = "2026-03-11T16:50:24.672Z" }, ] [[package]] From 64debc87a594ae339a5ac725550de3d4c9bcfaeb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 Apr 2026 08:16:36 -0700 Subject: [PATCH 06/11] Chore(deps): Bump djangorestframework in the django-ecosystem group (#12488) Bumps the django-ecosystem group with 1 update: [djangorestframework](https://github.com/encode/django-rest-framework). Updates `djangorestframework` from 3.16.1 to 3.17.1 - [Release notes](https://github.com/encode/django-rest-framework/releases) - [Commits](https://github.com/encode/django-rest-framework/compare/3.16.1...3.17.1) --- updated-dependencies: - dependency-name: djangorestframework dependency-version: 3.17.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: django-ecosystem ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index 8b72cbcc3..180448efe 100644 --- a/uv.lock +++ b/uv.lock @@ -1108,14 +1108,14 @@ wheels = [ [[package]] name = "djangorestframework" -version = "3.16.1" +version = "3.17.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "django", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/8a/95/5376fe618646fde6899b3cdc85fd959716bb67542e273a76a80d9f326f27/djangorestframework-3.16.1.tar.gz", hash = "sha256:166809528b1aced0a17dc66c24492af18049f2c9420dbd0be29422029cfc3ff7", size = 1089735, upload-time = "2025-08-06T17:50:53.251Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/d7/c016e69fac19ff8afdc89db9d31d9ae43ae031e4d1993b20aca179b8301a/djangorestframework-3.17.1.tar.gz", hash = "sha256:a6def5f447fe78ff853bff1d47a3c59bf38f5434b031780b351b0c73a62db1a5", size = 905742, upload-time = "2026-03-24T16:58:33.705Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/ce/bf8b9d3f415be4ac5588545b5fcdbbb841977db1c1d923f7568eeabe1689/djangorestframework-3.16.1-py3-none-any.whl", hash = "sha256:33a59f47fb9c85ede792cbf88bde71893bcda0667bc573f784649521f1102cec", size = 1080442, upload-time = "2025-08-06T17:50:50.667Z" }, + { url = "https://files.pythonhosted.org/packages/5a/e1/2c516bdc83652b1a60c6119366ac2c0607b479ed05cd6093f916ca8928f8/djangorestframework-3.17.1-py3-none-any.whl", hash = "sha256:c3c74dd3e83a5a3efc37b3c18d92bd6f86a6791c7b7d4dff62bb068500e76457", size = 898844, upload-time = "2026-03-24T16:58:31.845Z" }, ] [[package]] From f32ad98d8e4bba2e92d5eecc5bfb32ef4b7dff54 Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Fri, 3 Apr 2026 13:31:40 -0700 Subject: [PATCH 07/11] Feature: Update consumer logging to include task ID for log correlation (#12510) --- src/documents/tasks.py | 126 +++++++++++++++------------- src/paperless/logging.py | 33 ++++++++ src/paperless/settings/__init__.py | 3 +- src/paperless/tests/test_logging.py | 34 ++++++++ 4 files changed, 136 insertions(+), 60 deletions(-) create mode 100644 src/paperless/logging.py create mode 100644 src/paperless/tests/test_logging.py diff --git a/src/documents/tasks.py b/src/documents/tasks.py index ae65a5fbe..c40b1ff3f 100644 --- a/src/documents/tasks.py +++ b/src/documents/tasks.py @@ -61,6 +61,7 @@ from documents.utils import compute_checksum from documents.utils import identity from documents.workflows.utils import get_workflows_for_trigger from paperless.config import AIConfig +from paperless.logging import consume_task_id from paperless.parsers import ParserContext from paperless.parsers.registry import get_parser_registry from paperless_ai.indexing import llm_index_add_or_update_document @@ -147,76 +148,85 @@ def consume_file( input_doc: ConsumableDocument, overrides: DocumentMetadataOverrides | None = None, ): - # Default no overrides - if overrides is None: - overrides = DocumentMetadataOverrides() + token = consume_task_id.set((self.request.id or "")[:8]) + try: + # Default no overrides + if overrides is None: + overrides = DocumentMetadataOverrides() - plugins: list[type[ConsumeTaskPlugin]] = ( - [ - ConsumerPreflightPlugin, - ConsumerPlugin, - ] - if input_doc.root_document_id is not None - else [ - ConsumerPreflightPlugin, - AsnCheckPlugin, - CollatePlugin, - BarcodePlugin, - AsnCheckPlugin, # Re-run ASN check after barcode reading - WorkflowTriggerPlugin, - ConsumerPlugin, - ] - ) + plugins: list[type[ConsumeTaskPlugin]] = ( + [ + ConsumerPreflightPlugin, + ConsumerPlugin, + ] + if input_doc.root_document_id is not None + else [ + ConsumerPreflightPlugin, + AsnCheckPlugin, + CollatePlugin, + BarcodePlugin, + AsnCheckPlugin, # Re-run ASN check after barcode reading + WorkflowTriggerPlugin, + ConsumerPlugin, + ] + ) - with ( - ProgressManager( - overrides.filename or input_doc.original_file.name, - self.request.id, - ) as status_mgr, - TemporaryDirectory(dir=settings.SCRATCH_DIR) as tmp_dir, - ): - tmp_dir = Path(tmp_dir) - for plugin_class in plugins: - plugin_name = plugin_class.NAME - - plugin = plugin_class( - input_doc, - overrides, - status_mgr, - tmp_dir, + with ( + ProgressManager( + overrides.filename or input_doc.original_file.name, self.request.id, - ) + ) as status_mgr, + TemporaryDirectory(dir=settings.SCRATCH_DIR) as tmp_dir, + ): + tmp_dir = Path(tmp_dir) + for plugin_class in plugins: + plugin_name = plugin_class.NAME - if not plugin.able_to_run: - logger.debug(f"Skipping plugin {plugin_name}") - continue + plugin = plugin_class( + input_doc, + overrides, + status_mgr, + tmp_dir, + self.request.id, + ) - try: - logger.debug(f"Executing plugin {plugin_name}") - plugin.setup() + if not plugin.able_to_run: + logger.debug(f"Skipping plugin {plugin_name}") + continue - msg = plugin.run() + try: + logger.debug(f"Executing plugin {plugin_name}") + plugin.setup() - if msg is not None: - logger.info(f"{plugin_name} completed with: {msg}") - else: - logger.info(f"{plugin_name} completed with no message") + msg = plugin.run() - overrides = plugin.metadata + if msg is not None: + logger.info(f"{plugin_name} completed with: {msg}") + else: + logger.info(f"{plugin_name} completed with no message") - except StopConsumeTaskError as e: - logger.info(f"{plugin_name} requested task exit: {e.message}") - return e.message + overrides = plugin.metadata - except Exception as e: - logger.exception(f"{plugin_name} failed: {e}") - status_mgr.send_progress(ProgressStatusOptions.FAILED, f"{e}", 100, 100) - raise + except StopConsumeTaskError as e: + logger.info(f"{plugin_name} requested task exit: {e.message}") + return e.message - finally: - plugin.cleanup() + except Exception as e: + logger.exception(f"{plugin_name} failed: {e}") + status_mgr.send_progress( + ProgressStatusOptions.FAILED, + f"{e}", + 100, + 100, + ) + raise - return msg + finally: + plugin.cleanup() + + return msg + finally: + consume_task_id.reset(token) @shared_task diff --git a/src/paperless/logging.py b/src/paperless/logging.py new file mode 100644 index 000000000..ce2eff4fc --- /dev/null +++ b/src/paperless/logging.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import logging +from contextvars import ContextVar + +consume_task_id: ContextVar[str] = ContextVar("consume_task_id", default="") + + +class ConsumeTaskFormatter(logging.Formatter): + """ + Logging formatter that prepends a short task correlation ID to messages + emitted during document consumption. + + The ID is the first 8 characters of the Celery task UUID, set via the + ``consume_task_id`` ContextVar at the entry of ``consume_file``. When + the ContextVar is empty (any log outside a consume task) no prefix is + added and the output is identical to the standard verbose format. + """ + + def __init__(self) -> None: + super().__init__( + fmt="[{asctime}] [{levelname}] [{name}] {task_prefix}{message}", + style="{", + validate=False, # {task_prefix} is not a standard LogRecord attribute, so Python's + # init-time format-string validation would raise ValueError without + # this. Runtime safety comes from format() always setting + # record.task_prefix before calling super().format(). + ) + + def format(self, record: logging.LogRecord) -> str: + task_id = consume_task_id.get() + record.task_prefix = f"[{task_id}] " if task_id else "" + return super().format(record) diff --git a/src/paperless/settings/__init__.py b/src/paperless/settings/__init__.py index 964295020..fac1391b3 100644 --- a/src/paperless/settings/__init__.py +++ b/src/paperless/settings/__init__.py @@ -592,8 +592,7 @@ LOGGING = { "disable_existing_loggers": False, "formatters": { "verbose": { - "format": "[{asctime}] [{levelname}] [{name}] {message}", - "style": "{", + "()": "paperless.logging.ConsumeTaskFormatter", }, "simple": { "format": "{levelname} {message}", diff --git a/src/paperless/tests/test_logging.py b/src/paperless/tests/test_logging.py new file mode 100644 index 000000000..dbd36c7d0 --- /dev/null +++ b/src/paperless/tests/test_logging.py @@ -0,0 +1,34 @@ +import logging + +from paperless.logging import ConsumeTaskFormatter +from paperless.logging import consume_task_id + + +def _make_record(msg: str = "Test message") -> logging.LogRecord: + return logging.LogRecord( + name="paperless.consumer", + level=logging.INFO, + pathname="", + lineno=0, + msg=msg, + args=(), + exc_info=None, + ) + + +def test_formatter_includes_task_id_when_set(): + token = consume_task_id.set("a8098c1a") + try: + formatter = ConsumeTaskFormatter() + output = formatter.format(_make_record()) + assert "[a8098c1a] Test message" in output + finally: + consume_task_id.reset(token) + + +def test_formatter_omits_prefix_when_no_task_id(): + # ContextVar default is "" — no task active + formatter = ConsumeTaskFormatter() + output = formatter.format(_make_record()) + assert "[] " not in output + assert "Test message" in output From 566afdffcaaf90eb5a63ca9ae3d6f49128fb623c Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Fri, 3 Apr 2026 13:53:45 -0700 Subject: [PATCH 08/11] Enhancement: unify text search to use tantivy (#12485) --- docs/api.md | 10 +- .../e2e/document-list/document-list.spec.ts | 4 +- .../requests/api-document-list2.har | 8 +- .../global-search.component.spec.ts | 4 +- .../global-search/global-search.component.ts | 4 +- ...storage-path-edit-dialog.component.spec.ts | 4 +- .../storage-path-edit-dialog.component.ts | 4 +- .../document-link.component.spec.ts | 4 +- .../document-link/document-link.component.ts | 4 +- .../bulk-editor/bulk-editor.component.spec.ts | 2 +- .../filter-editor.component.spec.ts | 140 ++++++++-- .../filter-editor/filter-editor.component.ts | 41 ++- src-ui/src/app/data/filter-rule-type.ts | 24 +- .../services/rest/document.service.spec.ts | 6 +- src-ui/src/app/utils/query-params.spec.ts | 48 ++++ src-ui/src/app/utils/query-params.ts | 21 +- src/documents/filters.py | 12 + .../0018_saved_view_simple_search_rules.py | 92 +++++++ src/documents/models.py | 2 + src/documents/search/__init__.py | 2 + src/documents/search/_backend.py | 40 ++- src/documents/search/_normalize.py | 8 + src/documents/search/_query.py | 82 ++++++ src/documents/search/_schema.py | 12 + src/documents/search/_tokenizer.py | 14 + src/documents/tests/search/test_backend.py | 253 ++++++++++++++++++ src/documents/tests/search/test_tokenizer.py | 33 +++ src/documents/tests/test_api_search.py | 154 +++++++++++ src/documents/views.py | 84 ++++-- 29 files changed, 1019 insertions(+), 97 deletions(-) create mode 100644 src/documents/migrations/0018_saved_view_simple_search_rules.py create mode 100644 src/documents/search/_normalize.py diff --git a/docs/api.md b/docs/api.md index 2284d9d29..af1190f3d 100644 --- a/docs/api.md +++ b/docs/api.md @@ -62,10 +62,14 @@ The REST api provides five different forms of authentication. ## Searching for documents -Full text searching is available on the `/api/documents/` endpoint. Two -specific query parameters cause the API to return full text search +Full text searching is available on the `/api/documents/` endpoint. The +following query parameters cause the API to return Tantivy-backed search results: +- `/api/documents/?text=your%20search%20query`: Search title and content + using simple substring-style search. +- `/api/documents/?title_search=your%20search%20query`: Search title only + using simple substring-style search. - `/api/documents/?query=your%20search%20query`: Search for a document using a full text query. For details on the syntax, see [Basic Usage - Searching](usage.md#basic-usage_searching). - `/api/documents/?more_like_id=1234`: Search for documents similar to @@ -439,3 +443,5 @@ Initial API version. - The `all` parameter of list endpoints is now deprecated and will be removed in a future version. - The bulk edit objects endpoint now supports `all` and `filters` parameters to avoid having to send large lists of object IDs for operations affecting many objects. +- The legacy `title_content` document search parameter is deprecated and will be removed in a future version. + Clients should use `text` for simple title-and-content search and `title_search` for title-only search. diff --git a/src-ui/e2e/document-list/document-list.spec.ts b/src-ui/e2e/document-list/document-list.spec.ts index 700304186..0cea8effa 100644 --- a/src-ui/e2e/document-list/document-list.spec.ts +++ b/src-ui/e2e/document-list/document-list.spec.ts @@ -49,11 +49,11 @@ test('text filtering', async ({ page }) => { await page.getByRole('main').getByRole('combobox').click() await page.getByRole('main').getByRole('combobox').fill('test') await expect(page.locator('pngx-document-list')).toHaveText(/32 documents/) - await expect(page).toHaveURL(/title_content=test/) + await expect(page).toHaveURL(/text=test/) await page.getByRole('button', { name: 'Title & content' }).click() await page.getByRole('button', { name: 'Title', exact: true }).click() await expect(page.locator('pngx-document-list')).toHaveText(/9 documents/) - await expect(page).toHaveURL(/title__icontains=test/) + await expect(page).toHaveURL(/title_search=test/) await page.getByRole('button', { name: 'Title', exact: true }).click() await page.getByRole('button', { name: 'Advanced search' }).click() await expect(page).toHaveURL(/query=test/) diff --git a/src-ui/e2e/document-list/requests/api-document-list2.har b/src-ui/e2e/document-list/requests/api-document-list2.har index 3cbc9e8a6..f6a488b26 100644 --- a/src-ui/e2e/document-list/requests/api-document-list2.har +++ b/src-ui/e2e/document-list/requests/api-document-list2.har @@ -3545,7 +3545,7 @@ "time": 1.091, "request": { "method": "GET", - "url": "http://localhost:8000/api/documents/?page=1&page_size=50&ordering=-created&truncate_content=true&include_selection_data=true&title_content=test", + "url": "http://localhost:8000/api/documents/?page=1&page_size=50&ordering=-created&truncate_content=true&include_selection_data=true&text=test", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [ @@ -3579,7 +3579,7 @@ "value": "true" }, { - "name": "title_content", + "name": "text", "value": "test" } ], @@ -4303,7 +4303,7 @@ "time": 0.603, "request": { "method": "GET", - "url": "http://localhost:8000/api/documents/?page=1&page_size=50&ordering=-created&truncate_content=true&include_selection_data=true&title__icontains=test", + "url": "http://localhost:8000/api/documents/?page=1&page_size=50&ordering=-created&truncate_content=true&include_selection_data=true&title_search=test", "httpVersion": "HTTP/1.1", "cookies": [], "headers": [ @@ -4337,7 +4337,7 @@ "value": "true" }, { - "name": "title__icontains", + "name": "title_search", "value": "test" } ], diff --git a/src-ui/src/app/components/app-frame/global-search/global-search.component.spec.ts b/src-ui/src/app/components/app-frame/global-search/global-search.component.spec.ts index eaae4a814..1be801478 100644 --- a/src-ui/src/app/components/app-frame/global-search/global-search.component.spec.ts +++ b/src-ui/src/app/components/app-frame/global-search/global-search.component.spec.ts @@ -24,7 +24,7 @@ import { FILTER_HAS_DOCUMENT_TYPE_ANY, FILTER_HAS_STORAGE_PATH_ANY, FILTER_HAS_TAGS_ALL, - FILTER_TITLE_CONTENT, + FILTER_SIMPLE_TEXT, } from 'src/app/data/filter-rule-type' import { GlobalSearchType, SETTINGS_KEYS } from 'src/app/data/ui-settings' import { DocumentListViewService } from 'src/app/services/document-list-view.service' @@ -545,7 +545,7 @@ describe('GlobalSearchComponent', () => { component.query = 'test' component.runFullSearch() expect(qfSpy).toHaveBeenCalledWith([ - { rule_type: FILTER_TITLE_CONTENT, value: 'test' }, + { rule_type: FILTER_SIMPLE_TEXT, value: 'test' }, ]) settingsService.set( diff --git a/src-ui/src/app/components/app-frame/global-search/global-search.component.ts b/src-ui/src/app/components/app-frame/global-search/global-search.component.ts index 4f9a2467c..e95b52cfc 100644 --- a/src-ui/src/app/components/app-frame/global-search/global-search.component.ts +++ b/src-ui/src/app/components/app-frame/global-search/global-search.component.ts @@ -25,7 +25,7 @@ import { FILTER_HAS_DOCUMENT_TYPE_ANY, FILTER_HAS_STORAGE_PATH_ANY, FILTER_HAS_TAGS_ALL, - FILTER_TITLE_CONTENT, + FILTER_SIMPLE_TEXT, } from 'src/app/data/filter-rule-type' import { ObjectWithId } from 'src/app/data/object-with-id' import { GlobalSearchType, SETTINGS_KEYS } from 'src/app/data/ui-settings' @@ -410,7 +410,7 @@ export class GlobalSearchComponent implements OnInit { public runFullSearch() { const ruleType = this.useAdvancedForFullSearch ? FILTER_FULLTEXT_QUERY - : FILTER_TITLE_CONTENT + : FILTER_SIMPLE_TEXT this.documentService.searchQuery = this.useAdvancedForFullSearch ? this.query : '' diff --git a/src-ui/src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.spec.ts b/src-ui/src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.spec.ts index 89e7b1fee..2466ced73 100644 --- a/src-ui/src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.spec.ts +++ b/src-ui/src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.spec.ts @@ -4,7 +4,7 @@ import { ComponentFixture, TestBed } from '@angular/core/testing' import { By } from '@angular/platform-browser' import { NgbAccordionButton, NgbActiveModal } from '@ng-bootstrap/ng-bootstrap' import { of, throwError } from 'rxjs' -import { FILTER_TITLE } from 'src/app/data/filter-rule-type' +import { FILTER_SIMPLE_TITLE } from 'src/app/data/filter-rule-type' import { DocumentService } from 'src/app/services/rest/document.service' import { StoragePathService } from 'src/app/services/rest/storage-path.service' import { SettingsService } from 'src/app/services/settings.service' @@ -105,7 +105,7 @@ describe('StoragePathEditDialogComponent', () => { null, 'created', true, - [{ rule_type: FILTER_TITLE, value: 'bar' }], + [{ rule_type: FILTER_SIMPLE_TITLE, value: 'bar' }], { truncate_content: true } ) listSpy.mockReturnValueOnce( diff --git a/src-ui/src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.ts b/src-ui/src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.ts index f06831588..68ce40f5e 100644 --- a/src-ui/src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.ts +++ b/src-ui/src/app/components/common/edit-dialog/storage-path-edit-dialog/storage-path-edit-dialog.component.ts @@ -23,7 +23,7 @@ import { } from 'rxjs' import { EditDialogComponent } from 'src/app/components/common/edit-dialog/edit-dialog.component' import { Document } from 'src/app/data/document' -import { FILTER_TITLE } from 'src/app/data/filter-rule-type' +import { FILTER_SIMPLE_TITLE } from 'src/app/data/filter-rule-type' import { DEFAULT_MATCHING_ALGORITHM } from 'src/app/data/matching-model' import { StoragePath } from 'src/app/data/storage-path' import { IfOwnerDirective } from 'src/app/directives/if-owner.directive' @@ -146,7 +146,7 @@ export class StoragePathEditDialogComponent null, 'created', true, - [{ rule_type: FILTER_TITLE, value: title }], + [{ rule_type: FILTER_SIMPLE_TITLE, value: title }], { truncate_content: true } ) .pipe( diff --git a/src-ui/src/app/components/common/input/document-link/document-link.component.spec.ts b/src-ui/src/app/components/common/input/document-link/document-link.component.spec.ts index 7021012ab..f8a8f3817 100644 --- a/src-ui/src/app/components/common/input/document-link/document-link.component.spec.ts +++ b/src-ui/src/app/components/common/input/document-link/document-link.component.spec.ts @@ -3,7 +3,7 @@ import { provideHttpClientTesting } from '@angular/common/http/testing' import { ComponentFixture, TestBed } from '@angular/core/testing' import { NG_VALUE_ACCESSOR } from '@angular/forms' import { of, throwError } from 'rxjs' -import { FILTER_TITLE } from 'src/app/data/filter-rule-type' +import { FILTER_SIMPLE_TITLE } from 'src/app/data/filter-rule-type' import { DocumentService } from 'src/app/services/rest/document.service' import { DocumentLinkComponent } from './document-link.component' @@ -99,7 +99,7 @@ describe('DocumentLinkComponent', () => { null, 'created', true, - [{ rule_type: FILTER_TITLE, value: 'bar' }], + [{ rule_type: FILTER_SIMPLE_TITLE, value: 'bar' }], { truncate_content: true } ) listSpy.mockReturnValueOnce(throwError(() => new Error())) diff --git a/src-ui/src/app/components/common/input/document-link/document-link.component.ts b/src-ui/src/app/components/common/input/document-link/document-link.component.ts index b50f5701d..9bfb60063 100644 --- a/src-ui/src/app/components/common/input/document-link/document-link.component.ts +++ b/src-ui/src/app/components/common/input/document-link/document-link.component.ts @@ -28,7 +28,7 @@ import { tap, } from 'rxjs' import { Document } from 'src/app/data/document' -import { FILTER_TITLE } from 'src/app/data/filter-rule-type' +import { FILTER_SIMPLE_TITLE } from 'src/app/data/filter-rule-type' import { CustomDatePipe } from 'src/app/pipes/custom-date.pipe' import { DocumentService } from 'src/app/services/rest/document.service' import { AbstractInputComponent } from '../abstract-input' @@ -121,7 +121,7 @@ export class DocumentLinkComponent null, 'created', true, - [{ rule_type: FILTER_TITLE, value: title }], + [{ rule_type: FILTER_SIMPLE_TITLE, value: title }], { truncate_content: true } ) .pipe( diff --git a/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.spec.ts b/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.spec.ts index f283a75f3..8f82be1ab 100644 --- a/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.spec.ts +++ b/src-ui/src/app/components/document-list/bulk-editor/bulk-editor.component.spec.ts @@ -428,7 +428,7 @@ describe('BulkEditorComponent', () => { req.flush(true) expect(req.request.body).toEqual({ all: true, - filters: { title__icontains: 'apple' }, + filters: { title_search: 'apple' }, method: 'modify_tags', parameters: { add_tags: [101], remove_tags: [] }, }) diff --git a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.spec.ts b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.spec.ts index bf5240f1b..d75e38630 100644 --- a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.spec.ts +++ b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.spec.ts @@ -67,6 +67,8 @@ import { FILTER_OWNER_DOES_NOT_INCLUDE, FILTER_OWNER_ISNULL, FILTER_SHARED_BY_USER, + FILTER_SIMPLE_TEXT, + FILTER_SIMPLE_TITLE, FILTER_STORAGE_PATH, FILTER_TITLE, FILTER_TITLE_CONTENT, @@ -312,7 +314,7 @@ describe('FilterEditorComponent', () => { expect(component.textFilter).toEqual(null) component.filterRules = [ { - rule_type: FILTER_TITLE_CONTENT, + rule_type: FILTER_SIMPLE_TEXT, value: 'foo', }, ] @@ -320,6 +322,18 @@ describe('FilterEditorComponent', () => { expect(component.textFilterTarget).toEqual('title-content') // TEXT_FILTER_TARGET_TITLE_CONTENT })) + it('should ingest legacy text filter rules for doc title + content', fakeAsync(() => { + expect(component.textFilter).toEqual(null) + component.filterRules = [ + { + rule_type: FILTER_TITLE_CONTENT, + value: 'legacy foo', + }, + ] + expect(component.textFilter).toEqual('legacy foo') + expect(component.textFilterTarget).toEqual('title-content') // TEXT_FILTER_TARGET_TITLE_CONTENT + })) + it('should ingest text filter rules for doc asn', fakeAsync(() => { expect(component.textFilter).toEqual(null) component.filterRules = [ @@ -1117,7 +1131,7 @@ describe('FilterEditorComponent', () => { expect(component.textFilter).toEqual('foo') expect(component.filterRules).toEqual([ { - rule_type: FILTER_TITLE_CONTENT, + rule_type: FILTER_SIMPLE_TEXT, value: 'foo', }, ]) @@ -1136,7 +1150,7 @@ describe('FilterEditorComponent', () => { expect(component.textFilterTarget).toEqual('title') expect(component.filterRules).toEqual([ { - rule_type: FILTER_TITLE, + rule_type: FILTER_SIMPLE_TITLE, value: 'foo', }, ]) @@ -1250,30 +1264,12 @@ describe('FilterEditorComponent', () => { ]) })) - it('should convert user input to correct filter rules on custom fields query', fakeAsync(() => { - component.textFilterInput.nativeElement.value = 'foo' - component.textFilterInput.nativeElement.dispatchEvent(new Event('input')) - const textFieldTargetDropdown = fixture.debugElement.queryAll( - By.directive(NgbDropdownItem) - )[3] - textFieldTargetDropdown.triggerEventHandler('click') // TEXT_FILTER_TARGET_CUSTOM_FIELDS - fixture.detectChanges() - tick(400) - expect(component.textFilterTarget).toEqual('custom-fields') - expect(component.filterRules).toEqual([ - { - rule_type: FILTER_CUSTOM_FIELDS_TEXT, - value: 'foo', - }, - ]) - })) - it('should convert user input to correct filter rules on mime type', fakeAsync(() => { component.textFilterInput.nativeElement.value = 'pdf' component.textFilterInput.nativeElement.dispatchEvent(new Event('input')) const textFieldTargetDropdown = fixture.debugElement.queryAll( By.directive(NgbDropdownItem) - )[4] + )[3] textFieldTargetDropdown.triggerEventHandler('click') // TEXT_FILTER_TARGET_MIME_TYPE fixture.detectChanges() tick(400) @@ -1291,8 +1287,8 @@ describe('FilterEditorComponent', () => { component.textFilterInput.nativeElement.dispatchEvent(new Event('input')) const textFieldTargetDropdown = fixture.debugElement.queryAll( By.directive(NgbDropdownItem) - )[5] - textFieldTargetDropdown.triggerEventHandler('click') // TEXT_FILTER_TARGET_ASN + )[4] + textFieldTargetDropdown.triggerEventHandler('click') // TEXT_FILTER_TARGET_FULLTEXT_QUERY fixture.detectChanges() tick(400) expect(component.textFilterTarget).toEqual('fulltext-query') @@ -1696,12 +1692,56 @@ describe('FilterEditorComponent', () => { ]) })) + it('should convert legacy title filters into full text query when adding a created relative date', fakeAsync(() => { + component.filterRules = [ + { + rule_type: FILTER_TITLE, + value: 'foo', + }, + ] + const dateCreatedDropdown = fixture.debugElement.queryAll( + By.directive(DatesDropdownComponent) + )[0] + component.dateCreatedRelativeDate = RelativeDate.WITHIN_1_WEEK + dateCreatedDropdown.triggerEventHandler('datesSet') + fixture.detectChanges() + tick(400) + expect(component.filterRules).toEqual([ + { + rule_type: FILTER_FULLTEXT_QUERY, + value: 'foo,created:[-1 week to now]', + }, + ]) + })) + + it('should convert simple title filters into full text query when adding a created relative date', fakeAsync(() => { + component.filterRules = [ + { + rule_type: FILTER_SIMPLE_TITLE, + value: 'foo', + }, + ] + const dateCreatedDropdown = fixture.debugElement.queryAll( + By.directive(DatesDropdownComponent) + )[0] + component.dateCreatedRelativeDate = RelativeDate.WITHIN_1_WEEK + dateCreatedDropdown.triggerEventHandler('datesSet') + fixture.detectChanges() + tick(400) + expect(component.filterRules).toEqual([ + { + rule_type: FILTER_FULLTEXT_QUERY, + value: 'foo,created:[-1 week to now]', + }, + ]) + })) + it('should leave relative dates not in quick list intact', fakeAsync(() => { component.textFilterInput.nativeElement.value = 'created:[-2 week to now]' component.textFilterInput.nativeElement.dispatchEvent(new Event('input')) const textFieldTargetDropdown = fixture.debugElement.queryAll( By.directive(NgbDropdownItem) - )[5] + )[4] textFieldTargetDropdown.triggerEventHandler('click') fixture.detectChanges() tick(400) @@ -2031,12 +2071,30 @@ describe('FilterEditorComponent', () => { component.filterRules = [ { - rule_type: FILTER_TITLE, + rule_type: FILTER_SIMPLE_TITLE, value: 'foo', }, ] expect(component.generateFilterName()).toEqual('Title: foo') + component.filterRules = [ + { + rule_type: FILTER_TITLE_CONTENT, + value: 'legacy foo', + }, + ] + expect(component.generateFilterName()).toEqual( + 'Title & content: legacy foo' + ) + + component.filterRules = [ + { + rule_type: FILTER_SIMPLE_TEXT, + value: 'foo', + }, + ] + expect(component.generateFilterName()).toEqual('Title & content: foo') + component.filterRules = [ { rule_type: FILTER_ASN, @@ -2156,6 +2214,36 @@ describe('FilterEditorComponent', () => { }) }) + it('should hide deprecated custom fields target from default text filter targets', () => { + expect(component.textFilterTargets).not.toContainEqual({ + id: 'custom-fields', + name: $localize`Custom fields (Deprecated)`, + }) + }) + + it('should keep deprecated custom fields target available for legacy filters', fakeAsync(() => { + component.filterRules = [ + { + rule_type: FILTER_CUSTOM_FIELDS_TEXT, + value: 'foo', + }, + ] + fixture.detectChanges() + tick() + + expect(component.textFilterTarget).toEqual('custom-fields') + expect(component.textFilterTargets).toContainEqual({ + id: 'custom-fields', + name: $localize`Custom fields (Deprecated)`, + }) + expect(component.filterRules).toEqual([ + { + rule_type: FILTER_CUSTOM_FIELDS_TEXT, + value: 'foo', + }, + ]) + })) + it('should call autocomplete endpoint on input', fakeAsync(() => { component.textFilterTarget = 'fulltext-query' // TEXT_FILTER_TARGET_FULLTEXT_QUERY const autocompleteSpy = jest.spyOn(searchService, 'autocomplete') diff --git a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts index f7b50181b..b4e63317a 100644 --- a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts +++ b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts @@ -71,6 +71,8 @@ import { FILTER_OWNER_DOES_NOT_INCLUDE, FILTER_OWNER_ISNULL, FILTER_SHARED_BY_USER, + FILTER_SIMPLE_TEXT, + FILTER_SIMPLE_TITLE, FILTER_STORAGE_PATH, FILTER_TITLE, FILTER_TITLE_CONTENT, @@ -195,10 +197,6 @@ const DEFAULT_TEXT_FILTER_TARGET_OPTIONS = [ name: $localize`Title & content`, }, { id: TEXT_FILTER_TARGET_ASN, name: $localize`ASN` }, - { - id: TEXT_FILTER_TARGET_CUSTOM_FIELDS, - name: $localize`Custom fields`, - }, { id: TEXT_FILTER_TARGET_MIME_TYPE, name: $localize`File type` }, { id: TEXT_FILTER_TARGET_FULLTEXT_QUERY, @@ -206,6 +204,12 @@ const DEFAULT_TEXT_FILTER_TARGET_OPTIONS = [ }, ] +const DEPRECATED_CUSTOM_FIELDS_TEXT_FILTER_TARGET_OPTION = { + // Kept only so legacy saved views can render and be edited away from, remove me eventually + id: TEXT_FILTER_TARGET_CUSTOM_FIELDS, + name: $localize`Custom fields (Deprecated)`, +} + const TEXT_FILTER_TARGET_MORELIKE_OPTION = { id: TEXT_FILTER_TARGET_FULLTEXT_MORELIKE, name: $localize`More like`, @@ -318,8 +322,13 @@ export class FilterEditorComponent return $localize`Custom fields query` case FILTER_TITLE: + case FILTER_SIMPLE_TITLE: return $localize`Title: ${rule.value}` + case FILTER_TITLE_CONTENT: + case FILTER_SIMPLE_TEXT: + return $localize`Title & content: ${rule.value}` + case FILTER_ASN: return $localize`ASN: ${rule.value}` @@ -353,12 +362,16 @@ export class FilterEditorComponent _moreLikeDoc: Document get textFilterTargets() { + let targets = DEFAULT_TEXT_FILTER_TARGET_OPTIONS if (this.textFilterTarget == TEXT_FILTER_TARGET_FULLTEXT_MORELIKE) { - return DEFAULT_TEXT_FILTER_TARGET_OPTIONS.concat([ - TEXT_FILTER_TARGET_MORELIKE_OPTION, + targets = targets.concat([TEXT_FILTER_TARGET_MORELIKE_OPTION]) + } + if (this.textFilterTarget == TEXT_FILTER_TARGET_CUSTOM_FIELDS) { + targets = targets.concat([ + DEPRECATED_CUSTOM_FIELDS_TEXT_FILTER_TARGET_OPTION, ]) } - return DEFAULT_TEXT_FILTER_TARGET_OPTIONS + return targets } textFilterTarget = TEXT_FILTER_TARGET_TITLE_CONTENT @@ -437,10 +450,12 @@ export class FilterEditorComponent value.forEach((rule) => { switch (rule.rule_type) { case FILTER_TITLE: + case FILTER_SIMPLE_TITLE: this._textFilter = rule.value this.textFilterTarget = TEXT_FILTER_TARGET_TITLE break case FILTER_TITLE_CONTENT: + case FILTER_SIMPLE_TEXT: this._textFilter = rule.value this.textFilterTarget = TEXT_FILTER_TARGET_TITLE_CONTENT break @@ -762,12 +777,15 @@ export class FilterEditorComponent this.textFilterTarget == TEXT_FILTER_TARGET_TITLE_CONTENT ) { filterRules.push({ - rule_type: FILTER_TITLE_CONTENT, + rule_type: FILTER_SIMPLE_TEXT, value: this._textFilter.trim(), }) } if (this._textFilter && this.textFilterTarget == TEXT_FILTER_TARGET_TITLE) { - filterRules.push({ rule_type: FILTER_TITLE, value: this._textFilter }) + filterRules.push({ + rule_type: FILTER_SIMPLE_TITLE, + value: this._textFilter, + }) } if (this.textFilterTarget == TEXT_FILTER_TARGET_ASN) { if ( @@ -1009,7 +1027,10 @@ export class FilterEditorComponent ) { existingRule = filterRules.find( (fr) => - fr.rule_type == FILTER_TITLE_CONTENT || fr.rule_type == FILTER_TITLE + fr.rule_type == FILTER_TITLE_CONTENT || + fr.rule_type == FILTER_SIMPLE_TEXT || + fr.rule_type == FILTER_TITLE || + fr.rule_type == FILTER_SIMPLE_TITLE ) existingRule.rule_type = FILTER_FULLTEXT_QUERY } diff --git a/src-ui/src/app/data/filter-rule-type.ts b/src-ui/src/app/data/filter-rule-type.ts index 7f0f0d56d..6330eb44c 100644 --- a/src-ui/src/app/data/filter-rule-type.ts +++ b/src-ui/src/app/data/filter-rule-type.ts @@ -3,7 +3,7 @@ import { DataType } from './datatype' export const NEGATIVE_NULL_FILTER_VALUE = -1 // These correspond to src/documents/models.py and changes here require a DB migration (and vice versa) -export const FILTER_TITLE = 0 +export const FILTER_TITLE = 0 // Deprecated in favor of Tantivy-backed `title_search`. Keep for now for existing saved views export const FILTER_CONTENT = 1 export const FILTER_ASN = 2 @@ -46,7 +46,9 @@ export const FILTER_ADDED_FROM = 46 export const FILTER_MODIFIED_BEFORE = 15 export const FILTER_MODIFIED_AFTER = 16 -export const FILTER_TITLE_CONTENT = 19 +export const FILTER_TITLE_CONTENT = 19 // Deprecated in favor of Tantivy-backed `text` filtervar. Keep for now for existing saved views +export const FILTER_SIMPLE_TITLE = 48 +export const FILTER_SIMPLE_TEXT = 49 export const FILTER_FULLTEXT_QUERY = 20 export const FILTER_FULLTEXT_MORELIKE = 21 @@ -56,7 +58,7 @@ export const FILTER_OWNER_ISNULL = 34 export const FILTER_OWNER_DOES_NOT_INCLUDE = 35 export const FILTER_SHARED_BY_USER = 37 -export const FILTER_CUSTOM_FIELDS_TEXT = 36 +export const FILTER_CUSTOM_FIELDS_TEXT = 36 // Deprecated. UI no longer includes CF text-search mode. Keep for now for existing saved views export const FILTER_HAS_CUSTOM_FIELDS_ALL = 38 export const FILTER_HAS_CUSTOM_FIELDS_ANY = 39 export const FILTER_DOES_NOT_HAVE_CUSTOM_FIELDS = 40 @@ -66,6 +68,9 @@ export const FILTER_CUSTOM_FIELDS_QUERY = 42 export const FILTER_MIME_TYPE = 47 +export const SIMPLE_TEXT_PARAMETER = 'text' +export const SIMPLE_TITLE_PARAMETER = 'title_search' + export const FILTER_RULE_TYPES: FilterRuleType[] = [ { id: FILTER_TITLE, @@ -74,6 +79,13 @@ export const FILTER_RULE_TYPES: FilterRuleType[] = [ multi: false, default: '', }, + { + id: FILTER_SIMPLE_TITLE, + filtervar: SIMPLE_TITLE_PARAMETER, + datatype: 'string', + multi: false, + default: '', + }, { id: FILTER_CONTENT, filtervar: 'content__icontains', @@ -279,6 +291,12 @@ export const FILTER_RULE_TYPES: FilterRuleType[] = [ datatype: 'string', multi: false, }, + { + id: FILTER_SIMPLE_TEXT, + filtervar: SIMPLE_TEXT_PARAMETER, + datatype: 'string', + multi: false, + }, { id: FILTER_FULLTEXT_QUERY, filtervar: 'query', diff --git a/src-ui/src/app/services/rest/document.service.spec.ts b/src-ui/src/app/services/rest/document.service.spec.ts index 711aab743..03375e367 100644 --- a/src-ui/src/app/services/rest/document.service.spec.ts +++ b/src-ui/src/app/services/rest/document.service.spec.ts @@ -10,7 +10,7 @@ import { DOCUMENT_SORT_FIELDS, DOCUMENT_SORT_FIELDS_FULLTEXT, } from 'src/app/data/document' -import { FILTER_TITLE } from 'src/app/data/filter-rule-type' +import { FILTER_SIMPLE_TITLE } from 'src/app/data/filter-rule-type' import { SETTINGS_KEYS } from 'src/app/data/ui-settings' import { environment } from 'src/environments/environment' import { PermissionsService } from '../permissions.service' @@ -138,13 +138,13 @@ describe(`DocumentService`, () => { subscription = service .listAllFilteredIds([ { - rule_type: FILTER_TITLE, + rule_type: FILTER_SIMPLE_TITLE, value: 'apple', }, ]) .subscribe() const req = httpTestingController.expectOne( - `${environment.apiBaseUrl}${endpoint}/?page=1&page_size=100000&fields=id&title__icontains=apple` + `${environment.apiBaseUrl}${endpoint}/?page=1&page_size=100000&fields=id&title_search=apple` ) expect(req.request.method).toEqual('GET') }) diff --git a/src-ui/src/app/utils/query-params.spec.ts b/src-ui/src/app/utils/query-params.spec.ts index c22c90d11..7fd8f6808 100644 --- a/src-ui/src/app/utils/query-params.spec.ts +++ b/src-ui/src/app/utils/query-params.spec.ts @@ -8,6 +8,10 @@ import { FILTER_HAS_CUSTOM_FIELDS_ALL, FILTER_HAS_CUSTOM_FIELDS_ANY, FILTER_HAS_TAGS_ALL, + FILTER_SIMPLE_TEXT, + FILTER_SIMPLE_TITLE, + FILTER_TITLE, + FILTER_TITLE_CONTENT, NEGATIVE_NULL_FILTER_VALUE, } from '../data/filter-rule-type' import { @@ -128,6 +132,26 @@ describe('QueryParams Utils', () => { is_tagged: 0, }) + params = queryParamsFromFilterRules([ + { + rule_type: FILTER_TITLE_CONTENT, + value: 'bank statement', + }, + ]) + expect(params).toEqual({ + text: 'bank statement', + }) + + params = queryParamsFromFilterRules([ + { + rule_type: FILTER_TITLE, + value: 'invoice', + }, + ]) + expect(params).toEqual({ + title_search: 'invoice', + }) + params = queryParamsFromFilterRules([ { rule_type: FILTER_HAS_TAGS_ALL, @@ -148,6 +172,30 @@ describe('QueryParams Utils', () => { it('should convert filter rules to query params', () => { let rules = filterRulesFromQueryParams( + convertToParamMap({ + text: 'bank statement', + }) + ) + expect(rules).toEqual([ + { + rule_type: FILTER_SIMPLE_TEXT, + value: 'bank statement', + }, + ]) + + rules = filterRulesFromQueryParams( + convertToParamMap({ + title_search: 'invoice', + }) + ) + expect(rules).toEqual([ + { + rule_type: FILTER_SIMPLE_TITLE, + value: 'invoice', + }, + ]) + + rules = filterRulesFromQueryParams( convertToParamMap({ tags__id__all, }) diff --git a/src-ui/src/app/utils/query-params.ts b/src-ui/src/app/utils/query-params.ts index 27716cc2d..be33ba724 100644 --- a/src-ui/src/app/utils/query-params.ts +++ b/src-ui/src/app/utils/query-params.ts @@ -9,8 +9,14 @@ import { FILTER_HAS_CUSTOM_FIELDS_ALL, FILTER_HAS_CUSTOM_FIELDS_ANY, FILTER_RULE_TYPES, + FILTER_SIMPLE_TEXT, + FILTER_SIMPLE_TITLE, + FILTER_TITLE, + FILTER_TITLE_CONTENT, FilterRuleType, NEGATIVE_NULL_FILTER_VALUE, + SIMPLE_TEXT_PARAMETER, + SIMPLE_TITLE_PARAMETER, } from '../data/filter-rule-type' import { ListViewState } from '../services/document-list-view.service' @@ -97,6 +103,8 @@ export function transformLegacyFilterRules( export function filterRulesFromQueryParams( queryParams: ParamMap ): FilterRule[] { + let filterRulesFromQueryParams: FilterRule[] = [] + const allFilterRuleQueryParams: string[] = FILTER_RULE_TYPES.map( (rt) => rt.filtervar ) @@ -104,7 +112,6 @@ export function filterRulesFromQueryParams( .filter((rt) => rt !== undefined) // transform query params to filter rules - let filterRulesFromQueryParams: FilterRule[] = [] allFilterRuleQueryParams .filter((frqp) => queryParams.has(frqp)) .forEach((filterQueryParamName) => { @@ -146,7 +153,17 @@ export function queryParamsFromFilterRules(filterRules: FilterRule[]): Params { let params = {} for (let rule of filterRules) { let ruleType = FILTER_RULE_TYPES.find((t) => t.id == rule.rule_type) - if (ruleType.isnull_filtervar && rule.value == null) { + if ( + rule.rule_type === FILTER_TITLE_CONTENT || + rule.rule_type === FILTER_SIMPLE_TEXT + ) { + params[SIMPLE_TEXT_PARAMETER] = rule.value + } else if ( + rule.rule_type === FILTER_TITLE || + rule.rule_type === FILTER_SIMPLE_TITLE + ) { + params[SIMPLE_TITLE_PARAMETER] = rule.value + } else if (ruleType.isnull_filtervar && rule.value == null) { params[ruleType.isnull_filtervar] = 1 } else if ( ruleType.isnull_filtervar && diff --git a/src/documents/filters.py b/src/documents/filters.py index 2f7de1cd4..b2b226ee1 100644 --- a/src/documents/filters.py +++ b/src/documents/filters.py @@ -3,6 +3,7 @@ from __future__ import annotations import functools import inspect import json +import logging import operator from contextlib import contextmanager from typing import TYPE_CHECKING @@ -77,6 +78,8 @@ DATETIME_KWARGS = [ CUSTOM_FIELD_QUERY_MAX_DEPTH = 10 CUSTOM_FIELD_QUERY_MAX_ATOMS = 20 +logger = logging.getLogger("paperless.api") + class CorrespondentFilterSet(FilterSet): class Meta: @@ -162,9 +165,13 @@ class InboxFilter(Filter): @extend_schema_field(serializers.CharField) class TitleContentFilter(Filter): + # Deprecated but retained for existing saved views. UI uses Tantivy-backed `text` / `title_search` params. def filter(self, qs: Any, value: Any) -> Any: value = value.strip() if isinstance(value, str) else value if value: + logger.warning( + "Deprecated document filter parameter 'title_content' used; use `text` instead.", + ) try: return qs.filter( Q(title__icontains=value) | Q(effective_content__icontains=value), @@ -243,6 +250,9 @@ class CustomFieldsFilter(Filter): def filter(self, qs, value): value = value.strip() if isinstance(value, str) else value if value: + logger.warning( + "Deprecated document filter parameter 'custom_fields__icontains' used; use `custom_field_query` or advanced Tantivy field syntax instead.", + ) fields_with_matching_selects = CustomField.objects.filter( extra_data__icontains=value, ) @@ -747,6 +757,7 @@ class DocumentFilterSet(FilterSet): is_in_inbox = InboxFilter() + # Deprecated, but keep for now for existing saved views title_content = TitleContentFilter() content__istartswith = EffectiveContentFilter(lookup_expr="istartswith") @@ -756,6 +767,7 @@ class DocumentFilterSet(FilterSet): owner__id__none = ObjectFilter(field_name="owner", exclude=True) + # Deprecated, UI no longer includes CF text-search mode, but keep for now for existing saved views custom_fields__icontains = CustomFieldsFilter() custom_fields__id__all = ObjectFilter(field_name="custom_fields__field") diff --git a/src/documents/migrations/0018_saved_view_simple_search_rules.py b/src/documents/migrations/0018_saved_view_simple_search_rules.py new file mode 100644 index 000000000..6d128c593 --- /dev/null +++ b/src/documents/migrations/0018_saved_view_simple_search_rules.py @@ -0,0 +1,92 @@ +# Generated by Django 5.2.12 on 2026-04-01 18:20 + +from django.db import migrations +from django.db import models + +OLD_TITLE_RULE = 0 +OLD_TITLE_CONTENT_RULE = 19 +NEW_SIMPLE_TITLE_RULE = 48 +NEW_SIMPLE_TEXT_RULE = 49 + + +# See documents/models.py SavedViewFilterRule +def migrate_saved_view_rules_forward(apps, schema_editor): + SavedViewFilterRule = apps.get_model("documents", "SavedViewFilterRule") + SavedViewFilterRule.objects.filter(rule_type=OLD_TITLE_RULE).update( + rule_type=NEW_SIMPLE_TITLE_RULE, + ) + SavedViewFilterRule.objects.filter(rule_type=OLD_TITLE_CONTENT_RULE).update( + rule_type=NEW_SIMPLE_TEXT_RULE, + ) + + +class Migration(migrations.Migration): + dependencies = [ + ("documents", "0017_migrate_fulltext_query_field_prefixes"), + ] + + operations = [ + migrations.AlterField( + model_name="savedviewfilterrule", + name="rule_type", + field=models.PositiveSmallIntegerField( + choices=[ + (0, "title contains"), + (1, "content contains"), + (2, "ASN is"), + (3, "correspondent is"), + (4, "document type is"), + (5, "is in inbox"), + (6, "has tag"), + (7, "has any tag"), + (8, "created before"), + (9, "created after"), + (10, "created year is"), + (11, "created month is"), + (12, "created day is"), + (13, "added before"), + (14, "added after"), + (15, "modified before"), + (16, "modified after"), + (17, "does not have tag"), + (18, "does not have ASN"), + (19, "title or content contains"), + (20, "fulltext query"), + (21, "more like this"), + (22, "has tags in"), + (23, "ASN greater than"), + (24, "ASN less than"), + (25, "storage path is"), + (26, "has correspondent in"), + (27, "does not have correspondent in"), + (28, "has document type in"), + (29, "does not have document type in"), + (30, "has storage path in"), + (31, "does not have storage path in"), + (32, "owner is"), + (33, "has owner in"), + (34, "does not have owner"), + (35, "does not have owner in"), + (36, "has custom field value"), + (37, "is shared by me"), + (38, "has custom fields"), + (39, "has custom field in"), + (40, "does not have custom field in"), + (41, "does not have custom field"), + (42, "custom fields query"), + (43, "created to"), + (44, "created from"), + (45, "added to"), + (46, "added from"), + (47, "mime type is"), + (48, "simple title search"), + (49, "simple text search"), + ], + verbose_name="rule type", + ), + ), + migrations.RunPython( + migrate_saved_view_rules_forward, + migrations.RunPython.noop, + ), + ] diff --git a/src/documents/models.py b/src/documents/models.py index 96f027b94..9af5fbc23 100644 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -623,6 +623,8 @@ class SavedViewFilterRule(models.Model): (45, _("added to")), (46, _("added from")), (47, _("mime type is")), + (48, _("simple title search")), + (49, _("simple text search")), ] saved_view = models.ForeignKey( diff --git a/src/documents/search/__init__.py b/src/documents/search/__init__.py index b0a89f242..a4145d7ef 100644 --- a/src/documents/search/__init__.py +++ b/src/documents/search/__init__.py @@ -1,4 +1,5 @@ from documents.search._backend import SearchIndexLockError +from documents.search._backend import SearchMode from documents.search._backend import SearchResults from documents.search._backend import TantivyBackend from documents.search._backend import TantivyRelevanceList @@ -10,6 +11,7 @@ from documents.search._schema import wipe_index __all__ = [ "SearchIndexLockError", + "SearchMode", "SearchResults", "TantivyBackend", "TantivyRelevanceList", diff --git a/src/documents/search/_backend.py b/src/documents/search/_backend.py index a1bff8a9f..405c24360 100644 --- a/src/documents/search/_backend.py +++ b/src/documents/search/_backend.py @@ -2,11 +2,11 @@ from __future__ import annotations import logging import threading -import unicodedata from collections import Counter from dataclasses import dataclass from datetime import UTC from datetime import datetime +from enum import StrEnum from typing import TYPE_CHECKING from typing import Self from typing import TypedDict @@ -19,7 +19,10 @@ from django.conf import settings from django.utils.timezone import get_current_timezone from guardian.shortcuts import get_users_with_perms +from documents.search._normalize import ascii_fold from documents.search._query import build_permission_filter +from documents.search._query import parse_simple_text_query +from documents.search._query import parse_simple_title_query from documents.search._query import parse_user_query from documents.search._schema import _write_sentinels from documents.search._schema import build_schema @@ -45,14 +48,10 @@ _AUTOCOMPLETE_REGEX_TIMEOUT = 1.0 # seconds; guards against ReDoS on untrusted T = TypeVar("T") -def _ascii_fold(s: str) -> str: - """ - Normalize unicode to ASCII equivalent characters for search consistency. - - Converts accented characters (e.g., "café") to their ASCII base forms ("cafe") - to enable cross-language searching without requiring exact diacritic matching. - """ - return unicodedata.normalize("NFD", s).encode("ascii", "ignore").decode() +class SearchMode(StrEnum): + QUERY = "query" + TEXT = "text" + TITLE = "title" def _extract_autocomplete_words(text_sources: list[str]) -> set[str]: @@ -74,7 +73,7 @@ def _extract_autocomplete_words(text_sources: list[str]) -> set[str]: ) continue for token in tokens: - normalized = _ascii_fold(token.lower()) + normalized = ascii_fold(token.lower()) if normalized: words.add(normalized) return words @@ -294,8 +293,10 @@ class TantivyBackend: doc.add_text("checksum", document.checksum) doc.add_text("title", document.title) doc.add_text("title_sort", document.title) + doc.add_text("simple_title", document.title) doc.add_text("content", content) doc.add_text("bigram_content", content) + doc.add_text("simple_content", content) # Original filename - only add if not None/empty if document.original_filename: @@ -433,6 +434,7 @@ class TantivyBackend: sort_field: str | None, *, sort_reverse: bool, + search_mode: SearchMode = SearchMode.QUERY, ) -> SearchResults: """ Execute a search query against the document index. @@ -441,20 +443,32 @@ class TantivyBackend: permission filtering before executing against Tantivy. Supports both relevance-based and field-based sorting. + QUERY search mode supports natural date keywords, field filters, etc. + TITLE search mode treats the query as plain text to search for in title only + TEXT search mode treats the query as plain text to search for in title and content + Args: - query: User's search query (supports natural date keywords, field filters) + query: User's search query user: User for permission filtering (None for superuser/no filtering) page: Page number (1-indexed) for pagination page_size: Number of results per page sort_field: Field to sort by (None for relevance ranking) sort_reverse: Whether to reverse the sort order + search_mode: "query" for advanced Tantivy syntax, "text" for + plain-text search over title and content only, "title" for + plain-text search over title only Returns: SearchResults with hits, total count, and processed query """ self._ensure_open() tz = get_current_timezone() - user_query = parse_user_query(self._index, query, tz) + if search_mode is SearchMode.TEXT: + user_query = parse_simple_text_query(self._index, query) + elif search_mode is SearchMode.TITLE: + user_query = parse_simple_title_query(self._index, query) + else: + user_query = parse_user_query(self._index, query, tz) # Apply permission filter if user is not None (not superuser) if user is not None: @@ -594,7 +608,7 @@ class TantivyBackend: List of word suggestions ordered by frequency, then alphabetically """ self._ensure_open() - normalized_term = _ascii_fold(term.lower()) + normalized_term = ascii_fold(term.lower()) searcher = self._index.searcher() diff --git a/src/documents/search/_normalize.py b/src/documents/search/_normalize.py new file mode 100644 index 000000000..3d7b23f33 --- /dev/null +++ b/src/documents/search/_normalize.py @@ -0,0 +1,8 @@ +from __future__ import annotations + +import unicodedata + + +def ascii_fold(text: str) -> str: + """Normalize unicode text to ASCII equivalents for search consistency.""" + return unicodedata.normalize("NFD", text).encode("ascii", "ignore").decode() diff --git a/src/documents/search/_query.py b/src/documents/search/_query.py index 212df1516..b7bcbbe9c 100644 --- a/src/documents/search/_query.py +++ b/src/documents/search/_query.py @@ -12,6 +12,8 @@ import tantivy from dateutil.relativedelta import relativedelta from django.conf import settings +from documents.search._normalize import ascii_fold + if TYPE_CHECKING: from datetime import tzinfo @@ -51,6 +53,7 @@ _WHOOSH_REL_RANGE_RE = regex.compile( ) # Whoosh-style 8-digit date: field:YYYYMMDD — field-aware so timezone can be applied correctly _DATE8_RE = regex.compile(r"(?P\w+):(?P\d{8})\b") +_SIMPLE_QUERY_TOKEN_RE = regex.compile(r"\S+") def _fmt(dt: datetime) -> str: @@ -436,7 +439,37 @@ DEFAULT_SEARCH_FIELDS = [ "document_type", "tag", ] +SIMPLE_SEARCH_FIELDS = ["simple_title", "simple_content"] +TITLE_SEARCH_FIELDS = ["simple_title"] _FIELD_BOOSTS = {"title": 2.0} +_SIMPLE_FIELD_BOOSTS = {"simple_title": 2.0} + + +def _build_simple_field_query( + index: tantivy.Index, + field: str, + tokens: list[str], +) -> tantivy.Query: + patterns = [] + for idx, token in enumerate(tokens): + escaped = regex.escape(token) + # For multi-token substring search, only the first token can begin mid-word. + # Later tokens follow a whitespace boundary in the original query, so anchor + # them to the start of the next indexed token to reduce false positives like + # matching "Z-Berichte 16" for the query "Z-Berichte 6". + if idx == 0: + patterns.append(f".*{escaped}.*") + else: + patterns.append(f"{escaped}.*") + if len(patterns) == 1: + query = tantivy.Query.regex_query(index.schema, field, patterns[0]) + else: + query = tantivy.Query.regex_phrase_query(index.schema, field, patterns) + + boost = _SIMPLE_FIELD_BOOSTS.get(field, 1.0) + if boost > 1.0: + return tantivy.Query.boost_query(query, boost) + return query def parse_user_query( @@ -495,3 +528,52 @@ def parse_user_query( ) return exact + + +def parse_simple_query( + index: tantivy.Index, + raw_query: str, + fields: list[str], +) -> tantivy.Query: + """ + Parse a plain-text query using Tantivy over a restricted field set. + + Query string is escaped and normalized to be treated as "simple" text query. + """ + tokens = [ + ascii_fold(token.lower()) + for token in _SIMPLE_QUERY_TOKEN_RE.findall(raw_query, timeout=_REGEX_TIMEOUT) + ] + tokens = [token for token in tokens if token] + if not tokens: + return tantivy.Query.empty_query() + + field_queries = [ + (tantivy.Occur.Should, _build_simple_field_query(index, field, tokens)) + for field in fields + ] + if len(field_queries) == 1: + return field_queries[0][1] + return tantivy.Query.boolean_query(field_queries) + + +def parse_simple_text_query( + index: tantivy.Index, + raw_query: str, +) -> tantivy.Query: + """ + Parse a plain-text query over title/content for simple search inputs. + """ + + return parse_simple_query(index, raw_query, SIMPLE_SEARCH_FIELDS) + + +def parse_simple_title_query( + index: tantivy.Index, + raw_query: str, +) -> tantivy.Query: + """ + Parse a plain-text query over the title field only. + """ + + return parse_simple_query(index, raw_query, TITLE_SEARCH_FIELDS) diff --git a/src/documents/search/_schema.py b/src/documents/search/_schema.py index ba6646007..5e9404235 100644 --- a/src/documents/search/_schema.py +++ b/src/documents/search/_schema.py @@ -53,6 +53,18 @@ def build_schema() -> tantivy.Schema: # CJK support - not stored, indexed only sb.add_text_field("bigram_content", stored=False, tokenizer_name="bigram_analyzer") + # Simple substring search support for title/content - not stored, indexed only + sb.add_text_field( + "simple_title", + stored=False, + tokenizer_name="simple_search_analyzer", + ) + sb.add_text_field( + "simple_content", + stored=False, + tokenizer_name="simple_search_analyzer", + ) + # Autocomplete prefix scan - stored, not indexed sb.add_text_field("autocomplete_word", stored=True, tokenizer_name="raw") diff --git a/src/documents/search/_tokenizer.py b/src/documents/search/_tokenizer.py index e597a879e..2079ca4cc 100644 --- a/src/documents/search/_tokenizer.py +++ b/src/documents/search/_tokenizer.py @@ -70,6 +70,7 @@ def register_tokenizers(index: tantivy.Index, language: str | None) -> None: index.register_tokenizer("paperless_text", _paperless_text(language)) index.register_tokenizer("simple_analyzer", _simple_analyzer()) index.register_tokenizer("bigram_analyzer", _bigram_analyzer()) + index.register_tokenizer("simple_search_analyzer", _simple_search_analyzer()) # Fast-field tokenizer required for fast=True text fields in the schema index.register_fast_field_tokenizer("simple_analyzer", _simple_analyzer()) @@ -114,3 +115,16 @@ def _bigram_analyzer() -> tantivy.TextAnalyzer: .filter(tantivy.Filter.lowercase()) .build() ) + + +def _simple_search_analyzer() -> tantivy.TextAnalyzer: + """Tokenizer for simple substring search fields: non-whitespace chunks -> remove_long(65) -> lowercase -> ascii_fold.""" + return ( + tantivy.TextAnalyzerBuilder( + tantivy.Tokenizer.regex(r"\S+"), + ) + .filter(tantivy.Filter.remove_long(65)) + .filter(tantivy.Filter.lowercase()) + .filter(tantivy.Filter.ascii_fold()) + .build() + ) diff --git a/src/documents/tests/search/test_backend.py b/src/documents/tests/search/test_backend.py index 5c92da447..ff9638e63 100644 --- a/src/documents/tests/search/test_backend.py +++ b/src/documents/tests/search/test_backend.py @@ -5,6 +5,7 @@ from documents.models import CustomField from documents.models import CustomFieldInstance from documents.models import Document from documents.models import Note +from documents.search._backend import SearchMode from documents.search._backend import TantivyBackend from documents.search._backend import get_backend from documents.search._backend import reset_backend @@ -46,6 +47,258 @@ class TestWriteBatch: class TestSearch: """Test search functionality.""" + def test_text_mode_limits_default_search_to_title_and_content( + self, + backend: TantivyBackend, + ): + """Simple text mode must not match metadata-only fields.""" + doc = Document.objects.create( + title="Invoice document", + content="monthly statement", + checksum="TXT1", + pk=9, + ) + backend.add_or_update(doc) + + metadata_only = backend.search( + "document_type:invoice", + user=None, + page=1, + page_size=10, + sort_field=None, + sort_reverse=False, + search_mode=SearchMode.TEXT, + ) + assert metadata_only.total == 0 + + content_match = backend.search( + "monthly", + user=None, + page=1, + page_size=10, + sort_field=None, + sort_reverse=False, + search_mode=SearchMode.TEXT, + ) + assert content_match.total == 1 + + def test_title_mode_limits_default_search_to_title_only( + self, + backend: TantivyBackend, + ): + """Title mode must not match content-only terms.""" + doc = Document.objects.create( + title="Invoice document", + content="monthly statement", + checksum="TXT2", + pk=10, + ) + backend.add_or_update(doc) + + content_only = backend.search( + "monthly", + user=None, + page=1, + page_size=10, + sort_field=None, + sort_reverse=False, + search_mode=SearchMode.TITLE, + ) + assert content_only.total == 0 + + title_match = backend.search( + "invoice", + user=None, + page=1, + page_size=10, + sort_field=None, + sort_reverse=False, + search_mode=SearchMode.TITLE, + ) + assert title_match.total == 1 + + def test_text_mode_matches_partial_term_substrings( + self, + backend: TantivyBackend, + ): + """Simple text mode should support substring matching within tokens.""" + doc = Document.objects.create( + title="Account access", + content="password reset instructions", + checksum="TXT3", + pk=11, + ) + backend.add_or_update(doc) + + prefix_match = backend.search( + "pass", + user=None, + page=1, + page_size=10, + sort_field=None, + sort_reverse=False, + search_mode=SearchMode.TEXT, + ) + assert prefix_match.total == 1 + + infix_match = backend.search( + "sswo", + user=None, + page=1, + page_size=10, + sort_field=None, + sort_reverse=False, + search_mode=SearchMode.TEXT, + ) + assert infix_match.total == 1 + + phrase_match = backend.search( + "sswo re", + user=None, + page=1, + page_size=10, + sort_field=None, + sort_reverse=False, + search_mode=SearchMode.TEXT, + ) + assert phrase_match.total == 1 + + def test_text_mode_does_not_match_on_partial_term_overlap( + self, + backend: TantivyBackend, + ): + """Simple text mode should not match documents that merely share partial fragments.""" + doc = Document.objects.create( + title="Adobe Acrobat PDF Files", + content="Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + checksum="TXT7", + pk=13, + ) + backend.add_or_update(doc) + + non_match = backend.search( + "raptor", + user=None, + page=1, + page_size=10, + sort_field=None, + sort_reverse=False, + search_mode=SearchMode.TEXT, + ) + assert non_match.total == 0 + + def test_text_mode_anchors_later_query_tokens_to_token_starts( + self, + backend: TantivyBackend, + ): + """Multi-token simple search should not match later tokens in the middle of a word.""" + exact_doc = Document.objects.create( + title="Z-Berichte 6", + content="monthly report", + checksum="TXT9", + pk=15, + ) + prefix_doc = Document.objects.create( + title="Z-Berichte 60", + content="monthly report", + checksum="TXT10", + pk=16, + ) + false_positive = Document.objects.create( + title="Z-Berichte 16", + content="monthly report", + checksum="TXT11", + pk=17, + ) + backend.add_or_update(exact_doc) + backend.add_or_update(prefix_doc) + backend.add_or_update(false_positive) + + results = backend.search( + "Z-Berichte 6", + user=None, + page=1, + page_size=10, + sort_field=None, + sort_reverse=False, + search_mode=SearchMode.TEXT, + ) + result_ids = {hit["id"] for hit in results.hits} + + assert exact_doc.id in result_ids + assert prefix_doc.id in result_ids + assert false_positive.id not in result_ids + + def test_text_mode_ignores_queries_without_searchable_tokens( + self, + backend: TantivyBackend, + ): + """Simple text mode should safely return no hits for symbol-only strings.""" + doc = Document.objects.create( + title="Guide", + content="This is a guide.", + checksum="TXT8", + pk=14, + ) + backend.add_or_update(doc) + + no_tokens = backend.search( + "!!!", + user=None, + page=1, + page_size=10, + sort_field=None, + sort_reverse=False, + search_mode=SearchMode.TEXT, + ) + assert no_tokens.total == 0 + + def test_title_mode_matches_partial_term_substrings( + self, + backend: TantivyBackend, + ): + """Title mode should support substring matching within title tokens.""" + doc = Document.objects.create( + title="Password guide", + content="reset instructions", + checksum="TXT4", + pk=12, + ) + backend.add_or_update(doc) + + prefix_match = backend.search( + "pass", + user=None, + page=1, + page_size=10, + sort_field=None, + sort_reverse=False, + search_mode=SearchMode.TITLE, + ) + assert prefix_match.total == 1 + + infix_match = backend.search( + "sswo", + user=None, + page=1, + page_size=10, + sort_field=None, + sort_reverse=False, + search_mode=SearchMode.TITLE, + ) + assert infix_match.total == 1 + + phrase_match = backend.search( + "sswo gu", + user=None, + page=1, + page_size=10, + sort_field=None, + sort_reverse=False, + search_mode=SearchMode.TITLE, + ) + assert phrase_match.total == 1 + def test_scores_normalised_top_hit_is_one(self, backend: TantivyBackend): """Search scores must be normalized so top hit has score 1.0 for UI consistency.""" for i, title in enumerate(["bank invoice", "bank statement", "bank receipt"]): diff --git a/src/documents/tests/search/test_tokenizer.py b/src/documents/tests/search/test_tokenizer.py index aee52a567..fc2c41231 100644 --- a/src/documents/tests/search/test_tokenizer.py +++ b/src/documents/tests/search/test_tokenizer.py @@ -8,6 +8,7 @@ import tantivy from documents.search._tokenizer import _bigram_analyzer from documents.search._tokenizer import _paperless_text +from documents.search._tokenizer import _simple_search_analyzer from documents.search._tokenizer import register_tokenizers if TYPE_CHECKING: @@ -41,6 +42,20 @@ class TestTokenizers: idx.register_tokenizer("bigram_analyzer", _bigram_analyzer()) return idx + @pytest.fixture + def simple_search_index(self) -> tantivy.Index: + """Index with simple-search field for Latin substring tests.""" + sb = tantivy.SchemaBuilder() + sb.add_text_field( + "simple_content", + stored=False, + tokenizer_name="simple_search_analyzer", + ) + schema = sb.build() + idx = tantivy.Index(schema, path=None) + idx.register_tokenizer("simple_search_analyzer", _simple_search_analyzer()) + return idx + def test_ascii_fold_finds_accented_content( self, content_index: tantivy.Index, @@ -66,6 +81,24 @@ class TestTokenizers: q = bigram_index.parse_query("東京", ["bigram_content"]) assert bigram_index.searcher().search(q, limit=5).count == 1 + def test_simple_search_analyzer_supports_regex_substrings( + self, + simple_search_index: tantivy.Index, + ) -> None: + """Whitespace-preserving simple search analyzer supports substring regex matching.""" + writer = simple_search_index.writer() + doc = tantivy.Document() + doc.add_text("simple_content", "tag:invoice password-reset") + writer.add_document(doc) + writer.commit() + simple_search_index.reload() + q = tantivy.Query.regex_query( + simple_search_index.schema, + "simple_content", + ".*sswo.*", + ) + assert simple_search_index.searcher().search(q, limit=5).count == 1 + def test_unsupported_language_logs_warning(self, caplog: LogCaptureFixture) -> None: """Unsupported language codes should log a warning and disable stemming gracefully.""" sb = tantivy.SchemaBuilder() diff --git a/src/documents/tests/test_api_search.py b/src/documents/tests/test_api_search.py index 69bd65198..9e0879e89 100644 --- a/src/documents/tests/test_api_search.py +++ b/src/documents/tests/test_api_search.py @@ -91,6 +91,135 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): self.assertEqual(response.data["count"], 0) self.assertEqual(len(results), 0) + def test_simple_text_search(self) -> None: + tagged = Tag.objects.create(name="invoice") + matching_doc = Document.objects.create( + title="Quarterly summary", + content="Monthly bank report", + checksum="T1", + pk=11, + ) + matching_doc.tags.add(tagged) + + metadata_only_doc = Document.objects.create( + title="Completely unrelated", + content="No matching terms here", + checksum="T2", + pk=12, + ) + metadata_only_doc.tags.add(tagged) + + backend = get_backend() + backend.add_or_update(matching_doc) + backend.add_or_update(metadata_only_doc) + + response = self.client.get("/api/documents/?text=monthly") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["count"], 1) + self.assertEqual(response.data["results"][0]["id"], matching_doc.id) + + response = self.client.get("/api/documents/?text=tag:invoice") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["count"], 0) + + def test_simple_text_search_matches_substrings(self) -> None: + matching_doc = Document.objects.create( + title="Quarterly summary", + content="Password reset instructions", + checksum="T5", + pk=15, + ) + + backend = get_backend() + backend.add_or_update(matching_doc) + + response = self.client.get("/api/documents/?text=pass") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["count"], 1) + self.assertEqual(response.data["results"][0]["id"], matching_doc.id) + + response = self.client.get("/api/documents/?text=sswo") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["count"], 1) + self.assertEqual(response.data["results"][0]["id"], matching_doc.id) + + response = self.client.get("/api/documents/?text=sswo re") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["count"], 1) + self.assertEqual(response.data["results"][0]["id"], matching_doc.id) + + def test_simple_text_search_does_not_match_on_partial_term_overlap(self) -> None: + non_matching_doc = Document.objects.create( + title="Adobe Acrobat PDF Files", + content="Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + checksum="T7", + pk=17, + ) + + backend = get_backend() + backend.add_or_update(non_matching_doc) + + response = self.client.get("/api/documents/?text=raptor") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["count"], 0) + + def test_simple_title_search(self) -> None: + title_match = Document.objects.create( + title="Quarterly summary", + content="No matching content here", + checksum="T3", + pk=13, + ) + content_only = Document.objects.create( + title="Completely unrelated", + content="Quarterly summary appears only in content", + checksum="T4", + pk=14, + ) + + backend = get_backend() + backend.add_or_update(title_match) + backend.add_or_update(content_only) + + response = self.client.get("/api/documents/?title_search=quarterly") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["count"], 1) + self.assertEqual(response.data["results"][0]["id"], title_match.id) + + def test_simple_title_search_matches_substrings(self) -> None: + title_match = Document.objects.create( + title="Password handbook", + content="No matching content here", + checksum="T6", + pk=16, + ) + + backend = get_backend() + backend.add_or_update(title_match) + + response = self.client.get("/api/documents/?title_search=pass") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["count"], 1) + self.assertEqual(response.data["results"][0]["id"], title_match.id) + + response = self.client.get("/api/documents/?title_search=sswo") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["count"], 1) + self.assertEqual(response.data["results"][0]["id"], title_match.id) + + response = self.client.get("/api/documents/?title_search=sswo hand") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["count"], 1) + self.assertEqual(response.data["results"][0]["id"], title_match.id) + + def test_search_rejects_multiple_search_modes(self) -> None: + response = self.client.get("/api/documents/?text=bank&query=bank") + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual( + response.data["detail"], + "Specify only one of text, title_search, query, or more_like_id.", + ) + def test_search_returns_all_for_api_version_9(self) -> None: d1 = Document.objects.create( title="invoice", @@ -1493,6 +1622,31 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): self.assertEqual(results["custom_fields"][0]["id"], custom_field1.id) self.assertEqual(results["workflows"][0]["id"], workflow1.id) + def test_global_search_db_only_limits_documents_to_title_matches(self) -> None: + title_match = Document.objects.create( + title="bank statement", + content="no additional terms", + checksum="GS1", + pk=21, + ) + content_only = Document.objects.create( + title="not a title match", + content="bank appears only in content", + checksum="GS2", + pk=22, + ) + + backend = get_backend() + backend.add_or_update(title_match) + backend.add_or_update(content_only) + + self.client.force_authenticate(self.user) + + response = self.client.get("/api/search/?query=bank&db_only=true") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data["documents"]), 1) + self.assertEqual(response.data["documents"][0]["id"], title_match.id) + def test_global_search_filters_owned_mail_objects(self) -> None: user1 = User.objects.create_user("mail-search-user") user2 = User.objects.create_user("other-mail-search-user") diff --git a/src/documents/views.py b/src/documents/views.py index 024e846a0..68d2b7961 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -1995,11 +1995,23 @@ class ChatStreamingView(GenericAPIView): list=extend_schema( description="Document views including search", parameters=[ + OpenApiParameter( + name="text", + type=OpenApiTypes.STR, + location=OpenApiParameter.QUERY, + description="Simple Tantivy-backed text search query string", + ), + OpenApiParameter( + name="title_search", + type=OpenApiTypes.STR, + location=OpenApiParameter.QUERY, + description="Simple Tantivy-backed title-only search query string", + ), OpenApiParameter( name="query", type=OpenApiTypes.STR, location=OpenApiParameter.QUERY, - description="Advanced search query string", + description="Advanced Tantivy search query string", ), OpenApiParameter( name="full_perms", @@ -2025,22 +2037,28 @@ class ChatStreamingView(GenericAPIView): ), ) class UnifiedSearchViewSet(DocumentViewSet): + SEARCH_PARAM_NAMES = ("text", "title_search", "query", "more_like_id") + def get_serializer_class(self): if self._is_search_request(): return SearchResultSerializer else: return DocumentSerializer + def _get_active_search_params(self, request: Request | None = None) -> list[str]: + request = request or self.request + return [ + param for param in self.SEARCH_PARAM_NAMES if param in request.query_params + ] + def _is_search_request(self): - return ( - "query" in self.request.query_params - or "more_like_id" in self.request.query_params - ) + return bool(self._get_active_search_params()) def list(self, request, *args, **kwargs): if not self._is_search_request(): return super().list(request) + from documents.search import SearchMode from documents.search import TantivyRelevanceList from documents.search import get_backend @@ -2050,9 +2068,31 @@ class UnifiedSearchViewSet(DocumentViewSet): filtered_qs = self.filter_queryset(self.get_queryset()) user = None if request.user.is_superuser else request.user + active_search_params = self._get_active_search_params(request) - if "query" in request.query_params: - query_str = request.query_params["query"] + if len(active_search_params) > 1: + raise ValidationError( + { + "detail": _( + "Specify only one of text, title_search, query, or more_like_id.", + ), + }, + ) + + if ( + "text" in request.query_params + or "title_search" in request.query_params + or "query" in request.query_params + ): + if "text" in request.query_params: + search_mode = SearchMode.TEXT + query_str = request.query_params["text"] + elif "title_search" in request.query_params: + search_mode = SearchMode.TITLE + query_str = request.query_params["title_search"] + else: + search_mode = SearchMode.QUERY + query_str = request.query_params["query"] results = backend.search( query_str, user=user, @@ -2060,6 +2100,7 @@ class UnifiedSearchViewSet(DocumentViewSet): page_size=10000, sort_field=None, sort_reverse=False, + search_mode=search_mode, ) else: # more_like_id — validate permission on the seed document first @@ -2132,6 +2173,8 @@ class UnifiedSearchViewSet(DocumentViewSet): if str(e.detail) == str(invalid_more_like_id_message): return HttpResponseForbidden(invalid_more_like_id_message) return HttpResponseForbidden(_("Insufficient permissions.")) + except ValidationError: + raise except Exception as e: logger.warning(f"An error occurred listing search results: {e!s}") return HttpResponseBadRequest( @@ -3003,6 +3046,9 @@ class GlobalSearchView(PassUserMixin): serializer_class = SearchResultSerializer def get(self, request, *args, **kwargs): + from documents.search import SearchMode + from documents.search import get_backend + query = request.query_params.get("query", None) if query is None: return HttpResponseBadRequest("Query required") @@ -3019,25 +3065,25 @@ class GlobalSearchView(PassUserMixin): "view_document", Document, ) - # First search by title - docs = all_docs.filter(title__icontains=query) - if not db_only and len(docs) < OBJECT_LIMIT: - # If we don't have enough results, search by content. - # Over-fetch from Tantivy (no permission filter) and rely on - # the ORM all_docs queryset for authoritative permission gating. - from documents.search import get_backend - + if db_only: + docs = all_docs.filter(title__icontains=query)[:OBJECT_LIMIT] + else: + user = None if request.user.is_superuser else request.user fts_results = get_backend().search( query, - user=None, + user=user, page=1, page_size=1000, sort_field=None, sort_reverse=False, + search_mode=SearchMode.TEXT, ) - fts_ids = {h["id"] for h in fts_results.hits} - docs = docs | all_docs.filter(id__in=fts_ids) - docs = docs[:OBJECT_LIMIT] + docs_by_id = all_docs.in_bulk([hit["id"] for hit in fts_results.hits]) + docs = [ + docs_by_id[hit["id"]] + for hit in fts_results.hits + if hit["id"] in docs_by_id + ][:OBJECT_LIMIT] saved_views = ( get_objects_for_user_owner_aware( request.user, From d0f8a98a9a7b52ed02fa3fc52e1eb31f83940b5f Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 3 Apr 2026 20:55:14 +0000 Subject: [PATCH 09/11] Auto translate strings --- src-ui/messages.xlf | 68 ++-- src/locale/en_US/LC_MESSAGES/django.po | 452 +++++++++++++------------ 2 files changed, 271 insertions(+), 249 deletions(-) diff --git a/src-ui/messages.xlf b/src-ui/messages.xlf index 19b2f7ce2..f30605a4e 100644 --- a/src-ui/messages.xlf +++ b/src-ui/messages.xlf @@ -1081,7 +1081,7 @@ src/app/components/document-list/filter-editor/filter-editor.component.ts - 205 + 203 @@ -3027,10 +3027,6 @@ src/app/components/document-list/filter-editor/filter-editor.component.html 84 - - src/app/components/document-list/filter-editor/filter-editor.component.ts - 200 - src/app/components/manage/document-attributes/document-attributes.component.ts 129 @@ -7504,7 +7500,7 @@ src/app/components/document-list/filter-editor/filter-editor.component.ts - 192 + 194 src/app/data/document.ts @@ -8817,7 +8813,7 @@ src/app/components/document-list/filter-editor/filter-editor.component.ts - 197 + 199 src/app/data/document.ts @@ -9020,56 +9016,63 @@ Title & content src/app/components/document-list/filter-editor/filter-editor.component.ts - 195 + 197 File type src/app/components/document-list/filter-editor/filter-editor.component.ts - 202 + 200 + + + + Custom fields (Deprecated) + + src/app/components/document-list/filter-editor/filter-editor.component.ts + 210 More like src/app/components/document-list/filter-editor/filter-editor.component.ts - 211 + 215 equals src/app/components/document-list/filter-editor/filter-editor.component.ts - 217 + 221 is empty src/app/components/document-list/filter-editor/filter-editor.component.ts - 221 + 225 is not empty src/app/components/document-list/filter-editor/filter-editor.component.ts - 225 + 229 greater than src/app/components/document-list/filter-editor/filter-editor.component.ts - 229 + 233 less than src/app/components/document-list/filter-editor/filter-editor.component.ts - 233 + 237 @@ -9078,14 +9081,14 @@ )?.name"/> src/app/components/document-list/filter-editor/filter-editor.component.ts - 274,278 + 278,282 Without correspondent src/app/components/document-list/filter-editor/filter-editor.component.ts - 280 + 284 @@ -9094,14 +9097,14 @@ )?.name"/> src/app/components/document-list/filter-editor/filter-editor.component.ts - 286,290 + 290,294 Without document type src/app/components/document-list/filter-editor/filter-editor.component.ts - 292 + 296 @@ -9110,70 +9113,77 @@ )?.name"/> src/app/components/document-list/filter-editor/filter-editor.component.ts - 298,302 + 302,306 Without storage path src/app/components/document-list/filter-editor/filter-editor.component.ts - 304 + 308 Tag: src/app/components/document-list/filter-editor/filter-editor.component.ts - 308,310 + 312,314 Without any tag src/app/components/document-list/filter-editor/filter-editor.component.ts - 314 + 318 Custom fields query src/app/components/document-list/filter-editor/filter-editor.component.ts - 318 + 322 Title: src/app/components/document-list/filter-editor/filter-editor.component.ts - 321 + 326 + + + + Title & content: + + src/app/components/document-list/filter-editor/filter-editor.component.ts + 330 ASN: src/app/components/document-list/filter-editor/filter-editor.component.ts - 324 + 333 Owner: src/app/components/document-list/filter-editor/filter-editor.component.ts - 327 + 336 Owner not in: src/app/components/document-list/filter-editor/filter-editor.component.ts - 330 + 339 Without an owner src/app/components/document-list/filter-editor/filter-editor.component.ts - 333 + 342 diff --git a/src/locale/en_US/LC_MESSAGES/django.po b/src/locale/en_US/LC_MESSAGES/django.po index 57ade319a..03fdcc6e1 100644 --- a/src/locale/en_US/LC_MESSAGES/django.po +++ b/src/locale/en_US/LC_MESSAGES/django.po @@ -2,7 +2,7 @@ msgid "" msgstr "" "Project-Id-Version: paperless-ngx\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2026-04-03 03:25+0000\n" +"POT-Creation-Date: 2026-04-03 20:54+0000\n" "PO-Revision-Date: 2022-02-17 04:17\n" "Last-Translator: \n" "Language-Team: English\n" @@ -21,67 +21,67 @@ msgstr "" msgid "Documents" msgstr "" -#: documents/filters.py:421 +#: documents/filters.py:431 msgid "Value must be valid JSON." msgstr "" -#: documents/filters.py:440 +#: documents/filters.py:450 msgid "Invalid custom field query expression" msgstr "" -#: documents/filters.py:450 +#: documents/filters.py:460 msgid "Invalid expression list. Must be nonempty." msgstr "" -#: documents/filters.py:471 +#: documents/filters.py:481 msgid "Invalid logical operator {op!r}" msgstr "" -#: documents/filters.py:485 +#: documents/filters.py:495 msgid "Maximum number of query conditions exceeded." msgstr "" -#: documents/filters.py:550 +#: documents/filters.py:560 msgid "{name!r} is not a valid custom field." msgstr "" -#: documents/filters.py:587 +#: documents/filters.py:597 msgid "{data_type} does not support query expr {expr!r}." msgstr "" -#: documents/filters.py:695 documents/models.py:137 +#: documents/filters.py:705 documents/models.py:137 msgid "Maximum nesting depth exceeded." msgstr "" -#: documents/filters.py:907 +#: documents/filters.py:919 msgid "Custom field not found" msgstr "" -#: documents/models.py:40 documents/models.py:842 documents/models.py:890 +#: documents/models.py:40 documents/models.py:844 documents/models.py:892 msgid "owner" msgstr "" -#: documents/models.py:57 documents/models.py:1172 +#: documents/models.py:57 documents/models.py:1174 msgid "None" msgstr "" -#: documents/models.py:58 documents/models.py:1173 +#: documents/models.py:58 documents/models.py:1175 msgid "Any word" msgstr "" -#: documents/models.py:59 documents/models.py:1174 +#: documents/models.py:59 documents/models.py:1176 msgid "All words" msgstr "" -#: documents/models.py:60 documents/models.py:1175 +#: documents/models.py:60 documents/models.py:1177 msgid "Exact match" msgstr "" -#: documents/models.py:61 documents/models.py:1176 +#: documents/models.py:61 documents/models.py:1178 msgid "Regular expression" msgstr "" -#: documents/models.py:62 documents/models.py:1177 +#: documents/models.py:62 documents/models.py:1179 msgid "Fuzzy word" msgstr "" @@ -89,20 +89,20 @@ msgstr "" msgid "Automatic" msgstr "" -#: documents/models.py:66 documents/models.py:536 documents/models.py:1755 +#: documents/models.py:66 documents/models.py:536 documents/models.py:1757 #: paperless_mail/models.py:23 paperless_mail/models.py:143 msgid "name" msgstr "" -#: documents/models.py:68 documents/models.py:1241 +#: documents/models.py:68 documents/models.py:1243 msgid "match" msgstr "" -#: documents/models.py:71 documents/models.py:1244 +#: documents/models.py:71 documents/models.py:1246 msgid "matching algorithm" msgstr "" -#: documents/models.py:76 documents/models.py:1249 +#: documents/models.py:76 documents/models.py:1251 msgid "is insensitive" msgstr "" @@ -168,7 +168,7 @@ msgstr "" msgid "title" msgstr "" -#: documents/models.py:191 documents/models.py:756 +#: documents/models.py:191 documents/models.py:758 msgid "content" msgstr "" @@ -206,8 +206,8 @@ msgstr "" msgid "The number of pages of the document." msgstr "" -#: documents/models.py:246 documents/models.py:762 documents/models.py:800 -#: documents/models.py:862 documents/models.py:980 documents/models.py:1039 +#: documents/models.py:246 documents/models.py:764 documents/models.py:802 +#: documents/models.py:864 documents/models.py:982 documents/models.py:1041 msgid "created" msgstr "" @@ -271,12 +271,12 @@ msgstr "" msgid "Optional short label for a document version." msgstr "" -#: documents/models.py:340 documents/models.py:773 documents/models.py:827 -#: documents/models.py:1798 +#: documents/models.py:340 documents/models.py:775 documents/models.py:829 +#: documents/models.py:1800 msgid "document" msgstr "" -#: documents/models.py:341 documents/models.py:933 +#: documents/models.py:341 documents/models.py:935 msgid "documents" msgstr "" @@ -296,11 +296,11 @@ msgstr "" msgid "Title" msgstr "" -#: documents/models.py:523 documents/models.py:1193 +#: documents/models.py:523 documents/models.py:1195 msgid "Created" msgstr "" -#: documents/models.py:524 documents/models.py:1192 +#: documents/models.py:524 documents/models.py:1194 msgid "Added" msgstr "" @@ -360,7 +360,7 @@ msgstr "" msgid "Document display fields" msgstr "" -#: documents/models.py:569 documents/models.py:632 +#: documents/models.py:569 documents/models.py:634 msgid "saved view" msgstr "" @@ -560,748 +560,756 @@ msgstr "" msgid "mime type is" msgstr "" -#: documents/models.py:635 -msgid "rule type" +#: documents/models.py:626 +msgid "simple title search" +msgstr "" + +#: documents/models.py:627 +msgid "simple text search" msgstr "" #: documents/models.py:637 +msgid "rule type" +msgstr "" + +#: documents/models.py:639 msgid "value" msgstr "" -#: documents/models.py:640 +#: documents/models.py:642 msgid "filter rule" msgstr "" -#: documents/models.py:641 +#: documents/models.py:643 msgid "filter rules" msgstr "" -#: documents/models.py:665 +#: documents/models.py:667 msgid "Auto Task" msgstr "" -#: documents/models.py:666 +#: documents/models.py:668 msgid "Scheduled Task" msgstr "" -#: documents/models.py:667 +#: documents/models.py:669 msgid "Manual Task" msgstr "" -#: documents/models.py:670 +#: documents/models.py:672 msgid "Consume File" msgstr "" -#: documents/models.py:671 +#: documents/models.py:673 msgid "Train Classifier" msgstr "" -#: documents/models.py:672 +#: documents/models.py:674 msgid "Check Sanity" msgstr "" -#: documents/models.py:673 +#: documents/models.py:675 msgid "Index Optimize" msgstr "" -#: documents/models.py:674 +#: documents/models.py:676 msgid "LLM Index Update" msgstr "" -#: documents/models.py:679 +#: documents/models.py:681 msgid "Task ID" msgstr "" -#: documents/models.py:680 +#: documents/models.py:682 msgid "Celery ID for the Task that was run" msgstr "" -#: documents/models.py:685 +#: documents/models.py:687 msgid "Acknowledged" msgstr "" -#: documents/models.py:686 +#: documents/models.py:688 msgid "If the task is acknowledged via the frontend or API" msgstr "" -#: documents/models.py:692 +#: documents/models.py:694 msgid "Task Filename" msgstr "" -#: documents/models.py:693 +#: documents/models.py:695 msgid "Name of the file which the Task was run for" msgstr "" -#: documents/models.py:700 +#: documents/models.py:702 msgid "Task Name" msgstr "" -#: documents/models.py:701 +#: documents/models.py:703 msgid "Name of the task that was run" msgstr "" -#: documents/models.py:708 +#: documents/models.py:710 msgid "Task State" msgstr "" -#: documents/models.py:709 +#: documents/models.py:711 msgid "Current state of the task being run" msgstr "" -#: documents/models.py:715 +#: documents/models.py:717 msgid "Created DateTime" msgstr "" -#: documents/models.py:716 +#: documents/models.py:718 msgid "Datetime field when the task result was created in UTC" msgstr "" -#: documents/models.py:722 +#: documents/models.py:724 msgid "Started DateTime" msgstr "" -#: documents/models.py:723 +#: documents/models.py:725 msgid "Datetime field when the task was started in UTC" msgstr "" -#: documents/models.py:729 +#: documents/models.py:731 msgid "Completed DateTime" msgstr "" -#: documents/models.py:730 +#: documents/models.py:732 msgid "Datetime field when the task was completed in UTC" msgstr "" -#: documents/models.py:736 +#: documents/models.py:738 msgid "Result Data" msgstr "" -#: documents/models.py:738 +#: documents/models.py:740 msgid "The data returned by the task" msgstr "" -#: documents/models.py:746 +#: documents/models.py:748 msgid "Task Type" msgstr "" -#: documents/models.py:747 +#: documents/models.py:749 msgid "The type of task that was run" msgstr "" -#: documents/models.py:758 +#: documents/models.py:760 msgid "Note for the document" msgstr "" -#: documents/models.py:782 +#: documents/models.py:784 msgid "user" msgstr "" -#: documents/models.py:787 +#: documents/models.py:789 msgid "note" msgstr "" -#: documents/models.py:788 +#: documents/models.py:790 msgid "notes" msgstr "" -#: documents/models.py:796 +#: documents/models.py:798 msgid "Archive" msgstr "" -#: documents/models.py:797 +#: documents/models.py:799 msgid "Original" msgstr "" -#: documents/models.py:808 documents/models.py:870 paperless_mail/models.py:75 +#: documents/models.py:810 documents/models.py:872 paperless_mail/models.py:75 msgid "expiration" msgstr "" -#: documents/models.py:815 documents/models.py:877 +#: documents/models.py:817 documents/models.py:879 msgid "slug" msgstr "" -#: documents/models.py:847 +#: documents/models.py:849 msgid "share link" msgstr "" -#: documents/models.py:848 +#: documents/models.py:850 msgid "share links" msgstr "" -#: documents/models.py:856 +#: documents/models.py:858 msgid "Pending" msgstr "" -#: documents/models.py:857 +#: documents/models.py:859 msgid "Processing" msgstr "" -#: documents/models.py:858 +#: documents/models.py:860 msgid "Ready" msgstr "" -#: documents/models.py:859 +#: documents/models.py:861 msgid "Failed" msgstr "" -#: documents/models.py:906 +#: documents/models.py:908 msgid "size (bytes)" msgstr "" -#: documents/models.py:912 +#: documents/models.py:914 msgid "last error" msgstr "" -#: documents/models.py:919 +#: documents/models.py:921 msgid "file path" msgstr "" -#: documents/models.py:925 +#: documents/models.py:927 msgid "built at" msgstr "" -#: documents/models.py:938 +#: documents/models.py:940 msgid "share link bundle" msgstr "" -#: documents/models.py:939 +#: documents/models.py:941 msgid "share link bundles" msgstr "" -#: documents/models.py:942 +#: documents/models.py:944 #, python-format msgid "Share link bundle %(slug)s" msgstr "" -#: documents/models.py:968 +#: documents/models.py:970 msgid "String" msgstr "" -#: documents/models.py:969 +#: documents/models.py:971 msgid "URL" msgstr "" -#: documents/models.py:970 +#: documents/models.py:972 msgid "Date" msgstr "" -#: documents/models.py:971 +#: documents/models.py:973 msgid "Boolean" msgstr "" -#: documents/models.py:972 +#: documents/models.py:974 msgid "Integer" msgstr "" -#: documents/models.py:973 +#: documents/models.py:975 msgid "Float" msgstr "" -#: documents/models.py:974 +#: documents/models.py:976 msgid "Monetary" msgstr "" -#: documents/models.py:975 +#: documents/models.py:977 msgid "Document Link" msgstr "" -#: documents/models.py:976 +#: documents/models.py:978 msgid "Select" msgstr "" -#: documents/models.py:977 +#: documents/models.py:979 msgid "Long Text" msgstr "" -#: documents/models.py:989 +#: documents/models.py:991 msgid "data type" msgstr "" -#: documents/models.py:996 +#: documents/models.py:998 msgid "extra data" msgstr "" -#: documents/models.py:1000 +#: documents/models.py:1002 msgid "Extra data for the custom field, such as select options" msgstr "" -#: documents/models.py:1006 +#: documents/models.py:1008 msgid "custom field" msgstr "" -#: documents/models.py:1007 +#: documents/models.py:1009 msgid "custom fields" msgstr "" -#: documents/models.py:1107 +#: documents/models.py:1109 msgid "custom field instance" msgstr "" -#: documents/models.py:1108 +#: documents/models.py:1110 msgid "custom field instances" msgstr "" -#: documents/models.py:1180 +#: documents/models.py:1182 msgid "Consumption Started" msgstr "" -#: documents/models.py:1181 +#: documents/models.py:1183 msgid "Document Added" msgstr "" -#: documents/models.py:1182 +#: documents/models.py:1184 msgid "Document Updated" msgstr "" -#: documents/models.py:1183 +#: documents/models.py:1185 msgid "Scheduled" msgstr "" -#: documents/models.py:1186 +#: documents/models.py:1188 msgid "Consume Folder" msgstr "" -#: documents/models.py:1187 +#: documents/models.py:1189 msgid "Api Upload" msgstr "" -#: documents/models.py:1188 +#: documents/models.py:1190 msgid "Mail Fetch" msgstr "" -#: documents/models.py:1189 +#: documents/models.py:1191 msgid "Web UI" msgstr "" -#: documents/models.py:1194 +#: documents/models.py:1196 msgid "Modified" msgstr "" -#: documents/models.py:1195 +#: documents/models.py:1197 msgid "Custom Field" msgstr "" -#: documents/models.py:1198 +#: documents/models.py:1200 msgid "Workflow Trigger Type" msgstr "" -#: documents/models.py:1210 +#: documents/models.py:1212 msgid "filter path" msgstr "" -#: documents/models.py:1215 +#: documents/models.py:1217 msgid "" "Only consume documents with a path that matches this if specified. Wildcards " "specified as * are allowed. Case insensitive." msgstr "" -#: documents/models.py:1222 +#: documents/models.py:1224 msgid "filter filename" msgstr "" -#: documents/models.py:1227 paperless_mail/models.py:200 +#: documents/models.py:1229 paperless_mail/models.py:200 msgid "" "Only consume documents which entirely match this filename if specified. " "Wildcards such as *.pdf or *invoice* are allowed. Case insensitive." msgstr "" -#: documents/models.py:1238 +#: documents/models.py:1240 msgid "filter documents from this mail rule" msgstr "" -#: documents/models.py:1254 +#: documents/models.py:1256 msgid "has these tag(s)" msgstr "" -#: documents/models.py:1261 +#: documents/models.py:1263 msgid "has all of these tag(s)" msgstr "" -#: documents/models.py:1268 +#: documents/models.py:1270 msgid "does not have these tag(s)" msgstr "" -#: documents/models.py:1276 +#: documents/models.py:1278 msgid "has this document type" msgstr "" -#: documents/models.py:1283 +#: documents/models.py:1285 msgid "has one of these document types" msgstr "" -#: documents/models.py:1290 +#: documents/models.py:1292 msgid "does not have these document type(s)" msgstr "" -#: documents/models.py:1298 +#: documents/models.py:1300 msgid "has this correspondent" msgstr "" -#: documents/models.py:1305 +#: documents/models.py:1307 msgid "does not have these correspondent(s)" msgstr "" -#: documents/models.py:1312 +#: documents/models.py:1314 msgid "has one of these correspondents" msgstr "" -#: documents/models.py:1320 +#: documents/models.py:1322 msgid "has this storage path" msgstr "" -#: documents/models.py:1327 +#: documents/models.py:1329 msgid "has one of these storage paths" msgstr "" -#: documents/models.py:1334 +#: documents/models.py:1336 msgid "does not have these storage path(s)" msgstr "" -#: documents/models.py:1338 +#: documents/models.py:1340 msgid "filter custom field query" msgstr "" -#: documents/models.py:1341 +#: documents/models.py:1343 msgid "JSON-encoded custom field query expression." msgstr "" -#: documents/models.py:1345 +#: documents/models.py:1347 msgid "schedule offset days" msgstr "" -#: documents/models.py:1348 +#: documents/models.py:1350 msgid "The number of days to offset the schedule trigger by." msgstr "" -#: documents/models.py:1353 +#: documents/models.py:1355 msgid "schedule is recurring" msgstr "" -#: documents/models.py:1356 +#: documents/models.py:1358 msgid "If the schedule should be recurring." msgstr "" -#: documents/models.py:1361 +#: documents/models.py:1363 msgid "schedule recurring delay in days" msgstr "" -#: documents/models.py:1365 +#: documents/models.py:1367 msgid "The number of days between recurring schedule triggers." msgstr "" -#: documents/models.py:1370 +#: documents/models.py:1372 msgid "schedule date field" msgstr "" -#: documents/models.py:1375 +#: documents/models.py:1377 msgid "The field to check for a schedule trigger." msgstr "" -#: documents/models.py:1384 +#: documents/models.py:1386 msgid "schedule date custom field" msgstr "" -#: documents/models.py:1388 +#: documents/models.py:1390 msgid "workflow trigger" msgstr "" -#: documents/models.py:1389 +#: documents/models.py:1391 msgid "workflow triggers" msgstr "" -#: documents/models.py:1397 +#: documents/models.py:1399 msgid "email subject" msgstr "" -#: documents/models.py:1401 +#: documents/models.py:1403 msgid "" "The subject of the email, can include some placeholders, see documentation." msgstr "" -#: documents/models.py:1407 +#: documents/models.py:1409 msgid "email body" msgstr "" -#: documents/models.py:1410 +#: documents/models.py:1412 msgid "" "The body (message) of the email, can include some placeholders, see " "documentation." msgstr "" -#: documents/models.py:1416 +#: documents/models.py:1418 msgid "emails to" msgstr "" -#: documents/models.py:1419 +#: documents/models.py:1421 msgid "The destination email addresses, comma separated." msgstr "" -#: documents/models.py:1425 +#: documents/models.py:1427 msgid "include document in email" msgstr "" -#: documents/models.py:1436 +#: documents/models.py:1438 msgid "webhook url" msgstr "" -#: documents/models.py:1439 +#: documents/models.py:1441 msgid "The destination URL for the notification." msgstr "" -#: documents/models.py:1444 +#: documents/models.py:1446 msgid "use parameters" msgstr "" -#: documents/models.py:1449 +#: documents/models.py:1451 msgid "send as JSON" msgstr "" -#: documents/models.py:1453 +#: documents/models.py:1455 msgid "webhook parameters" msgstr "" -#: documents/models.py:1456 +#: documents/models.py:1458 msgid "The parameters to send with the webhook URL if body not used." msgstr "" -#: documents/models.py:1460 +#: documents/models.py:1462 msgid "webhook body" msgstr "" -#: documents/models.py:1463 +#: documents/models.py:1465 msgid "The body to send with the webhook URL if parameters not used." msgstr "" -#: documents/models.py:1467 +#: documents/models.py:1469 msgid "webhook headers" msgstr "" -#: documents/models.py:1470 +#: documents/models.py:1472 msgid "The headers to send with the webhook URL." msgstr "" -#: documents/models.py:1475 +#: documents/models.py:1477 msgid "include document in webhook" msgstr "" -#: documents/models.py:1486 +#: documents/models.py:1488 msgid "Assignment" msgstr "" -#: documents/models.py:1490 +#: documents/models.py:1492 msgid "Removal" msgstr "" -#: documents/models.py:1494 documents/templates/account/password_reset.html:15 +#: documents/models.py:1496 documents/templates/account/password_reset.html:15 msgid "Email" msgstr "" -#: documents/models.py:1498 +#: documents/models.py:1500 msgid "Webhook" msgstr "" -#: documents/models.py:1502 +#: documents/models.py:1504 msgid "Password removal" msgstr "" -#: documents/models.py:1506 +#: documents/models.py:1508 msgid "Move to trash" msgstr "" -#: documents/models.py:1510 +#: documents/models.py:1512 msgid "Workflow Action Type" msgstr "" -#: documents/models.py:1515 documents/models.py:1757 +#: documents/models.py:1517 documents/models.py:1759 #: paperless_mail/models.py:145 msgid "order" msgstr "" -#: documents/models.py:1518 +#: documents/models.py:1520 msgid "assign title" msgstr "" -#: documents/models.py:1522 +#: documents/models.py:1524 msgid "Assign a document title, must be a Jinja2 template, see documentation." msgstr "" -#: documents/models.py:1530 paperless_mail/models.py:274 +#: documents/models.py:1532 paperless_mail/models.py:274 msgid "assign this tag" msgstr "" -#: documents/models.py:1539 paperless_mail/models.py:282 +#: documents/models.py:1541 paperless_mail/models.py:282 msgid "assign this document type" msgstr "" -#: documents/models.py:1548 paperless_mail/models.py:296 +#: documents/models.py:1550 paperless_mail/models.py:296 msgid "assign this correspondent" msgstr "" -#: documents/models.py:1557 +#: documents/models.py:1559 msgid "assign this storage path" msgstr "" -#: documents/models.py:1566 +#: documents/models.py:1568 msgid "assign this owner" msgstr "" -#: documents/models.py:1573 +#: documents/models.py:1575 msgid "grant view permissions to these users" msgstr "" -#: documents/models.py:1580 +#: documents/models.py:1582 msgid "grant view permissions to these groups" msgstr "" -#: documents/models.py:1587 +#: documents/models.py:1589 msgid "grant change permissions to these users" msgstr "" -#: documents/models.py:1594 +#: documents/models.py:1596 msgid "grant change permissions to these groups" msgstr "" -#: documents/models.py:1601 +#: documents/models.py:1603 msgid "assign these custom fields" msgstr "" -#: documents/models.py:1605 +#: documents/models.py:1607 msgid "custom field values" msgstr "" -#: documents/models.py:1609 +#: documents/models.py:1611 msgid "Optional values to assign to the custom fields." msgstr "" -#: documents/models.py:1618 +#: documents/models.py:1620 msgid "remove these tag(s)" msgstr "" -#: documents/models.py:1623 +#: documents/models.py:1625 msgid "remove all tags" msgstr "" -#: documents/models.py:1630 +#: documents/models.py:1632 msgid "remove these document type(s)" msgstr "" -#: documents/models.py:1635 +#: documents/models.py:1637 msgid "remove all document types" msgstr "" -#: documents/models.py:1642 +#: documents/models.py:1644 msgid "remove these correspondent(s)" msgstr "" -#: documents/models.py:1647 +#: documents/models.py:1649 msgid "remove all correspondents" msgstr "" -#: documents/models.py:1654 +#: documents/models.py:1656 msgid "remove these storage path(s)" msgstr "" -#: documents/models.py:1659 +#: documents/models.py:1661 msgid "remove all storage paths" msgstr "" -#: documents/models.py:1666 +#: documents/models.py:1668 msgid "remove these owner(s)" msgstr "" -#: documents/models.py:1671 +#: documents/models.py:1673 msgid "remove all owners" msgstr "" -#: documents/models.py:1678 +#: documents/models.py:1680 msgid "remove view permissions for these users" msgstr "" -#: documents/models.py:1685 +#: documents/models.py:1687 msgid "remove view permissions for these groups" msgstr "" -#: documents/models.py:1692 +#: documents/models.py:1694 msgid "remove change permissions for these users" msgstr "" -#: documents/models.py:1699 +#: documents/models.py:1701 msgid "remove change permissions for these groups" msgstr "" -#: documents/models.py:1704 +#: documents/models.py:1706 msgid "remove all permissions" msgstr "" -#: documents/models.py:1711 +#: documents/models.py:1713 msgid "remove these custom fields" msgstr "" -#: documents/models.py:1716 +#: documents/models.py:1718 msgid "remove all custom fields" msgstr "" -#: documents/models.py:1725 +#: documents/models.py:1727 msgid "email" msgstr "" -#: documents/models.py:1734 +#: documents/models.py:1736 msgid "webhook" msgstr "" -#: documents/models.py:1738 +#: documents/models.py:1740 msgid "passwords" msgstr "" -#: documents/models.py:1742 +#: documents/models.py:1744 msgid "" "Passwords to try when removing PDF protection. Separate with commas or new " "lines." msgstr "" -#: documents/models.py:1747 +#: documents/models.py:1749 msgid "workflow action" msgstr "" -#: documents/models.py:1748 +#: documents/models.py:1750 msgid "workflow actions" msgstr "" -#: documents/models.py:1763 +#: documents/models.py:1765 msgid "triggers" msgstr "" -#: documents/models.py:1770 +#: documents/models.py:1772 msgid "actions" msgstr "" -#: documents/models.py:1773 paperless_mail/models.py:154 +#: documents/models.py:1775 paperless_mail/models.py:154 msgid "enabled" msgstr "" -#: documents/models.py:1784 +#: documents/models.py:1786 msgid "workflow" msgstr "" -#: documents/models.py:1788 +#: documents/models.py:1790 msgid "workflow trigger type" msgstr "" -#: documents/models.py:1802 +#: documents/models.py:1804 msgid "date run" msgstr "" -#: documents/models.py:1808 +#: documents/models.py:1810 msgid "workflow run" msgstr "" -#: documents/models.py:1809 +#: documents/models.py:1811 msgid "workflow runs" msgstr "" #: documents/serialisers.py:463 documents/serialisers.py:815 -#: documents/serialisers.py:2545 documents/views.py:2079 -#: documents/views.py:2134 paperless_mail/serialisers.py:143 +#: documents/serialisers.py:2545 documents/views.py:2120 +#: documents/views.py:2175 paperless_mail/serialisers.py:143 msgid "Insufficient permissions." msgstr "" @@ -1341,7 +1349,7 @@ msgstr "" msgid "Duplicate document identifiers are not allowed." msgstr "" -#: documents/serialisers.py:2631 documents/views.py:3738 +#: documents/serialisers.py:2631 documents/views.py:3784 #, python-format msgid "Documents not found: %(ids)s" msgstr "" @@ -1609,24 +1617,28 @@ msgstr "" msgid "Unable to parse URI {value}" msgstr "" -#: documents/views.py:2072 documents/views.py:2131 +#: documents/views.py:2077 +msgid "Specify only one of text, title_search, query, or more_like_id." +msgstr "" + +#: documents/views.py:2113 documents/views.py:2172 msgid "Invalid more_like_id" msgstr "" -#: documents/views.py:3750 +#: documents/views.py:3796 #, python-format msgid "Insufficient permissions to share document %(id)s." msgstr "" -#: documents/views.py:3793 +#: documents/views.py:3839 msgid "Bundle is already being processed." msgstr "" -#: documents/views.py:3850 +#: documents/views.py:3896 msgid "The share link bundle is still being prepared. Please try again later." msgstr "" -#: documents/views.py:3860 +#: documents/views.py:3906 msgid "The share link bundle is unavailable." msgstr "" From c2f02851da5417817a21b6d237b5a215a7fcec22 Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Fri, 3 Apr 2026 14:18:01 -0700 Subject: [PATCH 10/11] Chore: Better typed status manager messages (#12509) --- src/documents/consumer.py | 14 ++-- src/documents/plugins/helpers.py | 91 +++++++++++++++++++++----- src/documents/tests/utils.py | 13 ++-- src/paperless/consumers.py | 25 +++++-- src/paperless/tests/test_websockets.py | 10 ++- 5 files changed, 114 insertions(+), 39 deletions(-) diff --git a/src/documents/consumer.py b/src/documents/consumer.py index 6ae5914b7..f68fa0685 100644 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -139,14 +139,12 @@ class ConsumerPluginMixin: message, current_progress, max_progress, - extra_args={ - "document_id": document_id, - "owner_id": self.metadata.owner_id if self.metadata.owner_id else None, - "users_can_view": (self.metadata.view_users or []) - + (self.metadata.change_users or []), - "groups_can_view": (self.metadata.view_groups or []) - + (self.metadata.change_groups or []), - }, + document_id=document_id, + owner_id=self.metadata.owner_id if self.metadata.owner_id else None, + users_can_view=(self.metadata.view_users or []) + + (self.metadata.change_users or []), + groups_can_view=(self.metadata.view_groups or []) + + (self.metadata.change_groups or []), ) def _fail( diff --git a/src/documents/plugins/helpers.py b/src/documents/plugins/helpers.py index e5cfde3b8..e30591125 100644 --- a/src/documents/plugins/helpers.py +++ b/src/documents/plugins/helpers.py @@ -1,6 +1,9 @@ import enum -from collections.abc import Mapping from typing import TYPE_CHECKING +from typing import Literal +from typing import Self +from typing import TypeAlias +from typing import TypedDict from asgiref.sync import async_to_sync from channels.layers import get_channel_layer @@ -16,6 +19,59 @@ class ProgressStatusOptions(enum.StrEnum): FAILED = "FAILED" +class PermissionsData(TypedDict, total=False): + """Permission fields included in status messages for access control.""" + + owner_id: int | None + users_can_view: list[int] + groups_can_view: list[int] + + +class ProgressUpdateData(TypedDict): + filename: str | None + task_id: str | None + current_progress: int + max_progress: int + status: str + message: str + document_id: int | None + owner_id: int | None + users_can_view: list[int] + groups_can_view: list[int] + + +class StatusUpdatePayload(TypedDict): + type: Literal["status_update"] + data: ProgressUpdateData + + +class DocumentsDeletedData(TypedDict): + documents: list[int] + + +class DocumentsDeletedPayload(TypedDict): + type: Literal["documents_deleted"] + data: DocumentsDeletedData + + +class DocumentUpdatedData(TypedDict): + document_id: int + modified: str + owner_id: int | None + users_can_view: list[int] + groups_can_view: list[int] + + +class DocumentUpdatedPayload(TypedDict): + type: Literal["document_updated"] + data: DocumentUpdatedData + + +WebsocketPayload: TypeAlias = ( + StatusUpdatePayload | DocumentsDeletedPayload | DocumentUpdatedPayload +) + + class BaseStatusManager: """ Handles sending of progress information via the channel layer, with proper management @@ -25,11 +81,11 @@ class BaseStatusManager: def __init__(self) -> None: self._channel: RedisPubSubChannelLayer | None = None - def __enter__(self): + def __enter__(self) -> Self: self.open() return self - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None: self.close() def open(self) -> None: @@ -48,7 +104,7 @@ class BaseStatusManager: async_to_sync(self._channel.flush) self._channel = None - def send(self, payload: Mapping[str, object]) -> None: + def send(self, payload: WebsocketPayload) -> None: # Ensure the layer is open self.open() @@ -72,36 +128,36 @@ class ProgressManager(BaseStatusManager): message: str, current_progress: int, max_progress: int, - extra_args: dict[str, str | int | None] | None = None, + *, + document_id: int | None = None, + owner_id: int | None = None, + users_can_view: list[int] | None = None, + groups_can_view: list[int] | None = None, ) -> None: - data: dict[str, object] = { + data: ProgressUpdateData = { "filename": self.filename, "task_id": self.task_id, "current_progress": current_progress, "max_progress": max_progress, "status": status, "message": message, + "document_id": document_id, + "owner_id": owner_id, + "users_can_view": users_can_view or [], + "groups_can_view": groups_can_view or [], } - if extra_args is not None: - data.update(extra_args) - - payload: dict[str, object] = { - "type": "status_update", - "data": data, - } - + payload: StatusUpdatePayload = {"type": "status_update", "data": data} self.send(payload) class DocumentsStatusManager(BaseStatusManager): def send_documents_deleted(self, documents: list[int]) -> None: - payload: dict[str, object] = { + payload: DocumentsDeletedPayload = { "type": "documents_deleted", "data": { "documents": documents, }, } - self.send(payload) def send_document_updated( @@ -113,7 +169,7 @@ class DocumentsStatusManager(BaseStatusManager): users_can_view: list[int] | None = None, groups_can_view: list[int] | None = None, ) -> None: - payload: dict[str, object] = { + payload: DocumentUpdatedPayload = { "type": "document_updated", "data": { "document_id": document_id, @@ -123,5 +179,4 @@ class DocumentsStatusManager(BaseStatusManager): "groups_can_view": groups_can_view or [], }, } - self.send(payload) diff --git a/src/documents/tests/utils.py b/src/documents/tests/utils.py index cc4190974..98c8258b8 100644 --- a/src/documents/tests/utils.py +++ b/src/documents/tests/utils.py @@ -435,7 +435,11 @@ class DummyProgressManager: message: str, current_progress: int, max_progress: int, - extra_args: dict[str, str | int] | None = None, + *, + document_id: int | None = None, + owner_id: int | None = None, + users_can_view: list[int] | None = None, + groups_can_view: list[int] | None = None, ) -> None: # Ensure the layer is open self.open() @@ -449,9 +453,10 @@ class DummyProgressManager: "max_progress": max_progress, "status": status, "message": message, + "document_id": document_id, + "owner_id": owner_id, + "users_can_view": users_can_view or [], + "groups_can_view": groups_can_view or [], }, } - if extra_args is not None: - payload["data"].update(extra_args) - self.payloads.append(payload) diff --git a/src/paperless/consumers.py b/src/paperless/consumers.py index 9d59a1a5a..4a3cda8fe 100644 --- a/src/paperless/consumers.py +++ b/src/paperless/consumers.py @@ -1,16 +1,27 @@ +from __future__ import annotations + import json -from typing import Any +from typing import TYPE_CHECKING from channels.generic.websocket import AsyncWebsocketConsumer +if TYPE_CHECKING: + from django.contrib.auth.base_user import AbstractBaseUser + from django.contrib.auth.models import AnonymousUser + + from documents.plugins.helpers import DocumentsDeletedPayload + from documents.plugins.helpers import DocumentUpdatedPayload + from documents.plugins.helpers import PermissionsData + from documents.plugins.helpers import StatusUpdatePayload + class StatusConsumer(AsyncWebsocketConsumer): def _authenticated(self) -> bool: - user: Any = self.scope.get("user") + user: AbstractBaseUser | AnonymousUser | None = self.scope.get("user") return user is not None and user.is_authenticated - async def _can_view(self, data: dict[str, Any]) -> bool: - user: Any = self.scope.get("user") + async def _can_view(self, data: PermissionsData) -> bool: + user: AbstractBaseUser | AnonymousUser | None = self.scope.get("user") if user is None: return False owner_id = data.get("owner_id") @@ -32,19 +43,19 @@ class StatusConsumer(AsyncWebsocketConsumer): async def disconnect(self, code: int) -> None: await self.channel_layer.group_discard("status_updates", self.channel_name) - async def status_update(self, event: dict[str, Any]) -> None: + async def status_update(self, event: StatusUpdatePayload) -> None: if not self._authenticated(): await self.close() elif await self._can_view(event["data"]): await self.send(json.dumps(event)) - async def documents_deleted(self, event: dict[str, Any]) -> None: + async def documents_deleted(self, event: DocumentsDeletedPayload) -> None: if not self._authenticated(): await self.close() else: await self.send(json.dumps(event)) - async def document_updated(self, event: dict[str, Any]) -> None: + async def document_updated(self, event: DocumentUpdatedPayload) -> None: if not self._authenticated(): await self.close() elif await self._can_view(event["data"]): diff --git a/src/paperless/tests/test_websockets.py b/src/paperless/tests/test_websockets.py index bffc44f82..9f7c9a652 100644 --- a/src/paperless/tests/test_websockets.py +++ b/src/paperless/tests/test_websockets.py @@ -200,7 +200,10 @@ class TestWebSockets: "Test message", 1, 10, - extra_args={"foo": "bar"}, + document_id=42, + owner_id=1, + users_can_view=[2, 3], + groups_can_view=[4], ) assert mock_group_send.call_args[0][1] == { @@ -212,7 +215,10 @@ class TestWebSockets: "max_progress": 10, "status": ProgressStatusOptions.STARTED, "message": "Test message", - "foo": "bar", + "document_id": 42, + "owner_id": 1, + "users_can_view": [2, 3], + "groups_can_view": [4], }, } From b807b107adcb3020b16ad1112d588046eb6b37b9 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Fri, 3 Apr 2026 14:51:57 -0700 Subject: [PATCH 11/11] Enhancement: include sharelinks + bundles in export/import (#12479) --- .../management/commands/document_exporter.py | 76 +++++++++++++ .../management/commands/document_importer.py | 60 +++++++++- src/documents/settings.py | 1 + .../tests/test_management_exporter.py | 107 ++++++++++++++++++ 4 files changed, 243 insertions(+), 1 deletion(-) diff --git a/src/documents/management/commands/document_exporter.py b/src/documents/management/commands/document_exporter.py index ee3b44e0c..562a2ca8d 100644 --- a/src/documents/management/commands/document_exporter.py +++ b/src/documents/management/commands/document_exporter.py @@ -45,6 +45,8 @@ from documents.models import DocumentType from documents.models import Note from documents.models import SavedView from documents.models import SavedViewFilterRule +from documents.models import ShareLink +from documents.models import ShareLinkBundle from documents.models import StoragePath from documents.models import Tag from documents.models import UiSettings @@ -55,6 +57,7 @@ from documents.models import WorkflowActionWebhook from documents.models import WorkflowTrigger from documents.settings import EXPORTER_ARCHIVE_NAME from documents.settings import EXPORTER_FILE_NAME +from documents.settings import EXPORTER_SHARE_LINK_BUNDLE_NAME from documents.settings import EXPORTER_THUMBNAIL_NAME from documents.utils import compute_checksum from documents.utils import copy_file_with_basic_stats @@ -389,6 +392,8 @@ class Command(CryptMixin, PaperlessCommand): "app_configs": ApplicationConfiguration.objects.all(), "notes": Note.global_objects.all(), "documents": Document.global_objects.order_by("id").all(), + "share_links": ShareLink.global_objects.all(), + "share_link_bundles": ShareLinkBundle.objects.order_by("id").all(), "social_accounts": SocialAccount.objects.all(), "social_apps": SocialApp.objects.all(), "social_tokens": SocialToken.objects.all(), @@ -409,6 +414,7 @@ class Command(CryptMixin, PaperlessCommand): ) document_manifest: list[dict] = [] + share_link_bundle_manifest: list[dict] = [] manifest_path = (self.target / "manifest.json").resolve() with StreamingManifestWriter( @@ -427,6 +433,15 @@ class Command(CryptMixin, PaperlessCommand): for record in batch: self._encrypt_record_inline(record) document_manifest.extend(batch) + elif key == "share_link_bundles": + # Accumulate for file-copy loop; written to manifest after + for batch in serialize_queryset_batched( + qs, + batch_size=self.batch_size, + ): + for record in batch: + self._encrypt_record_inline(record) + share_link_bundle_manifest.extend(batch) elif self.split_manifest and key in ( "notes", "custom_field_instances", @@ -445,6 +460,12 @@ class Command(CryptMixin, PaperlessCommand): document_map: dict[int, Document] = { d.pk: d for d in Document.global_objects.order_by("id") } + share_link_bundle_map: dict[int, ShareLinkBundle] = { + b.pk: b + for b in ShareLinkBundle.objects.order_by("id").prefetch_related( + "documents", + ) + } # 3. Export files from each document for index, document_dict in enumerate( @@ -478,6 +499,19 @@ class Command(CryptMixin, PaperlessCommand): else: writer.write_record(document_dict) + for bundle_dict in share_link_bundle_manifest: + bundle = share_link_bundle_map[bundle_dict["pk"]] + + bundle_target = self.generate_share_link_bundle_target( + bundle, + bundle_dict, + ) + + if not self.data_only and bundle_target is not None: + self.copy_share_link_bundle_file(bundle, bundle_target) + + writer.write_record(bundle_dict) + # 4.2 write version information to target folder extra_metadata_path = (self.target / "metadata.json").resolve() metadata: dict[str, str | int | dict[str, str | int]] = { @@ -598,6 +632,48 @@ class Command(CryptMixin, PaperlessCommand): archive_target, ) + def generate_share_link_bundle_target( + self, + bundle: ShareLinkBundle, + bundle_dict: dict, + ) -> Path | None: + """ + Generates the export target for a share link bundle file, when present. + """ + if not bundle.file_path: + return None + + stored_bundle_path = Path(bundle.file_path) + portable_bundle_path = ( + stored_bundle_path + if not stored_bundle_path.is_absolute() + else Path(stored_bundle_path.name) + ) + export_bundle_path = Path("share_link_bundles") / portable_bundle_path + + bundle_dict["fields"]["file_path"] = portable_bundle_path.as_posix() + bundle_dict[EXPORTER_SHARE_LINK_BUNDLE_NAME] = export_bundle_path.as_posix() + + return (self.target / export_bundle_path).resolve() + + def copy_share_link_bundle_file( + self, + bundle: ShareLinkBundle, + bundle_target: Path, + ) -> None: + """ + Copies a share link bundle ZIP into the export directory. + """ + bundle_source_path = bundle.absolute_file_path + if bundle_source_path is None: + raise FileNotFoundError(f"Share link bundle {bundle.pk} has no file path") + + self.check_and_copy( + bundle_source_path, + None, + bundle_target, + ) + def _encrypt_record_inline(self, record: dict) -> None: """Encrypt sensitive fields in a single record, if passphrase is set.""" if not self.passphrase: diff --git a/src/documents/management/commands/document_importer.py b/src/documents/management/commands/document_importer.py index 4572b4617..becdf7b76 100644 --- a/src/documents/management/commands/document_importer.py +++ b/src/documents/management/commands/document_importer.py @@ -32,10 +32,12 @@ from documents.models import CustomFieldInstance from documents.models import Document from documents.models import DocumentType from documents.models import Note +from documents.models import ShareLinkBundle from documents.models import Tag from documents.settings import EXPORTER_ARCHIVE_NAME from documents.settings import EXPORTER_CRYPTO_SETTINGS_NAME from documents.settings import EXPORTER_FILE_NAME +from documents.settings import EXPORTER_SHARE_LINK_BUNDLE_NAME from documents.settings import EXPORTER_THUMBNAIL_NAME from documents.signals.handlers import check_paths_and_prune_custom_fields from documents.signals.handlers import update_filename_and_move_files @@ -348,18 +350,42 @@ class Command(CryptMixin, PaperlessCommand): f"Failed to read from archive file {doc_archive_path}", ) from e + def check_share_link_bundle_validity(bundle_record: dict) -> None: + if EXPORTER_SHARE_LINK_BUNDLE_NAME not in bundle_record: + return + + bundle_file = bundle_record[EXPORTER_SHARE_LINK_BUNDLE_NAME] + bundle_path: Path = self.source / bundle_file + if not bundle_path.exists(): + raise CommandError( + f'The manifest file refers to "{bundle_file}" which does not ' + "appear to be in the source directory.", + ) + try: + with bundle_path.open(mode="rb"): + pass + except Exception as e: + raise CommandError( + f"Failed to read from share link bundle file {bundle_path}", + ) from e + self.stdout.write("Checking the manifest") for manifest_path in self.manifest_paths: for record in iter_manifest_records(manifest_path): # Only check if the document files exist if this is not data only # We don't care about documents for a data only import - if not self.data_only and record["model"] == "documents.document": + if self.data_only: + continue + if record["model"] == "documents.document": check_document_validity(record) + elif record["model"] == "documents.sharelinkbundle": + check_share_link_bundle_validity(record) def _import_files_from_manifest(self) -> None: settings.ORIGINALS_DIR.mkdir(parents=True, exist_ok=True) settings.THUMBNAIL_DIR.mkdir(parents=True, exist_ok=True) settings.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True) + settings.SHARE_LINK_BUNDLE_DIR.mkdir(parents=True, exist_ok=True) self.stdout.write("Copy files into paperless...") @@ -374,6 +400,18 @@ class Command(CryptMixin, PaperlessCommand): for record in iter_manifest_records(manifest_path) if record["model"] == "documents.document" ] + share_link_bundle_records = [ + { + "pk": record["pk"], + EXPORTER_SHARE_LINK_BUNDLE_NAME: record.get( + EXPORTER_SHARE_LINK_BUNDLE_NAME, + ), + } + for manifest_path in self.manifest_paths + for record in iter_manifest_records(manifest_path) + if record["model"] == "documents.sharelinkbundle" + and record.get(EXPORTER_SHARE_LINK_BUNDLE_NAME) + ] for record in self.track(document_records, description="Copying files..."): document = Document.global_objects.get(pk=record["pk"]) @@ -416,6 +454,26 @@ class Command(CryptMixin, PaperlessCommand): document.save() + for record in self.track( + share_link_bundle_records, + description="Copying share link bundles...", + ): + bundle = ShareLinkBundle.objects.get(pk=record["pk"]) + bundle_file = record[EXPORTER_SHARE_LINK_BUNDLE_NAME] + bundle_source_path = (self.source / bundle_file).resolve() + bundle_target_path = bundle.absolute_file_path + if bundle_target_path is None: + raise CommandError( + f"Share link bundle {bundle.pk} does not have a valid file path.", + ) + + with FileLock(settings.MEDIA_LOCK): + bundle_target_path.parent.mkdir(parents=True, exist_ok=True) + copy_file_with_basic_stats( + bundle_source_path, + bundle_target_path, + ) + def _decrypt_record_if_needed(self, record: dict) -> dict: fields = self.CRYPT_FIELDS_BY_MODEL.get(record.get("model", "")) if fields: diff --git a/src/documents/settings.py b/src/documents/settings.py index 9dff44c95..c4c87b8a7 100644 --- a/src/documents/settings.py +++ b/src/documents/settings.py @@ -3,6 +3,7 @@ EXPORTER_FILE_NAME = "__exported_file_name__" EXPORTER_THUMBNAIL_NAME = "__exported_thumbnail_name__" EXPORTER_ARCHIVE_NAME = "__exported_archive_name__" +EXPORTER_SHARE_LINK_BUNDLE_NAME = "__exported_share_link_bundle_name__" EXPORTER_CRYPTO_SETTINGS_NAME = "__crypto__" EXPORTER_CRYPTO_SALT_NAME = "__salt_hex__" diff --git a/src/documents/tests/test_management_exporter.py b/src/documents/tests/test_management_exporter.py index a214ef51d..4ee7677ca 100644 --- a/src/documents/tests/test_management_exporter.py +++ b/src/documents/tests/test_management_exporter.py @@ -2,6 +2,7 @@ import hashlib import json import shutil import tempfile +from datetime import timedelta from io import StringIO from pathlib import Path from unittest import mock @@ -11,6 +12,7 @@ import pytest from allauth.socialaccount.models import SocialAccount from allauth.socialaccount.models import SocialApp from allauth.socialaccount.models import SocialToken +from django.conf import settings from django.contrib.auth.models import Group from django.contrib.auth.models import Permission from django.contrib.contenttypes.models import ContentType @@ -31,6 +33,8 @@ from documents.models import CustomFieldInstance from documents.models import Document from documents.models import DocumentType from documents.models import Note +from documents.models import ShareLink +from documents.models import ShareLinkBundle from documents.models import StoragePath from documents.models import Tag from documents.models import User @@ -39,6 +43,7 @@ from documents.models import WorkflowAction from documents.models import WorkflowTrigger from documents.sanity_checker import check_sanity from documents.settings import EXPORTER_FILE_NAME +from documents.settings import EXPORTER_SHARE_LINK_BUNDLE_NAME from documents.tests.utils import DirectoriesMixin from documents.tests.utils import FileSystemAssertsMixin from documents.tests.utils import SampleDirMixin @@ -306,6 +311,108 @@ class TestExportImport( ): self.test_exporter(use_filename_format=True) + def test_exporter_includes_share_links_and_bundles(self) -> None: + shutil.rmtree(Path(self.dirs.media_dir) / "documents") + shutil.copytree( + Path(__file__).parent / "samples" / "documents", + Path(self.dirs.media_dir) / "documents", + ) + + share_link = ShareLink.objects.create( + slug="share-link-slug", + document=self.d1, + owner=self.user, + file_version=ShareLink.FileVersion.ORIGINAL, + expiration=timezone.now() + timedelta(days=7), + ) + + bundle_relative_path = Path("nested") / "share-bundle.zip" + bundle_source_path = settings.SHARE_LINK_BUNDLE_DIR / bundle_relative_path + bundle_source_path.parent.mkdir(parents=True, exist_ok=True) + bundle_source_path.write_bytes(b"share-bundle-contents") + bundle = ShareLinkBundle.objects.create( + slug="share-bundle-slug", + owner=self.user, + file_version=ShareLink.FileVersion.ARCHIVE, + expiration=timezone.now() + timedelta(days=7), + status=ShareLinkBundle.Status.READY, + size_bytes=bundle_source_path.stat().st_size, + file_path=str(bundle_relative_path), + built_at=timezone.now(), + ) + bundle.documents.set([self.d1, self.d2]) + + manifest = self._do_export() + + share_link_records = [ + record for record in manifest if record["model"] == "documents.sharelink" + ] + self.assertEqual(len(share_link_records), 1) + self.assertEqual(share_link_records[0]["pk"], share_link.pk) + self.assertEqual(share_link_records[0]["fields"]["document"], self.d1.pk) + self.assertEqual(share_link_records[0]["fields"]["owner"], self.user.pk) + + share_link_bundle_records = [ + record + for record in manifest + if record["model"] == "documents.sharelinkbundle" + ] + self.assertEqual(len(share_link_bundle_records), 1) + bundle_record = share_link_bundle_records[0] + self.assertEqual(bundle_record["pk"], bundle.pk) + self.assertEqual( + bundle_record["fields"]["documents"], + [self.d1.pk, self.d2.pk], + ) + self.assertEqual( + bundle_record[EXPORTER_SHARE_LINK_BUNDLE_NAME], + "share_link_bundles/nested/share-bundle.zip", + ) + self.assertEqual( + bundle_record["fields"]["file_path"], + "nested/share-bundle.zip", + ) + self.assertIsFile(self.target / bundle_record[EXPORTER_SHARE_LINK_BUNDLE_NAME]) + + with paperless_environment(): + ShareLink.objects.all().delete() + ShareLinkBundle.objects.all().delete() + shutil.rmtree(settings.SHARE_LINK_BUNDLE_DIR, ignore_errors=True) + + call_command( + "document_importer", + "--no-progress-bar", + self.target, + skip_checks=True, + ) + + imported_share_link = ShareLink.objects.get(pk=share_link.pk) + self.assertEqual(imported_share_link.document_id, self.d1.pk) + self.assertEqual(imported_share_link.owner_id, self.user.pk) + self.assertEqual( + imported_share_link.file_version, + ShareLink.FileVersion.ORIGINAL, + ) + + imported_bundle = ShareLinkBundle.objects.get(pk=bundle.pk) + imported_bundle_path = imported_bundle.absolute_file_path + self.assertEqual(imported_bundle.owner_id, self.user.pk) + self.assertEqual( + list( + imported_bundle.documents.order_by("pk").values_list( + "pk", + flat=True, + ), + ), + [self.d1.pk, self.d2.pk], + ) + self.assertEqual(imported_bundle.file_path, "nested/share-bundle.zip") + self.assertIsNotNone(imported_bundle_path) + self.assertEqual( + imported_bundle_path.read_bytes(), + b"share-bundle-contents", + ) + def test_update_export_changed_time(self) -> None: shutil.rmtree(Path(self.dirs.media_dir) / "documents") shutil.copytree(