diff --git a/src/documents/barcodes.py b/src/documents/barcodes.py index 65cba9a71..976861a64 100644 --- a/src/documents/barcodes.py +++ b/src/documents/barcodes.py @@ -10,9 +10,6 @@ from typing import TYPE_CHECKING import regex as regex_mod from django.conf import settings from pdf2image import convert_from_path -from pikepdf import Page -from pikepdf import PasswordError -from pikepdf import Pdf from documents.converters import convert_from_tiff_to_pdf from documents.data_models import ConsumableDocument @@ -34,6 +31,7 @@ from documents.utils import maybe_override_pixel_limit from paperless.config import BarcodeConfig if TYPE_CHECKING: + from pikepdf import Page from PIL import Image logger = logging.getLogger("paperless.barcodes") @@ -282,6 +280,8 @@ class BarcodePlugin(ConsumeTaskPlugin): # No op if not a TIFF self.convert_from_tiff_to_pdf() + from pikepdf import PasswordError + try: self.barcodes = scan_pdf( self.pdf_file, @@ -433,6 +433,8 @@ class BarcodePlugin(ConsumeTaskPlugin): document_paths = [] fname: str = self.input_doc.original_file.stem + from pikepdf import Pdf + with Pdf.open(self.pdf_file) as input_pdf: # Start with an empty document current_document: list[Page] = [] @@ -516,6 +518,8 @@ def scan_pdf(pdf_path: Path, settings: BarcodeConfig, work_dir: Path) -> list[Ba """ barcodes: list[Barcode] = [] + from pikepdf import Pdf + with Pdf.open(pdf_path) as pdf: num_of_pages = len(pdf.pages) logger.debug(f"PDF has {num_of_pages} pages") diff --git a/src/documents/converters.py b/src/documents/converters.py index 76f518d7e..bc66925d3 100644 --- a/src/documents/converters.py +++ b/src/documents/converters.py @@ -1,6 +1,5 @@ from pathlib import Path -import img2pdf from PIL import Image from documents.utils import copy_basic_file_stats @@ -17,6 +16,8 @@ def convert_from_tiff_to_pdf(tiff_path: Path, target_directory: Path) -> Path: Returns the path of the PDF created. """ + import img2pdf + # override pixel setting if needed maybe_override_pixel_limit() diff --git a/src/documents/double_sided.py b/src/documents/double_sided.py index 3c3ec4723..9452a4469 100644 --- a/src/documents/double_sided.py +++ b/src/documents/double_sided.py @@ -6,7 +6,6 @@ from pathlib import Path from typing import Final from django.conf import settings -from pikepdf import Pdf from documents.consumer import ConsumerError from documents.converters import convert_from_tiff_to_pdf @@ -85,6 +84,8 @@ class CollatePlugin(NoCleanupPluginMixin, NoSetupPluginMixin, ConsumeTaskPlugin) valid_staging_exists = True if valid_staging_exists: + from pikepdf import Pdf + try: # Collate pages from second PDF in reverse order with Pdf.open(staging) as pdf1, Pdf.open(pdf_file) as pdf2: diff --git a/src/documents/filters.py b/src/documents/filters.py index 00a47552d..2c6cfb269 100644 --- a/src/documents/filters.py +++ b/src/documents/filters.py @@ -11,7 +11,6 @@ from decimal import InvalidOperation from typing import TYPE_CHECKING from typing import Any -import dateparser from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.db.models import Case @@ -350,6 +349,8 @@ class DateQueryField(serializers.DateField): if not isinstance(value, str): raise + import dateparser + try: parsed = dateparser.parse( value, diff --git a/src/documents/pdf_ops.py b/src/documents/pdf_ops.py index 0f5b696f6..d31a6b31f 100644 --- a/src/documents/pdf_ops.py +++ b/src/documents/pdf_ops.py @@ -14,8 +14,6 @@ from __future__ import annotations from typing import TYPE_CHECKING from typing import NamedTuple -import pikepdf - if TYPE_CHECKING: from collections.abc import Callable from collections.abc import Iterable @@ -43,6 +41,8 @@ def rotate_pdf(src: Path, dst: Path, degrees: int) -> None: Rotate every page relatively on the opened document, not a rebuild, so Info, XMP and outlines are kept. ``src`` is not modified. """ + import pikepdf + with pikepdf.open(src) as pdf: for page in pdf.pages: page.rotate(degrees, relative=True) @@ -58,6 +58,8 @@ def remove_pages(src: Path, dst: Path, pages: Iterable[int]) -> None: Duplicates are ignored. Pages are removed highest first so earlier removals never shift the index of later ones. """ + import pikepdf + unique = sorted(set(pages)) _require_positive(unique) with pikepdf.open(src) as pdf: @@ -81,6 +83,8 @@ def build_pdfs( (Info, XMP, outlines) is not carried over. Returns the written paths in output order. """ + import pikepdf + for specs, _ in outputs: _require_positive(spec.page for spec in specs) @@ -140,6 +144,8 @@ def needs_decrypt(src: Path) -> bool: True if ``src`` is encrypted. A PDF that needs a password to open at all counts as encrypted. """ + import pikepdf + try: with pikepdf.open(src) as pdf: return bool(pdf.is_encrypted) @@ -154,6 +160,8 @@ def decrypt_pdf(src: Path, make_dst: Callable[[], Path], password: str) -> Path: ``make_dst`` is only called once the password has been accepted, so a wrong password never leaves a destination behind. """ + import pikepdf + with pikepdf.open(src, password=password) as pdf: pdf.remove_unreferenced_resources() dst = make_dst() @@ -170,6 +178,8 @@ class PdfMerger: """ def __init__(self) -> None: + import pikepdf + self._pdf = pikepdf.new() self._version: str = self._pdf.pdf_version @@ -185,6 +195,8 @@ class PdfMerger: self._pdf.close() def add(self, path: Path) -> None: + import pikepdf + with pikepdf.open(str(path)) as pdf: self._version = max(self._version, pdf.pdf_version) self._pdf.pages.extend(pdf.pages) diff --git a/src/documents/plugins/date_parsing/base.py b/src/documents/plugins/date_parsing/base.py index fd45ceec8..af393b577 100644 --- a/src/documents/plugins/date_parsing/base.py +++ b/src/documents/plugins/date_parsing/base.py @@ -7,8 +7,6 @@ from dataclasses import dataclass from types import TracebackType from typing import Self -import dateparser - logger = logging.getLogger(__name__) @@ -80,6 +78,8 @@ class DateParserPluginBase(ABC): Uses configuration from `self.config`. """ + import dateparser + try: return dateparser.parse( date_string, diff --git a/src/documents/tasks.py b/src/documents/tasks.py index 778ee9673..8f0e85f16 100644 --- a/src/documents/tasks.py +++ b/src/documents/tasks.py @@ -76,9 +76,6 @@ from paperless.logging import consume_task_id from paperless.parsers import ParserContext from paperless.parsers.registry import get_parser_registry from paperless_ai.exceptions import LLMTimeoutError -from paperless_ai.indexing import llm_index_add_or_update_document -from paperless_ai.indexing import llm_index_remove_document -from paperless_ai.indexing import update_llm_index if settings.AUDIT_LOG_ENABLED: from auditlog.models import LogEntry @@ -339,6 +336,8 @@ def bulk_update_documents(document_ids) -> None: ai_config = AIConfig() if ai_config.llm_index_enabled: + from paperless_ai.indexing import update_llm_index + update_llm_index( rebuild=False, document_ids=document_ids, @@ -499,6 +498,8 @@ def update_document_content_maybe_archive_file( ai_config = AIConfig() if ai_config.llm_index_enabled: + from paperless_ai.indexing import llm_index_add_or_update_document + llm_index_add_or_update_document(document) clear_document_caches(document.pk) @@ -800,11 +801,15 @@ def apply_ai_suggestions(self, action_id: int, document_id: int) -> None: @shared_task def update_document_in_llm_index(document) -> None: + from paperless_ai.indexing import llm_index_add_or_update_document + llm_index_add_or_update_document(document) @shared_task def remove_document_from_llm_index(document) -> None: + from paperless_ai.indexing import llm_index_remove_document + llm_index_remove_document(document) diff --git a/src/documents/tests/date_parsing/test_date_parsing.py b/src/documents/tests/date_parsing/test_date_parsing.py index f43303d22..c7de0c370 100644 --- a/src/documents/tests/date_parsing/test_date_parsing.py +++ b/src/documents/tests/date_parsing/test_date_parsing.py @@ -71,7 +71,7 @@ class TestParseString: ): # We still need to mock dateparser.parse to force the exception mocker.patch( - "documents.plugins.date_parsing.base.dateparser.parse", + "dateparser.parse", side_effect=ValueError( "Parsing error: 01/01/2024", ), @@ -260,7 +260,7 @@ class TestRegexDateParser: parser = RegexDateParserPlugin(base_config) # Patch the dateparser.parse - target = "documents.plugins.date_parsing.base.dateparser.parse" + target = "dateparser.parse" def fake_parse( date_string: str, @@ -335,7 +335,7 @@ class TestRegexDateParser: ) parser = RegexDateParserPlugin(cfg) - target = "documents.plugins.date_parsing.base.dateparser.parse" + target = "dateparser.parse" def fake_parse( date_string: str, @@ -400,7 +400,7 @@ class TestRegexDateParser: parser = RegexDateParserPlugin(cfg) # Patch the module's dateparser.parse so we can inspect calls - target = "documents.plugins.date_parsing.base.dateparser.parse" + target = "dateparser.parse" def fake_parse( date_string: str, diff --git a/src/documents/tests/test_api_app_config.py b/src/documents/tests/test_api_app_config.py index f617e646a..b06c6f67d 100644 --- a/src/documents/tests/test_api_app_config.py +++ b/src/documents/tests/test_api_app_config.py @@ -1070,7 +1070,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase): with ( patch("documents.tasks.llmindex_index.apply_async") as mock_update, - patch("paperless.views.llm_index_exists") as mock_exists, + patch("paperless_ai.indexing.llm_index_exists") as mock_exists, ): mock_exists.return_value = False self.client.patch( @@ -1095,7 +1095,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase): with ( patch("documents.tasks.llmindex_index.apply_async") as mock_update, - patch("paperless.views.llm_index_exists", return_value=False), + patch("paperless_ai.indexing.llm_index_exists", return_value=False), ): self.client.patch( f"{self.ENDPOINT}1/", @@ -1115,7 +1115,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase): with ( patch("documents.tasks.llmindex_index.apply_async") as mock_update, - patch("paperless.views.llm_index_exists") as mock_exists, + patch("paperless_ai.indexing.llm_index_exists") as mock_exists, ): mock_exists.return_value = True self.client.patch( @@ -1136,7 +1136,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase): with ( patch("documents.tasks.llmindex_index.apply_async") as mock_update, - patch("paperless.views.llm_index_exists") as mock_exists, + patch("paperless_ai.indexing.llm_index_exists") as mock_exists, ): mock_exists.return_value = True self.client.patch( @@ -1174,7 +1174,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase): with ( patch("documents.tasks.llmindex_index.apply_async") as mock_update, - patch("paperless.views.llm_index_exists") as mock_exists, + patch("paperless_ai.indexing.llm_index_exists") as mock_exists, ): mock_exists.return_value = True self.client.patch( diff --git a/src/documents/tests/test_api_chat.py b/src/documents/tests/test_api_chat.py index f19c4aa43..11703dfd9 100644 --- a/src/documents/tests/test_api_chat.py +++ b/src/documents/tests/test_api_chat.py @@ -61,7 +61,7 @@ class TestChatStreamingViewInputValidation(APITestCase): return_value=self._mock_ai_enabled(), ), mock.patch( - "documents.views.stream_chat_with_documents", + "paperless_ai.chat.stream_chat_with_documents", return_value=iter(chunks), ), ): @@ -104,7 +104,7 @@ class TestChatStreamingViewUnrestrictedFlag: """ mocker.patch("documents.views.AIConfig").return_value.ai_enabled = True return mocker.patch( - "documents.views.stream_chat_with_documents", + "paperless_ai.chat.stream_chat_with_documents", return_value=iter(()), ) diff --git a/src/documents/tests/test_api_document_versions.py b/src/documents/tests/test_api_document_versions.py index b1eb3c737..ba99a7b39 100644 --- a/src/documents/tests/test_api_document_versions.py +++ b/src/documents/tests/test_api_document_versions.py @@ -1110,7 +1110,7 @@ class TestVersionActionPermissions(DirectoriesMixin, APITestCase): self.assertEqual(response.status_code, 403) with ( mock.patch("documents.views.AIConfig") as ai_config, - mock.patch("documents.views.stream_chat_with_documents") as chat, + mock.patch("paperless_ai.chat.stream_chat_with_documents") as chat, ): ai_config.return_value.ai_enabled = True response = self.client.post( diff --git a/src/documents/tests/test_api_documents.py b/src/documents/tests/test_api_documents.py index eeb1ab59a..9a502b2cb 100644 --- a/src/documents/tests/test_api_documents.py +++ b/src/documents/tests/test_api_documents.py @@ -2686,7 +2686,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase): response = self.client.get("/api/documents/34676/suggestions/") self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) - @mock.patch("documents.views.get_ai_document_classification") + @mock.patch("paperless_ai.ai_classifier.get_ai_document_classification") @override_settings(AI_ENABLED=True) def test_suggestions_still_uses_classifier_when_ai_enabled( self, diff --git a/src/documents/tests/test_permission_filtering_security.py b/src/documents/tests/test_permission_filtering_security.py index 3c2ff3bff..8b1b0169a 100644 --- a/src/documents/tests/test_permission_filtering_security.py +++ b/src/documents/tests/test_permission_filtering_security.py @@ -572,7 +572,7 @@ class TestAiChatAllDocumentsPermissionBoundary: ENDPOINT = "/api/documents/chat/" @override_settings(AI_ENABLED=True) - @patch("documents.views.stream_chat_with_documents") + @patch("paperless_ai.chat.stream_chat_with_documents") def test_chat_all_documents_excludes_unshared_document(self, mock_stream_chat): mock_stream_chat.return_value = iter([b"data"]) diff --git a/src/documents/tests/test_tasks.py b/src/documents/tests/test_tasks.py index a5def0944..2e2b541a9 100644 --- a/src/documents/tests/test_tasks.py +++ b/src/documents/tests/test_tasks.py @@ -322,7 +322,7 @@ class TestUpdateContent(DirectoriesMixin, TestCase): ) @override_settings(AI_ENABLED=True, LLM_EMBEDDING_BACKEND="huggingface") - @mock.patch("documents.tasks.llm_index_add_or_update_document") + @mock.patch("paperless_ai.indexing.llm_index_add_or_update_document") @mock.patch("documents.search.get_backend") def test_update_content_version_updates_llm_index( self, @@ -450,7 +450,7 @@ class TestAIIndex(DirectoriesMixin, TestCase): checksum="wow", ) with mock.patch( - "documents.tasks.llm_index_add_or_update_document", + "paperless_ai.indexing.llm_index_add_or_update_document", ) as llm_index_add_or_update_document: tasks.update_document_in_llm_index(doc) llm_index_add_or_update_document.assert_called_once_with(doc) @@ -470,7 +470,7 @@ class TestAIIndex(DirectoriesMixin, TestCase): checksum="wow", ) with mock.patch( - "documents.tasks.llm_index_remove_document", + "paperless_ai.indexing.llm_index_remove_document", ) as llm_index_remove_document: tasks.remove_document_from_llm_index(doc) llm_index_remove_document.assert_called_once_with(doc) @@ -495,7 +495,7 @@ class TestAIIndex(DirectoriesMixin, TestCase): "documents.tasks.update_document_in_llm_index", ) as update_document_in_llm_index, mock.patch( - "documents.tasks.update_llm_index", + "paperless_ai.indexing.update_llm_index", ) as update_llm_index, ): doc_ids = [doc.pk for doc in docs] diff --git a/src/documents/tests/test_views.py b/src/documents/tests/test_views.py index d91ced30a..d6672f5b1 100644 --- a/src/documents/tests/test_views.py +++ b/src/documents/tests/test_views.py @@ -474,7 +474,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase): self.assertEqual(response.json()["tags"], []) self.assertEqual(response.json()["suggested_tags"], []) - @patch("documents.views.get_ai_document_classification") + @patch("paperless_ai.ai_classifier.get_ai_document_classification") @override_settings( AI_ENABLED=True, LLM_BACKEND="mock_backend", @@ -524,7 +524,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase): None, ) - @patch("documents.views.get_ai_document_classification") + @patch("paperless_ai.ai_classifier.get_ai_document_classification") @override_settings( AI_ENABLED=True, LLM_BACKEND="mock_backend", @@ -562,7 +562,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase): "KI Title", ) - @patch("documents.views.get_ai_document_classification") + @patch("paperless_ai.ai_classifier.get_ai_document_classification") @override_settings( AI_ENABLED=True, LLM_BACKEND="mock_backend", @@ -601,7 +601,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase): "Titre IA", ) - @patch("documents.views.get_ai_document_classification") + @patch("paperless_ai.ai_classifier.get_ai_document_classification") @override_settings( AI_ENABLED=True, LLM_BACKEND="mock_backend", @@ -637,7 +637,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase): ), ) - @patch("documents.views.get_ai_document_classification") + @patch("paperless_ai.ai_classifier.get_ai_document_classification") @override_settings( AI_ENABLED=True, LLM_BACKEND="mock_backend", @@ -709,7 +709,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase): ), ) - @patch("documents.views.get_ai_document_classification") + @patch("paperless_ai.ai_classifier.get_ai_document_classification") @override_settings( AI_ENABLED=True, LLM_BACKEND="openai-like", @@ -738,7 +738,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase): get_llm_suggestion_cache(self.document.pk, backend="openai-like"), ) - @patch("documents.views.get_ai_document_classification") + @patch("paperless_ai.ai_classifier.get_ai_document_classification") @override_settings( AI_ENABLED=True, LLM_BACKEND="openai-like", @@ -765,7 +765,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase): get_llm_suggestion_cache(self.document.pk, backend="openai-like"), ) - @patch("documents.views.get_ai_document_classification") + @patch("paperless_ai.ai_classifier.get_ai_document_classification") @override_settings( AI_ENABLED=True, LLM_BACKEND="openai-like", @@ -797,7 +797,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase): get_llm_suggestion_cache(self.document.pk, backend="openai-like"), ) - @patch("documents.views.get_ai_document_classification") + @patch("paperless_ai.ai_classifier.get_ai_document_classification") @override_settings( AI_ENABLED=True, LLM_BACKEND="mock_backend", @@ -835,7 +835,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase): self.assertEqual(response.json()["tags"], [self.tag1.pk]) self.assertEqual(response.json()["suggested_tags"], ["Follow-up"]) - @patch("documents.views.get_ai_document_classification") + @patch("paperless_ai.ai_classifier.get_ai_document_classification") @override_settings( AI_ENABLED=True, LLM_BACKEND="mock_backend", @@ -874,7 +874,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase): self.assertEqual(response.json()["tags"], [self.tag1.pk]) self.assertEqual(response.json()["suggested_tags"], []) - @patch("documents.views.get_ai_document_classification") + @patch("paperless_ai.ai_classifier.get_ai_document_classification") @override_settings( AI_ENABLED=True, LLM_BACKEND="mock_backend", @@ -996,7 +996,7 @@ class TestAIChatStreamingView(DirectoriesMixin, TestCase): self.assertEqual(response.status_code, 400) self.assertIn(b"AI is required for this feature", response.content) - @patch("documents.views.stream_chat_with_documents") + @patch("paperless_ai.chat.stream_chat_with_documents") @patch("documents.views.permitted_document_ids") @override_settings(AI_ENABLED=True) def test_post_no_document_id(self, mock_permitted_ids, mock_stream_chat) -> None: @@ -1016,7 +1016,7 @@ class TestAIChatStreamingView(DirectoriesMixin, TestCase): self.assertEqual(list(call_kwargs["documents"]), [self.document]) self.assertIsNone(call_kwargs["output_language"]) - @patch("documents.views.stream_chat_with_documents") + @patch("paperless_ai.chat.stream_chat_with_documents") @patch("documents.views.permitted_document_ids") @override_settings(AI_ENABLED=True) def test_post_uses_user_display_language( @@ -1042,7 +1042,7 @@ class TestAIChatStreamingView(DirectoriesMixin, TestCase): self.assertEqual(list(call_kwargs["documents"]), [self.document]) self.assertEqual(call_kwargs["output_language"], "de-de") - @patch("documents.views.stream_chat_with_documents") + @patch("paperless_ai.chat.stream_chat_with_documents") @override_settings(AI_ENABLED=True) def test_post_with_document_id(self, mock_stream_chat) -> None: self.grant_view_document_permission() @@ -1079,7 +1079,7 @@ class TestAIChatStreamingView(DirectoriesMixin, TestCase): self.assertEqual(response.status_code, 403) self.assertIn(b"Insufficient permissions", response.content) - @patch("documents.views.stream_chat_with_documents") + @patch("paperless_ai.chat.stream_chat_with_documents") @override_settings(AI_ENABLED=True) def test_post_no_document_id_requires_view_document_permission( self, @@ -1093,7 +1093,7 @@ class TestAIChatStreamingView(DirectoriesMixin, TestCase): self.assertEqual(response.status_code, 403) mock_stream_chat.assert_not_called() - @patch("documents.views.stream_chat_with_documents") + @patch("paperless_ai.chat.stream_chat_with_documents") @override_settings(AI_ENABLED=True) def test_post_with_document_id_requires_view_document_permission( self, diff --git a/src/documents/views.py b/src/documents/views.py index 5e17c67cb..5a4a78296 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -250,20 +250,8 @@ from paperless.parsers.remote import RemoteEngineConfig from paperless.serialisers import GroupSerializer from paperless.serialisers import UserSerializer from paperless.views import StandardPagination -from paperless_ai.ai_classifier import get_ai_document_classification -from paperless_ai.ai_classifier import get_llm_output_language -from paperless_ai.chat import stream_chat_with_documents from paperless_ai.exceptions import LLMProviderError from paperless_ai.exceptions import LLMTimeoutError -from paperless_ai.matching import extract_unmatched_names -from paperless_ai.matching import match_correspondents_by_name -from paperless_ai.matching import match_document_types_by_name -from paperless_ai.matching import match_storage_paths_by_name -from paperless_ai.matching import match_tags_by_name -from paperless_ai.matching import resolve_correspondent_ids -from paperless_ai.matching import resolve_document_type_ids -from paperless_ai.matching import resolve_storage_path_ids -from paperless_ai.matching import resolve_tag_ids from paperless_mail.models import MailAccount from paperless_mail.models import MailRule from paperless_mail.oauth import PaperlessMailOAuth2Manager @@ -1613,6 +1601,18 @@ class DocumentViewSet( if not ai_config.ai_enabled: return HttpResponseBadRequest("AI is required for this feature") + from paperless_ai.ai_classifier import get_ai_document_classification + from paperless_ai.ai_classifier import get_llm_output_language + from paperless_ai.matching import extract_unmatched_names + from paperless_ai.matching import match_correspondents_by_name + from paperless_ai.matching import match_document_types_by_name + from paperless_ai.matching import match_storage_paths_by_name + from paperless_ai.matching import match_tags_by_name + from paperless_ai.matching import resolve_correspondent_ids + from paperless_ai.matching import resolve_document_type_ids + from paperless_ai.matching import resolve_storage_path_ids + from paperless_ai.matching import resolve_tag_ids + output_language = get_llm_output_language( ai_config=ai_config, user=request.user, @@ -2446,6 +2446,9 @@ class ChatStreamingView(GenericAPIView[Any]): ) unrestricted = user_is_unrestricted(request.user) + from paperless_ai.ai_classifier import get_llm_output_language + from paperless_ai.chat import stream_chat_with_documents + output_language = get_llm_output_language( ai_config=ai_config, user=request.user, diff --git a/src/paperless/asgi.py b/src/paperless/asgi.py index 3d74a1edb..611b8a4a5 100644 --- a/src/paperless/asgi.py +++ b/src/paperless/asgi.py @@ -17,7 +17,7 @@ from paperless.urls import websocket_urlpatterns # noqa: E402 application = ProtocolTypeRouter( { - "http": get_asgi_application(), + "http": django_asgi_app, "websocket": AuthMiddlewareStack(URLRouter(websocket_urlpatterns)), }, ) diff --git a/src/paperless/parsers/mail.py b/src/paperless/parsers/mail.py index 198a2b544..7fac9c669 100644 --- a/src/paperless/parsers/mail.py +++ b/src/paperless/parsers/mail.py @@ -36,9 +36,6 @@ from humanize import naturalsize from imap_tools import MailAttachment from imap_tools import MailMessage from tika_client import TikaClient -from turbohtml.clean import Linkify -from turbohtml.clean import linkify -from turbohtml.migration.bleach import clean from documents.parsers import ParseError from documents.parsers import make_thumbnail_from_pdf @@ -622,6 +619,10 @@ class MailDocumentParser: def clean_html(text: str) -> str: """Attempt to clean, escape, and linkify the given HTML string.""" + from turbohtml.clean import Linkify + from turbohtml.clean import linkify + from turbohtml.migration.bleach import clean + if isinstance(text, list): text = "\n".join([str(e) for e in text]) if not isinstance(text, str): diff --git a/src/paperless/parsers/tesseract.py b/src/paperless/parsers/tesseract.py index 5a3b77f88..e8ff0f3b5 100644 --- a/src/paperless/parsers/tesseract.py +++ b/src/paperless/parsers/tesseract.py @@ -1,5 +1,6 @@ from __future__ import annotations +import functools import importlib.resources import logging import os @@ -39,9 +40,12 @@ if TYPE_CHECKING: logger = logging.getLogger("paperless.parsing.tesseract") -_SRGB_ICC_DATA: Final[bytes] = ( - importlib.resources.files("ocrmypdf.data").joinpath("sRGB.icc").read_bytes() -) + +@functools.cache +def _srgb_icc_data() -> bytes: + # Read on first use, importing ocrmypdf.data pulls in the whole ocrmypdf package + return importlib.resources.files("ocrmypdf.data").joinpath("sRGB.icc").read_bytes() + _SUPPORTED_MIME_TYPES: Final[dict[str, str]] = { "application/pdf": ".pdf", @@ -403,7 +407,7 @@ class RasterisedDocumentParser: pdfa_path = Path(self.tempdir) / "archive.pdf" try: with pikepdf.open(plain_pdf_path) as pdf: - cs = pdf.make_stream(_SRGB_ICC_DATA) + cs = pdf.make_stream(_srgb_icc_data()) cs["/N"] = 3 output_intent = pikepdf.Dictionary( Type=pikepdf.Name("/OutputIntent"), diff --git a/src/paperless/settings/custom.py b/src/paperless/settings/custom.py index 2bf53a8cc..7b5b5a749 100644 --- a/src/paperless/settings/custom.py +++ b/src/paperless/settings/custom.py @@ -6,7 +6,6 @@ from pathlib import Path from typing import Any from celery.schedules import crontab -from dateparser.languages.loader import LocaleDataLoader from paperless.settings.parsers import get_choice_from_env from paperless.settings.parsers import get_int_from_env @@ -352,6 +351,8 @@ def parse_dateparser_languages(languages: str | None) -> list[str]: ) language_list.append("zh") + from dateparser.languages.loader import LocaleDataLoader + return list(LocaleDataLoader().get_locale_map(locales=language_list)) diff --git a/src/paperless/tests/test_utils.py b/src/paperless/tests/test_utils.py index 152913867..8dedc8854 100644 --- a/src/paperless/tests/test_utils.py +++ b/src/paperless/tests/test_utils.py @@ -59,7 +59,10 @@ def test_ocr_to_dateparser_languages_exception( raise RuntimeError("Simulated error") with caplog.at_level(logging.WARNING): - monkeypatch.setattr(utils, "LocaleDataLoader", lambda: DummyLoader()) + monkeypatch.setattr( + "dateparser.languages.loader.LocaleDataLoader", + lambda: DummyLoader(), + ) result = utils.ocr_to_dateparser_languages("eng+fra") assert result == [] assert ( diff --git a/src/paperless/utils.py b/src/paperless/utils.py index 46269588c..9380901bf 100644 --- a/src/paperless/utils.py +++ b/src/paperless/utils.py @@ -1,7 +1,5 @@ import logging -from dateparser.languages.loader import LocaleDataLoader - logger = logging.getLogger("paperless.utils") OCR_TO_DATEPARSER_LANGUAGES = { @@ -125,6 +123,8 @@ def ocr_to_dateparser_languages(ocr_languages: str) -> list[str]: - If a language cannot be mapped or validated, it is skipped with a warning. - Returns a list of valid locales, or an empty list if none could be converted. """ + from dateparser.languages.loader import LocaleDataLoader + loader = LocaleDataLoader() result = [] try: diff --git a/src/paperless/validators.py b/src/paperless/validators.py index 6e3c78f79..fa907baa6 100644 --- a/src/paperless/validators.py +++ b/src/paperless/validators.py @@ -3,7 +3,6 @@ from io import BytesIO from django.conf import settings from django.core.exceptions import ValidationError from django.core.files.uploadedfile import UploadedFile -from lxml import etree from PIL import Image ALLOWED_SVG_TAGS: set[str] = { @@ -186,6 +185,7 @@ def reject_dangerous_svg(file: UploadedFile) -> None: Raises ValidationError if unsafe content is found. See GHSA-6p53-hqqw-8j62 """ + from lxml import etree try: parser = etree.XMLParser(resolve_entities=False) diff --git a/src/paperless/views.py b/src/paperless/views.py index ba1e6738d..62b624294 100644 --- a/src/paperless/views.py +++ b/src/paperless/views.py @@ -49,7 +49,6 @@ from paperless.serialisers import GroupSerializer from paperless.serialisers import PaperlessAuthTokenSerializer from paperless.serialisers import ProfileSerializer from paperless.serialisers import UserSerializer -from paperless_ai.indexing import llm_index_exists class PaperlessObtainAuthTokenView(ObtainAuthToken): @@ -471,6 +470,8 @@ class ApplicationConfigurationViewSet(ModelViewSet[ApplicationConfiguration]): or old_llm_embedding_model != new_llm_embedding_model or old_llm_context_size != new_llm_context_size ) + from paperless_ai.indexing import llm_index_exists + rebuild_needed = new_ai_index_enabled and ( not llm_index_exists() or embedding_config_changed )