mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-11-20 04:26:53 +00:00
Merge branch 'dev' into feature/ocr-templates
This commit is contained in:
408 files changed
+6392
-5987
No files matched your search
@@ -10,9 +10,6 @@ from typing import TYPE_CHECKING
|
||||
import regex as regex_mod
|
||||
from django.conf import settings
|
||||
from pdf2image import convert_from_path
|
||||
from pikepdf import Page
|
||||
from pikepdf import PasswordError
|
||||
from pikepdf import Pdf
|
||||
|
||||
from documents.converters import convert_from_tiff_to_pdf
|
||||
from documents.data_models import ConsumableDocument
|
||||
@@ -34,6 +31,7 @@ from documents.utils import maybe_override_pixel_limit
|
||||
from paperless.config import BarcodeConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pikepdf import Page
|
||||
from PIL import Image
|
||||
|
||||
logger = logging.getLogger("paperless.barcodes")
|
||||
@@ -282,6 +280,8 @@ class BarcodePlugin(ConsumeTaskPlugin):
|
||||
# No op if not a TIFF
|
||||
self.convert_from_tiff_to_pdf()
|
||||
|
||||
from pikepdf import PasswordError
|
||||
|
||||
try:
|
||||
self.barcodes = scan_pdf(
|
||||
self.pdf_file,
|
||||
@@ -433,6 +433,8 @@ class BarcodePlugin(ConsumeTaskPlugin):
|
||||
|
||||
document_paths = []
|
||||
fname: str = self.input_doc.original_file.stem
|
||||
from pikepdf import Pdf
|
||||
|
||||
with Pdf.open(self.pdf_file) as input_pdf:
|
||||
# Start with an empty document
|
||||
current_document: list[Page] = []
|
||||
@@ -516,6 +518,8 @@ def scan_pdf(pdf_path: Path, settings: BarcodeConfig, work_dir: Path) -> list[Ba
|
||||
"""
|
||||
barcodes: list[Barcode] = []
|
||||
|
||||
from pikepdf import Pdf
|
||||
|
||||
with Pdf.open(pdf_path) as pdf:
|
||||
num_of_pages = len(pdf.pages)
|
||||
logger.debug(f"PDF has {num_of_pages} pages")
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
from pathlib import Path
|
||||
|
||||
import img2pdf
|
||||
from PIL import Image
|
||||
|
||||
from documents.utils import copy_basic_file_stats
|
||||
@@ -17,6 +16,8 @@ def convert_from_tiff_to_pdf(tiff_path: Path, target_directory: Path) -> Path:
|
||||
|
||||
Returns the path of the PDF created.
|
||||
"""
|
||||
import img2pdf
|
||||
|
||||
# override pixel setting if needed
|
||||
maybe_override_pixel_limit()
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@ from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
from django.conf import settings
|
||||
from pikepdf import Pdf
|
||||
|
||||
from documents.consumer import ConsumerError
|
||||
from documents.converters import convert_from_tiff_to_pdf
|
||||
@@ -85,6 +84,8 @@ class CollatePlugin(NoCleanupPluginMixin, NoSetupPluginMixin, ConsumeTaskPlugin)
|
||||
valid_staging_exists = True
|
||||
|
||||
if valid_staging_exists:
|
||||
from pikepdf import Pdf
|
||||
|
||||
try:
|
||||
# Collate pages from second PDF in reverse order
|
||||
with Pdf.open(staging) as pdf1, Pdf.open(pdf_file) as pdf2:
|
||||
|
||||
@@ -11,7 +11,6 @@ from decimal import InvalidOperation
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import Any
|
||||
|
||||
import dateparser
|
||||
from django.conf import settings
|
||||
from django.contrib.contenttypes.models import ContentType
|
||||
from django.db.models import Case
|
||||
@@ -350,6 +349,8 @@ class DateQueryField(serializers.DateField):
|
||||
if not isinstance(value, str):
|
||||
raise
|
||||
|
||||
import dateparser
|
||||
|
||||
try:
|
||||
parsed = dateparser.parse(
|
||||
value,
|
||||
|
||||
@@ -14,8 +14,6 @@ from __future__ import annotations
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import NamedTuple
|
||||
|
||||
import pikepdf
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Iterable
|
||||
@@ -43,6 +41,8 @@ def rotate_pdf(src: Path, dst: Path, degrees: int) -> None:
|
||||
Rotate every page relatively on the opened document, not a rebuild, so Info,
|
||||
XMP and outlines are kept. ``src`` is not modified.
|
||||
"""
|
||||
import pikepdf
|
||||
|
||||
with pikepdf.open(src) as pdf:
|
||||
for page in pdf.pages:
|
||||
page.rotate(degrees, relative=True)
|
||||
@@ -58,6 +58,8 @@ def remove_pages(src: Path, dst: Path, pages: Iterable[int]) -> None:
|
||||
Duplicates are ignored. Pages are removed highest first so earlier removals
|
||||
never shift the index of later ones.
|
||||
"""
|
||||
import pikepdf
|
||||
|
||||
unique = sorted(set(pages))
|
||||
_require_positive(unique)
|
||||
with pikepdf.open(src) as pdf:
|
||||
@@ -81,6 +83,8 @@ def build_pdfs(
|
||||
(Info, XMP, outlines) is not carried over. Returns the written paths in output
|
||||
order.
|
||||
"""
|
||||
import pikepdf
|
||||
|
||||
for specs, _ in outputs:
|
||||
_require_positive(spec.page for spec in specs)
|
||||
|
||||
@@ -140,6 +144,8 @@ def needs_decrypt(src: Path) -> bool:
|
||||
True if ``src`` is encrypted. A PDF that needs a password to open at all
|
||||
counts as encrypted.
|
||||
"""
|
||||
import pikepdf
|
||||
|
||||
try:
|
||||
with pikepdf.open(src) as pdf:
|
||||
return bool(pdf.is_encrypted)
|
||||
@@ -154,6 +160,8 @@ def decrypt_pdf(src: Path, make_dst: Callable[[], Path], password: str) -> Path:
|
||||
``make_dst`` is only called once the password has been accepted, so a wrong
|
||||
password never leaves a destination behind.
|
||||
"""
|
||||
import pikepdf
|
||||
|
||||
with pikepdf.open(src, password=password) as pdf:
|
||||
pdf.remove_unreferenced_resources()
|
||||
dst = make_dst()
|
||||
@@ -170,6 +178,8 @@ class PdfMerger:
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
import pikepdf
|
||||
|
||||
self._pdf = pikepdf.new()
|
||||
self._version: str = self._pdf.pdf_version
|
||||
|
||||
@@ -185,6 +195,8 @@ class PdfMerger:
|
||||
self._pdf.close()
|
||||
|
||||
def add(self, path: Path) -> None:
|
||||
import pikepdf
|
||||
|
||||
with pikepdf.open(str(path)) as pdf:
|
||||
self._version = max(self._version, pdf.pdf_version)
|
||||
self._pdf.pages.extend(pdf.pages)
|
||||
|
||||
@@ -7,8 +7,6 @@ from dataclasses import dataclass
|
||||
from types import TracebackType
|
||||
from typing import Self
|
||||
|
||||
import dateparser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -80,6 +78,8 @@ class DateParserPluginBase(ABC):
|
||||
|
||||
Uses configuration from `self.config`.
|
||||
"""
|
||||
import dateparser
|
||||
|
||||
try:
|
||||
return dateparser.parse(
|
||||
date_string,
|
||||
|
||||
@@ -476,13 +476,13 @@ def update_filename_and_move_files(
|
||||
return
|
||||
|
||||
with FileLock(settings.MEDIA_LOCK):
|
||||
try:
|
||||
# If this was waiting for the lock, the filename or archive_filename
|
||||
# of this document may have been updated. This happens if multiple updates
|
||||
# get queued from the UI for the same document
|
||||
# So freshen up the data before doing anything
|
||||
instance.refresh_from_db()
|
||||
# If this was waiting for the lock, the filename or archive_filename
|
||||
# of this document may have been updated. This happens if multiple updates
|
||||
# get queued from the UI for the same document
|
||||
# So freshen up the data before doing anything
|
||||
instance.refresh_from_db()
|
||||
|
||||
try:
|
||||
old_filename = instance.filename
|
||||
old_source_path = instance.source_path
|
||||
move_original = False
|
||||
|
||||
@@ -78,9 +78,6 @@ from paperless.logging import consume_task_id
|
||||
from paperless.parsers import ParserContext
|
||||
from paperless.parsers.registry import get_parser_registry
|
||||
from paperless_ai.exceptions import LLMTimeoutError
|
||||
from paperless_ai.indexing import llm_index_add_or_update_document
|
||||
from paperless_ai.indexing import llm_index_remove_document
|
||||
from paperless_ai.indexing import update_llm_index
|
||||
|
||||
if settings.AUDIT_LOG_ENABLED:
|
||||
from auditlog.models import LogEntry
|
||||
@@ -341,6 +338,8 @@ def bulk_update_documents(document_ids) -> None:
|
||||
|
||||
ai_config = AIConfig()
|
||||
if ai_config.llm_index_enabled:
|
||||
from paperless_ai.indexing import update_llm_index
|
||||
|
||||
update_llm_index(
|
||||
rebuild=False,
|
||||
document_ids=document_ids,
|
||||
@@ -526,6 +525,8 @@ def update_document_content_maybe_archive_file(
|
||||
|
||||
ai_config = AIConfig()
|
||||
if ai_config.llm_index_enabled:
|
||||
from paperless_ai.indexing import llm_index_add_or_update_document
|
||||
|
||||
llm_index_add_or_update_document(document)
|
||||
|
||||
clear_document_caches(document.pk)
|
||||
@@ -827,11 +828,15 @@ def apply_ai_suggestions(self, action_id: int, document_id: int) -> None:
|
||||
|
||||
@shared_task
|
||||
def update_document_in_llm_index(document) -> None:
|
||||
from paperless_ai.indexing import llm_index_add_or_update_document
|
||||
|
||||
llm_index_add_or_update_document(document)
|
||||
|
||||
|
||||
@shared_task
|
||||
def remove_document_from_llm_index(document) -> None:
|
||||
from paperless_ai.indexing import llm_index_remove_document
|
||||
|
||||
llm_index_remove_document(document)
|
||||
|
||||
|
||||
|
||||
@@ -44,7 +44,10 @@ def localize_date(value: date | datetime | str, format: str, locale: str) -> str
|
||||
|
||||
def format_datetime(value: str | datetime, format: str) -> str:
|
||||
if isinstance(value, str):
|
||||
value = parse_date(value)
|
||||
parsed = parse_datetime(value) or parse_date(value)
|
||||
if parsed is None:
|
||||
raise ValueError(f"Invalid date or datetime string: {value!r}")
|
||||
return parsed.strftime(format=format)
|
||||
return value.strftime(format=format)
|
||||
|
||||
|
||||
|
||||
@@ -71,7 +71,7 @@ class TestParseString:
|
||||
):
|
||||
# We still need to mock dateparser.parse to force the exception
|
||||
mocker.patch(
|
||||
"documents.plugins.date_parsing.base.dateparser.parse",
|
||||
"dateparser.parse",
|
||||
side_effect=ValueError(
|
||||
"Parsing error: 01/01/2024",
|
||||
),
|
||||
@@ -260,7 +260,7 @@ class TestRegexDateParser:
|
||||
parser = RegexDateParserPlugin(base_config)
|
||||
|
||||
# Patch the dateparser.parse
|
||||
target = "documents.plugins.date_parsing.base.dateparser.parse"
|
||||
target = "dateparser.parse"
|
||||
|
||||
def fake_parse(
|
||||
date_string: str,
|
||||
@@ -335,7 +335,7 @@ class TestRegexDateParser:
|
||||
)
|
||||
parser = RegexDateParserPlugin(cfg)
|
||||
|
||||
target = "documents.plugins.date_parsing.base.dateparser.parse"
|
||||
target = "dateparser.parse"
|
||||
|
||||
def fake_parse(
|
||||
date_string: str,
|
||||
@@ -400,7 +400,7 @@ class TestRegexDateParser:
|
||||
parser = RegexDateParserPlugin(cfg)
|
||||
|
||||
# Patch the module's dateparser.parse so we can inspect calls
|
||||
target = "documents.plugins.date_parsing.base.dateparser.parse"
|
||||
target = "dateparser.parse"
|
||||
|
||||
def fake_parse(
|
||||
date_string: str,
|
||||
|
||||
@@ -1070,7 +1070,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
|
||||
|
||||
with (
|
||||
patch("documents.tasks.llmindex_index.apply_async") as mock_update,
|
||||
patch("paperless.views.llm_index_exists") as mock_exists,
|
||||
patch("paperless_ai.indexing.llm_index_exists") as mock_exists,
|
||||
):
|
||||
mock_exists.return_value = False
|
||||
self.client.patch(
|
||||
@@ -1095,7 +1095,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
|
||||
|
||||
with (
|
||||
patch("documents.tasks.llmindex_index.apply_async") as mock_update,
|
||||
patch("paperless.views.llm_index_exists", return_value=False),
|
||||
patch("paperless_ai.indexing.llm_index_exists", return_value=False),
|
||||
):
|
||||
self.client.patch(
|
||||
f"{self.ENDPOINT}1/",
|
||||
@@ -1115,7 +1115,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
|
||||
|
||||
with (
|
||||
patch("documents.tasks.llmindex_index.apply_async") as mock_update,
|
||||
patch("paperless.views.llm_index_exists") as mock_exists,
|
||||
patch("paperless_ai.indexing.llm_index_exists") as mock_exists,
|
||||
):
|
||||
mock_exists.return_value = True
|
||||
self.client.patch(
|
||||
@@ -1136,7 +1136,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
|
||||
|
||||
with (
|
||||
patch("documents.tasks.llmindex_index.apply_async") as mock_update,
|
||||
patch("paperless.views.llm_index_exists") as mock_exists,
|
||||
patch("paperless_ai.indexing.llm_index_exists") as mock_exists,
|
||||
):
|
||||
mock_exists.return_value = True
|
||||
self.client.patch(
|
||||
@@ -1174,7 +1174,7 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
|
||||
|
||||
with (
|
||||
patch("documents.tasks.llmindex_index.apply_async") as mock_update,
|
||||
patch("paperless.views.llm_index_exists") as mock_exists,
|
||||
patch("paperless_ai.indexing.llm_index_exists") as mock_exists,
|
||||
):
|
||||
mock_exists.return_value = True
|
||||
self.client.patch(
|
||||
|
||||
@@ -61,7 +61,7 @@ class TestChatStreamingViewInputValidation(APITestCase):
|
||||
return_value=self._mock_ai_enabled(),
|
||||
),
|
||||
mock.patch(
|
||||
"documents.views.stream_chat_with_documents",
|
||||
"paperless_ai.chat.stream_chat_with_documents",
|
||||
return_value=iter(chunks),
|
||||
),
|
||||
):
|
||||
@@ -104,7 +104,7 @@ class TestChatStreamingViewUnrestrictedFlag:
|
||||
"""
|
||||
mocker.patch("documents.views.AIConfig").return_value.ai_enabled = True
|
||||
return mocker.patch(
|
||||
"documents.views.stream_chat_with_documents",
|
||||
"paperless_ai.chat.stream_chat_with_documents",
|
||||
return_value=iter(()),
|
||||
)
|
||||
|
||||
|
||||
@@ -1110,7 +1110,7 @@ class TestVersionActionPermissions(DirectoriesMixin, APITestCase):
|
||||
self.assertEqual(response.status_code, 403)
|
||||
with (
|
||||
mock.patch("documents.views.AIConfig") as ai_config,
|
||||
mock.patch("documents.views.stream_chat_with_documents") as chat,
|
||||
mock.patch("paperless_ai.chat.stream_chat_with_documents") as chat,
|
||||
):
|
||||
ai_config.return_value.ai_enabled = True
|
||||
response = self.client.post(
|
||||
|
||||
@@ -2686,7 +2686,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
|
||||
response = self.client.get("/api/documents/34676/suggestions/")
|
||||
self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
|
||||
|
||||
@mock.patch("documents.views.get_ai_document_classification")
|
||||
@mock.patch("paperless_ai.ai_classifier.get_ai_document_classification")
|
||||
@override_settings(AI_ENABLED=True)
|
||||
def test_suggestions_still_uses_classifier_when_ai_enabled(
|
||||
self,
|
||||
|
||||
@@ -308,3 +308,32 @@ class TestBulkDownloadSchema:
|
||||
def test_bulk_download_response_403(self, api_schema: SchemaGenerator) -> None:
|
||||
op = api_schema["paths"]["/api/documents/bulk_download/"]["post"]
|
||||
assert "403" in op["responses"]
|
||||
|
||||
|
||||
class TestDocumentListObjectFilterSchema:
|
||||
"""documents_list: comma-separated ObjectFilter params must be typed as integer arrays."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"param_name",
|
||||
[
|
||||
"tags__id__all",
|
||||
"tags__id__none",
|
||||
"tags__id__in",
|
||||
"correspondent__id__none",
|
||||
"document_type__id__none",
|
||||
"storage_path__id__none",
|
||||
"owner__id__none",
|
||||
"custom_fields__id__all",
|
||||
"custom_fields__id__none",
|
||||
"custom_fields__id__in",
|
||||
],
|
||||
)
|
||||
def test_object_filter_param_is_integer_array(
|
||||
self,
|
||||
api_schema: SchemaGenerator,
|
||||
param_name: str,
|
||||
) -> None:
|
||||
params = api_schema["paths"]["/api/documents/"]["get"]["parameters"]
|
||||
param = next(p for p in params if p["name"] == param_name)
|
||||
assert param["schema"] == {"type": "array", "items": {"type": "integer"}}
|
||||
assert param["explode"] is False
|
||||
@@ -539,3 +539,50 @@ class TestSystemStatus(APITestCase):
|
||||
self.assertEqual(response.data["tasks"]["summary"]["success_count"], 1)
|
||||
self.assertEqual(response.data["tasks"]["summary"]["failure_count"], 1)
|
||||
self.assertEqual(response.data["tasks"]["summary"]["pending_count"], 1)
|
||||
|
||||
def test_system_status_reports_database_error(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- Reading the migration state fails
|
||||
WHEN:
|
||||
- The system status is requested
|
||||
THEN:
|
||||
- The response reports the database error rather than a server error
|
||||
"""
|
||||
self.client.force_login(self.user)
|
||||
with mock.patch(
|
||||
"documents.views.MigrationLoader",
|
||||
side_effect=Exception("database is down"),
|
||||
):
|
||||
response = self.client.get(self.ENDPOINT)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
database = response.data["database"]
|
||||
self.assertEqual(database["status"], "ERROR")
|
||||
self.assertIsNotNone(database["error"])
|
||||
self.assertEqual(
|
||||
database["migration_status"],
|
||||
{"latest_migration": None, "unapplied_migrations": []},
|
||||
)
|
||||
|
||||
def test_system_status_migration_query_error(self) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- The migration graph loads but reading the applied migrations fails
|
||||
WHEN:
|
||||
- The system status is requested
|
||||
THEN:
|
||||
- No migrations are reported as unapplied, since the state is unknown
|
||||
"""
|
||||
self.client.force_login(self.user)
|
||||
with mock.patch("documents.views.MigrationRecorder") as recorder:
|
||||
recorder.Migration.objects.all.side_effect = Exception("query failed")
|
||||
response = self.client.get(self.ENDPOINT)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||
database = response.data["database"]
|
||||
self.assertEqual(database["status"], "ERROR")
|
||||
self.assertEqual(
|
||||
database["migration_status"],
|
||||
{"latest_migration": None, "unapplied_migrations": []},
|
||||
)
|
||||
@@ -31,6 +31,7 @@ from documents.serialisers import DocumentSerializer
|
||||
from documents.tasks import empty_trash
|
||||
from paperless_testing.assertions import FileSystemAssertsMixin
|
||||
from paperless_testing.dirs import DirectoriesMixin
|
||||
from paperless_testing.dirs import PaperlessDirs
|
||||
from paperless_testing.factories import DocumentFactory
|
||||
from paperless_testing.factories import UserFactory
|
||||
|
||||
@@ -2134,3 +2135,28 @@ class TestPathDateLocalization:
|
||||
with override_settings(FILENAME_FORMAT=filename_format):
|
||||
filename = generate_filename(document)
|
||||
assert filename == Path(expected_filename)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_refresh_failure_surfaces_database_error(
|
||||
paperless_dirs: PaperlessDirs,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- A document with a stored filename
|
||||
WHEN:
|
||||
- Saving it triggers a filename update and refreshing it from the database fails
|
||||
THEN:
|
||||
- The database error is raised, not an unbound variable error from the rollback
|
||||
"""
|
||||
document = DocumentFactory(filename="some.pdf")
|
||||
|
||||
with (
|
||||
mock.patch.object(
|
||||
Document,
|
||||
"refresh_from_db",
|
||||
side_effect=DatabaseError("database is locked"),
|
||||
),
|
||||
pytest.raises(DatabaseError, match="database is locked"),
|
||||
):
|
||||
document.save()
|
||||
@@ -4,6 +4,7 @@ from typing import Literal
|
||||
|
||||
import pytest
|
||||
|
||||
from documents.templating.filters import format_datetime
|
||||
from documents.templating.filters import localize_date
|
||||
|
||||
|
||||
@@ -294,3 +295,23 @@ class TestDateLocalization:
|
||||
Tests `localize_date` with `date` string across different locales and formats.
|
||||
"""
|
||||
assert localize_date(value, format_style, locale_str) == expected_output
|
||||
|
||||
|
||||
class TestFormatDatetime:
|
||||
@pytest.mark.parametrize(
|
||||
("value", "expected"),
|
||||
[
|
||||
pytest.param("2026-09-21", "2026-09-21 00:00", id="date"),
|
||||
pytest.param(
|
||||
"2026-09-21T13:45:00+00:00",
|
||||
"2026-09-21 13:45",
|
||||
id="datetime",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_parses_iso_strings(self, value: str, expected: str) -> None:
|
||||
assert format_datetime(value, "%Y-%m-%d %H:%M") == expected
|
||||
|
||||
def test_rejects_unparseable_string(self) -> None:
|
||||
with pytest.raises(ValueError, match="not a date"):
|
||||
format_datetime("not a date", "%Y")
|
||||
@@ -572,7 +572,7 @@ class TestAiChatAllDocumentsPermissionBoundary:
|
||||
ENDPOINT = "/api/documents/chat/"
|
||||
|
||||
@override_settings(AI_ENABLED=True)
|
||||
@patch("documents.views.stream_chat_with_documents")
|
||||
@patch("paperless_ai.chat.stream_chat_with_documents")
|
||||
def test_chat_all_documents_excludes_unshared_document(self, mock_stream_chat):
|
||||
mock_stream_chat.return_value = iter([b"data"])
|
||||
|
||||
|
||||
@@ -322,7 +322,7 @@ class TestUpdateContent(DirectoriesMixin, TestCase):
|
||||
)
|
||||
|
||||
@override_settings(AI_ENABLED=True, LLM_EMBEDDING_BACKEND="huggingface")
|
||||
@mock.patch("documents.tasks.llm_index_add_or_update_document")
|
||||
@mock.patch("paperless_ai.indexing.llm_index_add_or_update_document")
|
||||
@mock.patch("documents.search.get_backend")
|
||||
def test_update_content_version_updates_llm_index(
|
||||
self,
|
||||
@@ -450,7 +450,7 @@ class TestAIIndex(DirectoriesMixin, TestCase):
|
||||
checksum="wow",
|
||||
)
|
||||
with mock.patch(
|
||||
"documents.tasks.llm_index_add_or_update_document",
|
||||
"paperless_ai.indexing.llm_index_add_or_update_document",
|
||||
) as llm_index_add_or_update_document:
|
||||
tasks.update_document_in_llm_index(doc)
|
||||
llm_index_add_or_update_document.assert_called_once_with(doc)
|
||||
@@ -470,7 +470,7 @@ class TestAIIndex(DirectoriesMixin, TestCase):
|
||||
checksum="wow",
|
||||
)
|
||||
with mock.patch(
|
||||
"documents.tasks.llm_index_remove_document",
|
||||
"paperless_ai.indexing.llm_index_remove_document",
|
||||
) as llm_index_remove_document:
|
||||
tasks.remove_document_from_llm_index(doc)
|
||||
llm_index_remove_document.assert_called_once_with(doc)
|
||||
@@ -495,7 +495,7 @@ class TestAIIndex(DirectoriesMixin, TestCase):
|
||||
"documents.tasks.update_document_in_llm_index",
|
||||
) as update_document_in_llm_index,
|
||||
mock.patch(
|
||||
"documents.tasks.update_llm_index",
|
||||
"paperless_ai.indexing.update_llm_index",
|
||||
) as update_llm_index,
|
||||
):
|
||||
doc_ids = [doc.pk for doc in docs]
|
||||
|
||||
@@ -474,7 +474,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
|
||||
self.assertEqual(response.json()["tags"], [])
|
||||
self.assertEqual(response.json()["suggested_tags"], [])
|
||||
|
||||
@patch("documents.views.get_ai_document_classification")
|
||||
@patch("paperless_ai.ai_classifier.get_ai_document_classification")
|
||||
@override_settings(
|
||||
AI_ENABLED=True,
|
||||
LLM_BACKEND="mock_backend",
|
||||
@@ -524,7 +524,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
|
||||
None,
|
||||
)
|
||||
|
||||
@patch("documents.views.get_ai_document_classification")
|
||||
@patch("paperless_ai.ai_classifier.get_ai_document_classification")
|
||||
@override_settings(
|
||||
AI_ENABLED=True,
|
||||
LLM_BACKEND="mock_backend",
|
||||
@@ -562,7 +562,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
|
||||
"KI Title",
|
||||
)
|
||||
|
||||
@patch("documents.views.get_ai_document_classification")
|
||||
@patch("paperless_ai.ai_classifier.get_ai_document_classification")
|
||||
@override_settings(
|
||||
AI_ENABLED=True,
|
||||
LLM_BACKEND="mock_backend",
|
||||
@@ -601,7 +601,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
|
||||
"Titre IA",
|
||||
)
|
||||
|
||||
@patch("documents.views.get_ai_document_classification")
|
||||
@patch("paperless_ai.ai_classifier.get_ai_document_classification")
|
||||
@override_settings(
|
||||
AI_ENABLED=True,
|
||||
LLM_BACKEND="mock_backend",
|
||||
@@ -637,7 +637,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
|
||||
),
|
||||
)
|
||||
|
||||
@patch("documents.views.get_ai_document_classification")
|
||||
@patch("paperless_ai.ai_classifier.get_ai_document_classification")
|
||||
@override_settings(
|
||||
AI_ENABLED=True,
|
||||
LLM_BACKEND="mock_backend",
|
||||
@@ -709,7 +709,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
|
||||
),
|
||||
)
|
||||
|
||||
@patch("documents.views.get_ai_document_classification")
|
||||
@patch("paperless_ai.ai_classifier.get_ai_document_classification")
|
||||
@override_settings(
|
||||
AI_ENABLED=True,
|
||||
LLM_BACKEND="openai-like",
|
||||
@@ -738,7 +738,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
|
||||
get_llm_suggestion_cache(self.document.pk, backend="openai-like"),
|
||||
)
|
||||
|
||||
@patch("documents.views.get_ai_document_classification")
|
||||
@patch("paperless_ai.ai_classifier.get_ai_document_classification")
|
||||
@override_settings(
|
||||
AI_ENABLED=True,
|
||||
LLM_BACKEND="openai-like",
|
||||
@@ -765,7 +765,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
|
||||
get_llm_suggestion_cache(self.document.pk, backend="openai-like"),
|
||||
)
|
||||
|
||||
@patch("documents.views.get_ai_document_classification")
|
||||
@patch("paperless_ai.ai_classifier.get_ai_document_classification")
|
||||
@override_settings(
|
||||
AI_ENABLED=True,
|
||||
LLM_BACKEND="openai-like",
|
||||
@@ -797,7 +797,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
|
||||
get_llm_suggestion_cache(self.document.pk, backend="openai-like"),
|
||||
)
|
||||
|
||||
@patch("documents.views.get_ai_document_classification")
|
||||
@patch("paperless_ai.ai_classifier.get_ai_document_classification")
|
||||
@override_settings(
|
||||
AI_ENABLED=True,
|
||||
LLM_BACKEND="mock_backend",
|
||||
@@ -835,7 +835,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
|
||||
self.assertEqual(response.json()["tags"], [self.tag1.pk])
|
||||
self.assertEqual(response.json()["suggested_tags"], ["Follow-up"])
|
||||
|
||||
@patch("documents.views.get_ai_document_classification")
|
||||
@patch("paperless_ai.ai_classifier.get_ai_document_classification")
|
||||
@override_settings(
|
||||
AI_ENABLED=True,
|
||||
LLM_BACKEND="mock_backend",
|
||||
@@ -874,7 +874,7 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
|
||||
self.assertEqual(response.json()["tags"], [self.tag1.pk])
|
||||
self.assertEqual(response.json()["suggested_tags"], [])
|
||||
|
||||
@patch("documents.views.get_ai_document_classification")
|
||||
@patch("paperless_ai.ai_classifier.get_ai_document_classification")
|
||||
@override_settings(
|
||||
AI_ENABLED=True,
|
||||
LLM_BACKEND="mock_backend",
|
||||
@@ -996,7 +996,7 @@ class TestAIChatStreamingView(DirectoriesMixin, TestCase):
|
||||
self.assertEqual(response.status_code, 400)
|
||||
self.assertIn(b"AI is required for this feature", response.content)
|
||||
|
||||
@patch("documents.views.stream_chat_with_documents")
|
||||
@patch("paperless_ai.chat.stream_chat_with_documents")
|
||||
@patch("documents.views.permitted_document_ids")
|
||||
@override_settings(AI_ENABLED=True)
|
||||
def test_post_no_document_id(self, mock_permitted_ids, mock_stream_chat) -> None:
|
||||
@@ -1016,7 +1016,7 @@ class TestAIChatStreamingView(DirectoriesMixin, TestCase):
|
||||
self.assertEqual(list(call_kwargs["documents"]), [self.document])
|
||||
self.assertIsNone(call_kwargs["output_language"])
|
||||
|
||||
@patch("documents.views.stream_chat_with_documents")
|
||||
@patch("paperless_ai.chat.stream_chat_with_documents")
|
||||
@patch("documents.views.permitted_document_ids")
|
||||
@override_settings(AI_ENABLED=True)
|
||||
def test_post_uses_user_display_language(
|
||||
@@ -1042,7 +1042,7 @@ class TestAIChatStreamingView(DirectoriesMixin, TestCase):
|
||||
self.assertEqual(list(call_kwargs["documents"]), [self.document])
|
||||
self.assertEqual(call_kwargs["output_language"], "de-de")
|
||||
|
||||
@patch("documents.views.stream_chat_with_documents")
|
||||
@patch("paperless_ai.chat.stream_chat_with_documents")
|
||||
@override_settings(AI_ENABLED=True)
|
||||
def test_post_with_document_id(self, mock_stream_chat) -> None:
|
||||
self.grant_view_document_permission()
|
||||
@@ -1079,7 +1079,7 @@ class TestAIChatStreamingView(DirectoriesMixin, TestCase):
|
||||
self.assertEqual(response.status_code, 403)
|
||||
self.assertIn(b"Insufficient permissions", response.content)
|
||||
|
||||
@patch("documents.views.stream_chat_with_documents")
|
||||
@patch("paperless_ai.chat.stream_chat_with_documents")
|
||||
@override_settings(AI_ENABLED=True)
|
||||
def test_post_no_document_id_requires_view_document_permission(
|
||||
self,
|
||||
@@ -1093,7 +1093,7 @@ class TestAIChatStreamingView(DirectoriesMixin, TestCase):
|
||||
self.assertEqual(response.status_code, 403)
|
||||
mock_stream_chat.assert_not_called()
|
||||
|
||||
@patch("documents.views.stream_chat_with_documents")
|
||||
@patch("paperless_ai.chat.stream_chat_with_documents")
|
||||
@override_settings(AI_ENABLED=True)
|
||||
def test_post_with_document_id_requires_view_document_permission(
|
||||
self,
|
||||
|
||||
+36
-28
@@ -259,20 +259,8 @@ from paperless.parsers.remote import RemoteEngineConfig
|
||||
from paperless.serialisers import GroupSerializer
|
||||
from paperless.serialisers import UserSerializer
|
||||
from paperless.views import StandardPagination
|
||||
from paperless_ai.ai_classifier import get_ai_document_classification
|
||||
from paperless_ai.ai_classifier import get_llm_output_language
|
||||
from paperless_ai.chat import stream_chat_with_documents
|
||||
from paperless_ai.exceptions import LLMProviderError
|
||||
from paperless_ai.exceptions import LLMTimeoutError
|
||||
from paperless_ai.matching import extract_unmatched_names
|
||||
from paperless_ai.matching import match_correspondents_by_name
|
||||
from paperless_ai.matching import match_document_types_by_name
|
||||
from paperless_ai.matching import match_storage_paths_by_name
|
||||
from paperless_ai.matching import match_tags_by_name
|
||||
from paperless_ai.matching import resolve_correspondent_ids
|
||||
from paperless_ai.matching import resolve_document_type_ids
|
||||
from paperless_ai.matching import resolve_storage_path_ids
|
||||
from paperless_ai.matching import resolve_tag_ids
|
||||
from paperless_mail.models import MailAccount
|
||||
from paperless_mail.models import MailRule
|
||||
from paperless_mail.oauth import PaperlessMailOAuth2Manager
|
||||
@@ -708,19 +696,6 @@ class EmailDocumentDetailSchema(EmailSerializer):
|
||||
|
||||
|
||||
@extend_schema_view(
|
||||
list=extend_schema(
|
||||
parameters=[
|
||||
OpenApiParameter(
|
||||
name=name,
|
||||
type=OpenApiTypes.INT,
|
||||
many=True,
|
||||
style="form",
|
||||
explode=False,
|
||||
)
|
||||
for name, field in DocumentFilterSet.base_filters.items()
|
||||
if isinstance(field, ObjectFilter)
|
||||
],
|
||||
),
|
||||
retrieve=extend_schema(
|
||||
description="Retrieve a single document",
|
||||
responses={
|
||||
@@ -1622,6 +1597,18 @@ class DocumentViewSet(
|
||||
if not ai_config.ai_enabled:
|
||||
return HttpResponseBadRequest("AI is required for this feature")
|
||||
|
||||
from paperless_ai.ai_classifier import get_ai_document_classification
|
||||
from paperless_ai.ai_classifier import get_llm_output_language
|
||||
from paperless_ai.matching import extract_unmatched_names
|
||||
from paperless_ai.matching import match_correspondents_by_name
|
||||
from paperless_ai.matching import match_document_types_by_name
|
||||
from paperless_ai.matching import match_storage_paths_by_name
|
||||
from paperless_ai.matching import match_tags_by_name
|
||||
from paperless_ai.matching import resolve_correspondent_ids
|
||||
from paperless_ai.matching import resolve_document_type_ids
|
||||
from paperless_ai.matching import resolve_storage_path_ids
|
||||
from paperless_ai.matching import resolve_tag_ids
|
||||
|
||||
output_language = get_llm_output_language(
|
||||
ai_config=ai_config,
|
||||
user=request.user,
|
||||
@@ -2525,6 +2512,9 @@ class ChatStreamingView(GenericAPIView[Any]):
|
||||
)
|
||||
unrestricted = user_is_unrestricted(request.user)
|
||||
|
||||
from paperless_ai.ai_classifier import get_llm_output_language
|
||||
from paperless_ai.chat import stream_chat_with_documents
|
||||
|
||||
output_language = get_llm_output_language(
|
||||
ai_config=ai_config,
|
||||
user=request.user,
|
||||
@@ -2546,6 +2536,17 @@ class ChatStreamingView(GenericAPIView[Any]):
|
||||
list=extend_schema(
|
||||
description="Document views including search",
|
||||
parameters=[
|
||||
*(
|
||||
OpenApiParameter(
|
||||
name=name,
|
||||
type=OpenApiTypes.INT,
|
||||
many=True,
|
||||
style="form",
|
||||
explode=False,
|
||||
)
|
||||
for name, field in DocumentFilterSet.base_filters.items()
|
||||
if isinstance(field, ObjectFilter)
|
||||
),
|
||||
OpenApiParameter(
|
||||
name="text",
|
||||
type=OpenApiTypes.STR,
|
||||
@@ -5380,7 +5381,9 @@ class CustomFieldViewSet(PermissionsAwareDocumentCountMixin, ModelViewSet[Custom
|
||||
"migration_status": inline_serializer(
|
||||
name="MigrationStatus",
|
||||
fields={
|
||||
"latest_migration": serializers.CharField(),
|
||||
"latest_migration": serializers.CharField(
|
||||
allow_null=True,
|
||||
),
|
||||
"unapplied_migrations": serializers.ListSerializer(
|
||||
child=serializers.CharField(),
|
||||
),
|
||||
@@ -5455,6 +5458,8 @@ class SystemStatusView(PassUserMixin):
|
||||
db_conn = connections["default"]
|
||||
db_url = str(db_conn.settings_dict["NAME"])
|
||||
db_error = None
|
||||
all_migrations: list[str] = []
|
||||
applied_migrations: list[str] = []
|
||||
|
||||
try:
|
||||
db_conn.ensure_connection()
|
||||
@@ -5465,7 +5470,8 @@ class SystemStatusView(PassUserMixin):
|
||||
f"{m.app}.{m.name}"
|
||||
for m in MigrationRecorder.Migration.objects.all().order_by("id")
|
||||
]
|
||||
except Exception as e: # pragma: no cover
|
||||
except Exception as e:
|
||||
all_migrations = []
|
||||
applied_migrations = []
|
||||
db_status = "ERROR"
|
||||
logger.exception(
|
||||
@@ -5661,7 +5667,9 @@ class SystemStatusView(PassUserMixin):
|
||||
"status": db_status,
|
||||
"error": db_error,
|
||||
"migration_status": {
|
||||
"latest_migration": applied_migrations[-1],
|
||||
"latest_migration": (
|
||||
applied_migrations[-1] if applied_migrations else None
|
||||
),
|
||||
"unapplied_migrations": [
|
||||
m for m in all_migrations if m not in applied_migrations
|
||||
],
|
||||
|
||||
Reference in new issue
Block a user