Fix: Blacklist certain OCR user arguments that paperless itself controls/uses

docs/administration.md

@@ -565,8 +565,6 @@ Specify `rebuild` to build the index from scratch from all documents in the data
 this the first time you enable the feature, or after changing the embedding backend or
 model.
 
-When using the `huggingface` embedding backend, a model that is already downloaded is loaded from the local cache without contacting the Hugging Face Hub. Only `rebuild` checks the Hub for a newer revision of the model, so a model update is picked up together with re-embedding every document. If the model is not cached yet, it is downloaded on first use.
-
 Specify `update` to incrementally index new and changed documents. This is what the
 scheduled task runs.
 

docs/configuration.md

@@ -873,6 +873,14 @@ If both the [PAPERLESS_ACCOUNT_DEFAULT_GROUPS](#PAPERLESS_ACCOUNT_DEFAULT_GROUPS
 
     Defaults to 1209600 (2 weeks)
 
+#### [`PAPERLESS_SESSION_CLEANUP_TASK_CRON=<cron expression>`](#PAPERLESS_SESSION_CLEANUP_TASK_CRON) {#PAPERLESS_SESSION_CLEANUP_TASK_CRON}
+
+: Configures the schedule to remove expired sessions from the session store.
+Django does not prune these automatically, so without this they would
+accumulate indefinitely.
+
+    Defaults to `0 3 * * *`, once per day at 03:00.
+
 #### [`PAPERLESS_TOKEN_THROTTLE_RATE=<rate>`](#PAPERLESS_TOKEN_THROTTLE_RATE) {#PAPERLESS_TOKEN_THROTTLE_RATE}
 
 : Rate limit for the API token authentication endpoint (`/api/token/`), used to mitigate brute-force login attempts.
@@ -1149,6 +1157,9 @@ they use underscores instead of dashes.
         be changed by trusted users. This applies to the `AppConfig` permission as well,
         which allows setting these arguments from the UI.
 
+        Arguments paperless manages itself (`input_file_or_options`, `output_file`
+        and `sidecar`) are rejected by the API and ignored if set here.
+
     Specify arguments as a JSON dictionary. Keep note of lower case
     booleans and double quoted parameter names and strings. Examples:
 

src/documents/tasks.py

@@ -12,6 +12,7 @@ from celery import Task
 from celery import shared_task
 from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
+from django.contrib.sessions.models import Session
 from django.db import models
 from django.db import transaction
 from django.db.models.signals import post_save
@@ -925,3 +926,17 @@ def cleanup_expired_share_link_bundles() -> None:
             )
     if count:
         logger.info("Deleted %s expired share link bundle(s)", count)
+
+
+@shared_task
+def clear_expired_sessions() -> None:
+    """
+    Remove expired database sessions in batches to limit the size of each delete.
+    """
+    expired = Session.objects.filter(expire_date__lt=timezone.now())
+    while session_keys := list(
+        expired.order_by("expire_date").values_list("pk", flat=True)[:500],
+    ):
+        expired.filter(pk__in=session_keys).delete()
+
+    logger.info("Cleared expired sessions")

src/documents/tests/test_api_app_config.py

@@ -287,6 +287,28 @@ class TestApiAppConfig(DirectoriesMixin, APITestCase):
         self.assertEqual(config.user_args, user_args)
         self.assertEqual(config.barcode_tag_mapping, barcode_tag_mapping)
 
+    def test_api_update_config_rejects_reserved_user_args(self) -> None:
+        """
+        GIVEN:
+            - API request to update app config with user_args which set an
+              argument the parser manages itself
+        WHEN:
+            - API is called
+        THEN:
+            - Request is rejected with a 400 naming the argument
+            - Config is not updated
+        """
+        response = self.client.patch(
+            f"{self.ENDPOINT}1/",
+            json.dumps({"user_args": json.dumps({"sidecar": "/tmp/x.py"})}),
+            content_type="application/json",
+        )
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+        self.assertIn("sidecar", str(response.data["user_args"]))
+        config = ApplicationConfiguration.objects.first()
+        assert config is not None
+        self.assertIsNone(config.user_args)
+
     def test_api_update_config_rejects_invalid_json_objects(self) -> None:
         """
         GIVEN:

src/documents/tests/test_tasks.py

@@ -5,8 +5,11 @@ from unittest import mock
 
 import pytest
 from django.conf import settings
+from django.contrib.sessions.models import Session
+from django.db import connection
 from django.test import TestCase
 from django.test import override_settings
+from django.test.utils import CaptureQueriesContext
 from django.utils import timezone
 
 from documents import tasks
@@ -612,3 +615,36 @@ class TestApplyAISuggestionsTask(DirectoriesMixin, TestCase):
 
         apply_suggestions.assert_not_called()
         self.assertIn("no longer exists", "".join(cm.output))
+
+
+@pytest.mark.django_db
+class TestClearExpiredSessions:
+    def test_clears_only_expired_sessions_in_batches(self) -> None:
+        """
+        GIVEN:
+            - More than one batch of expired sessions and one still-valid session
+        WHEN:
+            - The clear_expired_sessions task runs
+        THEN:
+            - Only expired sessions are removed, using multiple deletes
+        """
+        now = timezone.now()
+        Session.objects.bulk_create(
+            Session(
+                session_key=f"expired-{i}",
+                session_data="",
+                expire_date=now - timedelta(days=1),
+            )
+            for i in range(501)
+        )
+        Session.objects.create(
+            session_key="valid",
+            session_data="",
+            expire_date=now + timedelta(days=1),
+        )
+
+        with CaptureQueriesContext(connection) as queries:
+            tasks.clear_expired_sessions()
+
+        assert set(Session.objects.values_list("session_key", flat=True)) == {"valid"}
+        assert sum(query["sql"].startswith("DELETE FROM") for query in queries) == 2

src/paperless/config.py

@@ -12,6 +12,17 @@ from paperless.models import ModeChoices
 from paperless.models import OutputTypeChoices
 from paperless.models import RemoteOCRMode
 
+# OCRmyPDF arguments the parser sets itself, pointing into its own temporary
+# directory. The parser reads these files back after OCR, so user supplied
+# values can only break consumption (or write to arbitrary paths), never help.
+OCR_RESERVED_USER_ARGS: frozenset[str] = frozenset(
+    {
+        "input_file_or_options",
+        "output_file",
+        "sidecar",
+    },
+)
+
 
 @dataclasses.dataclass
 class BaseConfig:

src/paperless/parsers/tesseract.py

@@ -21,6 +21,7 @@ from documents.parsers import make_thumbnail_from_pdf
 from documents.utils import copy_file_with_basic_stats
 from documents.utils import maybe_override_pixel_limit
 from documents.utils import remove_alpha_channel
+from paperless.config import OCR_RESERVED_USER_ARGS
 from paperless.config import OcrConfig
 from paperless.models import CleanChoices
 from paperless.models import ModeChoices
@@ -351,7 +352,22 @@ class RasterisedDocumentParser:
 
         if self.settings.user_args is not None:
             try:
-                ocrmypdf_args = {**ocrmypdf_args, **self.settings.user_args}
+                reserved = sorted(
+                    OCR_RESERVED_USER_ARGS & self.settings.user_args.keys(),
+                )
+                if reserved:
+                    self.log.warning(
+                        f"Ignoring OCR user arguments managed by paperless: "
+                        f"{', '.join(reserved)}",
+                    )
+                ocrmypdf_args = {
+                    **ocrmypdf_args,
+                    **{
+                        k: v
+                        for k, v in self.settings.user_args.items()
+                        if k not in OCR_RESERVED_USER_ARGS
+                    },
+                }
             except Exception as e:
                 self.log.warning(
                     f"There is an issue with PAPERLESS_OCR_USER_ARGS, so "

src/paperless/serialisers.py

@@ -19,6 +19,7 @@ from PIL import Image
 from rest_framework import serializers
 from rest_framework.authtoken.serializers import AuthTokenSerializer
 
+from paperless.config import OCR_RESERVED_USER_ARGS
 from paperless.models import ApplicationConfiguration
 from paperless.network import validate_outbound_http_url
 from paperless.validators import reject_dangerous_svg
@@ -251,6 +252,13 @@ class ApplicationConfigurationSerializer(
 
     def validate_user_args(self, value):
         self._require_json_object("user_args", value)
+        if value is not None:
+            reserved = sorted(OCR_RESERVED_USER_ARGS & value.keys())
+            if reserved:
+                raise serializers.ValidationError(
+                    f"user_args may not set arguments managed by paperless: "
+                    f"{', '.join(reserved)}.",
+                )
         return value
 
     def validate_barcode_tag_mapping(self, value):

src/paperless/settings/custom.py

@@ -165,6 +165,17 @@ def parse_beat_schedule() -> dict:
                 "expires": 23.0 * 60.0 * 60.0,
             },
         },
+        {
+            "name": "Clear expired sessions",
+            "env_key": "PAPERLESS_SESSION_CLEANUP_TASK_CRON",
+            # Default daily at 03:00
+            "env_default": "0 3 * * *",
+            "task": "documents.tasks.clear_expired_sessions",
+            "options": {
+                # 1 hour before default schedule sends again
+                "expires": 23.0 * 60.0 * 60.0,
+            },
+        },
     ]
     for task in tasks:
         # Either get the environment setting or use the default

src/paperless/tests/parsers/test_tesseract_custom_settings.py

@@ -243,6 +243,34 @@ class TestParserSettingsFromDb(DirectoriesMixin, FileSystemAssertsMixin, TestCas
             "UseDeviceIndependentColor",
         )
 
+    def test_ocr_user_args_cannot_override_reserved(self) -> None:
+        """
+        GIVEN:
+            - User arguments which attempt to override arguments the parser
+              manages itself, alongside a legitimate argument
+        WHEN:
+            - OCR parameters are constructed
+        THEN:
+            - Reserved arguments keep the parser supplied values
+            - The legitimate argument is still applied
+        """
+        instance = ApplicationConfiguration.objects.all().first()
+        assert instance is not None
+        instance.user_args = {
+            "sidecar": "/tmp/evil.py",
+            "output_file": "/tmp/out.pdf",
+            "input_file_or_options": "/etc/passwd",
+            "unpaper_args": "--pre-rotate 90",
+        }
+        instance.save()
+
+        params = self.get_params()
+
+        self.assertEqual(params["sidecar"], "sidecar.txt")
+        self.assertEqual(params["output_file"], "output.pdf")
+        self.assertEqual(params["input_file_or_options"], "input.pdf")
+        self.assertEqual(params["unpaper_args"], "--pre-rotate 90")
+
     def test_ocr_user_args(self) -> None:
         """
         GIVEN:

src/paperless/tests/settings/test_custom_parsers.py

@@ -182,6 +182,7 @@ def make_expected_schedule(
     workflow_expire = 59.0 * 60.0
     llm_index_expire = 23.0 * 60.0 * 60.0
     share_link_cleanup_expire = 23.0 * 60.0 * 60.0
+    session_cleanup_expire = 23.0 * 60.0 * 60.0
 
     schedule: dict[str, Any] = {
         "Check all e-mail accounts": {
@@ -248,6 +249,14 @@ def make_expected_schedule(
                 "headers": {"trigger_source": "scheduled"},
             },
         },
+        "Clear expired sessions": {
+            "task": "documents.tasks.clear_expired_sessions",
+            "schedule": crontab(minute=0, hour="3"),
+            "options": {
+                "expires": session_cleanup_expire,
+                "headers": {"trigger_source": "scheduled"},
+            },
+        },
     }
 
     overrides = overrides or {}
@@ -298,6 +307,7 @@ class TestParseBeatSchedule:
                     "PAPERLESS_WORKFLOW_SCHEDULED_TASK_CRON": "disable",
                     "PAPERLESS_LLM_INDEX_TASK_CRON": "disable",
                     "PAPERLESS_SHARE_LINK_BUNDLE_CLEANUP_CRON": "disable",
+                    "PAPERLESS_SESSION_CLEANUP_TASK_CRON": "disable",
                 },
                 {},
                 id="all-disabled",

src/paperless_ai/embedding.py

@@ -1,5 +1,4 @@
 import re
-from contextlib import suppress
 from typing import TYPE_CHECKING
 
 from django.conf import settings
@@ -21,17 +20,7 @@ OCR_LEADER_REGEX = re.compile(r"[._\-\u00b7]{4,}")
 HORIZONTAL_WHITESPACE_REGEX = re.compile(r"[ \t\u00a0]+")
 
 
-def get_embedding_model(
-    config: AIConfig,
-    *,
-    refresh: bool = False,
-) -> "BaseEmbedding":
-    """
-    Build the embedding model for the configured backend.
-
-    For Hugging Face, a cached model loads offline by default, saving seconds per
-    call. Pass ``refresh=True`` to check the Hub for a newer model revision.
-    """
+def get_embedding_model(config: AIConfig) -> "BaseEmbedding":
     match config.llm_embedding_backend:
         case LLMEmbeddingBackend.OPENAI_LIKE:
             from llama_index.embeddings.openai_like import OpenAILikeEmbedding
@@ -63,15 +52,11 @@ def get_embedding_model(
         case LLMEmbeddingBackend.HUGGINGFACE:
             from llama_index.embeddings.huggingface import HuggingFaceEmbedding
 
-            hf_kwargs = {
-                "model_name": config.llm_embedding_model
+            return HuggingFaceEmbedding(
+                model_name=config.llm_embedding_model
                 or "sentence-transformers/all-MiniLM-L6-v2",
-                "cache_folder": str(settings.DATA_DIR / "hf_cache"),
-            }
-            if not refresh:
-                with suppress(OSError):
-                    return HuggingFaceEmbedding(**hf_kwargs, local_files_only=True)
-            return HuggingFaceEmbedding(**hf_kwargs)
+                cache_folder=str(settings.DATA_DIR / "hf_cache"),
+            )
         case LLMEmbeddingBackend.OLLAMA:
             from llama_index.embeddings.ollama import OllamaEmbedding
             from ollama import AsyncClient

src/paperless_ai/indexing.py

@@ -470,7 +470,7 @@ def update_llm_index(
         logger.warning("No documents found to index.")
 
     chunk_size = config.llm_embedding_chunk_size
-    embed_model = get_embedding_model(config, refresh=rebuild)
+    embed_model = get_embedding_model(config)
 
     with write_store(embed_model_name=model_name) as store:
         if rebuild or not store.table_exists():

src/paperless_ai/tests/test_ai_indexing.py

@@ -212,10 +212,6 @@ def test_update_llm_index(
 
         ai_config.assert_called_once()
         build_document_node.assert_called_once_with(real_document, chunk_size=512)
-        indexing.get_embedding_model.assert_called_once_with(
-            mock_config,
-            refresh=True,
-        )
 
 
 @pytest.mark.django_db
@@ -340,8 +336,6 @@ def test_update_llm_index_partial_update(
 
         indexing.update_llm_index(rebuild=False)
 
-    assert indexing.get_embedding_model.call_args.kwargs == {"refresh": False}
-
     with indexing.get_vector_store() as store:
         assert store.table_exists(), (
             "Expected the vector store table to exist after incremental update"

src/paperless_ai/tests/test_embedding.py

@@ -1,6 +1,5 @@
 from unittest.mock import ANY
 from unittest.mock import MagicMock
-from unittest.mock import call
 from unittest.mock import patch
 
 import pytest
@@ -168,45 +167,10 @@ def test_get_embedding_model_huggingface(mock_ai_config):
         MockHuggingFaceEmbedding.assert_called_once_with(
             model_name="sentence-transformers/all-MiniLM-L6-v2",
             cache_folder=str(settings.DATA_DIR / "hf_cache"),
-            local_files_only=True,
         )
         assert model == MockHuggingFaceEmbedding.return_value
 
 
-def test_get_embedding_model_huggingface_downloads_when_not_cached(mock_ai_config):
-    mock_ai_config.return_value.llm_embedding_backend = LLMEmbeddingBackend.HUGGINGFACE
-    mock_ai_config.return_value.llm_embedding_model = "some/model"
-    downloaded = MagicMock()
-
-    with patch(
-        "llama_index.embeddings.huggingface.HuggingFaceEmbedding",
-        side_effect=[OSError("not cached"), downloaded],
-    ) as MockHuggingFaceEmbedding:
-        model = get_embedding_model(mock_ai_config.return_value)
-
-    cache_folder = str(settings.DATA_DIR / "hf_cache")
-    assert MockHuggingFaceEmbedding.call_args_list == [
-        call(model_name="some/model", cache_folder=cache_folder, local_files_only=True),
-        call(model_name="some/model", cache_folder=cache_folder),
-    ]
-    assert model is downloaded
-
-
-def test_get_embedding_model_huggingface_refresh_checks_the_hub(mock_ai_config):
-    mock_ai_config.return_value.llm_embedding_backend = LLMEmbeddingBackend.HUGGINGFACE
-    mock_ai_config.return_value.llm_embedding_model = "some/model"
-
-    with patch(
-        "llama_index.embeddings.huggingface.HuggingFaceEmbedding",
-    ) as MockHuggingFaceEmbedding:
-        get_embedding_model(mock_ai_config.return_value, refresh=True)
-
-    MockHuggingFaceEmbedding.assert_called_once_with(
-        model_name="some/model",
-        cache_folder=str(settings.DATA_DIR / "hf_cache"),
-    )
-
-
 def test_get_embedding_model_ollama(mock_ai_config):
     mock_ai_config.return_value.llm_embedding_backend = LLMEmbeddingBackend.OLLAMA
     mock_ai_config.return_value.llm_embedding_model = "embeddinggemma"