Merge branch 'dev' into chore/clear-sessions

SECURITY.md

@@ -56,6 +56,9 @@ We may close reports that are:
 
 The following are not generally considered vulnerabilities unless accompanied by a concrete, reproducible impact in Paperless-ngx:
 
+- password length inferred from masked mail-account passwords by users who already have permission to view the account, without disclosure of the password or token itself
+- disclosure limited to document IDs, existence, page counts, checksums, or modification/deletion timestamps, without access to document contents or an unintended privileged action
+- authenticated access to non-secret application settings, such as service endpoint URLs, model names, or configured environment variable names, without disclosure of credentials or secret values
 - large uploads or resource usage that do not bypass documented limits or privileges
 - IDOR / access control claims regarding the ability to attach an un-viewable object to a document. This is expected behavior.
 - claims based solely on the presence of a library, framework feature or code pattern without a working exploit

docs/configuration.md

@@ -1871,6 +1871,14 @@ processing. This only has an effect if
 
     Defaults to `0 1 * * *`, once per day.
 
+#### [`PAPERLESS_SESSION_CLEANUP_TASK_CRON=<cron expression>`](#PAPERLESS_SESSION_CLEANUP_TASK_CRON) {#PAPERLESS_SESSION_CLEANUP_TASK_CRON}
+
+: Configures the schedule to remove expired sessions from the session store.
+Django does not prune these automatically, so without this they would
+accumulate indefinitely.
+
+    Defaults to `0 3 * * *`, once per day at 03:00.
+
 ## Share links
 
 #### [`PAPERLESS_SHARE_LINK_BUNDLE_CLEANUP_CRON=<cron expression>`](#PAPERLESS_SHARE_LINK_BUNDLE_CLEANUP_CRON) {#PAPERLESS_SHARE_LINK_BUNDLE_CLEANUP_CRON}

src-ui/messages.xlf

@@ -4030,14 +4030,21 @@
         <source>Sources</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/chat/chat/chat.component.html</context>
-          <context context-type="linenumber">46,47</context>
+          <context context-type="linenumber">47,48</context>
+        </context-group>
+      </trans-unit>
+      <trans-unit id="6025275430560167531" datatype="html">
+        <source>+<x id="INTERPOLATION" equiv-text="{{ hiddenReferenceCount(message) }}"/> more</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/components/chat/chat/chat.component.html</context>
+          <context context-type="linenumber">49,50</context>
         </context-group>
       </trans-unit>
       <trans-unit id="8066608938393600549" datatype="html">
         <source>Message</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/chat/chat/chat.component.html</context>
-          <context context-type="linenumber">61,62</context>
+          <context context-type="linenumber">66,67</context>
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/email-document-dialog/email-document-dialog.component.html</context>
@@ -4048,22 +4055,22 @@
         <source>Stop</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/chat/chat/chat.component.html</context>
-          <context context-type="linenumber">74</context>
+          <context context-type="linenumber">79</context>
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/chat/chat/chat.component.html</context>
-          <context context-type="linenumber">74</context>
+          <context context-type="linenumber">79</context>
         </context-group>
       </trans-unit>
       <trans-unit id="6490688569532630280" datatype="html">
         <source>Send</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/chat/chat/chat.component.html</context>
-          <context context-type="linenumber">78</context>
+          <context context-type="linenumber">83</context>
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/chat/chat/chat.component.html</context>
-          <context context-type="linenumber">78</context>
+          <context context-type="linenumber">83</context>
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/document-detail/document-detail.component.html</context>
@@ -4090,14 +4097,14 @@
         <source>Ask a question about this document...</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/chat/chat/chat.component.ts</context>
-          <context context-type="linenumber">61</context>
+          <context context-type="linenumber">83</context>
         </context-group>
       </trans-unit>
       <trans-unit id="224049052163028286" datatype="html">
         <source>Ask a question about a document...</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/chat/chat/chat.component.ts</context>
-          <context context-type="linenumber">62</context>
+          <context context-type="linenumber">84</context>
         </context-group>
       </trans-unit>
       <trans-unit id="8700121026680200191" datatype="html">
@@ -6984,7 +6991,7 @@
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/input/select/select.component.ts</context>
-          <context context-type="linenumber">179</context>
+          <context context-type="linenumber">185</context>
         </context-group>
       </trans-unit>
       <trans-unit id="1880237574877817137" datatype="html">
@@ -7091,7 +7098,7 @@
         <source>Private</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/input/select/select.component.ts</context>
-          <context context-type="linenumber">72</context>
+          <context context-type="linenumber">73</context>
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/tag/tag.component.html</context>
@@ -7114,7 +7121,7 @@
         <source>No items found</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/common/input/select/select.component.ts</context>
-          <context context-type="linenumber">113</context>
+          <context context-type="linenumber">114</context>
         </context-group>
       </trans-unit>
       <trans-unit id="6541407358060244620" datatype="html">

src-ui/src/app/components/chat/chat/chat.component.html

@@ -42,11 +42,16 @@
               <span class="text-break">{{ message.content }}</span>
             }
             @if (message.role === 'assistant' && !message.isStreaming && message.references?.length) {
-              <div class="chat-references list-group mt-3">
-                <div class="list-group-item text-muted small py-1" i18n>Sources</div>
-                @for (reference of message.references; track reference.id) {
-                  <a class="list-group-item list-group-item-action d-flex align-items-center text-primary" [routerLink]="['/documents', reference.id]" [title]="reference.title">
-                    <pngx-bs-icon width="0.9em" height="0.9em" name="file-text" class="me-2 flex-shrink-0"></pngx-bs-icon><span class="text-truncate">{{ reference.title }}</span>
+              <div class="chat-references mt-2 pt-2 border-top">
+                <div class="d-flex align-items-center text-muted mb-1">
+                  <span i18n>Sources</span>
+                  @if (hiddenReferenceCount(message) > 0 && !expandedReferences().has(message)) {
+                    <button type="button" class="badge rounded-pill bg-body-secondary text-body-secondary fw-normal border-0 ms-2" (click)="expandReferences(message)" i18n>+{{ hiddenReferenceCount(message) }} more</button>
+                  }
+                </div>
+                @for (reference of visibleReferences(message); track reference.id) {
+                  <a class="reference-link d-flex align-items-center lh-sm text-primary text-decoration-none py-1" [routerLink]="['/documents', reference.id]" [title]="reference.title">
+                    <pngx-bs-icon width="0.8em" height="0.8em" name="file-text" class="me-2 flex-shrink-0"></pngx-bs-icon><span class="text-truncate">{{ reference.title }}</span>
                   </a>
                 }
               </div>

src-ui/src/app/components/chat/chat/chat.component.scss

@@ -12,6 +12,10 @@
   display: none;
 }
 
+.message .flex-grow-1 {
+  min-width: 0;
+}
+
 .dropdown-item {
   white-space: initial;
 }

src-ui/src/app/components/chat/chat/chat.component.ts

@@ -14,6 +14,7 @@ import { BootstrapIconComponent } from 'src/app/components/common/bootstrap-icon
 import { MarkdownPipe } from 'src/app/pipes/markdown.pipe'
 import {
   ChatMessage,
+  ChatReference,
   ChatService,
   parseChatResponse,
 } from 'src/app/services/chat.service'
@@ -40,8 +41,29 @@ export class ChatComponent implements OnInit {
   readonly input = signal('')
   readonly documentId = signal<number>(undefined)
   readonly slowResponse = signal(false)
+  readonly expandedReferences = signal<Set<ChatMessage>>(new Set())
   public popperOptions = pngxPopperOptions
 
+  static readonly COLLAPSED_REFERENCE_COUNT = 2
+
+  public visibleReferences(message: ChatMessage): ChatReference[] {
+    if (this.expandedReferences().has(message)) {
+      return message.references
+    }
+    return message.references.slice(0, ChatComponent.COLLAPSED_REFERENCE_COUNT)
+  }
+
+  public hiddenReferenceCount(message: ChatMessage): number {
+    return Math.max(
+      0,
+      message.references.length - ChatComponent.COLLAPSED_REFERENCE_COUNT
+    )
+  }
+
+  public expandReferences(message: ChatMessage): void {
+    this.expandedReferences.update((expanded) => new Set(expanded).add(message))
+  }
+
   private chatService: ChatService = inject(ChatService)
   private router: Router = inject(Router)
 

src-ui/src/app/components/common/input/document-link/document-link.component.html

@@ -36,7 +36,7 @@
     [appendTo]="appendTo"
     [loading]="loading()"
     [typeahead]="documentsInput$"
-    (mousedown)="$event.stopImmediatePropagation()"
+    (mousedown)="appendTo && $event.stopImmediatePropagation()"
     (change)="onChange(selectedDocumentIDs)">
     <ng-template ng-label-tmp let-document="item">
       <div class="d-flex align-items-center">

src-ui/src/app/components/common/input/select/select.component.spec.ts

@@ -97,6 +97,7 @@ describe('SelectComponent', () => {
     let createNewVal
     component.createNew.subscribe((v) => (createNewVal = v))
     expect(component.allowCreateNew).toBeTruthy()
+    const closeSpy = jest.spyOn(component.ngSelectComponent, 'close')
     component.onSearch({ term: 'foo' })
     component.addItem(undefined)
     expect(createNewVal).toEqual('foo')
@@ -105,6 +106,8 @@ describe('SelectComponent', () => {
     component.onSearch({ term: 'baz' })
     component.clickNew()
     expect(createNewVal).toEqual('baz')
+    // dropdown should close when creating a new item so it isn't left open behind the modal
+    expect(closeSpy).toHaveBeenCalled()
   })
 
   it('should search items by independent normalized terms', () => {

src-ui/src/app/components/common/input/select/select.component.ts

@@ -4,6 +4,7 @@ import {
   forwardRef,
   Input,
   Output,
+  ViewChild,
 } from '@angular/core'
 import {
   FormsModule,
@@ -11,7 +12,7 @@ import {
   ReactiveFormsModule,
 } from '@angular/forms'
 import { RouterModule } from '@angular/router'
-import { NgSelectModule } from '@ng-select/ng-select'
+import { NgSelectComponent, NgSelectModule } from '@ng-select/ng-select'
 import { BootstrapIconComponent } from 'src/app/components/common/bootstrap-icon/bootstrap-icon.component'
 import { matchesSearchText } from 'src/app/utils/text-search'
 import { AbstractInputComponent } from '../abstract-input'
@@ -124,6 +125,9 @@ export class SelectComponent extends AbstractInputComponent<number | string> {
   @Output()
   filterDocuments = new EventEmitter<any[]>()
 
+  @ViewChild(NgSelectComponent)
+  ngSelectComponent: NgSelectComponent
+
   public addItemRef: (name) => void
 
   private _lastSearchTerm: string
@@ -150,11 +154,13 @@ export class SelectComponent extends AbstractInputComponent<number | string> {
     if (name) this.createNew.next(name)
     else this.createNew.next(this._lastSearchTerm)
     this.clearLastSearchTerm()
+    this.ngSelectComponent?.close()
   }
 
   clickNew() {
     this.createNew.next(this._lastSearchTerm)
     this.clearLastSearchTerm()
+    this.ngSelectComponent?.close()
   }
 
   clearLastSearchTerm() {

src/documents/tasks.py

@@ -4,6 +4,7 @@ import shutil
 import uuid
 import zipfile
 from collections.abc import Callable
+from importlib import import_module
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from tempfile import mkstemp
@@ -925,3 +926,19 @@ def cleanup_expired_share_link_bundles() -> None:
             )
     if count:
         logger.info("Deleted %s expired share link bundle(s)", count)
+
+
+@shared_task
+def clear_expired_sessions() -> None:
+    """
+    Removes expired sessions from the session store. Django does not prune these automatically.
+    """
+    engine = import_module(settings.SESSION_ENGINE)
+    try:
+        engine.SessionStore.clear_expired()
+        logger.info("Cleared expired sessions")
+    except NotImplementedError:
+        logger.warning(
+            "Session engine '%s' does not support clearing expired sessions",
+            settings.SESSION_ENGINE,
+        )

src/documents/tests/test_tasks.py

@@ -612,3 +612,57 @@ class TestApplyAISuggestionsTask(DirectoriesMixin, TestCase):
 
         apply_suggestions.assert_not_called()
         self.assertIn("no longer exists", "".join(cm.output))
+
+
+@pytest.mark.django_db
+class TestClearExpiredSessions:
+    def test_clears_only_expired_sessions(self) -> None:
+        """
+        GIVEN:
+            - One expired and one still-valid session in the session store
+        WHEN:
+            - The clear_expired_sessions task runs
+        THEN:
+            - Only the expired session is removed
+        """
+        from django.contrib.sessions.models import Session
+
+        now = timezone.now()
+        Session.objects.create(
+            session_key="expired",
+            session_data="",
+            expire_date=now - timedelta(days=1),
+        )
+        Session.objects.create(
+            session_key="valid",
+            session_data="",
+            expire_date=now + timedelta(days=1),
+        )
+
+        tasks.clear_expired_sessions()
+
+        assert set(Session.objects.values_list("session_key", flat=True)) == {"valid"}
+
+    def test_unsupported_engine_logs_warning(
+        self,
+        caplog: pytest.LogCaptureFixture,
+    ) -> None:
+        """
+        GIVEN:
+            - A session engine that does not support clearing expired sessions
+        WHEN:
+            - The clear_expired_sessions task runs
+        THEN:
+            - The error is swallowed and a warning is logged
+        """
+        fake_engine = mock.Mock()
+        fake_engine.SessionStore.clear_expired.side_effect = NotImplementedError
+
+        with (
+            mock.patch("documents.tasks.import_module", return_value=fake_engine),
+            override_settings(SESSION_ENGINE="some.unsupported.engine"),
+            caplog.at_level("WARNING", logger="paperless.tasks"),
+        ):
+            tasks.clear_expired_sessions()
+
+        assert "does not support clearing expired sessions" in caplog.text

src/paperless/settings/custom.py

@@ -165,6 +165,17 @@ def parse_beat_schedule() -> dict:
                 "expires": 23.0 * 60.0 * 60.0,
             },
         },
+        {
+            "name": "Clear expired sessions",
+            "env_key": "PAPERLESS_SESSION_CLEANUP_TASK_CRON",
+            # Default daily at 03:00
+            "env_default": "0 3 * * *",
+            "task": "documents.tasks.clear_expired_sessions",
+            "options": {
+                # 1 hour before default schedule sends again
+                "expires": 23.0 * 60.0 * 60.0,
+            },
+        },
     ]
     for task in tasks:
         # Either get the environment setting or use the default

src/paperless/tests/settings/test_custom_parsers.py

@@ -182,6 +182,7 @@ def make_expected_schedule(
     workflow_expire = 59.0 * 60.0
     llm_index_expire = 23.0 * 60.0 * 60.0
     share_link_cleanup_expire = 23.0 * 60.0 * 60.0
+    session_cleanup_expire = 23.0 * 60.0 * 60.0
 
     schedule: dict[str, Any] = {
         "Check all e-mail accounts": {
@@ -248,6 +249,14 @@ def make_expected_schedule(
                 "headers": {"trigger_source": "scheduled"},
             },
         },
+        "Clear expired sessions": {
+            "task": "documents.tasks.clear_expired_sessions",
+            "schedule": crontab(minute=0, hour="3"),
+            "options": {
+                "expires": session_cleanup_expire,
+                "headers": {"trigger_source": "scheduled"},
+            },
+        },
     }
 
     overrides = overrides or {}
@@ -298,6 +307,7 @@ class TestParseBeatSchedule:
                     "PAPERLESS_WORKFLOW_SCHEDULED_TASK_CRON": "disable",
                     "PAPERLESS_LLM_INDEX_TASK_CRON": "disable",
                     "PAPERLESS_SHARE_LINK_BUNDLE_CLEANUP_CRON": "disable",
+                    "PAPERLESS_SESSION_CLEANUP_TASK_CRON": "disable",
                 },
                 {},
                 id="all-disabled",

src/paperless_ai/chat.py

@@ -22,8 +22,8 @@ logger = logging.getLogger("paperless_ai.chat")
 CHAT_METADATA_DELIMITER = "\n\n__PAPERLESS_CHAT_METADATA__"
 CHAT_ERROR_MESSAGE = "Sorry, something went wrong while generating a response."
 CHAT_NO_CONTENT_MESSAGE = "Sorry, I couldn't find any content to answer your question."
-MAX_CHAT_REFERENCES = 3
 CHAT_RETRIEVER_TOP_K = 5
+MAX_CHAT_REFERENCES = CHAT_RETRIEVER_TOP_K
 
 
 def _build_chat_prompt(output_language: str | None) -> str: