Compare commits

..
10 changed files with 86 additions and 64 deletions
@@ -25,7 +25,7 @@
<input #textFilterInput class="form-control form-control-sm" type="text"
[disabled]="textFilterModifierIsNull"
[(ngModel)]="textFilter"
(keydown)="textFilterKeydown($event)"
(keyup)="textFilterKeyup($event)"
[ngbTypeahead]="searchAutoComplete"
(selectItem)="itemSelected($event)"
[readonly]="textFilterTarget === 'fulltext-morelike'">
@@ -2180,17 +2180,17 @@ describe('FilterEditorComponent', () => {
it('should support Enter / Esc key on text field', () => {
component.textFilterInput.nativeElement.value = 'foo'
component.textFilterInput.nativeElement.dispatchEvent(
new KeyboardEvent('keydown', { key: 'Enter' })
new KeyboardEvent('keyup', { key: 'Enter' })
)
expect(component.textFilter).toEqual('foo')
component.textFilterInput.nativeElement.value = 'foo bar'
component.textFilterInput.nativeElement.dispatchEvent(
new KeyboardEvent('keydown', { key: 'Escape' })
new KeyboardEvent('keyup', { key: 'Escape' })
)
expect(component.textFilter).toEqual('')
const blurSpy = jest.spyOn(component.textFilterInput.nativeElement, 'blur')
component.textFilterInput.nativeElement.dispatchEvent(
new KeyboardEvent('keydown', { key: 'Escape' })
new KeyboardEvent('keyup', { key: 'Escape' })
)
expect(blurSpy).toHaveBeenCalled()
})
@@ -1312,7 +1312,7 @@ export class FilterEditorComponent
}
}
textFilterKeydown(event: KeyboardEvent) {
textFilterKeyup(event: KeyboardEvent) {
if (event.key == 'Enter') {
const filterString = (
this.textFilterInput.nativeElement as HTMLInputElement
@@ -172,3 +172,39 @@ class TestApiUiSettings(DirectoriesMixin, APITestCase):
self.assertIsNotNone(
response.data["settings"]["outlook_oauth_url"],
)
@override_settings(
OAUTH_CALLBACK_BASE_URL="http://localhost:8000",
GMAIL_OAUTH_CLIENT_ID="abc123",
GMAIL_OAUTH_CLIENT_SECRET="def456",
GMAIL_OAUTH_ENABLED=True,
OUTLOOK_OAUTH_CLIENT_ID="ghi789",
OUTLOOK_OAUTH_CLIENT_SECRET="jkl012",
OUTLOOK_OAUTH_ENABLED=True,
)
def test_settings_oauth_state_reused_across_calls(self) -> None:
"""
GIVEN:
- A mail account oauth flow has been started, storing a state in the session
WHEN:
- The ui_settings endpoint is called again before the flow completes
THEN:
- The same oauth state is reused, not replaced with a new one
"""
response = self.client.get(self.ENDPOINT, format="json")
self.assertEqual(response.status_code, status.HTTP_200_OK)
first_state = self.client.session["oauth_state"]
first_gmail_url = response.data["settings"]["gmail_oauth_url"]
first_outlook_url = response.data["settings"]["outlook_oauth_url"]
response = self.client.get(self.ENDPOINT, format="json")
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(self.client.session["oauth_state"], first_state)
self.assertEqual(
response.data["settings"]["gmail_oauth_url"],
first_gmail_url,
)
self.assertEqual(
response.data["settings"]["outlook_oauth_url"],
first_outlook_url,
)
+7 -3
View File
@@ -4002,15 +4002,19 @@ class UiSettingsView(GenericAPIView[Any]):
ui_settings["auditlog_enabled"] = settings.AUDIT_LOG_ENABLED
if settings.GMAIL_OAUTH_ENABLED or settings.OUTLOOK_OAUTH_ENABLED:
manager = PaperlessMailOAuth2Manager()
# Reuse an in-flight state instead of minting a new one on every
# settings fetch, or a concurrent request can invalidate a login
# that's still in progress.
manager = PaperlessMailOAuth2Manager(
state=request.session.get("oauth_state"),
)
request.session["oauth_state"] = manager.state
if settings.GMAIL_OAUTH_ENABLED:
ui_settings["gmail_oauth_url"] = manager.get_gmail_authorization_url()
request.session["oauth_state"] = manager.state
if settings.OUTLOOK_OAUTH_ENABLED:
ui_settings["outlook_oauth_url"] = (
manager.get_outlook_authorization_url()
)
request.session["oauth_state"] = manager.state
ui_settings["email_enabled"] = settings.EMAIL_ENABLED
-2
View File
@@ -2,8 +2,6 @@ from pydantic import BaseModel
class DocumentClassifierSchema(BaseModel):
"""Schema for document classification suggestions."""
title: str
tags: list[str]
correspondents: list[str]
+7 -17
View File
@@ -5,7 +5,6 @@ from datetime import timedelta
from typing import TYPE_CHECKING
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from django.utils import timezone
from filelock import FileLock
from filelock import ReadWriteLock
@@ -168,19 +167,6 @@ def write_store(embed_model_name: str | None = None):
yield store
def _safe_related_name(document: Document, field: str) -> str | None:
"""
Returns the ``name`` of a related object (correspondent, document_type,
storage_path), or None if the FK is unset or points at a row that has
since been deleted (e.g. concurrently with this call).
"""
try:
related = getattr(document, field)
except ObjectDoesNotExist:
return None
return related.name if related else None
def build_document_node(
document: Document,
*,
@@ -194,10 +180,14 @@ def build_document_node(
"document_id": str(document.id),
"title": document.title,
"tags": [t.name for t in document.tags.all()],
"correspondent": _safe_related_name(document, "correspondent"),
"document_type": _safe_related_name(document, "document_type"),
"correspondent": document.correspondent.name
if document.correspondent
else None,
"document_type": document.document_type.name
if document.document_type
else None,
"filename": document.filename,
"storage_path": _safe_related_name(document, "storage_path"),
"storage_path": document.storage_path.name if document.storage_path else None,
"archive_serial_number": document.archive_serial_number,
"created": document.created.isoformat() if document.created else None,
"added": document.added.isoformat() if document.added else None,
@@ -8,9 +8,7 @@ from django.test import override_settings
from django.utils import timezone
from llama_index.core.schema import MetadataMode
from documents.models import Correspondent
from documents.models import Document
from documents.models import DocumentType
from documents.models import PaperlessTask
from documents.signals import document_consumption_finished
from documents.signals import document_updated
@@ -97,38 +95,6 @@ def test_build_document_node_structured_fields_in_metadata(
assert "modified" in node.metadata
@pytest.mark.django_db
def test_build_document_node_survives_concurrently_deleted_correspondent(
real_document: Document,
) -> None:
"""Regression test for #13314.
If a document's correspondent (or document type) is deleted after the
in-memory Document instance was loaded but before build_document_node
resolves the relation, accessing the FK must not raise -- it should
behave like an unset FK and produce None in the metadata instead of
aborting the whole indexing pass.
"""
correspondent = Correspondent.objects.create(name="Stale Correspondent")
document_type = DocumentType.objects.create(name="Stale Type")
real_document.correspondent = correspondent
real_document.document_type = document_type
real_document.save()
# Re-fetch to get an instance whose correspondent/document_type relations
# are unresolved (not yet cached), mirroring a task that loaded the
# document before the concurrent deletion below.
stale_document = Document.objects.get(pk=real_document.pk)
correspondent.delete()
document_type.delete()
nodes = indexing.build_document_node(stale_document)
assert len(nodes) > 0
assert nodes[0].metadata["correspondent"] is None
assert nodes[0].metadata["document_type"] is None
@pytest.mark.django_db
def test_build_document_node_excludes_document_id_from_llm_context(
real_document: Document,
@@ -138,6 +138,16 @@ class TestMailOAuth(
MailAccount.objects.filter(imap_server="imap.gmail.com").exists(),
)
# State is single-use and was cleared by the callback above, so a new
# flow needs a new state stored in the session.
session = self.client.session
session.update(
{
"oauth_state": "test_state",
},
)
session.save()
# Test Outlook OAuth callback
response = self.client.get(
"/api/oauth/callback/?code=test_code&state=test_state",
@@ -180,6 +190,16 @@ class TestMailOAuth(
MailAccount.objects.filter(imap_server="imap.gmail.com").exists(),
)
# State is single-use and was cleared by the callback above, so a
# new flow needs a new state stored in the session.
session = self.client.session
session.update(
{
"oauth_state": "test_state",
},
)
session.save()
# Test Outlook OAuth callback
response = self.client.get(
"/api/oauth/callback/?code=test_code&state=test_state",
+11 -3
View File
@@ -4,6 +4,7 @@ from datetime import timedelta
from http import HTTPStatus
from typing import Any
from django.conf import settings
from django.http import HttpResponseBadRequest
from django.http import HttpResponseForbidden
from django.http import HttpResponseRedirect
@@ -256,9 +257,14 @@ class OauthCallbackView(GenericAPIView[Any]):
)
return HttpResponseBadRequest("Invalid request, see logs for more detail")
oauth_manager = PaperlessMailOAuth2Manager(
state=request.session.get("oauth_state"),
)
session_state = request.session.get("oauth_state")
if not session_state and not settings.DEBUG:
logger.error(
"Invalid oauth callback request: no state in session",
)
return HttpResponseBadRequest("Invalid request, see logs for more detail")
oauth_manager = PaperlessMailOAuth2Manager(state=session_state)
state = request.query_params.get("state", "")
if not oauth_manager.validate_state(state):
@@ -315,3 +321,5 @@ class OauthCallbackView(GenericAPIView[Any]):
return HttpResponseRedirect(
f"{oauth_manager.oauth_redirect_url}?oauth_success=0",
)
finally:
request.session.pop("oauth_state", None)