Compare commits

..
Author SHA1 Message Date
shamoon 01a0880e6f prevent overwriting index
Nothing actually passed in something different for the effective_content args, so drop them!
2026-08-19 16:50:17 -07:00
shamoon ac5a39a3d4 Fix: always index the root for a version 2026-08-19 16:40:31 -07:00
15 changed files with 162 additions and 277 deletions
+1 -1
View File
@@ -1215,7 +1215,7 @@ should be a valid crontab(5) expression describing when to run.
: If set to the string "disable", no emails will be fetched automatically. : If set to the string "disable", no emails will be fetched automatically.
Defaults to every ten minutes, with an installation-specific minute offset. Defaults to `*/10 * * * *` or every ten minutes.
#### [`PAPERLESS_TRAIN_TASK_CRON=<cron expression>`](#PAPERLESS_TRAIN_TASK_CRON) {#PAPERLESS_TRAIN_TASK_CRON} #### [`PAPERLESS_TRAIN_TASK_CRON=<cron expression>`](#PAPERLESS_TRAIN_TASK_CRON) {#PAPERLESS_TRAIN_TASK_CRON}
+17 -17
View File
@@ -2195,7 +2195,7 @@
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context> <context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context>
<context context-type="linenumber">67,68</context> <context context-type="linenumber">64,65</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="2134950584701094962" datatype="html"> <trans-unit id="2134950584701094962" datatype="html">
@@ -6180,7 +6180,7 @@
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context> <context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context>
<context context-type="linenumber">38,39</context> <context context-type="linenumber">35,36</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="5554528553553249088" datatype="html"> <trans-unit id="5554528553553249088" datatype="html">
@@ -8361,11 +8361,11 @@
<source>An error occurred loading tiff: <x id="PH" equiv-text="err.toString()"/></source> <source>An error occurred loading tiff: <x id="PH" equiv-text="err.toString()"/></source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context> <context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">2025</context> <context context-type="linenumber">2023</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context> <context context-type="sourcefile">src/app/components/document-detail/document-detail.component.ts</context>
<context context-type="linenumber">2031</context> <context context-type="linenumber">2029</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="4958946940233632319" datatype="html"> <trans-unit id="4958946940233632319" datatype="html">
@@ -8407,88 +8407,88 @@
<source>Label</source> <source>Label</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context> <context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context>
<context context-type="linenumber">13,14</context> <context context-type="linenumber">10,11</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="3915966149686975421" datatype="html"> <trans-unit id="3915966149686975421" datatype="html">
<source>Optional</source> <source>Optional</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context> <context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context>
<context context-type="linenumber">20,21</context> <context context-type="linenumber">17,18</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="3009569956897708604" datatype="html"> <trans-unit id="3009569956897708604" datatype="html">
<source>Upload a new version</source> <source>Upload a new version</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context> <context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context>
<context context-type="linenumber">35,36</context> <context context-type="linenumber">32,33</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="278763911843514709" datatype="html"> <trans-unit id="278763911843514709" datatype="html">
<source>Use an existing document</source> <source>Use an existing document</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context> <context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context>
<context context-type="linenumber">44,45</context> <context context-type="linenumber">41,42</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="1906507263389090131" datatype="html"> <trans-unit id="1906507263389090131" datatype="html">
<source>Existing</source> <source>Existing</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context> <context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context>
<context context-type="linenumber">47,48</context> <context context-type="linenumber">44,45</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="3119565892291077820" datatype="html"> <trans-unit id="3119565892291077820" datatype="html">
<source>Uploading version...</source> <source>Uploading version...</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context> <context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context>
<context context-type="linenumber">55,56</context> <context context-type="linenumber">52,53</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="8538204518127762311" datatype="html"> <trans-unit id="8538204518127762311" datatype="html">
<source>Processing version...</source> <source>Processing version...</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context> <context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context>
<context context-type="linenumber">61,62</context> <context context-type="linenumber">58,59</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="6587183611950398104" datatype="html"> <trans-unit id="6587183611950398104" datatype="html">
<source>Version upload failed.</source> <source>Version upload failed.</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context> <context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context>
<context context-type="linenumber">66,67</context> <context context-type="linenumber">63,64</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="6328986243040954523" datatype="html"> <trans-unit id="6328986243040954523" datatype="html">
<source>Version label</source> <source>Version label</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context> <context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context>
<context context-type="linenumber">91,92</context> <context context-type="linenumber">88,89</context>
</context-group> </context-group>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context> <context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context>
<context context-type="linenumber">101,102</context> <context context-type="linenumber">98,99</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="2724055831234181057" datatype="html"> <trans-unit id="2724055831234181057" datatype="html">
<source>Version</source> <source>Version</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context> <context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context>
<context context-type="linenumber">112</context> <context context-type="linenumber">109</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="2491751767883178298" datatype="html"> <trans-unit id="2491751767883178298" datatype="html">
<source>Delete this version?</source> <source>Delete this version?</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context> <context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context>
<context context-type="linenumber">143,144</context> <context context-type="linenumber">140,141</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="894108684749537200" datatype="html"> <trans-unit id="894108684749537200" datatype="html">
<source>Delete version</source> <source>Delete version</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context> <context context-type="sourcefile">src/app/components/document-detail/document-version-dropdown/document-version-dropdown.component.html</context>
<context context-type="linenumber">149,150</context> <context context-type="linenumber">146,147</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="9111284310237278846" datatype="html"> <trans-unit id="9111284310237278846" datatype="html">
@@ -2171,11 +2171,6 @@ describe('DocumentDetailComponent', () => {
).toBe(10) ).toBe(10)
component.openEmailDocument() component.openEmailDocument()
expect(modalSpy).toHaveBeenCalled() expect(modalSpy).toHaveBeenCalled()
expect(
(
modalSpy.mock.results[1].value as NgbModalRef
).componentInstance.documentIds()
).toEqual([10])
}) })
it('should set previewText', () => { it('should set previewText', () => {
@@ -1973,9 +1973,7 @@ export class DocumentDetailComponent
const modal = this.modalService.open(EmailDocumentDialogComponent, { const modal = this.modalService.open(EmailDocumentDialogComponent, {
backdrop: 'static', backdrop: 'static',
}) })
modal.componentInstance.documentIds.set([ modal.componentInstance.documentIds.set([this.document().id])
this.selectedVersionId() ?? this.document().id,
])
modal.componentInstance.hasArchiveVersion.set( modal.componentInstance.hasArchiveVersion.set(
this.metadata()?.has_archive_version ?? this.metadata()?.has_archive_version ??
!!this.document()?.archived_file_name !!this.document()?.archived_file_name
@@ -2,9 +2,6 @@
<button class="btn btn-sm btn-outline-secondary dropdown-toggle" ngbDropdownToggle> <button class="btn btn-sm btn-outline-secondary dropdown-toggle" ngbDropdownToggle>
<i-bs name="file-earmark-diff"></i-bs> <i-bs name="file-earmark-diff"></i-bs>
<span class="d-none d-lg-inline ps-1" i18n>Versions</span> <span class="d-none d-lg-inline ps-1" i18n>Versions</span>
@if (versions.length > 1) {
<span class="badge text-bg-secondary ms-1">{{ versions.length }}</span>
}
</button> </button>
<div class="dropdown-menu shadow" ngbDropdownMenu> <div class="dropdown-menu shadow" ngbDropdownMenu>
<div class="px-3 py-2 mb-2"> <div class="px-3 py-2 mb-2">
+9 -22
View File
@@ -266,11 +266,7 @@ class WriteBatch:
if self._lock is not None: if self._lock is not None:
self._lock.release() self._lock.release()
def add_or_update( def add_or_update(self, document: Document) -> None:
self,
document: Document,
effective_content: str | None = None,
) -> None:
""" """
Add or update a document in the batch. Add or update a document in the batch.
@@ -280,11 +276,9 @@ class WriteBatch:
Args: Args:
document: Django Document instance to index document: Django Document instance to index
effective_content: Override document.content for indexing (used when
re-indexing with newer OCR text from document versions)
""" """
self.remove(document.pk) self.remove(document.pk)
doc = self._backend._build_tantivy_doc(document, effective_content) doc = self._backend._build_tantivy_doc(document)
self._writer.add_document(doc) self._writer.add_document(doc)
def remove(self, doc_id: int) -> None: def remove(self, doc_id: int) -> None:
@@ -425,18 +419,17 @@ class TantivyBackend:
def _build_tantivy_doc( def _build_tantivy_doc(
self, self,
document: Document, document: Document,
effective_content: str | None = None,
viewer_ids: list[int] | None = None, viewer_ids: list[int] | None = None,
viewer_group_ids: list[int] | None = None, viewer_group_ids: list[int] | None = None,
) -> tantivy.Document: ) -> tantivy.Document:
"""Build a tantivy Document from a Django Document instance. """Build a tantivy Document from a Django Document instance.
``effective_content`` overrides ``document.content`` for indexing — A root document is indexed with its effective content, i.e. the newest
used when re-indexing a root document with a newer version's OCR text. version's OCR text, so it is never indexed with its own outdated text.
Annotate the queryset with ``annotate_effective_content`` when indexing
more than a couple of documents, to resolve that without a query each.
""" """
content = ( content = document.get_effective_content() or ""
effective_content if effective_content is not None else document.content
)
doc = tantivy.Document() doc = tantivy.Document()
@@ -584,11 +577,7 @@ class TantivyBackend:
return doc return doc
def add_or_update( def add_or_update(self, document: Document) -> None:
self,
document: Document,
effective_content: str | None = None,
) -> None:
""" """
Add or update a single document with file locking. Add or update a single document with file locking.
@@ -601,12 +590,11 @@ class TantivyBackend:
Args: Args:
document: Django Document instance to index document: Django Document instance to index
effective_content: Override document.content for indexing
""" """
self._ensure_open() self._ensure_open()
try: try:
with self.batch_update(lock_timeout=_LOCK_TIMEOUT_SECONDS) as batch: with self.batch_update(lock_timeout=_LOCK_TIMEOUT_SECONDS) as batch:
batch.add_or_update(document, effective_content) batch.add_or_update(document)
except SearchIndexLockError: except SearchIndexLockError:
logger.error( logger.error(
"Search index lock exhausted for document %d after %d attempts; " "Search index lock exhausted for document %d after %d attempts; "
@@ -1027,7 +1015,6 @@ class TantivyBackend:
): ):
doc = self._build_tantivy_doc( doc = self._build_tantivy_doc(
document, document,
document.get_effective_content(),
viewer_ids=viewer_ids, viewer_ids=viewer_ids,
viewer_group_ids=viewer_group_ids, viewer_group_ids=viewer_group_ids,
) )
+6 -4
View File
@@ -794,10 +794,12 @@ def cleanup_user_deletion(sender, instance: User | Group, **kwargs) -> None:
def add_to_index(sender, document, **kwargs) -> None: def add_to_index(sender, document, **kwargs) -> None:
from documents.search import get_backend from documents.search import get_backend
get_backend().add_or_update( # A newly consumed version is not searchable on its own, its content
document, # becomes the effective_content of the root document
effective_content=document.get_effective_content(), if document.root_document_id:
) document = document.root_document
get_backend().add_or_update(document)
def run_workflows_added( def run_workflows_added(
+6 -5
View File
@@ -64,6 +64,7 @@ from documents.signals.handlers import send_websocket_document_updated
from documents.utils import IterWrapper from documents.utils import IterWrapper
from documents.utils import compute_checksum from documents.utils import compute_checksum
from documents.utils import identity from documents.utils import identity
from documents.versioning import annotate_effective_content
from documents.workflows.utils import get_workflows_for_trigger from documents.workflows.utils import get_workflows_for_trigger
from paperless.config import AIConfig from paperless.config import AIConfig
from paperless.logging import consume_task_id from paperless.logging import consume_task_id
@@ -114,10 +115,7 @@ def index_document(self, document_id: int) -> None:
) )
return return
with get_backend().batch_update() as batch: with get_backend().batch_update() as batch:
batch.add_or_update( batch.add_or_update(document)
document,
effective_content=document.get_effective_content(),
)
@shared_task( @shared_task(
@@ -312,7 +310,10 @@ def bulk_update_documents(document_ids) -> None:
from documents.search import get_backend from documents.search import get_backend
document_ids = list(document_ids) document_ids = list(document_ids)
documents = Document.objects.filter(id__in=document_ids) # Annotated so indexing below doesn't query the versions of each document
documents = annotate_effective_content(
Document.objects.filter(id__in=document_ids),
)
for doc in documents: for doc in documents:
clear_document_caches(doc.pk) clear_document_caches(doc.pk)
@@ -16,6 +16,7 @@ from documents.search._backend import TantivyBackend
from documents.search._backend import WriteBatch from documents.search._backend import WriteBatch
from documents.search._backend import get_backend from documents.search._backend import get_backend
from documents.search._backend import reset_backend from documents.search._backend import reset_backend
from documents.signals.handlers import add_to_index
from documents.tests.factories import CorrespondentFactory from documents.tests.factories import CorrespondentFactory
from documents.tests.factories import DocumentFactory from documents.tests.factories import DocumentFactory
from documents.tests.factories import DocumentTypeFactory from documents.tests.factories import DocumentTypeFactory
@@ -1030,6 +1031,81 @@ class TestHighlightHits:
assert len(hits) == 0 assert len(hits) == 0
class TestVersionIndexing:
"""
GIVEN:
- A root document whose new version has just been consumed, e.g. by
the password removal workflow action
WHEN:
- The consumption finished signal is handled
THEN:
- The root document is indexed with the new version's content, since
versions are not searchable on their own
"""
def test_consumed_version_updates_root_entry(
self,
backend: TantivyBackend,
mocker: MockerFixture,
) -> None:
root = Document.objects.create(
title="Statement",
content="",
checksum="VER1",
pk=90,
)
backend.add_or_update(root)
version = Document.objects.create(
title="Statement",
content="unprotected statement text",
checksum="VER2",
pk=91,
root_document=root,
version_index=1,
)
mocker.patch("documents.search.get_backend", return_value=backend)
add_to_index(sender=None, document=version)
assert backend.search_ids("unprotected", user=None) == [root.pk]
class TestEffectiveContentIndexing:
"""
GIVEN:
- A root document with a newer version
WHEN:
- The root document is indexed
THEN:
- The newest version's content is indexed, never the root's own
outdated text
"""
def test_root_is_indexed_with_latest_version_content(
self,
backend: TantivyBackend,
) -> None:
root = Document.objects.create(
title="Statement",
content="stale original text",
checksum="EFF1",
pk=95,
)
Document.objects.create(
title="Statement",
content="latest version text",
checksum="EFF2",
pk=96,
root_document=root,
version_index=1,
)
backend.add_or_update(root)
assert backend.search_ids("latest", user=None) == [root.pk]
assert backend.search_ids("stale", user=None) == []
class TestIndexDirectoryGarbageCollection: class TestIndexDirectoryGarbageCollection:
"""Regression tests for Tantivy segment files leaking on disk when """Regression tests for Tantivy segment files leaking on disk when
multiple long-lived worker processes (Granian/Celery) take turns writing multiple long-lived worker processes (Granian/Celery) take turns writing
-139
View File
@@ -1360,145 +1360,6 @@ class TestBulkEditObjectPermissions(APITestCase):
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
self.assertEqual(response.content, b"Insufficient permissions") self.assertEqual(response.content, b"Insufficient permissions")
def test_bulk_edit_object_permissions_shared_object_not_owner(self) -> None:
"""
GIVEN:
- Object owned by another user, shared with the logged in user with
change permissions
WHEN:
- bulk_edit_objects API endpoint is called with set_permissions operation
THEN:
- User is not able to take ownership or change permissions, consistent
with the single object API
"""
self.t1.owner = self.user2
self.t1.save()
assign_perm("view_tag", self.user1, self.t1)
assign_perm("change_tag", self.user1, self.t1)
self.user1.user_permissions.add(
*Permission.objects.filter(
codename__in=["view_tag", "change_tag"],
),
)
user1 = User.objects.get(pk=self.user1.pk)
self.client.force_authenticate(user=user1)
response = self.client.post(
"/api/bulk_edit_objects/",
json.dumps(
{
"objects": [self.t1.id],
"object_type": "tags",
"operation": "set_permissions",
"owner": user1.id,
"permissions": {
"view": {"users": [user1.id], "groups": []},
"change": {"users": [user1.id], "groups": []},
},
"merge": False,
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
self.assertEqual(Tag.objects.get(pk=self.t1.id).owner, self.user2)
# the single object endpoint refuses the same request
response = self.client.patch(
f"/api/tags/{self.t1.id}/",
{"owner": user1.id},
format="json",
)
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
self.assertEqual(Tag.objects.get(pk=self.t1.id).owner, self.user2)
def test_bulk_edit_object_permissions_all_with_shared_objects(self) -> None:
"""
GIVEN:
- Objects owned by the logged in user, unowned objects and objects owned
by another user but shared with the logged in user
WHEN:
- bulk_edit_objects API endpoint is called with set_permissions operation
and all = True
THEN:
- The request is refused and no objects are changed
"""
owned = Tag.objects.create(name="owned", owner=self.user1)
shared = Tag.objects.create(name="shared", owner=self.user2)
assign_perm("view_tag", self.user1, shared)
assign_perm("change_tag", self.user1, shared)
self.user1.user_permissions.add(
*Permission.objects.filter(
codename__in=["view_tag", "change_tag"],
),
)
user1 = User.objects.get(pk=self.user1.pk)
self.client.force_authenticate(user=user1)
response = self.client.post(
"/api/bulk_edit_objects/",
json.dumps(
{
"objects": [],
"all": True,
"object_type": "tags",
"operation": "set_permissions",
"permissions": {
"view": {"users": [self.user3.id], "groups": []},
"change": {"users": [self.user3.id], "groups": []},
},
"merge": False,
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
# nothing was changed, including the objects the user does own
self.assertNotIn(self.user3, get_users_with_perms(owned))
self.assertNotIn(self.user3, get_users_with_perms(self.t1))
self.assertNotIn(self.user3, get_users_with_perms(shared))
self.assertEqual(Tag.objects.get(pk=shared.pk).owner, self.user2)
def test_bulk_edit_object_delete_shared_object_not_owner(self) -> None:
"""
GIVEN:
- Object owned by another user, shared with the logged in user with
change and delete permissions
WHEN:
- bulk_edit_objects API endpoint is called with delete operation
THEN:
- User is not able to delete the object, consistent with documents
"""
self.t1.owner = self.user2
self.t1.save()
assign_perm("view_tag", self.user1, self.t1)
assign_perm("change_tag", self.user1, self.t1)
assign_perm("delete_tag", self.user1, self.t1)
self.user1.user_permissions.add(
*Permission.objects.filter(
codename__in=["view_tag", "change_tag", "delete_tag"],
),
)
user1 = User.objects.get(pk=self.user1.pk)
self.client.force_authenticate(user=user1)
response = self.client.post(
"/api/bulk_edit_objects/",
json.dumps(
{
"objects": [self.t1.id],
"object_type": "tags",
"operation": "delete",
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
self.assertTrue(Tag.objects.filter(pk=self.t1.id).exists())
def test_bulk_edit_object_permissions_validation(self) -> None: def test_bulk_edit_object_permissions_validation(self) -> None:
""" """
GIVEN: GIVEN:
@@ -677,54 +677,16 @@ class TestBulkEditObjectsApplyToAllPermissionBoundary:
def test_apply_to_all_tags_excludes_unpermitted_tag(self, rest_api_client): def test_apply_to_all_tags_excludes_unpermitted_tag(self, rest_api_client):
owner = User.objects.create_user(username="tags_owner") owner = User.objects.create_user(username="tags_owner")
requester = User.objects.create_user(username="tags_requester") requester = User.objects.create_user(username="tags_requester")
new_owner = User.objects.create_user(username="tags_new_owner")
# grant the global change_tag permission so the object-level # grant the global change_tag permission so the object-level
# filtering (not the global has_perm check) is what's under test # filtering (not the global has_perm check) is what's under test
requester.user_permissions.add( requester.user_permissions.add(
Permission.objects.get(codename="change_tag"), Permission.objects.get(codename="change_tag"),
) )
rest_api_client.force_authenticate(user=requester) rest_api_client.force_authenticate(user=requester)
visible = TagFactory(owner=requester) visible = TagFactory(owner=owner)
hidden = TagFactory(owner=owner) hidden = TagFactory(owner=owner)
assign_perm("view_tag", requester, visible)
response = rest_api_client.post( assign_perm("change_tag", requester, visible)
"/api/bulk_edit_objects/",
{
"object_type": "tags",
"operation": "set_permissions",
"all": True,
"filters": {},
"owner": new_owner.pk,
},
format="json",
)
assert response.status_code == HTTPStatus.OK
# The apply_to_all dispatch must resolve permitted objects up front:
# the requester's own tag gets its owner reassigned, while the tag
# owned by someone else is excluded entirely and keeps its owner.
visible.refresh_from_db()
hidden.refresh_from_db()
assert visible.owner == new_owner
assert hidden.owner == owner
def test_apply_to_all_tags_refuses_shared_but_unowned_tag(self, rest_api_client):
"""
A tag owned by someone else but shared with the requester is inside the
permitted set, so it reaches the ownership gate and fails the whole
request rather than being silently skipped. Editing permissions is
limited to the owner, same as documents.
"""
owner = User.objects.create_user(username="shared_tags_owner")
requester = User.objects.create_user(username="shared_tags_requester")
requester.user_permissions.add(
Permission.objects.get(codename="change_tag"),
)
rest_api_client.force_authenticate(user=requester)
owned = TagFactory(owner=requester)
shared = TagFactory(owner=owner)
assign_perm("view_tag", requester, shared)
assign_perm("change_tag", requester, shared)
response = rest_api_client.post( response = rest_api_client.post(
"/api/bulk_edit_objects/", "/api/bulk_edit_objects/",
@@ -737,12 +699,16 @@ class TestBulkEditObjectsApplyToAllPermissionBoundary:
}, },
format="json", format="json",
) )
assert response.status_code == HTTPStatus.FORBIDDEN assert response.status_code == HTTPStatus.OK
owned.refresh_from_db() # The apply_to_all dispatch must resolve permitted objects up front:
shared.refresh_from_db() # the visible tag (object-level change_tag granted) gets its owner
assert shared.owner == owner # reassigned, while the hidden tag (no object-level grant) is
assert owned.owner == requester # excluded entirely and keeps its original owner.
visible.refresh_from_db()
hidden.refresh_from_db()
assert visible.owner == requester
assert hidden.owner == owner
@pytest.mark.django_db @pytest.mark.django_db
@@ -754,8 +720,8 @@ class TestBulkEditObjectsTagDescendantPartialPermission:
""" """
GIVEN: GIVEN:
- A tag hierarchy (parent -> permitted_child, unpermitted_child) - A tag hierarchy (parent -> permitted_child, unpermitted_child)
- A non-superuser requester who owns the parent and only ONE of - A non-superuser requester with object-level change_tag granted
the two children on the parent and on only ONE of the two children
WHEN: WHEN:
- bulk_edit_objects is called with all=True and a filter that - bulk_edit_objects is called with all=True and a filter that
matches only the root (parent) tag, engaging the matches only the root (parent) tag, engaging the
@@ -777,7 +743,6 @@ class TestBulkEditObjectsTagDescendantPartialPermission:
""" """
owner = User.objects.create_user(username="tag_hierarchy_owner") owner = User.objects.create_user(username="tag_hierarchy_owner")
requester = User.objects.create_user(username="tag_hierarchy_requester") requester = User.objects.create_user(username="tag_hierarchy_requester")
new_owner = User.objects.create_user(username="tag_hierarchy_new_owner")
# global change_tag permission so the has_perm() gate passes and the # global change_tag permission so the has_perm() gate passes and the
# object-level permitted_object_ids filtering is what's under test # object-level permitted_object_ids filtering is what's under test
requester.user_permissions.add( requester.user_permissions.add(
@@ -785,9 +750,9 @@ class TestBulkEditObjectsTagDescendantPartialPermission:
) )
rest_api_client.force_authenticate(user=requester) rest_api_client.force_authenticate(user=requester)
parent = TagFactory(owner=requester, name="parent-tag") parent = TagFactory(owner=owner, name="parent-tag")
permitted_child = TagFactory( permitted_child = TagFactory(
owner=requester, owner=owner,
name="permitted-child-tag", name="permitted-child-tag",
tn_parent=parent, tn_parent=parent,
) )
@@ -796,6 +761,9 @@ class TestBulkEditObjectsTagDescendantPartialPermission:
name="unpermitted-child-tag", name="unpermitted-child-tag",
tn_parent=parent, tn_parent=parent,
) )
assign_perm("change_tag", requester, parent)
assign_perm("change_tag", requester, permitted_child)
# unpermitted_child is intentionally NOT granted change_tag
response = rest_api_client.post( response = rest_api_client.post(
"/api/bulk_edit_objects/", "/api/bulk_edit_objects/",
@@ -804,7 +772,7 @@ class TestBulkEditObjectsTagDescendantPartialPermission:
"operation": "set_permissions", "operation": "set_permissions",
"all": True, "all": True,
"filters": {"is_root": True}, "filters": {"is_root": True},
"owner": new_owner.pk, "owner": requester.pk,
}, },
format="json", format="json",
) )
@@ -813,8 +781,8 @@ class TestBulkEditObjectsTagDescendantPartialPermission:
parent.refresh_from_db() parent.refresh_from_db()
permitted_child.refresh_from_db() permitted_child.refresh_from_db()
unpermitted_child.refresh_from_db() unpermitted_child.refresh_from_db()
assert parent.owner == new_owner assert parent.owner == requester
assert permitted_child.owner == new_owner assert permitted_child.owner == requester
assert unpermitted_child.owner == owner assert unpermitted_child.owner == owner
+21
View File
@@ -6,7 +6,10 @@ from typing import TYPE_CHECKING
from typing import Any from typing import Any
from django.db.models import F from django.db.models import F
from django.db.models import OuterRef
from django.db.models import QuerySet from django.db.models import QuerySet
from django.db.models import Subquery
from django.db.models.functions import Coalesce
from documents.models import Document from documents.models import Document
@@ -22,6 +25,24 @@ def versions_newest_first(documents: QuerySet[Document]) -> QuerySet[Document]:
return documents.order_by(F("version_index").desc(nulls_last=True), "-id") return documents.order_by(F("version_index").desc(nulls_last=True), "-id")
def annotate_effective_content(documents: QuerySet[Document]) -> QuerySet[Document]:
"""
Annotates documents with the content of their newest version, falling back
to their own, so get_effective_content() can answer from the row rather
than querying for the versions of each document
"""
return documents.annotate(
effective_content=Coalesce(
Subquery(
versions_newest_first(
Document.objects.filter(root_document=OuterRef("pk")),
).values("content")[:1],
),
F("content"),
),
)
def sort_versions_newest_first(documents: list[Document]) -> list[Document]: def sort_versions_newest_first(documents: list[Document]) -> list[Document]:
""" """
Same sorting as versions_newest_first() Same sorting as versions_newest_first()
+1 -2
View File
@@ -4879,11 +4879,10 @@ class BulkEditObjectsView(PassUserMixin):
if not user.is_superuser: if not user.is_superuser:
perm = f"documents.{perm_codename}" perm = f"documents.{perm_codename}"
# Limited to the owner (or unowned), same as documents, see BulkEditView
has_perms = ( has_perms = (
user.has_perm(perm) user.has_perm(perm)
and not objs.exclude( and not objs.exclude(
Q(owner=user) | Q(owner__isnull=True), pk__in=permitted_object_ids(user, object_class, perm_codename),
).exists() ).exists()
) )
-10
View File
@@ -1,7 +1,6 @@
import datetime import datetime
import logging import logging
import os import os
from hashlib import sha256
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
@@ -173,15 +172,6 @@ def parse_beat_schedule() -> dict:
# Don't add disabled tasks to the schedule # Don't add disabled tasks to the schedule
if value == "disable": if value == "disable":
continue continue
if (
task["env_key"] == "PAPERLESS_EMAIL_TASK_CRON"
and task["env_key"] not in os.environ
):
# Spread default polling across the ten-minute interval.
secret = os.environ["PAPERLESS_SECRET_KEY"].encode()
offset = int.from_bytes(sha256(secret).digest()) % 10
minutes = ",".join(str(minute) for minute in range(offset, 60, 10))
value = f"{minutes} * * * *"
# I find https://crontab.guru/ super helpful # I find https://crontab.guru/ super helpful
# crontab(5) format # crontab(5) format
# - five time-and-date fields # - five time-and-date fields
@@ -168,7 +168,6 @@ class TestParseHostingSettings:
def make_expected_schedule( def make_expected_schedule(
overrides: dict[str, dict[str, Any]] | None = None, overrides: dict[str, dict[str, Any]] | None = None,
disabled: set[str] | None = None, disabled: set[str] | None = None,
email_minute: str = "6,16,26,36,46,56",
) -> dict[str, Any]: ) -> dict[str, Any]:
""" """
Build the expected schedule with optional overrides and disabled tasks. Build the expected schedule with optional overrides and disabled tasks.
@@ -186,7 +185,7 @@ def make_expected_schedule(
schedule: dict[str, Any] = { schedule: dict[str, Any] = {
"Check all e-mail accounts": { "Check all e-mail accounts": {
"task": "paperless_mail.tasks.process_mail_accounts", "task": "paperless_mail.tasks.process_mail_accounts",
"schedule": crontab(minute=email_minute), "schedule": crontab(minute="*/10"),
"options": { "options": {
"expires": mail_expire, "expires": mail_expire,
"headers": {"trigger_source": "scheduled"}, "headers": {"trigger_source": "scheduled"},
@@ -267,11 +266,6 @@ class TestParseBeatSchedule:
("env", "expected"), ("env", "expected"),
[ [
pytest.param({}, make_expected_schedule(), id="defaults"), pytest.param({}, make_expected_schedule(), id="defaults"),
pytest.param(
{"PAPERLESS_EMAIL_TASK_CRON": "*/10 * * * *"},
make_expected_schedule(email_minute="*/10"),
id="email-explicit-default",
),
pytest.param( pytest.param(
{"PAPERLESS_EMAIL_TASK_CRON": "*/50 * * * mon"}, {"PAPERLESS_EMAIL_TASK_CRON": "*/50 * * * mon"},
make_expected_schedule( make_expected_schedule(
@@ -310,11 +304,7 @@ class TestParseBeatSchedule:
expected: dict[str, Any], expected: dict[str, Any],
mocker: MockerFixture, mocker: MockerFixture,
) -> None: ) -> None:
mocker.patch.dict( mocker.patch.dict(os.environ, env, clear=False)
os.environ,
{"PAPERLESS_SECRET_KEY": "test-secret", **env},
clear=False,
)
schedule = parse_beat_schedule() schedule = parse_beat_schedule()
assert schedule == expected assert schedule == expected