Compare commits

...
Author SHA1 Message Date
shamoon 320cf98254 documents_without_permitted_root helper used x4 2026-10-08 09:35:14 -07:00
shamoon 7588cdb952 include selectiondataview too 2026-10-08 09:28:44 -07:00
shamoon 0e27a1a327 documentfactory 2026-10-08 09:28:02 -07:00
shamoon 9821e4f73c Trash fix 2026-10-07 16:08:43 -07:00
shamoon 31776986e5 ai chat too 2026-10-07 16:03:13 -07:00
shamoon 94ae632920 Update test 2026-10-07 15:52:58 -07:00
shamoon 56d5bb7255 ai_suggestions 2026-10-07 15:52:31 -07:00
shamoon 2df81ce44e Fix: consistently use root doc for version action permissions 2026-10-07 15:37:01 -07:00
Trenton H d7a9894400 Fix: retry a search index rebuild that was interrupted (#14379)
An interrupted rebuild left an empty index stamped as current, so the next
start reported it as up to date. Mark the rebuild as in progress and only
clear the marker once it completes.
2026-10-07 08:54:36 -07:00
GitHub Actions ee34a6598e Auto translate strings 2026-10-06 15:13:08 +00:00
Trenton H 3a3b3ef66a Chore: Upgrade runners to Ubuntu 26.04 (#14347)
* Moves runners to 26.04 and a few jobs to -slim variant

* Probably fixing the imagemagik 7 problems and maybe the frontend playwright thing?

* Compare thumbnails, but allow a little difference in the perceptual hash
2026-10-06 08:11:44 -07:00
25 changed files with 507 additions and 158 deletions

No files matched your search

+3 -3
View File
@@ -80,7 +80,7 @@ jobs:
needs: changes needs: changes
if: needs.changes.outputs.backend_changed == 'true' if: needs.changes.outputs.backend_changed == 'true'
name: "Python ${{ matrix.python-version }}" name: "Python ${{ matrix.python-version }}"
runs-on: ubuntu-24.04 runs-on: ubuntu-26.04
permissions: permissions:
contents: read contents: read
strategy: strategy:
@@ -114,7 +114,7 @@ jobs:
packages: unpaper tesseract-ocr imagemagick ghostscript poppler-utils packages: unpaper tesseract-ocr imagemagick ghostscript poppler-utils
- name: Configure ImageMagick - name: Configure ImageMagick
run: | run: |
sudo cp docker/rootfs/etc/ImageMagick-6/paperless-policy.xml /etc/ImageMagick-6/policy.xml sudo cp docker/rootfs/etc/ImageMagick-6/paperless-policy.xml /etc/ImageMagick-7/policy.xml
- name: Install Python dependencies - name: Install Python dependencies
env: env:
PYTHON_VERSION: ${{ steps.setup-python.outputs.python-version }} PYTHON_VERSION: ${{ steps.setup-python.outputs.python-version }}
@@ -158,7 +158,7 @@ jobs:
needs: changes needs: changes
if: needs.changes.outputs.backend_changed == 'true' if: needs.changes.outputs.backend_changed == 'true'
name: Check project typing name: Check project typing
runs-on: ubuntu-24.04 runs-on: ubuntu-26.04
permissions: permissions:
contents: read contents: read
env: env:
+3 -3
View File
@@ -24,10 +24,10 @@ jobs:
fail-fast: false fail-fast: false
matrix: matrix:
include: include:
- runner: ubuntu-24.04 - runner: ubuntu-26.04
arch: amd64 arch: amd64
platform: linux/amd64 platform: linux/amd64
- runner: ubuntu-24.04-arm - runner: ubuntu-26.04-arm
arch: arm64 arch: arm64
platform: linux/arm64 platform: linux/arm64
runs-on: ${{ matrix.runner }} runs-on: ${{ matrix.runner }}
@@ -163,7 +163,7 @@ jobs:
archive: false archive: false
merge-and-push: merge-and-push:
name: Merge and Push Manifest name: Merge and Push Manifest
runs-on: ubuntu-24.04 runs-on: ubuntu-26.04
needs: build-arch needs: build-arch
if: needs.build-arch.outputs.should-push == 'true' if: needs.build-arch.outputs.should-push == 'true'
environment: image-publishing environment: image-publishing
+2 -2
View File
@@ -65,7 +65,7 @@ jobs:
needs: changes needs: changes
if: needs.changes.outputs.docs_changed == 'true' if: needs.changes.outputs.docs_changed == 'true'
name: Build Documentation name: Build Documentation
runs-on: ubuntu-24.04 runs-on: ubuntu-26.04
steps: steps:
- uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0 - uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0
- name: Checkout - name: Checkout
@@ -102,7 +102,7 @@ jobs:
name: Deploy Documentation name: Deploy Documentation
needs: [changes, build] needs: [changes, build]
if: github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.changes.outputs.docs_changed == 'true' if: github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.changes.outputs.docs_changed == 'true'
runs-on: ubuntu-24.04 runs-on: ubuntu-26.04
permissions: permissions:
pages: write pages: write
id-token: write id-token: write
+6 -6
View File
@@ -72,7 +72,7 @@ jobs:
needs: changes needs: changes
if: needs.changes.outputs.frontend_changed == 'true' if: needs.changes.outputs.frontend_changed == 'true'
name: Install Dependencies name: Install Dependencies
runs-on: ubuntu-24.04 runs-on: ubuntu-26.04
permissions: permissions:
contents: read contents: read
steps: steps:
@@ -104,7 +104,7 @@ jobs:
name: Lint name: Lint
needs: [changes, install-dependencies] needs: [changes, install-dependencies]
if: needs.changes.outputs.frontend_changed == 'true' if: needs.changes.outputs.frontend_changed == 'true'
runs-on: ubuntu-24.04 runs-on: ubuntu-26.04
permissions: permissions:
contents: read contents: read
steps: steps:
@@ -137,7 +137,7 @@ jobs:
name: "Unit Tests (${{ matrix.shard-index }}/${{ matrix.shard-count }})" name: "Unit Tests (${{ matrix.shard-index }}/${{ matrix.shard-count }})"
needs: [changes, install-dependencies] needs: [changes, install-dependencies]
if: needs.changes.outputs.frontend_changed == 'true' if: needs.changes.outputs.frontend_changed == 'true'
runs-on: ubuntu-24.04 runs-on: ubuntu-26.04
permissions: permissions:
contents: read contents: read
strategy: strategy:
@@ -188,10 +188,10 @@ jobs:
name: E2E Tests name: E2E Tests
needs: [changes, install-dependencies] needs: [changes, install-dependencies]
if: needs.changes.outputs.frontend_changed == 'true' if: needs.changes.outputs.frontend_changed == 'true'
runs-on: ubuntu-24.04 runs-on: ubuntu-26.04
permissions: permissions:
contents: read contents: read
container: mcr.microsoft.com/playwright:v1.62.1-noble container: mcr.microsoft.com/playwright:v1.62.1-resolute
env: env:
PLAYWRIGHT_BROWSERS_PATH: /ms-playwright PLAYWRIGHT_BROWSERS_PATH: /ms-playwright
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
@@ -246,7 +246,7 @@ jobs:
name: Frontend Build name: Frontend Build
needs: [changes, unit-tests, e2e-tests] needs: [changes, unit-tests, e2e-tests]
if: needs.changes.outputs.frontend_changed == 'true' if: needs.changes.outputs.frontend_changed == 'true'
runs-on: ubuntu-24.04 runs-on: ubuntu-26.04
permissions: permissions:
contents: read contents: read
steps: steps:
+5 -5
View File
@@ -14,7 +14,7 @@ permissions: {}
jobs: jobs:
wait-for-docker: wait-for-docker:
name: Wait for Docker Build name: Wait for Docker Build
runs-on: ubuntu-24.04 runs-on: ubuntu-26.04
permissions: permissions:
checks: read checks: read
statuses: read statuses: read
@@ -30,7 +30,7 @@ jobs:
build-release: build-release:
name: Build Release name: Build Release
needs: wait-for-docker needs: wait-for-docker
runs-on: ubuntu-24.04 runs-on: ubuntu-26.04
permissions: permissions:
contents: read contents: read
steps: steps:
@@ -73,7 +73,7 @@ jobs:
timeout-minutes: 12 timeout-minutes: 12
uses: $/.github/actions/apt-install uses: $/.github/actions/apt-install
with: with:
packages: gettext liblept5 packages: gettext libleptonica6
# ---- Build Documentation ---- # ---- Build Documentation ----
- name: Build documentation - name: Build documentation
env: env:
@@ -145,7 +145,7 @@ jobs:
publish-release: publish-release:
name: Publish Release name: Publish Release
needs: build-release needs: build-release
runs-on: ubuntu-24.04 runs-on: ubuntu-26.04
permissions: permissions:
contents: write contents: write
pull-requests: write pull-requests: write
@@ -197,7 +197,7 @@ jobs:
name: Append Changelog name: Append Changelog
needs: publish-release needs: publish-release
if: needs.publish-release.outputs.prerelease == 'false' if: needs.publish-release.outputs.prerelease == 'false'
runs-on: ubuntu-24.04 runs-on: ubuntu-26.04
permissions: permissions:
contents: write contents: write
pull-requests: write pull-requests: write
+2 -2
View File
@@ -15,7 +15,7 @@ permissions:
jobs: jobs:
zizmor: zizmor:
name: Run zizmor name: Run zizmor
runs-on: ubuntu-24.04 runs-on: ubuntu-26.04
permissions: permissions:
contents: read contents: read
actions: read actions: read
@@ -29,7 +29,7 @@ jobs:
uses: zizmorcore/zizmor-action@cc914d7f3750a2d13d75c7f184a1060aa0e9d482 # v0.6.4 uses: zizmorcore/zizmor-action@cc914d7f3750a2d13d75c7f184a1060aa0e9d482 # v0.6.4
semgrep: semgrep:
name: Semgrep CE name: Semgrep CE
runs-on: ubuntu-24.04 runs-on: ubuntu-26.04
container: container:
image: semgrep/semgrep:1.155.0@sha256:cc869c685dcc0fe497c86258da9f205397d8108e56d21a86082ea4886e52784d image: semgrep/semgrep:1.155.0@sha256:cc869c685dcc0fe497c86258da9f205397d8108e56d21a86082ea4886e52784d
if: github.actor != 'dependabot[bot]' if: github.actor != 'dependabot[bot]'
+2 -2
View File
@@ -17,7 +17,7 @@ jobs:
cleanup-images: cleanup-images:
name: Cleanup Image Tags for ${{ matrix.primary-name }} name: Cleanup Image Tags for ${{ matrix.primary-name }}
if: github.repository_owner == 'paperless-ngx' if: github.repository_owner == 'paperless-ngx'
runs-on: ubuntu-24.04 runs-on: ubuntu-26.04
environment: registry-maintenance environment: registry-maintenance
strategy: strategy:
fail-fast: false fail-fast: false
@@ -42,7 +42,7 @@ jobs:
cleanup-untagged-images: cleanup-untagged-images:
name: Cleanup Untagged Images Tags for ${{ matrix.primary-name }} name: Cleanup Untagged Images Tags for ${{ matrix.primary-name }}
if: github.repository_owner == 'paperless-ngx' if: github.repository_owner == 'paperless-ngx'
runs-on: ubuntu-24.04 runs-on: ubuntu-26.04
needs: needs:
- cleanup-images - cleanup-images
environment: registry-maintenance environment: registry-maintenance
+1 -1
View File
@@ -21,7 +21,7 @@ on:
jobs: jobs:
analyze: analyze:
name: Analyze name: Analyze
runs-on: ubuntu-24.04 runs-on: ubuntu-26.04
permissions: permissions:
actions: read actions: read
contents: read contents: read
+1 -1
View File
@@ -13,7 +13,7 @@ jobs:
synchronize-with-crowdin: synchronize-with-crowdin:
name: Crowdin Sync name: Crowdin Sync
if: github.repository_owner == 'paperless-ngx' if: github.repository_owner == 'paperless-ngx'
runs-on: ubuntu-24.04 runs-on: ubuntu-26.04
environment: translation-sync environment: translation-sync
steps: steps:
- name: Checkout - name: Checkout
+1 -1
View File
@@ -7,7 +7,7 @@ jobs:
# Note: peakoss/anti-slop does not support the `issues` event yet (all of its # Note: peakoss/anti-slop does not support the `issues` event yet (all of its
# issue inputs are still commented out upstream), so the checks that the PR Bot # issue inputs are still commented out upstream), so the checks that the PR Bot
# workflow gets from the action are implemented manually here. # workflow gets from the action are implemented manually here.
runs-on: ubuntu-latest runs-on: ubuntu-slim
permissions: permissions:
issues: write issues: write
steps: steps:
+2 -2
View File
@@ -4,7 +4,7 @@ on:
types: [opened] types: [opened]
jobs: jobs:
Anti-slop: Anti-slop:
runs-on: ubuntu-latest runs-on: ubuntu-slim
permissions: permissions:
contents: read contents: read
issues: read issues: read
@@ -24,7 +24,7 @@ jobs:
ASLOP-PR-VERIFY ASLOP-PR-VERIFY
pr-bot: pr-bot:
name: Automated PR Bot name: Automated PR Bot
runs-on: ubuntu-latest runs-on: ubuntu-slim
# Runs after Anti-slop so the welcome comment can see whether the PR was closed # Runs after Anti-slop so the welcome comment can see whether the PR was closed
# instead of racing it. Still runs if that job fails, so labeling is not lost. # instead of racing it. Still runs if that job fails, so labeling is not lost.
needs: Anti-slop needs: Anti-slop
+1 -1
View File
@@ -12,7 +12,7 @@ permissions:
jobs: jobs:
pr_opened_or_reopened: pr_opened_or_reopened:
name: pr_opened_or_reopened name: pr_opened_or_reopened
runs-on: ubuntu-24.04 runs-on: ubuntu-slim
permissions: permissions:
# write permission is required for autolabeler # write permission is required for autolabeler
pull-requests: write pull-requests: write
+5 -5
View File
@@ -9,7 +9,7 @@ jobs:
stale: stale:
name: 'Stale' name: 'Stale'
if: github.repository_owner == 'paperless-ngx' if: github.repository_owner == 'paperless-ngx'
runs-on: ubuntu-24.04 runs-on: ubuntu-26.04
permissions: permissions:
issues: write issues: write
pull-requests: write pull-requests: write
@@ -34,7 +34,7 @@ jobs:
lock-threads: lock-threads:
name: 'Lock Old Threads' name: 'Lock Old Threads'
if: github.repository_owner == 'paperless-ngx' if: github.repository_owner == 'paperless-ngx'
runs-on: ubuntu-24.04 runs-on: ubuntu-26.04
permissions: permissions:
issues: write issues: write
pull-requests: write pull-requests: write
@@ -58,7 +58,7 @@ jobs:
close-answered-discussions: close-answered-discussions:
name: 'Close Answered Discussions' name: 'Close Answered Discussions'
if: github.repository_owner == 'paperless-ngx' if: github.repository_owner == 'paperless-ngx'
runs-on: ubuntu-24.04 runs-on: ubuntu-slim
permissions: permissions:
discussions: write discussions: write
steps: steps:
@@ -117,7 +117,7 @@ jobs:
close-outdated-discussions: close-outdated-discussions:
name: 'Close Outdated Discussions' name: 'Close Outdated Discussions'
if: github.repository_owner == 'paperless-ngx' if: github.repository_owner == 'paperless-ngx'
runs-on: ubuntu-24.04 runs-on: ubuntu-slim
permissions: permissions:
discussions: write discussions: write
steps: steps:
@@ -211,7 +211,7 @@ jobs:
close-unsupported-feature-requests: close-unsupported-feature-requests:
name: 'Close Unsupported Feature Requests' name: 'Close Unsupported Feature Requests'
if: github.repository_owner == 'paperless-ngx' if: github.repository_owner == 'paperless-ngx'
runs-on: ubuntu-24.04 runs-on: ubuntu-slim
permissions: permissions:
discussions: write discussions: write
steps: steps:
+1 -1
View File
@@ -8,7 +8,7 @@ env:
jobs: jobs:
generate-translate-strings: generate-translate-strings:
name: Generate Translation Strings name: Generate Translation Strings
runs-on: ubuntu-latest runs-on: ubuntu-26.04
environment: translation-sync environment: translation-sync
permissions: permissions:
contents: write contents: write
+24
View File
@@ -14,6 +14,7 @@ from django.db.models import QuerySet
from django.db.models import Value from django.db.models import Value
from django.db.models import When from django.db.models import When
from django.db.models.functions import Cast from django.db.models.functions import Cast
from django.db.models.functions import Coalesce
from guardian.core import ObjectPermissionChecker from guardian.core import ObjectPermissionChecker
from guardian.models import GroupObjectPermission from guardian.models import GroupObjectPermission
from guardian.models import UserObjectPermission from guardian.models import UserObjectPermission
@@ -474,6 +475,29 @@ def permitted_document_ids(
return permitted_object_ids(user, Document, perm, include_deleted=include_deleted) return permitted_object_ids(user, Document, perm, include_deleted=include_deleted)
def documents_without_permitted_root(
documents: QuerySet[Document],
user: User | None,
*,
perm: str = "view_document",
include_deleted: bool = False,
) -> QuerySet[Document]:
"""
The documents the user lacks ``perm`` on. Versions are authorized by their
root document, so a version's own owner is ignored. A single query, without
loading the documents or joining the root.
"""
return documents.annotate(
root_id=Coalesce("root_document_id", "id"),
).exclude(
root_id__in=permitted_document_ids(
user,
perm=perm,
include_deleted=include_deleted,
),
)
def get_document_count_filter_for_user(user, related_name: str = "documents"): def get_document_count_filter_for_user(user, related_name: str = "documents"):
""" """
Return the Q object used to filter document counts for the given user. Return the Q object used to filter document counts for the given user.
+38 -32
View File
@@ -31,6 +31,7 @@ from documents.search._query import parse_user_query
from documents.search._schema import _write_sentinels from documents.search._schema import _write_sentinels
from documents.search._schema import build_schema from documents.search._schema import build_schema
from documents.search._schema import open_or_rebuild_index from documents.search._schema import open_or_rebuild_index
from documents.search._schema import rebuild_in_progress
from documents.search._schema import wipe_index from documents.search._schema import wipe_index
from documents.search._tokenizer import ascii_fold from documents.search._tokenizer import ascii_fold
from documents.search._tokenizer import autocomplete_tokens from documents.search._tokenizer import autocomplete_tokens
@@ -1110,39 +1111,44 @@ class TantivyBackend:
flushing a segment, deferring merge work; they do not avoid it. flushing a segment, deferring merge work; they do not avoid it.
""" """
wipe_index(self._path) wipe_index(self._path)
new_index = tantivy.Index(build_schema(), path=str(self._path)) # The marker covers the window where the empty index is already stamped
_write_sentinels(self._path) # as current but not yet populated, so an interrupted rebuild is retried.
register_tokenizers(new_index, settings.SEARCH_LANGUAGE) with rebuild_in_progress(self._path):
new_index = tantivy.Index(build_schema(), path=str(self._path))
_write_sentinels(self._path)
register_tokenizers(new_index, settings.SEARCH_LANGUAGE)
# Point instance at the new index so _build_tantivy_doc uses it # Point instance at the new index so _build_tantivy_doc uses it
old_index, old_schema = self._raw_index, self._raw_schema old_index, old_schema = self._raw_index, self._raw_schema
self._raw_index = new_index self._raw_index = new_index
self._raw_schema = new_index.schema self._raw_schema = new_index.schema
# Stream documents one-by-one (so the progress bar advances per # Stream documents one-by-one (so the progress bar advances per
# document) while fetching viewer permissions one SQL query per chunk. # document) while fetching viewer permissions one SQL query per
# The stream is Sized, so iter_wrapper can still discover the total. # chunk. The stream is Sized, so iter_wrapper can still discover
documents_stream = _DocumentViewerStream(documents, chunk_size=1000) # the total.
try: documents_stream = _DocumentViewerStream(documents, chunk_size=1000)
writer = new_index.writer(heap_size=writer_heap_bytes) try:
for document, (viewer_ids, viewer_group_ids) in iter_wrapper( writer = new_index.writer(heap_size=writer_heap_bytes)
documents_stream, for document, (viewer_ids, viewer_group_ids) in iter_wrapper(
): documents_stream,
doc = self._build_tantivy_doc( ):
document, doc = self._build_tantivy_doc(
viewer_ids=viewer_ids, document,
viewer_group_ids=viewer_group_ids, viewer_ids=viewer_ids,
) viewer_group_ids=viewer_group_ids,
writer.add_document(doc) )
writer.commit() writer.add_document(doc)
# Wait for background merge threads to finish so all segments are writer.commit()
# fully merged and persisted before the index is considered rebuilt. # Wait for background merge threads to finish so all segments
writer.wait_merging_threads() # are fully merged and persisted before the index is considered
new_index.reload() # rebuilt.
except BaseException: # pragma: no cover writer.wait_merging_threads()
# Restore old index on failure so the backend remains usable new_index.reload()
self._raw_index = old_index except BaseException: # pragma: no cover
self._raw_schema = old_schema # Restore old index on failure so the backend remains usable
raise self._raw_index = old_index
self._raw_schema = old_schema
raise
def chunked(iterable, size): def chunked(iterable, size):
+45 -4
View File
@@ -4,6 +4,7 @@ import hashlib
import json import json
import logging import logging
import shutil import shutil
from contextlib import contextmanager
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from typing import Final from typing import Final
from typing import NamedTuple from typing import NamedTuple
@@ -16,6 +17,7 @@ from whoosh_compat import FieldKind
from documents.search._fields import PUBLIC_FIELDS from documents.search._fields import PUBLIC_FIELDS
if TYPE_CHECKING: if TYPE_CHECKING:
from collections.abc import Iterator
from pathlib import Path from pathlib import Path
logger = logging.getLogger("paperless.search") logger = logging.getLogger("paperless.search")
@@ -28,6 +30,11 @@ logger = logging.getLogger("paperless.search")
# v3 - barcodes JSON field for stored barcode contents # v3 - barcodes JSON field for stored barcode contents
SCHEMA_VERSION: Final[int] = 3 SCHEMA_VERSION: Final[int] = 3
# Present in the index directory from the moment a full rebuild starts until it
# finishes. If a rebuild is interrupted it is left behind, so the half-built
# index is not mistaken for a complete one.
REBUILD_MARKER: Final[str] = ".rebuilding"
class FieldDescriptor(NamedTuple): class FieldDescriptor(NamedTuple):
"""One tantivy field, in declaration order. """One tantivy field, in declaration order.
@@ -255,9 +262,9 @@ def needs_rebuild(index_dir: Path) -> bool:
""" """
Check if the search index needs rebuilding. Check if the search index needs rebuilding.
Reads .index_settings.json to compare the stored schema version, search True if a previous full rebuild never finished (the rebuild marker is still
language and schema fingerprint against the current configuration. Returns present), or if the index's stamped settings no longer match the current
True if the file is missing, unparsable, or any value mismatches. configuration. See _settings_mismatch().
Args: Args:
index_dir: Path to the search index directory index_dir: Path to the search index directory
@@ -265,6 +272,40 @@ def needs_rebuild(index_dir: Path) -> bool:
Returns: Returns:
True if the index needs rebuilding, False if it's up to date True if the index needs rebuilding, False if it's up to date
""" """
if (index_dir / REBUILD_MARKER).exists():
logger.warning("Previous search index rebuild did not finish - rebuilding.")
return True
return _settings_mismatch(index_dir)
@contextmanager
def rebuild_in_progress(index_dir: Path) -> Iterator[None]:
"""
Flag the index as incomplete for the duration of a full rebuild.
The marker is cleared only if the block exits cleanly. There is deliberately
no try/finally: an exception must leave the marker behind so the next
needs_rebuild() check retries the rebuild.
"""
marker = index_dir / REBUILD_MARKER
marker.touch()
yield
marker.unlink(missing_ok=True)
def _settings_mismatch(index_dir: Path) -> bool:
"""
Check the stamped settings against the current configuration.
Reads .index_settings.json to compare the stored schema version, search
language and schema fingerprint. Returns True if the file is missing,
unparsable, or any value mismatches.
This deliberately ignores the rebuild marker: open_or_rebuild_index() uses it
so that a process opening the index while another process is mid-rebuild
(or after one died) does not wipe the partial index out from under it.
Repopulating is the job of ``document_index reindex``.
"""
settings_file = index_dir / ".index_settings.json" settings_file = index_dir / ".index_settings.json"
if not settings_file.exists(): if not settings_file.exists():
return True return True
@@ -333,7 +374,7 @@ def open_or_rebuild_index(index_dir: Path | None = None) -> tantivy.Index:
index_dir = cast("Path", settings.INDEX_DIR) index_dir = cast("Path", settings.INDEX_DIR)
if not index_dir.exists(): if not index_dir.exists():
return tantivy.Index(build_schema()) return tantivy.Index(build_schema())
if needs_rebuild(index_dir): if _settings_mismatch(index_dir):
wipe_index(index_dir) wipe_index(index_dir)
idx = tantivy.Index(build_schema(), path=str(index_dir)) idx = tantivy.Index(build_schema(), path=str(index_dir))
_write_sentinels(index_dir) _write_sentinels(index_dir)
+2 -1
View File
@@ -90,6 +90,7 @@ from documents.templating.utils import convert_format_str_to_template_format
from documents.templating.workflows import validate_workflow_template from documents.templating.workflows import validate_workflow_template
from documents.validators import uri_validator from documents.validators import uri_validator
from documents.validators import url_validator from documents.validators import url_validator
from documents.versioning import get_root_document
from documents.versioning import has_prefetched_effective_content from documents.versioning import has_prefetched_effective_content
from documents.versioning import sort_versions_newest_first from documents.versioning import sort_versions_newest_first
@@ -2894,7 +2895,7 @@ class ShareLinkSerializer(OwnedObjectSerializer):
and has_perms_owner_aware( and has_perms_owner_aware(
self.user, self.user,
"view_document", "view_document",
document, get_root_document(document),
) )
): ):
return document return document
@@ -16,7 +16,10 @@ from documents.search._backend import TantivyBackend
from documents.search._backend import WriteBatch from documents.search._backend import WriteBatch
from documents.search._backend import get_backend from documents.search._backend import get_backend
from documents.search._backend import reset_backend from documents.search._backend import reset_backend
from documents.search._schema import REBUILD_MARKER
from documents.search._schema import needs_rebuild
from documents.signals.handlers import add_to_index from documents.signals.handlers import add_to_index
from paperless_testing.dirs import PaperlessDirs
from paperless_testing.factories import CorrespondentFactory from paperless_testing.factories import CorrespondentFactory
from paperless_testing.factories import DocumentFactory from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import DocumentTypeFactory from paperless_testing.factories import DocumentTypeFactory
@@ -823,6 +826,53 @@ class TestRebuild:
backend.rebuild(Document.objects.all(), iter_wrapper=wrapper) backend.rebuild(Document.objects.all(), iter_wrapper=wrapper)
assert 30 in seen assert 30 in seen
def test_successful_rebuild_leaves_index_up_to_date(
self,
backend: TantivyBackend,
paperless_dirs: PaperlessDirs,
) -> None:
"""
GIVEN:
- A backend and one document
WHEN:
- rebuild() completes
THEN:
- needs_rebuild() is False and no rebuild marker remains
"""
DocumentFactory.create()
backend.rebuild(Document.objects.all())
assert needs_rebuild(paperless_dirs.index_dir) is False
assert not (paperless_dirs.index_dir / REBUILD_MARKER).exists()
def test_interrupted_rebuild_is_retried(
self,
backend: TantivyBackend,
paperless_dirs: PaperlessDirs,
) -> None:
"""
GIVEN:
- A rebuild that dies while indexing documents (e.g. the database
connection is lost)
WHEN:
- needs_rebuild() is checked afterwards
THEN:
- It is True, even though the empty index was already stamped with
current settings, so the next start rebuilds instead of reporting
the index as up to date
"""
DocumentFactory.create()
def die(pairs):
raise RuntimeError("terminating connection due to administrator command")
yield # pragma: no cover
with pytest.raises(RuntimeError):
backend.rebuild(Document.objects.all(), iter_wrapper=die)
assert needs_rebuild(paperless_dirs.index_dir) is True
def test_includes_group_granted_viewers(self, backend: TantivyBackend) -> None: def test_includes_group_granted_viewers(self, backend: TantivyBackend) -> None:
"""Rebuild must index viewer ids for group-only grants, not just direct ones. """Rebuild must index viewer ids for group-only grants, not just direct ones.
@@ -8,6 +8,7 @@ from auditlog.models import LogEntry # type: ignore[import-untyped]
from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes.models import ContentType
from django.core.files.uploadedfile import SimpleUploadedFile from django.core.files.uploadedfile import SimpleUploadedFile
from django.test import TestCase as DjangoTestCase from django.test import TestCase as DjangoTestCase
from django.test import override_settings
from django.utils import timezone from django.utils import timezone
from rest_framework import status from rest_framework import status
from rest_framework.test import APITestCase from rest_framework.test import APITestCase
@@ -16,13 +17,17 @@ from documents.data_models import DocumentSource
from documents.filters import EffectiveContentFilter from documents.filters import EffectiveContentFilter
from documents.filters import TitleContentFilter from documents.filters import TitleContentFilter
from documents.models import Document from documents.models import Document
from documents.models import Note
from documents.models import ShareLink
from documents.versioning import annotate_effective_content from documents.versioning import annotate_effective_content
from documents.views import DocumentSelectionMixin from documents.views import DocumentSelectionMixin
from paperless_testing.dirs import DirectoriesMixin from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import UserFactory from paperless_testing.factories import UserFactory
from paperless_testing.http import read_streaming_response from paperless_testing.http import read_streaming_response
from paperless_testing.permissions import grant_all_global
from paperless_testing.permissions import grant_global from paperless_testing.permissions import grant_global
from paperless_testing.permissions import grant_object
if TYPE_CHECKING: if TYPE_CHECKING:
from pathlib import Path from pathlib import Path
@@ -1043,3 +1048,152 @@ class TestBulkSelectionExcludesVersions(DjangoTestCase):
) )
self.assertEqual(selected, [root.id]) self.assertEqual(selected, [root.id])
class TestVersionActionPermissions(DirectoriesMixin, APITestCase):
def setUp(self):
super().setUp()
self.user = UserFactory()
grant_all_global(self.user)
self.client.force_authenticate(self.user)
self.root = DocumentFactory(owner=UserFactory())
self.version = DocumentFactory(root_document=self.root, owner=None)
@override_settings(AUDIT_LOG_ENABLED=True)
def test_actions_reject_stale_version_ownership(self):
note = Note.objects.create(document=self.version, note="Version note")
for owner in (None, self.user):
self.version.owner = owner
self.version.save(update_fields=["owner"])
for action in (
"notes",
"suggestions",
"ai_suggestions",
"history",
"share_links",
):
with self.subTest(owner=owner, action=action):
response = self.client.get(
f"/api/documents/{self.version.pk}/{action}/",
)
self.assertEqual(response.status_code, 403)
response = self.client.post(
f"/api/documents/{self.version.pk}/notes/",
{"note": "New note"},
)
self.assertEqual(response.status_code, 403)
response = self.client.delete(
f"/api/documents/{self.version.pk}/notes/?id={note.pk}",
)
self.assertEqual(response.status_code, 403)
response = self.client.post(
"/api/share_links/",
{"document": self.version.pk, "file_version": "original"},
)
self.assertEqual(response.status_code, 403)
response = self.client.post(
"/api/share_link_bundles/",
{"document_ids": [self.version.pk], "file_version": "original"},
format="json",
)
self.assertEqual(response.status_code, 400)
response = self.client.post(
"/api/documents/email/",
{
"documents": [self.version.pk],
"addresses": "recipient@example.com",
"subject": "Version",
"message": "Version",
},
format="json",
)
self.assertEqual(response.status_code, 403)
with (
mock.patch("documents.views.AIConfig") as ai_config,
mock.patch("documents.views.stream_chat_with_documents") as chat,
):
ai_config.return_value.ai_enabled = True
response = self.client.post(
"/api/documents/chat/",
{"q": "Version?", "document_id": self.version.pk},
format="json",
)
self.assertEqual(response.status_code, 403)
chat.assert_not_called()
self.assertTrue(Note.objects.filter(pk=note.pk).exists())
self.assertFalse(ShareLink.objects.exists())
@mock.patch("documents.views.build_share_link_bundle.apply_async")
def test_root_permissions_allow_sharing_a_private_version(self, build_mock):
self.version.owner = UserFactory()
self.version.save(update_fields=["owner"])
grant_object(self.user, self.root, "view_document", "change_document")
note = Note.objects.create(document=self.version, note="Version note")
response = self.client.get(f"/api/documents/{self.version.pk}/notes/")
self.assertEqual(response.status_code, 200)
self.assertEqual(response.data[0]["id"], note.pk)
response = self.client.post(
"/api/share_links/",
{"document": self.version.pk, "file_version": "original"},
)
self.assertEqual(response.status_code, 201)
self.assertEqual(ShareLink.objects.get().document_id, self.version.pk)
response = self.client.get(f"/api/documents/{self.version.pk}/share_links/")
self.assertEqual(response.status_code, 200)
self.assertEqual(len(response.data), 1)
response = self.client.post(
"/api/share_link_bundles/",
{"document_ids": [self.version.pk], "file_version": "original"},
format="json",
)
self.assertEqual(response.status_code, 201)
build_mock.assert_called_once()
def test_root_view_permission_does_not_allow_note_changes(self):
grant_object(self.user, self.root, "view_document")
note = Note.objects.create(document=self.version, note="Version note")
response = self.client.get(f"/api/documents/{self.version.pk}/notes/")
self.assertEqual(response.status_code, 200)
response = self.client.post(
f"/api/documents/{self.version.pk}/notes/",
{"note": "New note"},
)
self.assertEqual(response.status_code, 403)
response = self.client.delete(
f"/api/documents/{self.version.pk}/notes/?id={note.pk}",
)
self.assertEqual(response.status_code, 403)
self.assertTrue(Note.objects.filter(pk=note.pk).exists())
@override_settings(AUDIT_LOG_ENABLED=True)
def test_history_uses_root_ownership(self):
self.root.owner = self.user
self.root.save(update_fields=["owner"])
self.version.owner = UserFactory()
self.version.save(update_fields=["owner"])
response = self.client.get(f"/api/documents/{self.version.pk}/history/")
self.assertEqual(response.status_code, 200)
def test_selection_data_rejects_stale_version_ownership(self):
for owner in (None, self.user):
self.version.owner = owner
self.version.save(update_fields=["owner"])
with self.subTest(owner=owner):
response = self.client.post(
"/api/documents/selection_data/",
{"documents": [self.version.pk]},
format="json",
)
self.assertEqual(response.status_code, 403)
def test_selection_data_allows_private_version_of_permitted_root(self):
self.version.owner = UserFactory()
self.version.save(update_fields=["owner"])
grant_object(self.user, self.root, "view_document")
other = DocumentFactory(owner=self.user)
response = self.client.post(
"/api/documents/selection_data/",
{"documents": [self.version.pk, other.pk]},
format="json",
)
self.assertEqual(response.status_code, 200)
+51
View File
@@ -6,6 +6,7 @@ from rest_framework.test import APITestCase
from documents.models import Document from documents.models import Document
from paperless_testing.dirs import DirectoriesMixin from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import UserFactory from paperless_testing.factories import UserFactory
from paperless_testing.permissions import grant_all_global from paperless_testing.permissions import grant_all_global
@@ -279,3 +280,53 @@ class TestTrashAPI(DirectoriesMixin, APITestCase):
Document.objects.filter(root_document=root).values_list("id", flat=True), Document.objects.filter(root_document=root).values_list("id", flat=True),
[version.pk for version in versions], [version.pk for version in versions],
) )
def test_api_trash_version_follows_root_owner(self) -> None:
"""
GIVEN:
- A deleted version of user2's document, owned by nobody
- A deleted version of the user's document, owned by user2
WHEN:
- The user lists the trash and tries to restore or empty the versions
THEN:
- Only the version of the user's own document is listed
- The other version can't be restored or emptied
- The version of the user's own document can be restored
"""
user2 = UserFactory(username="user2")
other_version = DocumentFactory(
root_document=DocumentFactory(owner=user2),
version_index=1,
)
other_version.delete()
own_version = DocumentFactory(
owner=user2,
root_document=DocumentFactory(owner=self.user),
version_index=1,
)
own_version.delete()
resp = self.client.get("/api/trash/")
self.assertEqual(resp.status_code, status.HTTP_200_OK)
self.assertEqual(
[doc["id"] for doc in resp.data["results"]],
[own_version.pk],
)
for action in ("restore", "empty"):
with self.subTest(action=action):
resp = self.client.post(
"/api/trash/",
{"action": action, "documents": [other_version.pk]},
)
self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN)
self.assertTrue(
Document.deleted_objects.filter(pk=other_version.pk).exists(),
)
resp = self.client.post(
"/api/trash/",
{"action": "restore", "documents": [own_version.pk]},
)
self.assertEqual(resp.status_code, status.HTTP_200_OK)
self.assertTrue(Document.objects.filter(pk=own_version.pk).exists())
@@ -90,10 +90,13 @@ class ShareLinkBundleAPITests(DirectoriesMixin, APITestCase):
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn("document_ids", response.data) self.assertIn("document_ids", response.data)
@mock.patch("documents.views.permitted_document_ids", return_value=set()) def test_create_bundle_rejects_insufficient_permissions(self) -> None:
def test_create_bundle_rejects_insufficient_permissions(self, perms_mock) -> None: requester = UserFactory(username="bundle_creator")
grant_global(requester, "add_sharelinkbundle", "view_document")
self.client.force_authenticate(requester)
document = DocumentFactory(owner=UserFactory(username="document_owner"))
payload = { payload = {
"document_ids": [self.document.pk], "document_ids": [self.document.pk, document.pk],
"file_version": ShareLink.FileVersion.ARCHIVE, "file_version": ShareLink.FileVersion.ARCHIVE,
"expiration_days": 7, "expiration_days": 7,
} }
@@ -101,8 +104,8 @@ class ShareLinkBundleAPITests(DirectoriesMixin, APITestCase):
response = self.client.post(self.ENDPOINT, payload, format="json") response = self.client.post(self.ENDPOINT, payload, format="json")
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertIn("document_ids", response.data) self.assertIn(str(document.pk), str(response.data["document_ids"]))
perms_mock.assert_called() self.assertFalse(ShareLinkBundle.objects.exists())
@mock.patch("documents.views.build_share_link_bundle.apply_async") @mock.patch("documents.views.build_share_link_bundle.apply_async")
def test_rebuild_bundle_resets_state(self, delay_mock) -> None: def test_rebuild_bundle_resets_state(self, delay_mock) -> None:
+81 -41
View File
@@ -174,6 +174,7 @@ from documents.permissions import TrashPermissions
from documents.permissions import ViewDocumentsPermissions from documents.permissions import ViewDocumentsPermissions
from documents.permissions import annotate_document_count_by_ids from documents.permissions import annotate_document_count_by_ids
from documents.permissions import annotate_document_count_for_related_queryset from documents.permissions import annotate_document_count_for_related_queryset
from documents.permissions import documents_without_permitted_root
from documents.permissions import get_document_count_filter_for_user from documents.permissions import get_document_count_filter_for_user
from documents.permissions import get_objects_for_user_owner_aware from documents.permissions import get_objects_for_user_owner_aware
from documents.permissions import has_global_statistics_permission from documents.permissions import has_global_statistics_permission
@@ -1550,13 +1551,16 @@ class DocumentViewSet(
) )
def suggestions(self, request, pk=None): def suggestions(self, request, pk=None):
doc = get_object_or_404( doc = get_object_or_404(
Document.objects.select_related("owner").prefetch_related("versions"), Document.objects.select_related(
"owner",
"root_document__owner",
).prefetch_related("versions"),
pk=pk, pk=pk,
) )
if request.user is not None and not has_perms_owner_aware( if request.user is not None and not has_perms_owner_aware(
request.user, request.user,
"change_document", "change_document",
doc, get_root_document(doc),
): ):
return HttpResponseForbidden("Insufficient permissions") return HttpResponseForbidden("Insufficient permissions")
@@ -1610,13 +1614,16 @@ class DocumentViewSet(
@method_decorator(cache_control(no_cache=True)) @method_decorator(cache_control(no_cache=True))
def ai_suggestions(self, request, pk=None): def ai_suggestions(self, request, pk=None):
doc = get_object_or_404( doc = get_object_or_404(
Document.objects.select_related("owner").prefetch_related("versions"), Document.objects.select_related(
"owner",
"root_document__owner",
).prefetch_related("versions"),
pk=pk, pk=pk,
) )
if request.user is not None and not has_perms_owner_aware( if request.user is not None and not has_perms_owner_aware(
request.user, request.user,
"change_document", "change_document",
doc, get_root_document(doc),
): ):
return HttpResponseForbidden("Insufficient permissions") return HttpResponseForbidden("Insufficient permissions")
@@ -1856,15 +1863,20 @@ class DocumentViewSet(
currentUser = request.user currentUser = request.user
try: try:
doc = ( doc = (
Document.objects.select_related("owner") Document.objects.select_related("owner", "root_document__owner")
.prefetch_related("notes") .prefetch_related("notes")
.only("pk", "owner__id") .only(
"pk",
"owner__id",
"root_document__id",
"root_document__owner__id",
)
.get(pk=pk) .get(pk=pk)
) )
if currentUser is not None and not has_perms_owner_aware( if currentUser is not None and not has_perms_owner_aware(
currentUser, currentUser,
"view_document", "view_document",
doc, get_root_document(doc),
): ):
return HttpResponseForbidden("Insufficient permissions to view notes") return HttpResponseForbidden("Insufficient permissions to view notes")
except Document.DoesNotExist: except Document.DoesNotExist:
@@ -1886,7 +1898,7 @@ class DocumentViewSet(
if currentUser is not None and not has_perms_owner_aware( if currentUser is not None and not has_perms_owner_aware(
currentUser, currentUser,
"change_document", "change_document",
doc, get_root_document(doc),
): ):
return HttpResponseForbidden( return HttpResponseForbidden(
"Insufficient permissions to create notes", "Insufficient permissions to create notes",
@@ -1929,7 +1941,7 @@ class DocumentViewSet(
if currentUser is not None and not has_perms_owner_aware( if currentUser is not None and not has_perms_owner_aware(
currentUser, currentUser,
"change_document", "change_document",
doc, get_root_document(doc),
): ):
return HttpResponseForbidden("Insufficient permissions to delete notes") return HttpResponseForbidden("Insufficient permissions to delete notes")
@@ -1973,11 +1985,13 @@ class DocumentViewSet(
def share_links(self, request, pk=None): def share_links(self, request, pk=None):
currentUser = request.user currentUser = request.user
try: try:
doc = Document.objects.select_related("owner").get(pk=pk) doc = Document.objects.select_related("owner", "root_document__owner").get(
pk=pk,
)
if currentUser is not None and not has_perms_owner_aware( if currentUser is not None and not has_perms_owner_aware(
currentUser, currentUser,
"change_document", "change_document",
doc, get_root_document(doc),
): ):
return HttpResponseForbidden( return HttpResponseForbidden(
"Insufficient permissions to add share link", "Insufficient permissions to add share link",
@@ -2008,10 +2022,11 @@ class DocumentViewSet(
if not settings.AUDIT_LOG_ENABLED: if not settings.AUDIT_LOG_ENABLED:
return HttpResponseBadRequest("Audit log is disabled") return HttpResponseBadRequest("Audit log is disabled")
try: try:
doc = Document.objects.get(pk=pk) doc = Document.objects.select_related("root_document__owner").get(pk=pk)
root_doc = get_root_document(doc)
if not request.user.has_perm("auditlog.view_logentry") or ( if not request.user.has_perm("auditlog.view_logentry") or (
doc.owner is not None root_doc.owner is not None
and doc.owner != request.user and root_doc.owner != request.user
and not request.user.is_superuser and not request.user.is_superuser
): ):
return HttpResponseForbidden( return HttpResponseForbidden(
@@ -2102,9 +2117,7 @@ class DocumentViewSet(
documents = Document.objects.filter(pk__in=document_ids) documents = Document.objects.filter(pk__in=document_ids)
if ( if (
request.user is not None request.user is not None
and documents.exclude( and documents_without_permitted_root(documents, request.user).exists()
pk__in=permitted_document_ids(request.user),
).exists()
): ):
return HttpResponseForbidden("Insufficient permissions") return HttpResponseForbidden("Insufficient permissions")
@@ -2430,11 +2443,17 @@ class ChatStreamingView(GenericAPIView[Any]):
if doc_id: if doc_id:
try: try:
document = Document.objects.get(id=doc_id) document = Document.objects.select_related(
"root_document__owner",
).get(id=doc_id)
except Document.DoesNotExist: except Document.DoesNotExist:
return HttpResponseBadRequest("Document not found") return HttpResponseBadRequest("Document not found")
if not has_perms_owner_aware(request.user, "view_document", document): if not has_perms_owner_aware(
request.user,
"view_document",
get_root_document(document),
):
return HttpResponseForbidden("Insufficient permissions") return HttpResponseForbidden("Insufficient permissions")
documents = Document.objects.filter(pk=document.pk) documents = Document.objects.filter(pk=document.pk)
@@ -3605,10 +3624,11 @@ class SelectionDataView(DocumentSelectionMixin, GenericAPIView[Any]):
user=request.user, user=request.user,
validated_data=serializer.validated_data, validated_data=serializer.validated_data,
) )
permitted_documents = Document.objects.filter( documents = Document.objects.filter(pk__in=ids)
id__in=permitted_document_ids(request.user), if (
) documents.count() != len(ids)
if permitted_documents.filter(pk__in=ids).count() != len(ids): or documents_without_permitted_root(documents, request.user).exists()
):
return HttpResponseForbidden("Insufficient permissions") return HttpResponseForbidden("Insufficient permissions")
correspondents = Correspondent.objects.annotate( correspondents = Correspondent.objects.annotate(
@@ -4790,19 +4810,23 @@ class ShareLinkBundleViewSet(PassUserMixin, ModelViewSet[ShareLinkBundle]):
}, },
) )
documents = list(documents_qs) denied_id = (
permitted_ids = set(permitted_document_ids(request.user)) documents_without_permitted_root(documents_qs, request.user)
for document in documents: .order_by("pk")
if document.pk not in permitted_ids: .values_list("pk", flat=True)
raise ValidationError( .first()
{ )
"document_ids": _( if denied_id is not None:
"Insufficient permissions to share document %(id)s.", raise ValidationError(
) {
% {"id": document.pk}, "document_ids": _(
}, "Insufficient permissions to share document %(id)s.",
) )
% {"id": denied_id},
},
)
documents = list(documents_qs)
document_map = {document.pk: document for document in documents} document_map = {document.pk: document for document in documents}
ordered_documents = [document_map[doc_id] for doc_id in document_ids] ordered_documents = [document_map[doc_id] for doc_id in document_ids]
@@ -5587,6 +5611,23 @@ class TrashView(ListModelMixin, PassUserMixin):
class _TrashPermittedObjectsFilter(PermittedObjectsFilter): class _TrashPermittedObjectsFilter(PermittedObjectsFilter):
include_granted = False include_granted = False
def filter_queryset(self, request, queryset, view):
if request.user.is_superuser or not request.user.is_active:
return super().filter_queryset(request, queryset, view)
# A version belongs to whoever owns its root
def owned_or_unowned(prefix: str) -> Q:
return Q(**{f"{prefix}owner": request.user}) | Q(
**{f"{prefix}owner__isnull": True},
)
return queryset.filter(
(Q(root_document__isnull=True) & owned_or_unowned(""))
| (
Q(root_document__isnull=False) & owned_or_unowned("root_document__")
),
)
filter_backends = (_TrashPermittedObjectsFilter,) filter_backends = (_TrashPermittedObjectsFilter,)
pagination_class = StandardPagination pagination_class = StandardPagination
@@ -5616,12 +5657,11 @@ class TrashView(ListModelMixin, PassUserMixin):
if doc_ids is not None if doc_ids is not None
else self.filter_queryset(self.get_queryset()).all() else self.filter_queryset(self.get_queryset()).all()
) )
if docs.exclude( if documents_without_permitted_root(
pk__in=permitted_document_ids( docs,
request.user, request.user,
perm="delete_document", perm="delete_document",
include_deleted=True, include_deleted=True,
),
).exists(): ).exists():
return HttpResponseForbidden("Insufficient permissions") return HttpResponseForbidden("Insufficient permissions")
action = serializer.validated_data.get("action") action = serializer.validated_data.get("action")
+2 -19
View File
@@ -2,7 +2,7 @@ msgid ""
msgstr "" msgstr ""
"Project-Id-Version: paperless-ngx\n" "Project-Id-Version: paperless-ngx\n"
"Report-Msgid-Bugs-To: \n" "Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2026-10-05 16:26+0000\n" "POT-Creation-Date: 2026-10-06 15:12+0000\n"
"PO-Revision-Date: 2022-02-17 04:17\n" "PO-Revision-Date: 2022-02-17 04:17\n"
"Last-Translator: \n" "Last-Translator: \n"
"Language-Team: English\n" "Language-Team: English\n"
@@ -1941,25 +1941,8 @@ msgstr ""
msgid "As a final step, please complete the following form:" msgid "As a final step, please complete the following form:"
msgstr "" msgstr ""
#: documents/validators.py:24
#, python-brace-format
msgid "Unable to parse URI {value}, missing scheme"
msgstr ""
#: documents/validators.py:29
#, python-brace-format
msgid "Unable to parse URI {value}, missing net location or path"
msgstr ""
#: documents/validators.py:36 #: documents/validators.py:36
msgid "" msgid ", "
"URI scheme '{parts.scheme}' is not allowed. Allowed schemes: {', '."
"join(allowed_schemes)}"
msgstr ""
#: documents/validators.py:45
#, python-brace-format
msgid "Unable to parse URI {value}"
msgstr "" msgstr ""
#: documents/views.py:336 documents/views.py:2729 #: documents/views.py:336 documents/views.py:2729
@@ -137,9 +137,20 @@ class TestNginxService:
reason="No Gotenberg/Tika servers to test with", reason="No Gotenberg/Tika servers to test with",
) )
class TestParserLive: class TestParserLive:
@staticmethod # Rasterizer versions shift a few pixels, so compare perceptual hashes by
def imagehash(file: Path, hash_size: int = 18) -> str: # Hamming distance (out of 18 * 18 = 324 bits) rather than for equality
return f"{average_hash(Image.open(file), hash_size)}" MAX_HASH_DISTANCE = 8
@classmethod
def assert_thumbnails_similar(cls, generated: Path, expected: Path) -> None:
distance = average_hash(Image.open(generated), 18) - average_hash(
Image.open(expected),
18,
)
assert distance <= cls.MAX_HASH_DISTANCE, (
f"Thumbnail {generated} differs from {expected} by {distance} bits "
f"(max {cls.MAX_HASH_DISTANCE})"
)
def test_get_thumbnail( def test_get_thumbnail(
self, self,
@@ -168,12 +179,7 @@ class TestParserLive:
assert thumb.exists() assert thumb.exists()
assert thumb.is_file() assert thumb.is_file()
assert self.imagehash(thumb) == self.imagehash( self.assert_thumbnails_similar(thumb, simple_txt_email_thumbnail_file)
simple_txt_email_thumbnail_file,
), (
f"Created thumbnail {thumb} differs from expected file "
f"{simple_txt_email_thumbnail_file}"
)
def test_tika_parse_successful(self, mail_parser: MailDocumentParser) -> None: def test_tika_parse_successful(self, mail_parser: MailDocumentParser) -> None:
""" """
@@ -255,7 +261,7 @@ class TestParserLive:
THEN: THEN:
- Gotenberg shall be called to generate the PDF - Gotenberg shall be called to generate the PDF
- The archive PDF shall contain the expected content - The archive PDF shall contain the expected content
- The generated thumbnail shall match the expected image hash - The generated thumbnail shall be perceptually close to the expected image
""" """
util_call_with_backoff(mail_parser.parse, [html_email_file, "message/rfc822"]) util_call_with_backoff(mail_parser.parse, [html_email_file, "message/rfc822"])
@@ -272,14 +278,4 @@ class TestParserLive:
html_email_file, html_email_file,
"message/rfc822", "message/rfc822",
) )
generated_thumbnail_hash = self.imagehash(generated_thumbnail) self.assert_thumbnails_similar(generated_thumbnail, html_email_thumbnail_file)
# The created PDF is not reproducible, but the converted image
# should always look the same
expected_hash = self.imagehash(html_email_thumbnail_file)
assert generated_thumbnail_hash == expected_hash, (
f"PDF thumbnail differs from expected. "
f"Generated: {generated_thumbnail}, "
f"Hash: {generated_thumbnail_hash} vs {expected_hash}"
)