diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml index f92d1fb00..cff139e8c 100644 --- a/.github/workflows/ci-backend.yml +++ b/.github/workflows/ci-backend.yml @@ -24,6 +24,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 + persist-credentials: false - name: Decide run mode id: force run: | @@ -72,6 +73,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Start containers run: | docker compose --file docker/compose/docker-compose.ci-test.yml pull --quiet @@ -145,6 +148,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Set up Python id: setup-python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 diff --git a/.github/workflows/ci-docker.yml b/.github/workflows/ci-docker.yml index 48d258dca..43b79728d 100644 --- a/.github/workflows/ci-docker.yml +++ b/.github/workflows/ci-docker.yml @@ -42,6 +42,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Determine ref name id: ref run: | diff --git a/.github/workflows/ci-docs.yml b/.github/workflows/ci-docs.yml index 68f202264..a598a3c9d 100644 --- a/.github/workflows/ci-docs.yml +++ b/.github/workflows/ci-docs.yml @@ -26,6 +26,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 + persist-credentials: false - name: Decide run mode id: force run: | @@ -71,6 +72,8 @@ jobs: - uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0 - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Set up Python id: setup-python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 diff --git a/.github/workflows/ci-frontend.yml b/.github/workflows/ci-frontend.yml index dffb54e6b..9d4e23a1a 100644 --- a/.github/workflows/ci-frontend.yml +++ b/.github/workflows/ci-frontend.yml @@ -62,6 +62,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Install pnpm uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 with: @@ -90,6 +92,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Install pnpm uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 with: @@ -125,6 +129,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Install pnpm uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 with: @@ -176,6 +182,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Install pnpm uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 with: @@ -209,6 +217,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 2 + persist-credentials: false - name: Install pnpm uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 with: diff --git a/.github/workflows/ci-lint.yml b/.github/workflows/ci-lint.yml index bf1458e1d..314250719 100644 --- a/.github/workflows/ci-lint.yml +++ b/.github/workflows/ci-lint.yml @@ -16,6 +16,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Install Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: diff --git a/.github/workflows/ci-release.yml b/.github/workflows/ci-release.yml index b38ecbc40..030e3bcad 100644 --- a/.github/workflows/ci-release.yml +++ b/.github/workflows/ci-release.yml @@ -29,6 +29,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false # ---- Frontend Build ---- - name: Install pnpm uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 @@ -179,6 +181,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: main + persist-credentials: true # for pushing changelog branch - name: Set up Python id: setup-python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 diff --git a/.github/workflows/ci-static-analysis.yml b/.github/workflows/ci-static-analysis.yml new file mode 100644 index 000000000..99388354a --- /dev/null +++ b/.github/workflows/ci-static-analysis.yml @@ -0,0 +1,42 @@ +name: Static Analysis +on: + push: + branches-ignore: + - 'translations**' + pull_request: + branches-ignore: + - 'translations**' + workflow_dispatch: +concurrency: + group: static-analysis-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +permissions: + contents: read +jobs: + zizmor: + name: Run zizmor + runs-on: ubuntu-24.04 + permissions: + contents: read + actions: read + security-events: write + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Run zizmor + uses: zizmorcore/zizmor-action@71321a20a9ded102f6e9ce5718a2fcec2c4f70d8 # v0.5.2 + semgrep: + name: Semgrep CE + runs-on: ubuntu-24.04 + container: + image: semgrep/semgrep:1.155.0@sha256:cc869c685dcc0fe497c86258da9f205397d8108e56d21a86082ea4886e52784d + if: github.actor != 'dependabot[bot]' + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Run Semgrep + run: semgrep scan --config auto diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 08c2bc1a2..e295e938d 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -35,6 +35,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@c793b717bc78562f491db7b0e93a3a178b099162 # v4.32.5 diff --git a/.github/workflows/crowdin.yml b/.github/workflows/crowdin.yml index 38e73bbb5..29b4be02f 100644 --- a/.github/workflows/crowdin.yml +++ b/.github/workflows/crowdin.yml @@ -16,6 +16,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: token: ${{ secrets.PNGX_BOT_PAT }} + persist-credentials: false - name: crowdin action uses: crowdin/github-action@8818ff65bfc4322384f983ea37e3926948c11745 # v2.15.0 with: diff --git a/.github/workflows/translate-strings.yml b/.github/workflows/translate-strings.yml index c38886bc2..ad894abe7 100644 --- a/.github/workflows/translate-strings.yml +++ b/.github/workflows/translate-strings.yml @@ -17,6 +17,7 @@ jobs: with: token: ${{ secrets.PNGX_BOT_PAT }} ref: ${{ env.GH_REF }} + persist-credentials: true # for pushing translation branch - name: Set up Python id: setup-python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 diff --git a/docs/migration-v3.md b/docs/migration-v3.md index 1cfb212ff..afbc83186 100644 --- a/docs/migration-v3.md +++ b/docs/migration-v3.md @@ -1,5 +1,24 @@ # v3 Migration Guide +## Secret Key is Now Required + +The `PAPERLESS_SECRET_KEY` environment variable is now required. This is a critical security setting used for cryptographic signing and should be set to a long, random value. + +### Action Required + +If you are upgrading an existing installation, you must now set `PAPERLESS_SECRET_KEY` explicitly. + +If your installation was relying on the previous built-in default key, you have two options: + +- Set `PAPERLESS_SECRET_KEY` to that previous value to preserve existing sessions and tokens. +- Set `PAPERLESS_SECRET_KEY` to a new random value to improve security, understanding that this will invalidate existing sessions and other signed tokens. + +For new installations, or if you choose to rotate the key, you may generate a new secret key with: + +```bash +python3 -c "import secrets; print(secrets.token_urlsafe(64))" +``` + ## Consumer Settings Changes The v3 consumer command uses a [different library](https://watchfiles.helpmanual.io/) to unify diff --git a/src/documents/consumer.py b/src/documents/consumer.py index 6ae5914b7..f68fa0685 100644 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -139,14 +139,12 @@ class ConsumerPluginMixin: message, current_progress, max_progress, - extra_args={ - "document_id": document_id, - "owner_id": self.metadata.owner_id if self.metadata.owner_id else None, - "users_can_view": (self.metadata.view_users or []) - + (self.metadata.change_users or []), - "groups_can_view": (self.metadata.view_groups or []) - + (self.metadata.change_groups or []), - }, + document_id=document_id, + owner_id=self.metadata.owner_id if self.metadata.owner_id else None, + users_can_view=(self.metadata.view_users or []) + + (self.metadata.change_users or []), + groups_can_view=(self.metadata.view_groups or []) + + (self.metadata.change_groups or []), ) def _fail( diff --git a/src/documents/management/commands/document_exporter.py b/src/documents/management/commands/document_exporter.py index ee3b44e0c..562a2ca8d 100644 --- a/src/documents/management/commands/document_exporter.py +++ b/src/documents/management/commands/document_exporter.py @@ -45,6 +45,8 @@ from documents.models import DocumentType from documents.models import Note from documents.models import SavedView from documents.models import SavedViewFilterRule +from documents.models import ShareLink +from documents.models import ShareLinkBundle from documents.models import StoragePath from documents.models import Tag from documents.models import UiSettings @@ -55,6 +57,7 @@ from documents.models import WorkflowActionWebhook from documents.models import WorkflowTrigger from documents.settings import EXPORTER_ARCHIVE_NAME from documents.settings import EXPORTER_FILE_NAME +from documents.settings import EXPORTER_SHARE_LINK_BUNDLE_NAME from documents.settings import EXPORTER_THUMBNAIL_NAME from documents.utils import compute_checksum from documents.utils import copy_file_with_basic_stats @@ -389,6 +392,8 @@ class Command(CryptMixin, PaperlessCommand): "app_configs": ApplicationConfiguration.objects.all(), "notes": Note.global_objects.all(), "documents": Document.global_objects.order_by("id").all(), + "share_links": ShareLink.global_objects.all(), + "share_link_bundles": ShareLinkBundle.objects.order_by("id").all(), "social_accounts": SocialAccount.objects.all(), "social_apps": SocialApp.objects.all(), "social_tokens": SocialToken.objects.all(), @@ -409,6 +414,7 @@ class Command(CryptMixin, PaperlessCommand): ) document_manifest: list[dict] = [] + share_link_bundle_manifest: list[dict] = [] manifest_path = (self.target / "manifest.json").resolve() with StreamingManifestWriter( @@ -427,6 +433,15 @@ class Command(CryptMixin, PaperlessCommand): for record in batch: self._encrypt_record_inline(record) document_manifest.extend(batch) + elif key == "share_link_bundles": + # Accumulate for file-copy loop; written to manifest after + for batch in serialize_queryset_batched( + qs, + batch_size=self.batch_size, + ): + for record in batch: + self._encrypt_record_inline(record) + share_link_bundle_manifest.extend(batch) elif self.split_manifest and key in ( "notes", "custom_field_instances", @@ -445,6 +460,12 @@ class Command(CryptMixin, PaperlessCommand): document_map: dict[int, Document] = { d.pk: d for d in Document.global_objects.order_by("id") } + share_link_bundle_map: dict[int, ShareLinkBundle] = { + b.pk: b + for b in ShareLinkBundle.objects.order_by("id").prefetch_related( + "documents", + ) + } # 3. Export files from each document for index, document_dict in enumerate( @@ -478,6 +499,19 @@ class Command(CryptMixin, PaperlessCommand): else: writer.write_record(document_dict) + for bundle_dict in share_link_bundle_manifest: + bundle = share_link_bundle_map[bundle_dict["pk"]] + + bundle_target = self.generate_share_link_bundle_target( + bundle, + bundle_dict, + ) + + if not self.data_only and bundle_target is not None: + self.copy_share_link_bundle_file(bundle, bundle_target) + + writer.write_record(bundle_dict) + # 4.2 write version information to target folder extra_metadata_path = (self.target / "metadata.json").resolve() metadata: dict[str, str | int | dict[str, str | int]] = { @@ -598,6 +632,48 @@ class Command(CryptMixin, PaperlessCommand): archive_target, ) + def generate_share_link_bundle_target( + self, + bundle: ShareLinkBundle, + bundle_dict: dict, + ) -> Path | None: + """ + Generates the export target for a share link bundle file, when present. + """ + if not bundle.file_path: + return None + + stored_bundle_path = Path(bundle.file_path) + portable_bundle_path = ( + stored_bundle_path + if not stored_bundle_path.is_absolute() + else Path(stored_bundle_path.name) + ) + export_bundle_path = Path("share_link_bundles") / portable_bundle_path + + bundle_dict["fields"]["file_path"] = portable_bundle_path.as_posix() + bundle_dict[EXPORTER_SHARE_LINK_BUNDLE_NAME] = export_bundle_path.as_posix() + + return (self.target / export_bundle_path).resolve() + + def copy_share_link_bundle_file( + self, + bundle: ShareLinkBundle, + bundle_target: Path, + ) -> None: + """ + Copies a share link bundle ZIP into the export directory. + """ + bundle_source_path = bundle.absolute_file_path + if bundle_source_path is None: + raise FileNotFoundError(f"Share link bundle {bundle.pk} has no file path") + + self.check_and_copy( + bundle_source_path, + None, + bundle_target, + ) + def _encrypt_record_inline(self, record: dict) -> None: """Encrypt sensitive fields in a single record, if passphrase is set.""" if not self.passphrase: diff --git a/src/documents/management/commands/document_importer.py b/src/documents/management/commands/document_importer.py index 4572b4617..becdf7b76 100644 --- a/src/documents/management/commands/document_importer.py +++ b/src/documents/management/commands/document_importer.py @@ -32,10 +32,12 @@ from documents.models import CustomFieldInstance from documents.models import Document from documents.models import DocumentType from documents.models import Note +from documents.models import ShareLinkBundle from documents.models import Tag from documents.settings import EXPORTER_ARCHIVE_NAME from documents.settings import EXPORTER_CRYPTO_SETTINGS_NAME from documents.settings import EXPORTER_FILE_NAME +from documents.settings import EXPORTER_SHARE_LINK_BUNDLE_NAME from documents.settings import EXPORTER_THUMBNAIL_NAME from documents.signals.handlers import check_paths_and_prune_custom_fields from documents.signals.handlers import update_filename_and_move_files @@ -348,18 +350,42 @@ class Command(CryptMixin, PaperlessCommand): f"Failed to read from archive file {doc_archive_path}", ) from e + def check_share_link_bundle_validity(bundle_record: dict) -> None: + if EXPORTER_SHARE_LINK_BUNDLE_NAME not in bundle_record: + return + + bundle_file = bundle_record[EXPORTER_SHARE_LINK_BUNDLE_NAME] + bundle_path: Path = self.source / bundle_file + if not bundle_path.exists(): + raise CommandError( + f'The manifest file refers to "{bundle_file}" which does not ' + "appear to be in the source directory.", + ) + try: + with bundle_path.open(mode="rb"): + pass + except Exception as e: + raise CommandError( + f"Failed to read from share link bundle file {bundle_path}", + ) from e + self.stdout.write("Checking the manifest") for manifest_path in self.manifest_paths: for record in iter_manifest_records(manifest_path): # Only check if the document files exist if this is not data only # We don't care about documents for a data only import - if not self.data_only and record["model"] == "documents.document": + if self.data_only: + continue + if record["model"] == "documents.document": check_document_validity(record) + elif record["model"] == "documents.sharelinkbundle": + check_share_link_bundle_validity(record) def _import_files_from_manifest(self) -> None: settings.ORIGINALS_DIR.mkdir(parents=True, exist_ok=True) settings.THUMBNAIL_DIR.mkdir(parents=True, exist_ok=True) settings.ARCHIVE_DIR.mkdir(parents=True, exist_ok=True) + settings.SHARE_LINK_BUNDLE_DIR.mkdir(parents=True, exist_ok=True) self.stdout.write("Copy files into paperless...") @@ -374,6 +400,18 @@ class Command(CryptMixin, PaperlessCommand): for record in iter_manifest_records(manifest_path) if record["model"] == "documents.document" ] + share_link_bundle_records = [ + { + "pk": record["pk"], + EXPORTER_SHARE_LINK_BUNDLE_NAME: record.get( + EXPORTER_SHARE_LINK_BUNDLE_NAME, + ), + } + for manifest_path in self.manifest_paths + for record in iter_manifest_records(manifest_path) + if record["model"] == "documents.sharelinkbundle" + and record.get(EXPORTER_SHARE_LINK_BUNDLE_NAME) + ] for record in self.track(document_records, description="Copying files..."): document = Document.global_objects.get(pk=record["pk"]) @@ -416,6 +454,26 @@ class Command(CryptMixin, PaperlessCommand): document.save() + for record in self.track( + share_link_bundle_records, + description="Copying share link bundles...", + ): + bundle = ShareLinkBundle.objects.get(pk=record["pk"]) + bundle_file = record[EXPORTER_SHARE_LINK_BUNDLE_NAME] + bundle_source_path = (self.source / bundle_file).resolve() + bundle_target_path = bundle.absolute_file_path + if bundle_target_path is None: + raise CommandError( + f"Share link bundle {bundle.pk} does not have a valid file path.", + ) + + with FileLock(settings.MEDIA_LOCK): + bundle_target_path.parent.mkdir(parents=True, exist_ok=True) + copy_file_with_basic_stats( + bundle_source_path, + bundle_target_path, + ) + def _decrypt_record_if_needed(self, record: dict) -> dict: fields = self.CRYPT_FIELDS_BY_MODEL.get(record.get("model", "")) if fields: diff --git a/src/documents/plugins/helpers.py b/src/documents/plugins/helpers.py index e5cfde3b8..e30591125 100644 --- a/src/documents/plugins/helpers.py +++ b/src/documents/plugins/helpers.py @@ -1,6 +1,9 @@ import enum -from collections.abc import Mapping from typing import TYPE_CHECKING +from typing import Literal +from typing import Self +from typing import TypeAlias +from typing import TypedDict from asgiref.sync import async_to_sync from channels.layers import get_channel_layer @@ -16,6 +19,59 @@ class ProgressStatusOptions(enum.StrEnum): FAILED = "FAILED" +class PermissionsData(TypedDict, total=False): + """Permission fields included in status messages for access control.""" + + owner_id: int | None + users_can_view: list[int] + groups_can_view: list[int] + + +class ProgressUpdateData(TypedDict): + filename: str | None + task_id: str | None + current_progress: int + max_progress: int + status: str + message: str + document_id: int | None + owner_id: int | None + users_can_view: list[int] + groups_can_view: list[int] + + +class StatusUpdatePayload(TypedDict): + type: Literal["status_update"] + data: ProgressUpdateData + + +class DocumentsDeletedData(TypedDict): + documents: list[int] + + +class DocumentsDeletedPayload(TypedDict): + type: Literal["documents_deleted"] + data: DocumentsDeletedData + + +class DocumentUpdatedData(TypedDict): + document_id: int + modified: str + owner_id: int | None + users_can_view: list[int] + groups_can_view: list[int] + + +class DocumentUpdatedPayload(TypedDict): + type: Literal["document_updated"] + data: DocumentUpdatedData + + +WebsocketPayload: TypeAlias = ( + StatusUpdatePayload | DocumentsDeletedPayload | DocumentUpdatedPayload +) + + class BaseStatusManager: """ Handles sending of progress information via the channel layer, with proper management @@ -25,11 +81,11 @@ class BaseStatusManager: def __init__(self) -> None: self._channel: RedisPubSubChannelLayer | None = None - def __enter__(self): + def __enter__(self) -> Self: self.open() return self - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None: self.close() def open(self) -> None: @@ -48,7 +104,7 @@ class BaseStatusManager: async_to_sync(self._channel.flush) self._channel = None - def send(self, payload: Mapping[str, object]) -> None: + def send(self, payload: WebsocketPayload) -> None: # Ensure the layer is open self.open() @@ -72,36 +128,36 @@ class ProgressManager(BaseStatusManager): message: str, current_progress: int, max_progress: int, - extra_args: dict[str, str | int | None] | None = None, + *, + document_id: int | None = None, + owner_id: int | None = None, + users_can_view: list[int] | None = None, + groups_can_view: list[int] | None = None, ) -> None: - data: dict[str, object] = { + data: ProgressUpdateData = { "filename": self.filename, "task_id": self.task_id, "current_progress": current_progress, "max_progress": max_progress, "status": status, "message": message, + "document_id": document_id, + "owner_id": owner_id, + "users_can_view": users_can_view or [], + "groups_can_view": groups_can_view or [], } - if extra_args is not None: - data.update(extra_args) - - payload: dict[str, object] = { - "type": "status_update", - "data": data, - } - + payload: StatusUpdatePayload = {"type": "status_update", "data": data} self.send(payload) class DocumentsStatusManager(BaseStatusManager): def send_documents_deleted(self, documents: list[int]) -> None: - payload: dict[str, object] = { + payload: DocumentsDeletedPayload = { "type": "documents_deleted", "data": { "documents": documents, }, } - self.send(payload) def send_document_updated( @@ -113,7 +169,7 @@ class DocumentsStatusManager(BaseStatusManager): users_can_view: list[int] | None = None, groups_can_view: list[int] | None = None, ) -> None: - payload: dict[str, object] = { + payload: DocumentUpdatedPayload = { "type": "document_updated", "data": { "document_id": document_id, @@ -123,5 +179,4 @@ class DocumentsStatusManager(BaseStatusManager): "groups_can_view": groups_can_view or [], }, } - self.send(payload) diff --git a/src/documents/search/_backend.py b/src/documents/search/_backend.py index eb7e51391..56a5981d9 100644 --- a/src/documents/search/_backend.py +++ b/src/documents/search/_backend.py @@ -573,6 +573,7 @@ class TantivyBackend: # Build result hits with highlights hits: list[SearchHit] = [] snippet_generator = None + notes_snippet_generator = None # Determine which hits need highlights if highlight_page is not None and highlight_page_size is not None: @@ -609,13 +610,16 @@ class TantivyBackend: # Try notes highlights if "notes" in doc_dict: - notes_generator = tantivy.SnippetGenerator.create( - searcher, - final_query, - self._schema, - "notes", + if notes_snippet_generator is None: + notes_snippet_generator = tantivy.SnippetGenerator.create( + searcher, + final_query, + self._schema, + "notes", + ) + notes_snippet = notes_snippet_generator.snippet_from_doc( + actual_doc, ) - notes_snippet = notes_generator.snippet_from_doc(actual_doc) if notes_snippet: highlights["notes"] = str(notes_snippet) diff --git a/src/documents/settings.py b/src/documents/settings.py index 9dff44c95..c4c87b8a7 100644 --- a/src/documents/settings.py +++ b/src/documents/settings.py @@ -3,6 +3,7 @@ EXPORTER_FILE_NAME = "__exported_file_name__" EXPORTER_THUMBNAIL_NAME = "__exported_thumbnail_name__" EXPORTER_ARCHIVE_NAME = "__exported_archive_name__" +EXPORTER_SHARE_LINK_BUNDLE_NAME = "__exported_share_link_bundle_name__" EXPORTER_CRYPTO_SETTINGS_NAME = "__crypto__" EXPORTER_CRYPTO_SALT_NAME = "__salt_hex__" diff --git a/src/documents/tests/test_management_exporter.py b/src/documents/tests/test_management_exporter.py index a214ef51d..4ee7677ca 100644 --- a/src/documents/tests/test_management_exporter.py +++ b/src/documents/tests/test_management_exporter.py @@ -2,6 +2,7 @@ import hashlib import json import shutil import tempfile +from datetime import timedelta from io import StringIO from pathlib import Path from unittest import mock @@ -11,6 +12,7 @@ import pytest from allauth.socialaccount.models import SocialAccount from allauth.socialaccount.models import SocialApp from allauth.socialaccount.models import SocialToken +from django.conf import settings from django.contrib.auth.models import Group from django.contrib.auth.models import Permission from django.contrib.contenttypes.models import ContentType @@ -31,6 +33,8 @@ from documents.models import CustomFieldInstance from documents.models import Document from documents.models import DocumentType from documents.models import Note +from documents.models import ShareLink +from documents.models import ShareLinkBundle from documents.models import StoragePath from documents.models import Tag from documents.models import User @@ -39,6 +43,7 @@ from documents.models import WorkflowAction from documents.models import WorkflowTrigger from documents.sanity_checker import check_sanity from documents.settings import EXPORTER_FILE_NAME +from documents.settings import EXPORTER_SHARE_LINK_BUNDLE_NAME from documents.tests.utils import DirectoriesMixin from documents.tests.utils import FileSystemAssertsMixin from documents.tests.utils import SampleDirMixin @@ -306,6 +311,108 @@ class TestExportImport( ): self.test_exporter(use_filename_format=True) + def test_exporter_includes_share_links_and_bundles(self) -> None: + shutil.rmtree(Path(self.dirs.media_dir) / "documents") + shutil.copytree( + Path(__file__).parent / "samples" / "documents", + Path(self.dirs.media_dir) / "documents", + ) + + share_link = ShareLink.objects.create( + slug="share-link-slug", + document=self.d1, + owner=self.user, + file_version=ShareLink.FileVersion.ORIGINAL, + expiration=timezone.now() + timedelta(days=7), + ) + + bundle_relative_path = Path("nested") / "share-bundle.zip" + bundle_source_path = settings.SHARE_LINK_BUNDLE_DIR / bundle_relative_path + bundle_source_path.parent.mkdir(parents=True, exist_ok=True) + bundle_source_path.write_bytes(b"share-bundle-contents") + bundle = ShareLinkBundle.objects.create( + slug="share-bundle-slug", + owner=self.user, + file_version=ShareLink.FileVersion.ARCHIVE, + expiration=timezone.now() + timedelta(days=7), + status=ShareLinkBundle.Status.READY, + size_bytes=bundle_source_path.stat().st_size, + file_path=str(bundle_relative_path), + built_at=timezone.now(), + ) + bundle.documents.set([self.d1, self.d2]) + + manifest = self._do_export() + + share_link_records = [ + record for record in manifest if record["model"] == "documents.sharelink" + ] + self.assertEqual(len(share_link_records), 1) + self.assertEqual(share_link_records[0]["pk"], share_link.pk) + self.assertEqual(share_link_records[0]["fields"]["document"], self.d1.pk) + self.assertEqual(share_link_records[0]["fields"]["owner"], self.user.pk) + + share_link_bundle_records = [ + record + for record in manifest + if record["model"] == "documents.sharelinkbundle" + ] + self.assertEqual(len(share_link_bundle_records), 1) + bundle_record = share_link_bundle_records[0] + self.assertEqual(bundle_record["pk"], bundle.pk) + self.assertEqual( + bundle_record["fields"]["documents"], + [self.d1.pk, self.d2.pk], + ) + self.assertEqual( + bundle_record[EXPORTER_SHARE_LINK_BUNDLE_NAME], + "share_link_bundles/nested/share-bundle.zip", + ) + self.assertEqual( + bundle_record["fields"]["file_path"], + "nested/share-bundle.zip", + ) + self.assertIsFile(self.target / bundle_record[EXPORTER_SHARE_LINK_BUNDLE_NAME]) + + with paperless_environment(): + ShareLink.objects.all().delete() + ShareLinkBundle.objects.all().delete() + shutil.rmtree(settings.SHARE_LINK_BUNDLE_DIR, ignore_errors=True) + + call_command( + "document_importer", + "--no-progress-bar", + self.target, + skip_checks=True, + ) + + imported_share_link = ShareLink.objects.get(pk=share_link.pk) + self.assertEqual(imported_share_link.document_id, self.d1.pk) + self.assertEqual(imported_share_link.owner_id, self.user.pk) + self.assertEqual( + imported_share_link.file_version, + ShareLink.FileVersion.ORIGINAL, + ) + + imported_bundle = ShareLinkBundle.objects.get(pk=bundle.pk) + imported_bundle_path = imported_bundle.absolute_file_path + self.assertEqual(imported_bundle.owner_id, self.user.pk) + self.assertEqual( + list( + imported_bundle.documents.order_by("pk").values_list( + "pk", + flat=True, + ), + ), + [self.d1.pk, self.d2.pk], + ) + self.assertEqual(imported_bundle.file_path, "nested/share-bundle.zip") + self.assertIsNotNone(imported_bundle_path) + self.assertEqual( + imported_bundle_path.read_bytes(), + b"share-bundle-contents", + ) + def test_update_export_changed_time(self) -> None: shutil.rmtree(Path(self.dirs.media_dir) / "documents") shutil.copytree( diff --git a/src/documents/tests/utils.py b/src/documents/tests/utils.py index cc4190974..98c8258b8 100644 --- a/src/documents/tests/utils.py +++ b/src/documents/tests/utils.py @@ -435,7 +435,11 @@ class DummyProgressManager: message: str, current_progress: int, max_progress: int, - extra_args: dict[str, str | int] | None = None, + *, + document_id: int | None = None, + owner_id: int | None = None, + users_can_view: list[int] | None = None, + groups_can_view: list[int] | None = None, ) -> None: # Ensure the layer is open self.open() @@ -449,9 +453,10 @@ class DummyProgressManager: "max_progress": max_progress, "status": status, "message": message, + "document_id": document_id, + "owner_id": owner_id, + "users_can_view": users_can_view or [], + "groups_can_view": groups_can_view or [], }, } - if extra_args is not None: - payload["data"].update(extra_args) - self.payloads.append(payload) diff --git a/src/paperless/consumers.py b/src/paperless/consumers.py index 9d59a1a5a..4a3cda8fe 100644 --- a/src/paperless/consumers.py +++ b/src/paperless/consumers.py @@ -1,16 +1,27 @@ +from __future__ import annotations + import json -from typing import Any +from typing import TYPE_CHECKING from channels.generic.websocket import AsyncWebsocketConsumer +if TYPE_CHECKING: + from django.contrib.auth.base_user import AbstractBaseUser + from django.contrib.auth.models import AnonymousUser + + from documents.plugins.helpers import DocumentsDeletedPayload + from documents.plugins.helpers import DocumentUpdatedPayload + from documents.plugins.helpers import PermissionsData + from documents.plugins.helpers import StatusUpdatePayload + class StatusConsumer(AsyncWebsocketConsumer): def _authenticated(self) -> bool: - user: Any = self.scope.get("user") + user: AbstractBaseUser | AnonymousUser | None = self.scope.get("user") return user is not None and user.is_authenticated - async def _can_view(self, data: dict[str, Any]) -> bool: - user: Any = self.scope.get("user") + async def _can_view(self, data: PermissionsData) -> bool: + user: AbstractBaseUser | AnonymousUser | None = self.scope.get("user") if user is None: return False owner_id = data.get("owner_id") @@ -32,19 +43,19 @@ class StatusConsumer(AsyncWebsocketConsumer): async def disconnect(self, code: int) -> None: await self.channel_layer.group_discard("status_updates", self.channel_name) - async def status_update(self, event: dict[str, Any]) -> None: + async def status_update(self, event: StatusUpdatePayload) -> None: if not self._authenticated(): await self.close() elif await self._can_view(event["data"]): await self.send(json.dumps(event)) - async def documents_deleted(self, event: dict[str, Any]) -> None: + async def documents_deleted(self, event: DocumentsDeletedPayload) -> None: if not self._authenticated(): await self.close() else: await self.send(json.dumps(event)) - async def document_updated(self, event: dict[str, Any]) -> None: + async def document_updated(self, event: DocumentUpdatedPayload) -> None: if not self._authenticated(): await self.close() elif await self._can_view(event["data"]): diff --git a/src/paperless/tests/test_websockets.py b/src/paperless/tests/test_websockets.py index bffc44f82..9f7c9a652 100644 --- a/src/paperless/tests/test_websockets.py +++ b/src/paperless/tests/test_websockets.py @@ -200,7 +200,10 @@ class TestWebSockets: "Test message", 1, 10, - extra_args={"foo": "bar"}, + document_id=42, + owner_id=1, + users_can_view=[2, 3], + groups_can_view=[4], ) assert mock_group_send.call_args[0][1] == { @@ -212,7 +215,10 @@ class TestWebSockets: "max_progress": 10, "status": ProgressStatusOptions.STARTED, "message": "Test message", - "foo": "bar", + "document_id": 42, + "owner_id": 1, + "users_can_view": [2, 3], + "groups_can_view": [4], }, }