paperless-ngx/src/documents/tests/test_file_handling.py

import datetime
import hashlib
import logging
import tempfile
from pathlib import Path
from unittest import mock

import pytest
from auditlog.context import disable_auditlog
from django.conf import settings
from django.db import DatabaseError
from django.db import connection
from django.test import TestCase
from django.test import override_settings
from django.test.utils import CaptureQueriesContext
from django.utils import timezone

from documents.file_handling import UnsafeFilePathError
from documents.file_handling import create_source_path_directory
from documents.file_handling import delete_empty_directories
from documents.file_handling import generate_filename
from documents.file_handling import generate_unique_filename
from documents.file_handling import validate_path_in_root
from documents.models import Correspondent
from documents.models import CustomField
from documents.models import CustomFieldInstance
from documents.models import Document
from documents.models import DocumentType
from documents.models import StoragePath
from documents.serialisers import DocumentSerializer
from documents.tasks import empty_trash
from paperless_testing.assertions import FileSystemAssertsMixin
from paperless_testing.dirs import DirectoriesMixin
from paperless_testing.factories import DocumentFactory
from paperless_testing.factories import UserFactory


class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
    @override_settings(FILENAME_FORMAT="{title}")
    def test_generate_unique_filename_renders_template_once(self) -> None:
        document = Document.objects.create(
            title="collision",
            mime_type="application/pdf",
        )
        Document.objects.filter(pk=document.pk).update(filename="collision_03.pdf")
        document.refresh_from_db()

        for filename in (
            "collision.pdf",
            "collision_01.pdf",
            "collision_02.pdf",
            "collision_03.pdf",
        ):
            (settings.ORIGINALS_DIR / filename).touch()

        with CaptureQueriesContext(connection) as queries:
            generated = generate_unique_filename(document)

        relation_queries = [
            query["sql"]
            for query in queries.captured_queries
            if "documents_tag" in query["sql"]
            or "documents_customfieldinstance" in query["sql"]
        ]

        self.assertEqual(generated, Path("collision_03.pdf"))
        self.assertEqual(len(relation_queries), 2)

    @override_settings(FILENAME_FORMAT="")
    def test_generate_source_filename(self) -> None:
        document = Document()
        document.mime_type = "application/pdf"
        document.save()

        self.assertEqual(generate_filename(document), Path(f"{document.pk:07d}.pdf"))

    @override_settings(FILENAME_FORMAT="{correspondent}/{correspondent}")
    def test_file_renaming(self) -> None:
        document = Document()
        document.mime_type = "application/pdf"
        document.save()

        # Test default source_path
        self.assertEqual(
            document.source_path,
            settings.ORIGINALS_DIR / f"{document.pk:07d}.pdf",
        )

        document.filename = generate_filename(document)

        # Ensure that filename is properly generated
        self.assertEqual(document.filename, Path("none/none.pdf"))

        document.save()

        # test that creating dirs for the source_path creates the correct directory
        create_source_path_directory(document.source_path)
        Path(document.source_path).touch()
        self.assertIsDir(settings.ORIGINALS_DIR / "none")

        # Set a correspondent and save the document
        document.correspondent = Correspondent.objects.get_or_create(name="test")[0]
        document.save()

        # Check proper handling of files
        self.assertIsDir(
            settings.ORIGINALS_DIR / "test",
        )
        self.assertIsNotDir(
            settings.ORIGINALS_DIR / "none",
        )
        self.assertIsFile(
            settings.ORIGINALS_DIR / "test" / "test.pdf",
        )

    @override_settings(FILENAME_FORMAT=None)
    def test_root_storage_path_change_updates_version_files(self) -> None:
        old_storage_path = StoragePath.objects.create(
            name="old-path",
            path="old/{{title}}",
        )
        new_storage_path = StoragePath.objects.create(
            name="new-path",
            path="new/{{title}}",
        )

        root_doc = Document.objects.create(
            title="rootdoc",
            mime_type="application/pdf",
            checksum="root-checksum",
            storage_path=old_storage_path,
        )
        version_doc = Document.objects.create(
            title="version-title",
            mime_type="application/pdf",
            checksum="version-checksum",
            root_document=root_doc,
            version_index=1,
        )

        Document.objects.filter(pk=root_doc.pk).update(
            filename=generate_filename(root_doc),
        )
        Document.objects.filter(pk=version_doc.pk).update(
            filename=generate_filename(version_doc),
        )
        root_doc.refresh_from_db()
        version_doc.refresh_from_db()

        create_source_path_directory(root_doc.source_path)
        Path(root_doc.source_path).touch()
        create_source_path_directory(version_doc.source_path)
        Path(version_doc.source_path).touch()

        root_doc.storage_path = new_storage_path
        root_doc.save()

        root_doc.refresh_from_db()
        version_doc.refresh_from_db()

        self.assertEqual(root_doc.filename, "new/rootdoc.pdf")
        self.assertEqual(version_doc.filename, "new/rootdoc_v1.pdf")
        self.assertIsFile(root_doc.source_path)
        self.assertIsFile(version_doc.source_path)
        self.assertIsNotFile(settings.ORIGINALS_DIR / "old" / "rootdoc.pdf")
        self.assertIsNotFile(settings.ORIGINALS_DIR / "old" / "rootdoc_v1.pdf")

    @override_settings(FILENAME_FORMAT="{correspondent}/{correspondent}")
    def test_file_renaming_missing_permissions(self) -> None:
        document = Document()
        document.mime_type = "application/pdf"

        document.save()

        # Ensure that filename is properly generated
        document.filename = generate_filename(document)
        self.assertEqual(document.filename, Path("none/none.pdf"))
        create_source_path_directory(document.source_path)
        document.source_path.touch()

        # Test source_path
        self.assertEqual(
            document.source_path,
            settings.ORIGINALS_DIR / "none" / "none.pdf",
        )

        # Make the folder read- and execute-only (no writing and no renaming)
        (settings.ORIGINALS_DIR / "none").chmod(0o555)

        # Set a correspondent and save the document
        document.correspondent = Correspondent.objects.get_or_create(name="test")[0]
        document.save()

        # Check proper handling of files
        self.assertIsFile(
            settings.ORIGINALS_DIR / "none" / "none.pdf",
        )
        self.assertEqual(document.filename, "none/none.pdf")

        (settings.ORIGINALS_DIR / "none").chmod(0o777)

    @override_settings(FILENAME_FORMAT="{correspondent}/{correspondent}")
    def test_file_renaming_database_error(self) -> None:
        Document.objects.create(
            mime_type="application/pdf",
            checksum="AAAAA",
        )

        document = Document()
        document.mime_type = "application/pdf"
        document.checksum = "BBBBB"

        document.save()

        # Ensure that filename is properly generated
        document.filename = generate_filename(document)
        self.assertEqual(document.filename, Path("none/none.pdf"))
        create_source_path_directory(document.source_path)
        Path(document.source_path).touch()

        # Test source_path
        self.assertIsFile(document.source_path)

        # Set a correspondent and save the document
        document.correspondent = Correspondent.objects.get_or_create(name="test")[0]

        with (
            mock.patch(
                "documents.signals.handlers.Document.global_objects.filter",
            ) as m,
            disable_auditlog(),
        ):
            m.side_effect = DatabaseError()
            document.save()

            # Check proper handling of files
            self.assertIsFile(document.source_path)
            self.assertIsFile(
                settings.ORIGINALS_DIR / "none" / "none.pdf",
            )
            self.assertEqual(document.filename, "none/none.pdf")

    @override_settings(FILENAME_FORMAT=None)
    def test_stale_save_recovers_already_moved_files(self) -> None:
        old_storage_path = StoragePath.objects.create(
            name="old-path",
            path="old/{{title}}",
        )
        new_storage_path = StoragePath.objects.create(
            name="new-path",
            path="new/{{title}}",
        )
        original_bytes = b"original"
        archive_bytes = b"archive"

        doc = Document.objects.create(
            title="document",
            mime_type="application/pdf",
            checksum=hashlib.sha256(original_bytes).hexdigest(),
            archive_checksum=hashlib.sha256(archive_bytes).hexdigest(),
            filename="old/document.pdf",
            archive_filename="old/document.pdf",
            storage_path=old_storage_path,
        )
        create_source_path_directory(doc.source_path)
        doc.source_path.write_bytes(original_bytes)
        create_source_path_directory(doc.archive_path)
        doc.archive_path.write_bytes(archive_bytes)

        stale_doc = Document.objects.get(pk=doc.pk)
        fresh_doc = Document.objects.get(pk=doc.pk)
        fresh_doc.storage_path = new_storage_path
        fresh_doc.save()
        doc.refresh_from_db()
        self.assertEqual(doc.filename, "new/document.pdf")
        self.assertEqual(doc.archive_filename, "new/document.pdf")

        stale_doc.storage_path = new_storage_path
        stale_doc.save()

        doc.refresh_from_db()
        self.assertEqual(doc.filename, "new/document.pdf")
        self.assertEqual(doc.archive_filename, "new/document.pdf")
        self.assertIsFile(doc.source_path)
        self.assertIsFile(doc.archive_path)
        self.assertIsNotFile(settings.ORIGINALS_DIR / "old" / "document.pdf")
        self.assertIsNotFile(settings.ARCHIVE_DIR / "old" / "document.pdf")

    @override_settings(FILENAME_FORMAT="{title}")
    def test_serializer_stale_update_does_not_clobber_filename(self) -> None:
        old_path = settings.ORIGINALS_DIR / "original.pdf"
        old_path.touch()
        doc = Document.objects.create(
            title="original",
            mime_type="application/pdf",
            checksum=hashlib.sha256(b"").hexdigest(),
            filename="original.pdf",
        )

        first_instance = Document.objects.get(pk=doc.pk)
        stale_instance = Document.objects.get(pk=doc.pk)

        serializer = DocumentSerializer(
            first_instance,
            data={"title": "first"},
            partial=True,
        )
        self.assertTrue(serializer.is_valid(), serializer.errors)
        serializer.save()

        doc.refresh_from_db()
        self.assertEqual(doc.filename, "first.pdf")
        self.assertIsFile(settings.ORIGINALS_DIR / "first.pdf")

        serializer = DocumentSerializer(
            stale_instance,
            data={"title": "second"},
            partial=True,
        )
        self.assertTrue(serializer.is_valid(), serializer.errors)
        serializer.save()

        doc.refresh_from_db()
        self.assertEqual(doc.filename, "second.pdf")
        self.assertIsFile(settings.ORIGINALS_DIR / "second.pdf")
        self.assertIsNotFile(settings.ORIGINALS_DIR / "first.pdf")
        self.assertIsNotFile(old_path)

    @override_settings(FILENAME_FORMAT="{correspondent}/{correspondent}")
    def test_document_delete(self) -> None:
        document = Document()
        document.mime_type = "application/pdf"

        document.save()

        # Ensure that filename is properly generated
        document.filename = generate_filename(document)
        document.save()
        self.assertEqual(document.filename, "none/none.pdf")

        create_source_path_directory(document.source_path)
        Path(document.source_path).touch()

        # Ensure file deletion after delete
        document.delete()
        empty_trash([document.pk])
        self.assertIsNotFile(
            settings.ORIGINALS_DIR / "none" / "none.pdf",
        )
        self.assertIsNotDir(settings.ORIGINALS_DIR / "none")

    @override_settings(
        FILENAME_FORMAT="{correspondent}/{correspondent}",
        EMPTY_TRASH_DIR=Path(tempfile.mkdtemp()),
    )
    def test_document_delete_trash_dir(self) -> None:
        document = Document()
        document.mime_type = "application/pdf"

        document.save()

        # Ensure that filename is properly generated
        document.filename = generate_filename(document)
        document.save()
        self.assertEqual(document.filename, "none/none.pdf")

        create_source_path_directory(document.source_path)
        Path(document.source_path).touch()

        # Ensure file was moved to trash after delete
        self.assertIsNotFile(Path(settings.EMPTY_TRASH_DIR) / "none" / "none.pdf")
        document.delete()
        empty_trash([document.pk])
        self.assertIsNotFile(
            settings.ORIGINALS_DIR / "none" / "none.pdf",
        )
        self.assertIsNotDir(settings.ORIGINALS_DIR / "none")
        self.assertIsFile(Path(settings.EMPTY_TRASH_DIR) / "none.pdf")
        self.assertIsNotFile(Path(settings.EMPTY_TRASH_DIR) / "none_01.pdf")

        # Create an identical document and ensure it is trashed under a new name
        document = Document()
        document.mime_type = "application/pdf"

        document.save()
        document.filename = generate_filename(document)
        document.save()
        create_source_path_directory(document.source_path)
        Path(document.source_path).touch()
        document.delete()
        empty_trash([document.pk])
        self.assertIsFile(Path(settings.EMPTY_TRASH_DIR) / "none_01.pdf")

    @override_settings(FILENAME_FORMAT="{correspondent}/{correspondent}")
    def test_document_delete_nofile(self) -> None:
        document = Document()
        document.mime_type = "application/pdf"

        document.save()

        document.delete()
        empty_trash([document.pk])

    @override_settings(FILENAME_FORMAT="{correspondent}/{correspondent}")
    def test_directory_not_empty(self) -> None:
        document = Document()
        document.mime_type = "application/pdf"

        document.save()

        # Ensure that filename is properly generated
        document.filename = generate_filename(document)
        self.assertEqual(document.filename, Path("none/none.pdf"))

        create_source_path_directory(document.source_path)

        document.source_path.touch()
        important_file = document.source_path.with_suffix(".test")
        important_file.touch()

        # Set a correspondent and save the document
        document.correspondent = Correspondent.objects.get_or_create(name="test")[0]
        document.save()

        # Check proper handling of files
        self.assertIsDir(settings.ORIGINALS_DIR / "test")
        self.assertIsDir(settings.ORIGINALS_DIR / "none")
        self.assertIsFile(important_file)

    @override_settings(FILENAME_FORMAT="{document_type} - {title}")
    def test_document_type(self) -> None:
        dt = DocumentType.objects.create(name="my_doc_type")
        d = Document.objects.create(title="the_doc", mime_type="application/pdf")

        self.assertEqual(generate_filename(d), Path("none - the_doc.pdf"))

        d.document_type = dt

        self.assertEqual(generate_filename(d), Path("my_doc_type - the_doc.pdf"))

    @override_settings(FILENAME_FORMAT="{asn} - {title}")
    def test_asn(self) -> None:
        d1 = Document.objects.create(
            title="the_doc",
            mime_type="application/pdf",
            archive_serial_number=652,
            checksum="A",
        )
        d2 = Document.objects.create(
            title="the_doc",
            mime_type="application/pdf",
            archive_serial_number=None,
            checksum="B",
        )
        self.assertEqual(generate_filename(d1), Path("652 - the_doc.pdf"))
        self.assertEqual(generate_filename(d2), Path("none - the_doc.pdf"))

    @override_settings(FILENAME_FORMAT="{title} {tag_list}")
    def test_tag_list(self) -> None:
        doc = Document.objects.create(title="doc1", mime_type="application/pdf")
        doc.tags.create(name="tag2")
        doc.tags.create(name="tag1")

        self.assertEqual(generate_filename(doc), Path("doc1 tag1,tag2.pdf"))

        doc = Document.objects.create(
            title="doc2",
            checksum="B",
            mime_type="application/pdf",
        )

        self.assertEqual(generate_filename(doc), Path("doc2.pdf"))

    @override_settings(FILENAME_FORMAT="//etc/something/{title}")
    def test_filename_relative(self) -> None:
        doc = Document.objects.create(title="doc1", mime_type="application/pdf")
        doc.filename = generate_filename(doc)
        doc.save()

        self.assertEqual(
            doc.source_path,
            settings.ORIGINALS_DIR / "etc" / "something" / "doc1.pdf",
        )

    @override_settings(
        FILENAME_FORMAT="{created_year}-{created_month}-{created_day}",
    )
    def test_created_year_month_day(self) -> None:
        d1 = timezone.make_aware(datetime.datetime(2020, 3, 6, 1, 1, 1))
        doc1 = Document.objects.create(
            title="doc1",
            mime_type="application/pdf",
            created=d1,
        )

        self.assertEqual(generate_filename(doc1), Path("2020-03-06.pdf"))

        doc1.created = datetime.date(2020, 11, 16)

        self.assertEqual(generate_filename(doc1), Path("2020-11-16.pdf"))

    @override_settings(
        FILENAME_FORMAT="{added_year}-{added_month}-{added_day}",
    )
    def test_added_year_month_day(self) -> None:
        d1 = timezone.make_aware(datetime.datetime(1232, 1, 9, 1, 1, 1))
        doc1 = Document.objects.create(
            title="doc1",
            mime_type="application/pdf",
            added=d1,
        )

        # Account for 3.14 padding changes
        expected_year: str = d1.strftime("%Y")
        expected_filename: Path = Path(f"{expected_year}-01-09.pdf")

        self.assertEqual(generate_filename(doc1), expected_filename)

        doc1.added = timezone.make_aware(datetime.datetime(2020, 11, 16, 1, 1, 1))

        self.assertEqual(generate_filename(doc1), Path("2020-11-16.pdf"))

    @override_settings(
        FILENAME_FORMAT="{correspondent}/{correspondent}/{correspondent}",
    )
    def test_nested_directory_cleanup(self) -> None:
        document = Document()
        document.mime_type = "application/pdf"

        document.save()

        # Ensure that filename is properly generated
        document.filename = generate_filename(document)
        document.save()
        self.assertEqual(document.filename, "none/none/none.pdf")
        create_source_path_directory(document.source_path)
        Path(document.source_path).touch()

        # Check proper handling of files
        self.assertIsDir(settings.ORIGINALS_DIR / "none" / "none")

        document.delete()
        empty_trash([document.pk])

        self.assertIsNotFile(
            settings.ORIGINALS_DIR / "none" / "none" / "none.pdf",
        )
        self.assertIsNotDir(settings.ORIGINALS_DIR / "none" / "none")
        self.assertIsNotDir(settings.ORIGINALS_DIR / "none")
        self.assertIsDir(settings.ORIGINALS_DIR)

    @override_settings(FILENAME_FORMAT="{doc_pk}")
    def test_format_doc_pk(self) -> None:
        document = Document()
        document.pk = 1
        document.mime_type = "application/pdf"

        self.assertEqual(generate_filename(document), Path("0000001.pdf"))

        document.pk = 13579

        self.assertEqual(generate_filename(document), Path("0013579.pdf"))

    @override_settings(FILENAME_FORMAT=None)
    def test_format_none(self) -> None:
        document = Document()
        document.pk = 1
        document.mime_type = "application/pdf"

        self.assertEqual(generate_filename(document), Path("0000001.pdf"))

    def test_try_delete_empty_directories(self) -> None:
        # Create our working directory
        tmp: Path = settings.ORIGINALS_DIR / "test_delete_empty"
        tmp.mkdir(exist_ok=True, parents=True)

        (tmp / "notempty").mkdir(exist_ok=True, parents=True)
        (tmp / "notempty" / "file").touch()
        (tmp / "notempty" / "empty").mkdir(exist_ok=True, parents=True)

        delete_empty_directories(
            tmp / "notempty" / "empty",
            root=settings.ORIGINALS_DIR,
        )
        self.assertIsDir(tmp / "notempty")
        self.assertIsFile(tmp / "notempty" / "file")
        self.assertIsNotDir(tmp / "notempty" / "empty")

    @override_settings(FILENAME_FORMAT="{% if x is None %}/{title]")
    def test_invalid_format(self) -> None:
        document = Document()
        document.pk = 1
        document.mime_type = "application/pdf"

        self.assertEqual(generate_filename(document), Path("0000001.pdf"))

    @override_settings(FILENAME_FORMAT="{created__year}")
    def test_invalid_format_key(self) -> None:
        document = Document()
        document.pk = 1
        document.mime_type = "application/pdf"

        self.assertEqual(generate_filename(document), Path("0000001.pdf"))

    @override_settings(FILENAME_FORMAT="{title}")
    def test_duplicates(self) -> None:
        document = Document.objects.create(
            mime_type="application/pdf",
            title="qwe",
            checksum="A",
            pk=1,
        )
        document2 = Document.objects.create(
            mime_type="application/pdf",
            title="qwe",
            checksum="B",
            pk=2,
        )
        Path(document.source_path).touch()
        Path(document2.source_path).touch()
        document.filename = "0000001.pdf"
        document.save()

        self.assertIsFile(document.source_path)
        self.assertEqual(document.filename, "qwe.pdf")

        document2.filename = "0000002.pdf"
        document2.save()

        self.assertIsFile(document.source_path)
        self.assertEqual(document2.filename, "qwe_01.pdf")

        # saving should not change the file names.

        document.save()

        self.assertIsFile(document.source_path)
        self.assertEqual(document.filename, "qwe.pdf")

        document2.save()

        self.assertIsFile(document.source_path)
        self.assertEqual(document2.filename, "qwe_01.pdf")

        document.delete()
        empty_trash([document.pk])

        self.assertIsNotFile(document.source_path)

        # filename free, should remove _01 suffix

        document2.save()

        self.assertIsFile(document.source_path)
        self.assertEqual(document2.filename, "qwe.pdf")

    @override_settings(FILENAME_FORMAT="{title}")
    @mock.patch("documents.signals.handlers.Document.objects.filter")
    @mock.patch("documents.signals.handlers.shutil.move")
    def test_no_move_only_save(self, mock_move, mock_filter) -> None:
        """
        GIVEN:
            - A document with a filename
            - The document is saved
            - The filename is not changed
        WHEN:
            - The document is saved
        THEN:
            - The document modified date is updated
            - The document is not moved
        """
        with disable_auditlog():
            doc = Document.objects.create(
                title="document",
                filename="document.pdf",
                archive_filename="document.pdf",
                checksum="A",
                archive_checksum="B",
                mime_type="application/pdf",
            )
            original_modified = doc.modified
            Path(doc.source_path).touch()
            Path(doc.archive_path).touch()

            doc.save()
            doc.refresh_from_db()

            mock_filter.assert_called()
            self.assertNotEqual(original_modified, doc.modified)
            mock_move.assert_not_called()

    @override_settings(
        FILENAME_FORMAT="{{title}}_{{custom_fields|get_cf_value('test')}}",
        CELERY_TASK_ALWAYS_EAGER=True,
    )
    @mock.patch("documents.signals.handlers.update_filename_and_move_files")
    def test_select_cf_updated(self, m) -> None:
        """
        GIVEN:
            - A document with a select type custom field
        WHEN:
            - The custom field select options are updated
        THEN:
            - The update_filename_and_move_files handler is called and the document filename is updated
        """
        cf = CustomField.objects.create(
            name="test",
            data_type=CustomField.FieldDataType.SELECT,
            extra_data={
                "select_options": [
                    {"label": "apple", "id": "abc123"},
                    {"label": "banana", "id": "def456"},
                    {"label": "cherry", "id": "ghi789"},
                ],
            },
        )
        doc = Document.objects.create(
            title="document",
            filename="document.pdf",
            archive_filename="document.pdf",
            checksum="A",
            archive_checksum="B",
            mime_type="application/pdf",
        )
        CustomFieldInstance.objects.create(
            field=cf,
            document=doc,
            value_select="abc123",
        )

        self.assertEqual(generate_filename(doc), Path("document_apple.pdf"))

        # handler should not have been called
        self.assertEqual(m.call_count, 0)
        cf.extra_data = {
            "select_options": [
                {"label": "aubergine", "id": "abc123"},
                {"label": "banana", "id": "def456"},
                {"label": "cherry", "id": "ghi789"},
            ],
        }
        cf.save()
        self.assertEqual(generate_filename(doc), Path("document_aubergine.pdf"))
        # handler should have been called once via the async task
        self.assertEqual(m.call_count, 1)


class TestFileHandlingWithArchive(DirectoriesMixin, FileSystemAssertsMixin, TestCase):
    @override_settings(FILENAME_FORMAT=None)
    def test_create_no_format(self) -> None:
        original = settings.ORIGINALS_DIR / "0000001.pdf"
        archive = settings.ARCHIVE_DIR / "0000001.pdf"
        Path(original).touch()
        Path(archive).touch()
        doc = Document.objects.create(
            mime_type="application/pdf",
            filename="0000001.pdf",
            checksum="A",
            archive_filename="0000001.pdf",
            archive_checksum="B",
        )

        self.assertIsFile(original)
        self.assertIsFile(archive)
        self.assertIsFile(doc.source_path)
        self.assertIsFile(doc.archive_path)

    @override_settings(FILENAME_FORMAT="{correspondent}/{title}")
    def test_create_with_format(self) -> None:
        original = settings.ORIGINALS_DIR / "0000001.pdf"
        archive = settings.ARCHIVE_DIR / "0000001.pdf"
        Path(original).touch()
        Path(archive).touch()
        doc = Document.objects.create(
            mime_type="application/pdf",
            title="my_doc",
            filename="0000001.pdf",
            checksum="A",
            archive_checksum="B",
            archive_filename="0000001.pdf",
        )

        self.assertIsNotFile(original)
        self.assertIsNotFile(archive)
        self.assertIsFile(doc.source_path)
        self.assertIsFile(doc.archive_path)
        self.assertEqual(
            doc.source_path,
            settings.ORIGINALS_DIR / "none" / "my_doc.pdf",
        )
        self.assertEqual(
            doc.archive_path,
            settings.ARCHIVE_DIR / "none" / "my_doc.pdf",
        )

    @override_settings(FILENAME_FORMAT="{correspondent}/{title}")
    def test_move_archive_gone(self) -> None:
        original = settings.ORIGINALS_DIR / "0000001.pdf"
        archive = settings.ARCHIVE_DIR / "0000001.pdf"
        Path(original).touch()
        doc = Document.objects.create(
            mime_type="application/pdf",
            title="my_doc",
            filename="0000001.pdf",
            checksum="A",
            archive_checksum="B",
            archive_filename="0000001.pdf",
        )

        self.assertIsFile(original)
        self.assertIsNotFile(archive)
        self.assertIsFile(doc.source_path)
        self.assertIsNotFile(doc.archive_path)

    @override_settings(FILENAME_FORMAT="{correspondent}/{title}")
    def test_move_archive_exists(self) -> None:
        original = settings.ORIGINALS_DIR / "0000001.pdf"
        archive = settings.ARCHIVE_DIR / "0000001.pdf"
        existing_archive_file = settings.ARCHIVE_DIR / "none" / "my_doc.pdf"
        Path(original).touch()
        Path(archive).touch()
        (settings.ARCHIVE_DIR / "none").mkdir(parents=True, exist_ok=True)
        Path(existing_archive_file).touch()
        doc = Document.objects.create(
            mime_type="application/pdf",
            title="my_doc",
            filename="0000001.pdf",
            checksum="A",
            archive_checksum="B",
            archive_filename="0000001.pdf",
        )

        self.assertIsNotFile(original)
        self.assertIsNotFile(archive)
        self.assertIsFile(doc.source_path)
        self.assertIsFile(doc.archive_path)
        self.assertIsFile(existing_archive_file)
        self.assertEqual(doc.archive_filename, "none/my_doc_01.pdf")

    @override_settings(FILENAME_FORMAT="{title}")
    def test_move_original_only(self) -> None:
        original = settings.ORIGINALS_DIR / "document_01.pdf"
        archive = settings.ARCHIVE_DIR / "document.pdf"
        Path(original).touch()
        Path(archive).touch()

        doc = Document.objects.create(
            mime_type="application/pdf",
            title="document",
            filename="document_01.pdf",
            checksum="A",
            archive_checksum="B",
            archive_filename="document.pdf",
        )

        self.assertEqual(doc.filename, "document.pdf")
        self.assertEqual(doc.archive_filename, "document.pdf")

        self.assertIsFile(doc.source_path)
        self.assertIsFile(doc.archive_path)

    @override_settings(FILENAME_FORMAT="{title}")
    def test_move_archive_only(self) -> None:
        original = settings.ORIGINALS_DIR / "document.pdf"
        archive = settings.ARCHIVE_DIR / "document_01.pdf"
        Path(original).touch()
        Path(archive).touch()

        doc = Document.objects.create(
            mime_type="application/pdf",
            title="document",
            filename="document.pdf",
            checksum="A",
            archive_checksum="B",
            archive_filename="document_01.pdf",
        )

        self.assertEqual(doc.filename, "document.pdf")
        self.assertEqual(doc.archive_filename, "document.pdf")

        self.assertIsFile(doc.source_path)
        self.assertIsFile(doc.archive_path)

    @override_settings(FILENAME_FORMAT="{correspondent}/{title}")
    @mock.patch("documents.signals.handlers.shutil.move")
    def test_move_archive_error(self, m) -> None:
        def fake_rename(src, dst) -> None:
            if "archive" in str(src):
                raise OSError
            else:
                Path(src).unlink()
                Path(dst).touch()

        m.side_effect = fake_rename

        original = settings.ORIGINALS_DIR / "0000001.pdf"
        archive = settings.ARCHIVE_DIR / "0000001.pdf"
        Path(original).touch()
        Path(archive).touch()
        doc = Document.objects.create(
            mime_type="application/pdf",
            title="my_doc",
            filename="0000001.pdf",
            checksum="A",
            archive_checksum="B",
            archive_filename="0000001.pdf",
        )

        m.assert_called()
        self.assertIsFile(original)
        self.assertIsFile(archive)
        self.assertIsFile(doc.source_path)
        self.assertIsFile(doc.archive_path)

    @override_settings(FILENAME_FORMAT="{correspondent}/{title}")
    def test_move_file_gone(self) -> None:
        original = settings.ORIGINALS_DIR / "0000001.pdf"
        archive = settings.ARCHIVE_DIR / "0000001.pdf"
        # Path(original).touch()
        Path(archive).touch()
        doc = Document.objects.create(
            mime_type="application/pdf",
            title="my_doc",
            filename="0000001.pdf",
            archive_filename="0000001.pdf",
            checksum="A",
            archive_checksum="B",
        )

        self.assertIsNotFile(original)
        self.assertIsFile(archive)
        self.assertIsNotFile(doc.source_path)
        self.assertIsFile(doc.archive_path)

    @override_settings(FILENAME_FORMAT="{correspondent}/{title}")
    @mock.patch("documents.signals.handlers.shutil.move")
    def test_move_file_error(self, m) -> None:
        def fake_rename(src, dst) -> None:
            if "original" in str(src):
                raise OSError
            else:
                Path(src).unlink()
                Path(dst).touch()

        m.side_effect = fake_rename

        original = settings.ORIGINALS_DIR / "0000001.pdf"
        archive = settings.ARCHIVE_DIR / "0000001.pdf"
        Path(original).touch()
        Path(archive).touch()
        doc = Document.objects.create(
            mime_type="application/pdf",
            title="my_doc",
            filename="0000001.pdf",
            archive_filename="0000001.pdf",
            checksum="A",
            archive_checksum="B",
        )

        m.assert_called()
        self.assertIsFile(original)
        self.assertIsFile(archive)
        self.assertIsFile(doc.source_path)
        self.assertIsFile(doc.archive_path)

    @override_settings(FILENAME_FORMAT="")
    def test_archive_deleted(self) -> None:
        original = settings.ORIGINALS_DIR / "0000001.pdf"
        archive = settings.ARCHIVE_DIR / "0000001.pdf"
        Path(original).touch()
        Path(archive).touch()
        doc = Document.objects.create(
            mime_type="application/pdf",
            title="my_doc",
            filename="0000001.pdf",
            checksum="A",
            archive_checksum="B",
            archive_filename="0000001.pdf",
        )

        self.assertIsFile(original)
        self.assertIsFile(archive)
        self.assertIsFile(doc.source_path)
        self.assertIsFile(doc.archive_path)

        doc.delete()
        empty_trash([doc.pk])

        self.assertIsNotFile(original)
        self.assertIsNotFile(archive)
        self.assertIsNotFile(doc.source_path)
        self.assertIsNotFile(doc.archive_path)

    @override_settings(FILENAME_FORMAT="{title}")
    def test_archive_deleted2(self) -> None:
        original = settings.ORIGINALS_DIR / "document.webp"
        original2 = settings.ORIGINALS_DIR / "0000001.pdf"
        archive = settings.ARCHIVE_DIR / "0000001.pdf"
        Path(original).touch()
        Path(original2).touch()
        Path(archive).touch()

        doc1 = Document.objects.create(
            mime_type="image/webp",
            title="document",
            filename="document.webp",
            checksum="A",
            archive_checksum="B",
            archive_filename="0000001.pdf",
        )
        doc2 = Document.objects.create(
            mime_type="application/pdf",
            title="0000001",
            filename="0000001.pdf",
            checksum="C",
        )

        self.assertIsFile(doc1.source_path)
        self.assertIsFile(doc1.archive_path)
        self.assertIsFile(doc2.source_path)

        doc2.delete()
        empty_trash([doc2.pk])

        self.assertIsFile(doc1.source_path)
        self.assertIsFile(doc1.archive_path)
        self.assertIsNotFile(doc2.source_path)

    @override_settings(FILENAME_FORMAT="{correspondent}/{title}")
    def test_database_error(self) -> None:
        original = settings.ORIGINALS_DIR / "0000001.pdf"
        archive = settings.ARCHIVE_DIR / "0000001.pdf"
        Path(original).touch()
        Path(archive).touch()
        doc = Document(
            mime_type="application/pdf",
            title="my_doc",
            filename="0000001.pdf",
            checksum="A",
            archive_filename="0000001.pdf",
            archive_checksum="B",
        )
        with mock.patch(
            "documents.signals.handlers.Document.global_objects.filter",
        ) as m:
            m.side_effect = DatabaseError()
            doc.save()

        self.assertIsFile(original)
        self.assertIsFile(archive)
        self.assertIsFile(doc.source_path)
        self.assertIsFile(doc.archive_path)


class TestFilenameGeneration(DirectoriesMixin, TestCase):
    @override_settings(FILENAME_FORMAT="{title}")
    def test_invalid_characters(self) -> None:
        doc = Document.objects.create(
            title="This. is the title.",
            mime_type="application/pdf",
            pk=1,
            checksum="1",
        )
        self.assertEqual(generate_filename(doc), Path("This. is the title.pdf"))

        doc = Document.objects.create(
            title="my\\invalid/../title:yay",
            mime_type="application/pdf",
            pk=2,
            checksum="2",
        )
        self.assertEqual(generate_filename(doc), Path("my-invalid-..-title-yay.pdf"))

    @override_settings(FILENAME_FORMAT="{created}")
    def test_date(self) -> None:
        doc = Document.objects.create(
            title="does not matter",
            created=datetime.date(2020, 5, 21),
            mime_type="application/pdf",
            pk=2,
            checksum="2",
        )
        self.assertEqual(generate_filename(doc), Path("2020-05-21.pdf"))

    def test_dynamic_path(self) -> None:
        """
        GIVEN:
            - A document with a defined storage path
        WHEN:
            - the filename is generated for the document
        THEN:
            - the generated filename uses the defined storage path for the document
        """
        doc = Document.objects.create(
            title="does not matter",
            created=datetime.date(2020, 6, 25),
            mime_type="application/pdf",
            pk=2,
            checksum="2",
            storage_path=StoragePath.objects.create(path="TestFolder/{{created}}"),
        )
        self.assertEqual(generate_filename(doc), Path("TestFolder/2020-06-25.pdf"))

    def test_dynamic_path_with_none(self) -> None:
        """
        GIVEN:
            - A document with a defined storage path
            - The defined storage path uses an undefined field for the document
        WHEN:
            - the filename is generated for the document
        THEN:
            - the generated filename uses the defined storage path for the document
            - the generated filename includes "none" in the place undefined field
        """
        doc = Document.objects.create(
            title="does not matter",
            created=datetime.date(2020, 6, 25),
            mime_type="application/pdf",
            pk=2,
            checksum="2",
            storage_path=StoragePath.objects.create(path="{{asn}} - {{created}}"),
        )
        self.assertEqual(generate_filename(doc), Path("none - 2020-06-25.pdf"))

    @override_settings(
        FILENAME_FORMAT_REMOVE_NONE=True,
    )
    def test_dynamic_path_remove_none(self) -> None:
        """
        GIVEN:
            - A document with a defined storage path
            - The defined storage path uses an undefined field for the document
            - The setting for removing undefined fields is enabled
        WHEN:
            - the filename is generated for the document
        THEN:
            - the generated filename uses the defined storage path for the document
            - the generated filename does not include "none" in the place undefined field
        """
        sp = StoragePath.objects.create(
            path="TestFolder/{{asn}}/{{created}}",
        )
        doc = Document.objects.create(
            title="does not matter",
            created=datetime.date(2020, 6, 25),
            mime_type="application/pdf",
            pk=2,
            checksum="2",
            storage_path=sp,
        )
        self.assertEqual(generate_filename(doc), Path("TestFolder/2020-06-25.pdf"))

        # Special case, undefined variable, then defined at the start of the template
        # This could lead to an absolute path after we remove the leading -none-, but leave the leading /
        # -none-/2020/ -> /2020/
        sp.path = (
            "{{ owner_username }}/{{ created_year }}/{{ correspondent }}/{{ title }}"
        )
        sp.save()
        self.assertEqual(generate_filename(doc), Path("2020/does not matter.pdf"))

    def test_multiple_doc_paths(self) -> None:
        """
        GIVEN:
            - Two documents, each with different storage paths
        WHEN:
            - the filename is generated for the documents
        THEN:
            - Each document generated filename uses its storage path
        """
        doc_a = Document.objects.create(
            title="does not matter",
            created=datetime.date(2020, 6, 25),
            mime_type="application/pdf",
            pk=2,
            checksum="2",
            archive_serial_number=4,
            storage_path=StoragePath.objects.create(
                name="sp1",
                path="ThisIsAFolder/{{asn}}/{{created}}",
            ),
        )
        doc_b = Document.objects.create(
            title="does not matter",
            created=datetime.date(2020, 7, 25),
            mime_type="application/pdf",
            pk=5,
            checksum="abcde",
            storage_path=StoragePath.objects.create(
                name="sp2",
                path="SomeImportantNone/{{created}}",
            ),
        )

        self.assertEqual(
            generate_filename(doc_a),
            Path("ThisIsAFolder/4/2020-06-25.pdf"),
        )
        self.assertEqual(
            generate_filename(doc_b),
            Path("SomeImportantNone/2020-07-25.pdf"),
        )

    @override_settings(
        FILENAME_FORMAT=None,
    )
    def test_no_path_fallback(self) -> None:
        """
        GIVEN:
            - Two documents, one with defined storage path, the other not
        WHEN:
            - the filename is generated for the documents
        THEN:
            - Document with defined path uses its format
            - Document without defined path uses the default path
        """
        doc_a = Document.objects.create(
            title="does not matter",
            created=datetime.date(2020, 6, 25),
            mime_type="application/pdf",
            pk=2,
            checksum="2",
            archive_serial_number=4,
        )
        doc_b = Document.objects.create(
            title="does not matter",
            created=datetime.date(2020, 7, 25),
            mime_type="application/pdf",
            pk=5,
            checksum="abcde",
            storage_path=StoragePath.objects.create(
                name="sp2",
                path="SomeImportantNone/{{created}}",
            ),
        )

        self.assertEqual(generate_filename(doc_a), Path("0000002.pdf"))
        self.assertEqual(
            generate_filename(doc_b),
            Path("SomeImportantNone/2020-07-25.pdf"),
        )

    @override_settings(
        FILENAME_FORMAT=(
            "{% if correspondent == 'none' %}none/{% endif %}"
            "{% if correspondent == '-none-' %}dash/{% endif %}"
            "{% if not correspondent %}false/{% endif %}"
            "{% if correspondent != 'none' %}notnoneyes/{% else %}notnoneno/{% endif %}"
            "{{ correspondent or 'missing' }}/{{ title }}"
        ),
    )
    def test_placeholder_matches_none_variants_and_false(self) -> None:
        """
        GIVEN:
            - Templates that compare against 'none', '-none-' and rely on truthiness
        WHEN:
            - A document has or lacks a correspondent
        THEN:
            - Empty placeholders behave like both strings and evaluate False
        """
        doc_without_correspondent = Document.objects.create(
            title="does not matter",
            mime_type="application/pdf",
            checksum="abc",
        )
        doc_with_correspondent = Document.objects.create(
            title="does not matter",
            mime_type="application/pdf",
            checksum="def",
            correspondent=Correspondent.objects.create(name="Acme"),
        )

        self.assertEqual(
            generate_filename(doc_without_correspondent),
            Path(
                "none/dash/false/notnoneno/missing/does not matter.pdf",
            ),
        )
        self.assertEqual(
            generate_filename(doc_with_correspondent),
            Path("notnoneyes/Acme/does not matter.pdf"),
        )

    @override_settings(
        FILENAME_FORMAT="{created_year_short}/{created_month_name_short}/{created_month_name}/{title}",
    )
    def test_short_names_created(self) -> None:
        doc = Document.objects.create(
            title="The Title",
            created=datetime.date(1989, 12, 2),
            mime_type="application/pdf",
            pk=2,
            checksum="2",
        )
        self.assertEqual(generate_filename(doc), Path("89/Dec/December/The Title.pdf"))

    @override_settings(
        FILENAME_FORMAT="{added_year_short}/{added_month_name}/{added_month_name_short}/{title}",
    )
    def test_short_names_added(self) -> None:
        doc = Document.objects.create(
            title="The Title",
            added=timezone.make_aware(datetime.datetime(1984, 8, 21, 7, 36, 51, 153)),
            mime_type="application/pdf",
            pk=2,
            checksum="2",
        )
        self.assertEqual(generate_filename(doc), Path("84/August/Aug/The Title.pdf"))

    @override_settings(
        FILENAME_FORMAT="{owner_username}/{title}",
    )
    def test_document_owner_string(self) -> None:
        """
        GIVEN:
            - Document with an other
            - Document without an owner
            - Filename format string includes owner
        WHEN:
            - Filename is generated for each document
        THEN:
            - Owned document includes username
            - Document without owner returns "none"
        """

        u1 = UserFactory(username="user1")

        owned_doc = Document.objects.create(
            title="The Title",
            mime_type="application/pdf",
            checksum="2",
            owner=u1,
        )

        no_owner_doc = Document.objects.create(
            title="does matter",
            mime_type="application/pdf",
            checksum="3",
        )

        self.assertEqual(generate_filename(owned_doc), Path("user1/The Title.pdf"))
        self.assertEqual(generate_filename(no_owner_doc), Path("none/does matter.pdf"))

    @override_settings(
        FILENAME_FORMAT="{original_name}",
    )
    def test_document_original_filename(self) -> None:
        """
        GIVEN:
            - Document with an original filename
            - Document without an original filename
            - Document which was plain text document
            - Filename format string includes original filename
        WHEN:
            - Filename is generated for each document
        THEN:
            - Document with original name uses it, dropping suffix
            - Document without original name returns "none"
            - Text document returns extension of .txt
            - Text document archive returns extension of .pdf
            - No extensions are doubled
        """
        doc_with_original = Document.objects.create(
            title="does matter",
            mime_type="application/pdf",
            checksum="3",
            original_filename="someepdf.pdf",
        )
        tricky_with_original = Document.objects.create(
            title="does matter",
            mime_type="application/pdf",
            checksum="1",
            original_filename="some pdf with spaces and stuff.pdf",
        )
        no_original = Document.objects.create(
            title="does matter",
            mime_type="application/pdf",
            checksum="2",
        )

        text_doc = Document.objects.create(
            title="does matter",
            mime_type="text/plain",
            checksum="4",
            original_filename="logs.txt",
        )

        self.assertEqual(generate_filename(doc_with_original), Path("someepdf.pdf"))

        self.assertEqual(
            generate_filename(tricky_with_original),
            Path("some pdf with spaces and stuff.pdf"),
        )

        self.assertEqual(generate_filename(no_original), Path("none.pdf"))

        self.assertEqual(generate_filename(text_doc), Path("logs.txt"))
        self.assertEqual(
            generate_filename(text_doc, archive_filename=True),
            Path("logs.pdf"),
        )

    @override_settings(FILENAME_FORMAT="{title}")
    def test_version_index_suffix_for_template_filename(self) -> None:
        root_doc = Document.objects.create(
            title="the_doc",
            mime_type="application/pdf",
            checksum="root-checksum",
        )
        version_doc = Document.objects.create(
            title="the_doc",
            mime_type="application/pdf",
            checksum="version-checksum",
            root_document=root_doc,
            version_index=1,
        )

        self.assertEqual(generate_filename(version_doc), Path("the_doc_v1.pdf"))
        self.assertEqual(
            generate_filename(version_doc, counter=1),
            Path("the_doc_v1_01.pdf"),
        )

    @override_settings(FILENAME_FORMAT=None)
    def test_version_index_suffix_for_default_filename(self) -> None:
        root_doc = Document.objects.create(
            title="root",
            mime_type="text/plain",
            checksum="root-checksum",
        )
        version_doc = Document.objects.create(
            title="root",
            mime_type="text/plain",
            checksum="version-checksum",
            root_document=root_doc,
            version_index=2,
        )

        self.assertEqual(
            generate_filename(version_doc),
            Path(f"{root_doc.pk:07d}_v2.txt"),
        )
        self.assertEqual(
            generate_filename(version_doc, archive_filename=True),
            Path(f"{root_doc.pk:07d}_v2.pdf"),
        )

    @override_settings(FILENAME_FORMAT="{original_name}")
    def test_version_index_suffix_with_original_name_placeholder(self) -> None:
        root_doc = Document.objects.create(
            title="root",
            mime_type="application/pdf",
            checksum="root-checksum",
            original_filename="root-upload.pdf",
        )
        version_doc = Document.objects.create(
            title="root",
            mime_type="application/pdf",
            checksum="version-checksum",
            root_document=root_doc,
            version_index=1,
            original_filename="version-upload.pdf",
        )

        self.assertEqual(generate_filename(version_doc), Path("root-upload_v1.pdf"))

    def test_version_index_suffix_with_storage_path(self) -> None:
        storage_path = StoragePath.objects.create(
            name="vtest",
            path="folder/{{title}}",
        )
        root_doc = Document.objects.create(
            title="storage_doc",
            mime_type="application/pdf",
            checksum="root-checksum",
            storage_path=storage_path,
        )
        version_doc = Document.objects.create(
            title="version_title_should_not_be_used",
            mime_type="application/pdf",
            checksum="version-checksum",
            root_document=root_doc,
            version_index=3,
        )

        self.assertEqual(
            generate_filename(version_doc),
            Path("folder/storage_doc_v3.pdf"),
        )

    @override_settings(
        FILENAME_FORMAT="XX{correspondent}/{title}",
        FILENAME_FORMAT_REMOVE_NONE=True,
    )
    def test_remove_none_not_dir(self) -> None:
        """
        GIVEN:
            - A document with & filename format that includes correspondent as part of directory name
            - FILENAME_FORMAT_REMOVE_NONE is True
        WHEN:
            - the filename is generated for the document
        THEN:
            - the missing correspondent is removed but directory structure retained
        """
        document = Document.objects.create(
            title="doc1",
            mime_type="application/pdf",
        )

        document.save()

        # Ensure that filename is properly generated
        document.filename = generate_filename(document)
        self.assertEqual(document.filename, Path("XX/doc1.pdf"))

    @override_settings(FILENAME_FORMAT_REMOVE_NONE=True)
    def test_remove_none_cannot_create_traversal(self) -> None:
        """
        GIVEN:
            - A storage path whose components are safe when validated, but become
              ".." once the -none- placeholder is stripped out
            - FILENAME_FORMAT_REMOVE_NONE is True
        WHEN:
            - the filename is generated for the document
        THEN:
            - The unsafe filename is rejected and the default naming is used
        """
        sp = StoragePath.objects.create(
            name="sp1",
            path=".-none-./.-none-./tmp/pwned",
        )
        document = Document.objects.create(
            title="doc1",
            mime_type="application/pdf",
            storage_path=sp,
        )

        filename = generate_filename(document)

        self.assertNotIn("..", filename.parts)
        self.assertEqual(filename, Path(f"{document.pk:07}.pdf"))

    @override_settings(FILENAME_FORMAT_REMOVE_NONE=True)
    def test_remove_none_still_removes_placeholder(self) -> None:
        """
        GIVEN:
            - A storage path with a placeholder for a value the document does not have
            - FILENAME_FORMAT_REMOVE_NONE is True
        WHEN:
            - the filename is generated for the document
        THEN:
            - The placeholder is still removed as before
        """
        sp = StoragePath.objects.create(
            name="sp1",
            path="{{ correspondent }}/{{ title }}",
        )
        document = Document.objects.create(
            title="doc1",
            mime_type="application/pdf",
            storage_path=sp,
        )

        self.assertEqual(generate_filename(document), Path("doc1.pdf"))

    @override_settings(
        FILENAME_FORMAT="{{ correspondent }}/{{ title }}/{{ doc_pk }}",
        FILENAME_FORMAT_REMOVE_NONE=True,
    )
    def test_remove_none_cannot_create_traversal_from_metadata(self) -> None:
        """
        GIVEN:
            - A global filename format with directory components
            - A document whose title becomes ".." once -none- is stripped out
            - FILENAME_FORMAT_REMOVE_NONE is True
        WHEN:
            - the filename is generated for the document
        THEN:
            - The unsafe filename is rejected and the default naming is used
        """
        document = Document.objects.create(
            title=".-none-.",
            mime_type="application/pdf",
        )

        filename = generate_filename(document)

        self.assertNotIn("..", filename.parts)
        self.assertEqual(filename, Path(f"{document.pk:07}.pdf"))

    def test_validate_path_in_root(self) -> None:
        """
        GIVEN:
            - A path inside of the root and a path outside of it
        WHEN:
            - The path is validated against the root
        THEN:
            - Only the path outside of the root is rejected
        """
        validate_path_in_root(
            settings.ORIGINALS_DIR / "0000001.pdf",
            settings.ORIGINALS_DIR,
        )

        with self.assertRaises(UnsafeFilePathError):
            validate_path_in_root(
                (settings.ORIGINALS_DIR / ".." / ".." / "pwned.pdf"),
                settings.ORIGINALS_DIR,
            )

    def test_complex_template_strings(self) -> None:
        """
        GIVEN:
            - Storage paths with complex conditionals and logic
        WHEN:
            - Filepath for a document with this storage path is called
        THEN:
            - The filepath is rendered without error
            - The filepath is rendered as a single line string
        """
        sp = StoragePath.objects.create(
            name="sp1",
            path="""
                 somepath/
                 {% if document.checksum == '2' %}
                   some where/{{created}}
                 {% else %}
                   {{added}}
                 {% endif %}
                 /{{ title }}
                 """,
        )

        doc_a = Document.objects.create(
            title="Does Matter",
            created=datetime.date(2020, 6, 25),
            added=timezone.make_aware(datetime.datetime(2024, 10, 1, 7, 36, 51, 153)),
            mime_type="application/pdf",
            pk=2,
            checksum="2",
            archive_serial_number=25,
            storage_path=sp,
        )

        self.assertEqual(
            generate_filename(doc_a),
            Path("somepath/some where/2020-06-25/Does Matter.pdf"),
        )
        doc_a.checksum = "5"

        self.assertEqual(
            generate_filename(doc_a),
            Path("somepath/2024-10-01/Does Matter.pdf"),
        )

        sp.path = "{{ document.title|lower }}{{ document.archive_serial_number - 2 }}"
        sp.save()

        self.assertEqual(generate_filename(doc_a), Path("does matter23.pdf"))

        sp.path = """
                 somepath/
                 {% if document.archive_serial_number >= 0 and document.archive_serial_number <= 200 %}
                   asn-000-200/{{title}}
                 {% elif document.archive_serial_number >= 201 and document.archive_serial_number <= 400 %}
                   asn-201-400
                   {% if document.archive_serial_number >= 201 and document.archive_serial_number < 300 %}
                     /asn-2xx
                   {% elif document.archive_serial_number >= 300 and document.archive_serial_number < 400 %}
                     /asn-3xx
                   {% endif %}
                 {% endif %}
                 /{{ title }}
                 """
        sp.save()
        self.assertEqual(
            generate_filename(doc_a),
            Path("somepath/asn-000-200/Does Matter/Does Matter.pdf"),
        )
        doc_a.archive_serial_number = 301
        doc_a.save()
        self.assertEqual(
            generate_filename(doc_a),
            Path("somepath/asn-201-400/asn-3xx/Does Matter.pdf"),
        )

    def test_template_related_context_keeps_legacy_string_coercion(self) -> None:
        """
        GIVEN:
            - A storage path template that uses related objects directly as strings
        WHEN:
            - Filepath for a document with this format is called
        THEN:
            - Related objects coerce to their names (legacy behavior)
            - Explicit attribute access remains available for new templates
        """
        sp = StoragePath.objects.create(
            name="PARTNER",
            path=(
                "{{ document.storage_path|lower }} / "
                "{{ document.correspondent|lower|replace('mi:', 'mieter/') }} / "
                "{{ document_type|lower }} / "
                "{{ title|lower }}"
            ),
        )
        doc = Document.objects.create(
            title="scan_017562",
            created=datetime.date(2025, 7, 2),
            added=timezone.make_aware(datetime.datetime(2026, 3, 3, 11, 53, 16)),
            mime_type="application/pdf",
            checksum="test-checksum",
            storage_path=sp,
            correspondent=Correspondent.objects.create(name="mi:kochkach"),
            document_type=DocumentType.objects.create(name="Mietvertrag"),
        )

        self.assertEqual(
            generate_filename(doc),
            Path("partner/mieter/kochkach/mietvertrag/scan_017562.pdf"),
        )

    @override_settings(
        FILENAME_FORMAT="{{creation_date}}/{{ title_name_str }}",
    )
    def test_template_with_undefined_var(self) -> None:
        """
        GIVEN:
            - Filename format with one or more undefined variables
        WHEN:
            - Filepath for a document with this format is called
        THEN:
            - The first undefined variable is logged
            - The default format is used
        """
        doc_a = Document.objects.create(
            title="Does Matter",
            created=datetime.date(2020, 6, 25),
            added=timezone.make_aware(datetime.datetime(2024, 10, 1, 7, 36, 51, 153)),
            mime_type="application/pdf",
            pk=2,
            checksum="2",
            archive_serial_number=25,
        )

        with self.assertLogs(level=logging.WARNING) as capture:
            self.assertEqual(
                generate_filename(doc_a),
                Path("0000002.pdf"),
            )

            self.assertEqual(len(capture.output), 1)
            self.assertEqual(
                capture.output[0],
                "WARNING:paperless.templating:Template variable warning: 'creation_date' is undefined",
            )

    @override_settings(
        FILENAME_FORMAT="{{created}}/{{ document.save() }}",
    )
    def test_template_with_security(self) -> None:
        """
        GIVEN:
            - Filename format with an unavailable document attribute
        WHEN:
            - Filepath for a document with this format is called
        THEN:
            - The missing attribute is logged
            - The default format is used
        """
        doc_a = Document.objects.create(
            title="Does Matter",
            created=datetime.date(2020, 6, 25),
            added=timezone.make_aware(datetime.datetime(2024, 10, 1, 7, 36, 51, 153)),
            mime_type="application/pdf",
            pk=2,
            checksum="2",
            archive_serial_number=25,
        )

        with self.assertLogs(level=logging.WARNING) as capture:
            self.assertEqual(
                generate_filename(doc_a),
                Path("0000002.pdf"),
            )

            self.assertEqual(len(capture.output), 1)
            self.assertEqual(
                capture.output[0],
                "ERROR:paperless.templating:Template variable error: 'dict object' has no attribute 'save'",
            )

    def test_template_with_custom_fields(self) -> None:
        """
        GIVEN:
            - Filename format which accesses custom field data
        WHEN:
            - Filepath for a document with this format is called
        THEN:
            - The custom field data is rendered
            - If the field name is not defined, the default value is rendered, if any
        """
        doc_a = Document.objects.create(
            title="Some Title",
            created=datetime.date(2020, 6, 25),
            added=timezone.make_aware(datetime.datetime(2024, 10, 1, 7, 36, 51, 153)),
            mime_type="application/pdf",
            pk=2,
            checksum="2",
            archive_serial_number=25,
        )

        cf = CustomField.objects.create(
            name="Invoice",
            data_type=CustomField.FieldDataType.INT,
        )

        cf2 = CustomField.objects.create(
            name="Select Field",
            data_type=CustomField.FieldDataType.SELECT,
            extra_data={
                "select_options": [
                    {"label": "ChoiceOne", "id": "abc=123"},
                    {"label": "ChoiceTwo", "id": "def-456"},
                ],
            },
        )

        cfi1 = CustomFieldInstance.objects.create(
            document=doc_a,
            field=cf2,
            value_select="abc=123",
        )

        cfi = CustomFieldInstance.objects.create(
            document=doc_a,
            field=cf,
            value_int=1234,
        )

        with override_settings(
            FILENAME_FORMAT="""
                 {% if "Invoice" in custom_fields %}
                   invoices/{{ custom_fields | get_cf_value('Invoice') }}
                 {% else %}
                   not-invoices/{{ title }}
                 {% endif %}
                 """,
        ):
            self.assertEqual(
                generate_filename(doc_a),
                Path("invoices/1234.pdf"),
            )

        with override_settings(
            FILENAME_FORMAT="""
                 {% if "Select Field" in custom_fields %}
                   {{ title }}_{{ custom_fields | get_cf_value('Select Field', 'Default Value') }}
                 {% else %}
                   {{ title }}
                 {% endif %}
                 """,
        ):
            self.assertEqual(
                generate_filename(doc_a),
                Path("Some Title_ChoiceOne.pdf"),
            )

            # Check for handling Nones well
            cfi1.value_select = None
            cfi1.save()

            self.assertEqual(
                generate_filename(doc_a),
                Path("Some Title_Default Value.pdf"),
            )

        cf.name = "Invoice Number"
        cfi.value_int = 4567
        cfi.save()
        cf.save()

        with override_settings(
            FILENAME_FORMAT="invoices/{{ custom_fields | get_cf_value('Invoice Number') }}",
        ):
            self.assertEqual(
                generate_filename(doc_a),
                Path("invoices/4567.pdf"),
            )

        with override_settings(
            FILENAME_FORMAT="invoices/{{ custom_fields | get_cf_value('Ince Number', 0) }}",
        ):
            self.assertEqual(
                generate_filename(doc_a),
                Path("invoices/0.pdf"),
            )

    def test_datetime_filter(self) -> None:
        """
        GIVEN:
            - Filename format with datetime filter
        WHEN:
            - Filepath for a document with this format is called
        THEN:
            - The datetime filter is rendered
        """
        doc_a = Document.objects.create(
            title="Some Title",
            created=datetime.date(2020, 6, 25),
            added=timezone.make_aware(datetime.datetime(2024, 10, 1, 7, 36, 51, 153)),
            mime_type="application/pdf",
            pk=2,
            checksum="2",
            archive_serial_number=25,
        )

        CustomField.objects.create(
            name="Invoice Date",
            data_type=CustomField.FieldDataType.DATE,
        )
        CustomFieldInstance.objects.create(
            document=doc_a,
            field=CustomField.objects.get(name="Invoice Date"),
            value_date=timezone.make_aware(
                datetime.datetime(2024, 10, 1, 7, 36, 51, 153),
            ),
        )

        with override_settings(
            FILENAME_FORMAT="{{ created | datetime('%Y') }}/{{ title }}",
        ):
            self.assertEqual(
                generate_filename(doc_a),
                Path("2020/Some Title.pdf"),
            )

        with override_settings(
            FILENAME_FORMAT="{{ created | datetime('%Y-%m-%d') }}/{{ title }}",
        ):
            self.assertEqual(
                generate_filename(doc_a),
                Path("2020-06-25/Some Title.pdf"),
            )

        with override_settings(
            FILENAME_FORMAT="{{ custom_fields | get_cf_value('Invoice Date') | datetime('%Y-%m-%d') }}/{{ title }}",
        ):
            self.assertEqual(
                generate_filename(doc_a),
                Path("2024-10-01/Some Title.pdf"),
            )

    def test_slugify_filter(self) -> None:
        """
        GIVEN:
            - Filename format with slugify filter
        WHEN:
            - Filepath for a document with this format is called
        THEN:
            - The slugify filter properly converts strings to URL-friendly slugs
        """
        doc = Document.objects.create(
            title="Some Title! With @ Special # Characters",
            created=datetime.date(2020, 6, 25),
            added=timezone.make_aware(datetime.datetime(2024, 10, 1, 7, 36, 51, 153)),
            mime_type="application/pdf",
            pk=2,
            checksum="2",
            archive_serial_number=25,
        )

        with override_settings(
            FILENAME_FORMAT="{{ title | slugify }}",
        ):
            self.assertEqual(
                generate_filename(doc),
                Path("some-title-with-special-characters.pdf"),
            )

        # Test with correspondent name containing spaces and special chars
        doc.correspondent = Correspondent.objects.create(
            name="John's @ Office / Workplace",
        )
        doc.save()

        with override_settings(
            FILENAME_FORMAT="{{ correspondent | slugify }}/{{ title | slugify }}",
        ):
            self.assertEqual(
                generate_filename(doc),
                Path("johns-office-workplace/some-title-with-special-characters.pdf"),
            )

        # Test with custom fields
        cf = CustomField.objects.create(
            name="Location",
            data_type=CustomField.FieldDataType.STRING,
        )
        CustomFieldInstance.objects.create(
            document=doc,
            field=cf,
            value_text="Brussels @ Belgium!",
        )

        with override_settings(
            FILENAME_FORMAT="{{ custom_fields | get_cf_value('Location') | slugify }}/{{ title | slugify }}",
        ):
            self.assertEqual(
                generate_filename(doc),
                Path("brussels-belgium/some-title-with-special-characters.pdf"),
            )


class TestCustomFieldFilenameUpdates(
    DirectoriesMixin,
    FileSystemAssertsMixin,
    TestCase,
):
    def setUp(self):
        self.cf = CustomField.objects.create(
            name="flavor",
            data_type=CustomField.FieldDataType.STRING,
        )
        self.doc = Document.objects.create(
            title="document",
            mime_type="application/pdf",
            checksum="abc123",
        )
        self.cfi = CustomFieldInstance.objects.create(
            field=self.cf,
            document=self.doc,
            value_text="initial",
        )
        return super().setUp()

    @override_settings(FILENAME_FORMAT=None)
    def test_custom_field_not_in_template_skips_filename_work(self) -> None:
        storage_path = StoragePath.objects.create(path="{{created}}/{{ title }}")
        self.doc.storage_path = storage_path
        self.doc.save()
        initial_filename = generate_filename(self.doc)
        Document.objects.filter(pk=self.doc.pk).update(filename=str(initial_filename))
        self.doc.refresh_from_db()
        Path(self.doc.source_path).parent.mkdir(parents=True, exist_ok=True)
        Path(self.doc.source_path).touch()

        with mock.patch("documents.signals.handlers.generate_unique_filename") as m:
            m.side_effect = generate_unique_filename
            self.cfi.value_text = "updated"
            self.cfi.save()

        self.doc.refresh_from_db()
        self.assertEqual(Path(self.doc.filename), initial_filename)
        self.assertEqual(m.call_count, 0)

    @override_settings(FILENAME_FORMAT=None)
    def test_custom_field_in_template_triggers_filename_update(self) -> None:
        storage_path = StoragePath.objects.create(
            path="{{ custom_fields|get_cf_value('flavor') }}/{{ title }}",
        )
        self.doc.storage_path = storage_path
        self.doc.save()
        initial_filename = generate_filename(self.doc)
        Document.objects.filter(pk=self.doc.pk).update(filename=str(initial_filename))
        self.doc.refresh_from_db()
        Path(self.doc.source_path).parent.mkdir(parents=True, exist_ok=True)
        Path(self.doc.source_path).touch()

        with mock.patch("documents.signals.handlers.generate_unique_filename") as m:
            m.side_effect = generate_unique_filename
            self.cfi.value_text = "updated"
            self.cfi.save()

        self.doc.refresh_from_db()
        expected_filename = Path("updated/document.pdf")
        self.assertEqual(Path(self.doc.filename), expected_filename)
        self.assertTrue(Path(self.doc.source_path).is_file())
        self.assertLessEqual(m.call_count, 1)

    @override_settings(FILENAME_FORMAT=None)
    def test_overlong_storage_path_keeps_existing_filename(self) -> None:
        initial_filename = generate_filename(self.doc)
        Document.objects.filter(pk=self.doc.pk).update(filename=str(initial_filename))
        self.doc.refresh_from_db()
        Path(self.doc.source_path).parent.mkdir(parents=True, exist_ok=True)
        Path(self.doc.source_path).touch()

        self.doc.storage_path = StoragePath.objects.create(path="a" * 1100)
        self.doc.save()

        self.doc.refresh_from_db()
        self.assertEqual(Path(self.doc.filename), initial_filename)
        self.assertTrue(Path(self.doc.source_path).is_file())


class TestPathDateLocalization:
    """
    Groups all tests related to the `localize_date` function.
    """

    TEST_DATE = datetime.date(2023, 10, 26)

    @pytest.mark.django_db
    @pytest.mark.parametrize(
        "filename_format,expected_filename",
        [
            pytest.param(
                "{{title}}_{{ document.created | localize_date('MMMM', 'es_ES')}}",
                "My Document_octubre.pdf",
                id="spanish_month_name",
            ),
            pytest.param(
                "{{title}}_{{ document.created | localize_date('EEEE', 'fr_FR')}}",
                "My Document_jeudi.pdf",
                id="french_day_of_week",
            ),
            pytest.param(
                "{{title}}_{{ document.created | localize_date('dd/MM/yyyy', 'en_GB')}}",
                "My Document_26/10/2023.pdf",
                id="uk_date_format",
            ),
        ],
    )
    def test_localize_date_path_building(
        self,
        filename_format,
        expected_filename,
    ) -> None:
        document = DocumentFactory.create(
            title="My Document",
            mime_type="application/pdf",
            created=self.TEST_DATE,  # 2023-10-26 (which is a Thursday)
        )
        with override_settings(FILENAME_FORMAT=filename_format):
            filename = generate_filename(document)
            assert filename == Path(expected_filename)