diff --git a/pyproject.toml b/pyproject.toml index e1616b54a..66c8d462a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -186,6 +186,7 @@ line-ending = "lf" extend-select = [ "B", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b "COM", # https://docs.astral.sh/ruff/rules/#flake8-commas-com + "DTZ", # https://docs.astral.sh/ruff/rules/#flake8-datetimez-dtz "DJ", # https://docs.astral.sh/ruff/rules/#flake8-django-dj "EXE", # https://docs.astral.sh/ruff/rules/#flake8-executable-exe "FBT", # https://docs.astral.sh/ruff/rules/#flake8-boolean-trap-fbt diff --git a/src/documents/consumer.py b/src/documents/consumer.py index 7c01853c5..6cf196672 100644 --- a/src/documents/consumer.py +++ b/src/documents/consumer.py @@ -839,8 +839,9 @@ class ConsumerPlugin( self.log.debug(f"Creation date from parse_date: {create_date}") else: stats = Path(self.input_doc.original_file).stat() - create_date = timezone.make_aware( - datetime.datetime.fromtimestamp(stats.st_mtime), + create_date = datetime.datetime.fromtimestamp( + stats.st_mtime, + tz=datetime.UTC, ) self.log.debug(f"Creation date from st_mtime: {create_date}") diff --git a/src/documents/double_sided.py b/src/documents/double_sided.py index eb2f6aa84..b0c986588 100644 --- a/src/documents/double_sided.py +++ b/src/documents/double_sided.py @@ -1,4 +1,3 @@ -import datetime as dt import logging import os import shutil @@ -6,6 +5,7 @@ from pathlib import Path from typing import Final from django.conf import settings +from django.utils import timezone from pikepdf import Pdf from documents.consumer import ConsumerError @@ -78,7 +78,7 @@ class CollatePlugin(NoCleanupPluginMixin, NoSetupPluginMixin, ConsumeTaskPlugin) stats = staging.stat() # if the file is older than the timeout, we don't consider # it valid - if (dt.datetime.now().timestamp() - stats.st_mtime) > TIMEOUT_SECONDS: + if (timezone.now().timestamp() - stats.st_mtime) > TIMEOUT_SECONDS: logger.warning("Outdated double sided staging file exists, deleting it") staging.unlink() else: @@ -134,7 +134,7 @@ class CollatePlugin(NoCleanupPluginMixin, NoSetupPluginMixin, ConsumeTaskPlugin) shutil.move(pdf_file, staging) # update access to modification time so we know if the file # is outdated when another file gets uploaded - timestamp = dt.datetime.now().timestamp() + timestamp = timezone.now().timestamp() os.utime(staging, (timestamp, timestamp)) logger.info( "Got scan with odd numbered pages of double-sided scan, moved it to %s", diff --git a/src/documents/tests/test_api_bulk_download.py b/src/documents/tests/test_api_bulk_download.py index eae03a3ed..e1e25fc0b 100644 --- a/src/documents/tests/test_api_bulk_download.py +++ b/src/documents/tests/test_api_bulk_download.py @@ -6,7 +6,6 @@ import zipfile from django.contrib.auth.models import User from django.test import override_settings -from django.utils import timezone from rest_framework import status from rest_framework.test import APITestCase @@ -33,21 +32,21 @@ class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase): filename="docA.pdf", mime_type="application/pdf", checksum="B", - created=timezone.make_aware(datetime.datetime(2021, 1, 1)), + created=datetime.datetime(2021, 1, 1, tzinfo=datetime.UTC), ) self.doc2b = Document.objects.create( title="document A", filename="docA2.pdf", mime_type="application/pdf", checksum="D", - created=timezone.make_aware(datetime.datetime(2021, 1, 1)), + created=datetime.datetime(2021, 1, 1, tzinfo=datetime.UTC), ) self.doc3 = Document.objects.create( title="document B", filename="docB.jpg", mime_type="image/jpeg", checksum="C", - created=timezone.make_aware(datetime.datetime(2020, 3, 21)), + created=datetime.datetime(2020, 3, 21, tzinfo=datetime.UTC), archive_filename="docB.pdf", archive_checksum="D", ) diff --git a/src/documents/tests/test_api_custom_fields.py b/src/documents/tests/test_api_custom_fields.py index 8ad69dd0d..ba1490ce2 100644 --- a/src/documents/tests/test_api_custom_fields.py +++ b/src/documents/tests/test_api_custom_fields.py @@ -1,5 +1,5 @@ +import datetime import json -from datetime import date from unittest import mock from unittest.mock import ANY @@ -456,7 +456,7 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase): }, ) - date_value = date.today() + date_value = datetime.datetime.now(tz=datetime.UTC).date() resp = self.client.patch( f"/api/documents/{doc.id}/", @@ -618,7 +618,7 @@ class TestCustomFieldsAPI(DirectoriesMixin, APITestCase): data_type=CustomField.FieldDataType.DATE, ) - date_value = date.today() + date_value = datetime.datetime.now(tz=datetime.UTC).date() resp = self.client.patch( f"/api/documents/{doc.id}/", diff --git a/src/documents/tests/test_api_documents.py b/src/documents/tests/test_api_documents.py index 2e3e9cbfc..e241cd01c 100644 --- a/src/documents/tests/test_api_documents.py +++ b/src/documents/tests/test_api_documents.py @@ -265,7 +265,7 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase): created=date(2023, 1, 1), ) - created_datetime = datetime.datetime(2023, 2, 1, 12, 0, 0) + created_datetime = datetime.datetime(2023, 2, 1, 12, 0, 0, tzinfo=datetime.UTC) response = self.client.patch( f"/api/documents/{doc.pk}/", {"created": created_datetime}, diff --git a/src/documents/tests/test_api_search.py b/src/documents/tests/test_api_search.py index b42907132..6ff4a5811 100644 --- a/src/documents/tests/test_api_search.py +++ b/src/documents/tests/test_api_search.py @@ -700,7 +700,7 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): pk=3, checksum="C", # specific time zone aware date - added=timezone.make_aware(datetime.datetime(2023, 12, 1)), + added=datetime.datetime(2023, 12, 1, tzinfo=datetime.UTC), ) # refresh doc instance to ensure we operate on date objects that Django uses # Django converts dates to UTC @@ -1022,25 +1022,25 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): title="invoice", content="the thing i bought at a shop and paid with bank account", created=datetime.date(2018, 1, 1), - added=timezone.make_aware(datetime.datetime(2018, 1, 1)), + added=datetime.datetime(2018, 1, 1, tzinfo=datetime.UTC), ) d2 = DocumentFactory( title="bank statement 1", content="things i paid for in august", created=datetime.date(2019, 3, 4), - added=timezone.make_aware(datetime.datetime(2019, 3, 4)), + added=datetime.datetime(2019, 3, 4, tzinfo=datetime.UTC), ) d3 = DocumentFactory( title="bank statement 3", content="things i paid for in september", created=datetime.date(2020, 7, 9), - added=timezone.make_aware(datetime.datetime(2020, 7, 9)), + added=datetime.datetime(2020, 7, 9, tzinfo=datetime.UTC), ) d4 = DocumentFactory( title="Quarterly Report", content="quarterly revenue profit margin earnings growth", created=datetime.date(2021, 11, 30), - added=timezone.make_aware(datetime.datetime(2021, 11, 30)), + added=datetime.datetime(2021, 11, 30, tzinfo=datetime.UTC), ) backend = get_backend() backend.add_or_update(d1) @@ -1159,7 +1159,7 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): d4.tags.add(t2) d5 = Document.objects.create( checksum="5", - added=timezone.make_aware(datetime.datetime(2020, 7, 13)), + added=datetime.datetime(2020, 7, 13, tzinfo=datetime.UTC), content="test", original_filename="doc5.pdf", ) @@ -1269,14 +1269,18 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): d4.id, search_query( "&created__date__lt=" - + datetime.datetime(2020, 9, 2).strftime("%Y-%m-%d"), + + datetime.datetime(2020, 9, 2, tzinfo=datetime.UTC).strftime( + "%Y-%m-%d", + ), ), ) self.assertNotIn( d4.id, search_query( "&created__date__gt=" - + datetime.datetime(2020, 9, 2).strftime("%Y-%m-%d"), + + datetime.datetime(2020, 9, 2, tzinfo=datetime.UTC).strftime( + "%Y-%m-%d", + ), ), ) @@ -1284,14 +1288,18 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): d4.id, search_query( "&created__date__lt=" - + datetime.datetime(2020, 1, 2).strftime("%Y-%m-%d"), + + datetime.datetime(2020, 1, 2, tzinfo=datetime.UTC).strftime( + "%Y-%m-%d", + ), ), ) self.assertIn( d4.id, search_query( "&created__date__gt=" - + datetime.datetime(2020, 1, 2).strftime("%Y-%m-%d"), + + datetime.datetime(2020, 1, 2, tzinfo=datetime.UTC).strftime( + "%Y-%m-%d", + ), ), ) @@ -1299,14 +1307,18 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): d5.id, search_query( "&added__date__lt=" - + datetime.datetime(2020, 9, 2).strftime("%Y-%m-%d"), + + datetime.datetime(2020, 9, 2, tzinfo=datetime.UTC).strftime( + "%Y-%m-%d", + ), ), ) self.assertNotIn( d5.id, search_query( "&added__date__gt=" - + datetime.datetime(2020, 9, 2).strftime("%Y-%m-%d"), + + datetime.datetime(2020, 9, 2, tzinfo=datetime.UTC).strftime( + "%Y-%m-%d", + ), ), ) @@ -1314,7 +1326,9 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): d5.id, search_query( "&added__date__lt=" - + datetime.datetime(2020, 1, 2).strftime("%Y-%m-%d"), + + datetime.datetime(2020, 1, 2, tzinfo=datetime.UTC).strftime( + "%Y-%m-%d", + ), ), ) @@ -1322,7 +1336,9 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): d5.id, search_query( "&added__date__gt=" - + datetime.datetime(2020, 1, 2).strftime("%Y-%m-%d"), + + datetime.datetime(2020, 1, 2, tzinfo=datetime.UTC).strftime( + "%Y-%m-%d", + ), ), ) diff --git a/src/documents/tests/test_double_sided.py b/src/documents/tests/test_double_sided.py index 1189512df..a2263d2cc 100644 --- a/src/documents/tests/test_double_sided.py +++ b/src/documents/tests/test_double_sided.py @@ -59,7 +59,7 @@ class TestDoubleSided(DirectoriesMixin, FileSystemAssertsMixin, TestCase): def create_staging_file(self, src="double-sided-odd.pdf", datetime=None) -> None: shutil.copy(self.SAMPLE_DIR / src, self.staging_file) if datetime is None: - datetime = dt.datetime.now() + datetime = dt.datetime.now(tz=dt.UTC) os.utime(str(self.staging_file), (datetime.timestamp(),) * 2) def test_odd_numbered_moved_to_staging(self) -> None: @@ -79,8 +79,8 @@ class TestDoubleSided(DirectoriesMixin, FileSystemAssertsMixin, TestCase): self.assertIsFile(self.staging_file) self.assertAlmostEqual( - dt.datetime.fromtimestamp(self.staging_file.stat().st_mtime), - dt.datetime.now(), + dt.datetime.fromtimestamp(self.staging_file.stat().st_mtime, tz=dt.UTC), + dt.datetime.now(tz=dt.UTC), delta=dt.timedelta(seconds=5), ) self.assertIn("Received odd numbered pages", msg["reason"]) @@ -124,7 +124,7 @@ class TestDoubleSided(DirectoriesMixin, FileSystemAssertsMixin, TestCase): """ self.create_staging_file( - datetime=dt.datetime.now() + datetime=dt.datetime.now(tz=dt.UTC) - dt.timedelta(minutes=TIMEOUT_MINUTES, seconds=1), ) msg = self.consume_file("double-sided-odd.pdf") diff --git a/src/documents/tests/test_file_handling.py b/src/documents/tests/test_file_handling.py index 399d4ba7d..f881010e1 100644 --- a/src/documents/tests/test_file_handling.py +++ b/src/documents/tests/test_file_handling.py @@ -12,7 +12,6 @@ from django.contrib.auth.models import User from django.db import DatabaseError from django.test import TestCase from django.test import override_settings -from django.utils import timezone from documents.file_handling import create_source_path_directory from documents.file_handling import delete_empty_directories @@ -452,7 +451,7 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase): FILENAME_FORMAT="{created_year}-{created_month}-{created_day}", ) def test_created_year_month_day(self) -> None: - d1 = timezone.make_aware(datetime.datetime(2020, 3, 6, 1, 1, 1)) + d1 = datetime.datetime(2020, 3, 6, 1, 1, 1, tzinfo=datetime.UTC) doc1 = Document.objects.create( title="doc1", mime_type="application/pdf", @@ -469,7 +468,7 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase): FILENAME_FORMAT="{added_year}-{added_month}-{added_day}", ) def test_added_year_month_day(self) -> None: - d1 = timezone.make_aware(datetime.datetime(1232, 1, 9, 1, 1, 1)) + d1 = datetime.datetime(1232, 1, 9, 1, 1, 1, tzinfo=datetime.UTC) doc1 = Document.objects.create( title="doc1", mime_type="application/pdf", @@ -482,7 +481,7 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase): self.assertEqual(generate_filename(doc1), expected_filename) - doc1.added = timezone.make_aware(datetime.datetime(2020, 11, 16, 1, 1, 1)) + doc1.added = datetime.datetime(2020, 11, 16, 1, 1, 1, tzinfo=datetime.UTC) self.assertEqual(generate_filename(doc1), Path("2020-11-16.pdf")) @@ -1266,7 +1265,7 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase): def test_short_names_added(self) -> None: doc = Document.objects.create( title="The Title", - added=timezone.make_aware(datetime.datetime(1984, 8, 21, 7, 36, 51, 153)), + added=datetime.datetime(1984, 8, 21, 7, 36, 51, 153, tzinfo=datetime.UTC), mime_type="application/pdf", pk=2, checksum="2", @@ -1505,7 +1504,7 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase): doc_a = Document.objects.create( title="Does Matter", created=datetime.date(2020, 6, 25), - added=timezone.make_aware(datetime.datetime(2024, 10, 1, 7, 36, 51, 153)), + added=datetime.datetime(2024, 10, 1, 7, 36, 51, 153, tzinfo=datetime.UTC), mime_type="application/pdf", pk=2, checksum="2", @@ -1577,7 +1576,7 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase): doc = Document.objects.create( title="scan_017562", created=datetime.date(2025, 7, 2), - added=timezone.make_aware(datetime.datetime(2026, 3, 3, 11, 53, 16)), + added=datetime.datetime(2026, 3, 3, 11, 53, 16, tzinfo=datetime.UTC), mime_type="application/pdf", checksum="test-checksum", storage_path=sp, @@ -1606,7 +1605,7 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase): doc_a = Document.objects.create( title="Does Matter", created=datetime.date(2020, 6, 25), - added=timezone.make_aware(datetime.datetime(2024, 10, 1, 7, 36, 51, 153)), + added=datetime.datetime(2024, 10, 1, 7, 36, 51, 153, tzinfo=datetime.UTC), mime_type="application/pdf", pk=2, checksum="2", @@ -1641,7 +1640,7 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase): doc_a = Document.objects.create( title="Does Matter", created=datetime.date(2020, 6, 25), - added=timezone.make_aware(datetime.datetime(2024, 10, 1, 7, 36, 51, 153)), + added=datetime.datetime(2024, 10, 1, 7, 36, 51, 153, tzinfo=datetime.UTC), mime_type="application/pdf", pk=2, checksum="2", @@ -1673,7 +1672,7 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase): doc_a = Document.objects.create( title="Some Title", created=datetime.date(2020, 6, 25), - added=timezone.make_aware(datetime.datetime(2024, 10, 1, 7, 36, 51, 153)), + added=datetime.datetime(2024, 10, 1, 7, 36, 51, 153, tzinfo=datetime.UTC), mime_type="application/pdf", pk=2, checksum="2", @@ -1778,7 +1777,7 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase): doc_a = Document.objects.create( title="Some Title", created=datetime.date(2020, 6, 25), - added=timezone.make_aware(datetime.datetime(2024, 10, 1, 7, 36, 51, 153)), + added=datetime.datetime(2024, 10, 1, 7, 36, 51, 153, tzinfo=datetime.UTC), mime_type="application/pdf", pk=2, checksum="2", @@ -1792,8 +1791,15 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase): CustomFieldInstance.objects.create( document=doc_a, field=CustomField.objects.get(name="Invoice Date"), - value_date=timezone.make_aware( - datetime.datetime(2024, 10, 1, 7, 36, 51, 153), + value_date=datetime.datetime( + 2024, + 10, + 1, + 7, + 36, + 51, + 153, + tzinfo=datetime.UTC, ), ) @@ -1833,7 +1839,7 @@ class TestFilenameGeneration(DirectoriesMixin, TestCase): doc = Document.objects.create( title="Some Title! With @ Special # Characters", created=datetime.date(2020, 6, 25), - added=timezone.make_aware(datetime.datetime(2024, 10, 1, 7, 36, 51, 153)), + added=datetime.datetime(2024, 10, 1, 7, 36, 51, 153, tzinfo=datetime.UTC), mime_type="application/pdf", pk=2, checksum="2", diff --git a/src/documents/views.py b/src/documents/views.py index 2cf0afd9d..9ea6b3aeb 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -7,11 +7,11 @@ import tempfile import zipfile from collections import defaultdict from collections import deque +from datetime import UTC from datetime import datetime from datetime import timedelta from http import HTTPStatus from pathlib import Path -from time import mktime from time import sleep from typing import TYPE_CHECKING from typing import Any @@ -61,7 +61,6 @@ from django.http import StreamingHttpResponse from django.shortcuts import get_object_or_404 from django.utils import timezone from django.utils.decorators import method_decorator -from django.utils.timezone import make_aware from django.utils.translation import get_language from django.utils.translation import gettext_lazy as _ from django.views import View @@ -1993,7 +1992,7 @@ class DocumentViewSet( doc_name, doc_data = serializer.validated_data.get("document") version_label = serializer.validated_data.get("version_label") - t = int(mktime(datetime.now().timetuple())) + t = int(timezone.now().timestamp()) settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True) @@ -3218,7 +3217,7 @@ class PostDocumentView(GenericAPIView[Any]): cf = serializer.validated_data.get("custom_fields") from_webui = serializer.validated_data.get("from_webui") - t = int(mktime(datetime.now().timetuple())) + t = int(timezone.now().timestamp()) settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True) @@ -5120,7 +5119,7 @@ class SystemStatusView(PassUserMixin): index_dir = settings.INDEX_DIR mtimes = [p.stat().st_mtime for p in index_dir.iterdir() if p.is_file()] index_last_modified = ( - make_aware(datetime.fromtimestamp(max(mtimes))) if mtimes else None + datetime.fromtimestamp(max(mtimes), tz=UTC) if mtimes else None ) except Exception as e: index_status = "ERROR" diff --git a/src/paperless_mail/mail.py b/src/paperless_mail/mail.py index b3921108f..dad2542c7 100644 --- a/src/paperless_mail/mail.py +++ b/src/paperless_mail/mail.py @@ -5,7 +5,6 @@ import ssl import tempfile import traceback import unicodedata -from datetime import date from datetime import timedelta from fnmatch import fnmatch from pathlib import Path @@ -395,7 +394,7 @@ def make_criterias(rule: MailRule, *, supports_gmail_labels: bool): Returns criteria to be applied to MailBox.fetch for the given rule. """ - maximum_age = date.today() - timedelta(days=rule.maximum_age) + maximum_age = timezone.now().date() - timedelta(days=rule.maximum_age) criterias = {} if rule.maximum_age > 0: criterias["date_gte"] = maximum_age diff --git a/src/paperless_mail/views.py b/src/paperless_mail/views.py index 2e36f1b03..ac53c7181 100644 --- a/src/paperless_mail/views.py +++ b/src/paperless_mail/views.py @@ -1,4 +1,3 @@ -import datetime import logging from datetime import timedelta from http import HTTPStatus @@ -86,7 +85,7 @@ class MailAccountViewSet(PassUserMixin, ModelViewSet[MailAccount]): @action(methods=["post"], detail=False) def test(self, request): logger = logging.getLogger("paperless_mail") - request.data["name"] = datetime.datetime.now().isoformat() + request.data["name"] = timezone.now().isoformat() serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) existing_account = None