Compare commits

..
Author SHA1 Message Date
stumpylog d706d2a9fe Mark empty-pks early-return in set_permissions_for_objects as no-cover
Defensive guard for an edge case (all requested pks already gone/invalid)
rather than a path normal usage exercises; matches the existing
pragma: no cover convention elsewhere in this file.
2026-08-27 09:42:21 -07:00
stumpylog 5caa3327fd Fix: use .distinct() for existing-grant lookup, drop flaky query-count invariant tests
.distinct() lets the database dedupe identity ids server-side instead of
transferring one row per (object, grantee) match and deduping in Python --
was the dominant cost on a large selection with existing grants.

Also replaced the two query-count-equality tests (bulk_edit and the
bulk_edit_objects API path) with plain functional-correctness checks at
both batch sizes.  Hopefully stops that flake.
2026-08-27 09:42:21 -07:00
stumpylog c72c8d1574 Perf: avoid unnecessary full-row fetches in batch permission assignment
set_permissions_for_objects now takes a model + pks instead of instances,
and identity filtering resolves straight to ids, so bulk-editing
permissions no longer materializes full Document/User/Group rows just to
read their pk/id. Row construction for bulk_create is also chunked to
bound peak memory for very large "apply to all" operations.
2026-08-27 09:42:21 -07:00
stumpylog b1f5445689 Perf: batch guardian permission assignment in bulk-edit
bulk_edit.set_permissions and BulkEditObjectPermissionsView both
looped documents/objects and called set_permissions_for_object per
object, which itself calls guardian's assign_perm/remove_perm once
per (object, user) pair -- ~10-20+ queries per object, scaling with
selection size.

Added set_permissions_for_objects, a bulk equivalent that resolves
existing permission holders once across the whole batch (not once per
object) and applies changes with a small, batch-size-independent
number of queries per action instead of one per (object, user) pair.
2026-08-27 09:42:21 -07:00
54 changed files with 552 additions and 326 deletions
+23 -105
View File
@@ -186,110 +186,29 @@ line-ending = "lf"
# https://docs.astral.sh/ruff/rules/ # https://docs.astral.sh/ruff/rules/
select = [ "E4", "E7", "E9", "F" ] select = [ "E4", "E7", "E9", "F" ]
extend-select = [ extend-select = [
"ASYNC", # https://docs.astral.sh/ruff/rules/#flake8-async-async "COM", # https://docs.astral.sh/ruff/rules/#flake8-commas-com
"B002", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b "DJ", # https://docs.astral.sh/ruff/rules/#flake8-django-dj
"B003", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b "EXE", # https://docs.astral.sh/ruff/rules/#flake8-executable-exe
"B004", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b "FBT", # https://docs.astral.sh/ruff/rules/#flake8-boolean-trap-fbt
"B005", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b "FLY", # https://docs.astral.sh/ruff/rules/#flynt-fly
"B006", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b "G201", # https://docs.astral.sh/ruff/rules/#flake8-logging-format-g
"B008", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b "I", # https://docs.astral.sh/ruff/rules/#isort-i
"B009", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b "ICN", # https://docs.astral.sh/ruff/rules/#flake8-import-conventions-icn
"B010", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b "INP", # https://docs.astral.sh/ruff/rules/#flake8-no-pep420-inp
"B012", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b "ISC", # https://docs.astral.sh/ruff/rules/#flake8-implicit-str-concat-isc
"B013", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b "PIE", # https://docs.astral.sh/ruff/rules/#flake8-pie-pie
"B014", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b "PLC", # https://docs.astral.sh/ruff/rules/#pylint-pl
"B015", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b "PLE", # https://docs.astral.sh/ruff/rules/#pylint-pl
"B016", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b "PTH", # https://docs.astral.sh/ruff/rules/#flake8-use-pathlib-pth
"B017", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b "Q", # https://docs.astral.sh/ruff/rules/#flake8-quotes-q
"B018", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b "RSE", # https://docs.astral.sh/ruff/rules/#flake8-raise-rse
"B019", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b "RUF", # https://docs.astral.sh/ruff/rules/#ruff-specific-rules-ruf
"B020", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b "SIM", # https://docs.astral.sh/ruff/rules/#flake8-simplify-sim
"B021", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b "T20", # https://docs.astral.sh/ruff/rules/#flake8-print-t20
"B022", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b "TC", # https://docs.astral.sh/ruff/rules/#flake8-type-checking-tc
"B023", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b "TID", # https://docs.astral.sh/ruff/rules/#flake8-tidy-imports-tid
"B025", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b "UP", # https://docs.astral.sh/ruff/rules/#pyupgrade-up
"B026", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b "W", # https://docs.astral.sh/ruff/rules/#pycodestyle-e-w
"B029", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b
"B030", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b
"B031", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b
"B032", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b
"B033", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b
"B035", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b
"B039", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b
"C4", # https://docs.astral.sh/ruff/rules/#flake8-comprehensions-c4
"COM", # https://docs.astral.sh/ruff/rules/#flake8-commas-com
"D419", # https://docs.astral.sh/ruff/rules/#pydocstyle-d
"DJ", # https://docs.astral.sh/ruff/rules/#flake8-django-dj
"DTZ", # https://docs.astral.sh/ruff/rules/#flake8-datetimez-dtz
"EXE", # https://docs.astral.sh/ruff/rules/#flake8-executable-exe
"FA", # https://docs.astral.sh/ruff/rules/#flake8-future-annotations-fa
"FBT", # https://docs.astral.sh/ruff/rules/#flake8-boolean-trap-fbt
"FLY", # https://docs.astral.sh/ruff/rules/#flynt-fly
"FURB", # https://docs.astral.sh/ruff/rules/#refurb-furb
"G010", # https://docs.astral.sh/ruff/rules/#flake8-logging-format-g
"G101", # https://docs.astral.sh/ruff/rules/#flake8-logging-format-g
"G201", # https://docs.astral.sh/ruff/rules/#flake8-logging-format-g
"G202", # https://docs.astral.sh/ruff/rules/#flake8-logging-format-g
"I", # https://docs.astral.sh/ruff/rules/#isort-i
"ICN", # https://docs.astral.sh/ruff/rules/#flake8-import-conventions-icn
"INP", # https://docs.astral.sh/ruff/rules/#flake8-no-pep420-inp
"INT001", # https://docs.astral.sh/ruff/rules/#flake8-gettext-int
"INT002", # https://docs.astral.sh/ruff/rules/#flake8-gettext-int
"INT003", # https://docs.astral.sh/ruff/rules/#flake8-gettext-int
"ISC", # https://docs.astral.sh/ruff/rules/#flake8-implicit-str-concat-isc
"LOG001", # https://docs.astral.sh/ruff/rules/#flake8-logging-log
"LOG002", # https://docs.astral.sh/ruff/rules/#flake8-logging-log
"LOG009", # https://docs.astral.sh/ruff/rules/#flake8-logging-log
"LOG014", # https://docs.astral.sh/ruff/rules/#flake8-logging-log
"LOG015", # https://docs.astral.sh/ruff/rules/#flake8-logging-log
"N999", # https://docs.astral.sh/ruff/rules/#pep8-naming-n
"PERF101", # https://docs.astral.sh/ruff/rules/#perflint-perf
"PERF102", # https://docs.astral.sh/ruff/rules/#perflint-perf
"PERF402", # https://docs.astral.sh/ruff/rules/#perflint-perf
"PGH005", # https://docs.astral.sh/ruff/rules/#pygrep-hooks-pgh
"PIE", # https://docs.astral.sh/ruff/rules/#flake8-pie-pie
"PLC", # https://docs.astral.sh/ruff/rules/#pylint-pl
"PLE", # https://docs.astral.sh/ruff/rules/#pylint-pl
"PLR0124", # https://docs.astral.sh/ruff/rules/#pylint-pl
"PLR0133", # https://docs.astral.sh/ruff/rules/#pylint-pl
"PLR0206", # https://docs.astral.sh/ruff/rules/#pylint-pl
"PLR0402", # https://docs.astral.sh/ruff/rules/#pylint-pl
"PLR1704", # https://docs.astral.sh/ruff/rules/#pylint-pl
"PLR1708", # https://docs.astral.sh/ruff/rules/#pylint-pl
"PLR1711", # https://docs.astral.sh/ruff/rules/#pylint-pl
"PLR1716", # https://docs.astral.sh/ruff/rules/#pylint-pl
"PLR1722", # https://docs.astral.sh/ruff/rules/#pylint-pl
"PLR1730", # https://docs.astral.sh/ruff/rules/#pylint-pl
"PLR1733", # https://docs.astral.sh/ruff/rules/#pylint-pl
"PLR1736", # https://docs.astral.sh/ruff/rules/#pylint-pl
"PLR2044", # https://docs.astral.sh/ruff/rules/#pylint-pl
"PLW", # https://docs.astral.sh/ruff/rules/#pylint-pl
"PT010", # https://docs.astral.sh/ruff/rules/#flake8-pytest-style-pt
"PT014", # https://docs.astral.sh/ruff/rules/#flake8-pytest-style-pt
"PT020", # https://docs.astral.sh/ruff/rules/#flake8-pytest-style-pt
"PT025", # https://docs.astral.sh/ruff/rules/#flake8-pytest-style-pt
"PT026", # https://docs.astral.sh/ruff/rules/#flake8-pytest-style-pt
"PT031", # https://docs.astral.sh/ruff/rules/#flake8-pytest-style-pt
"PTH", # https://docs.astral.sh/ruff/rules/#flake8-use-pathlib-pth
"Q", # https://docs.astral.sh/ruff/rules/#flake8-quotes-q
"RSE", # https://docs.astral.sh/ruff/rules/#flake8-raise-rse
"RUF", # https://docs.astral.sh/ruff/rules/#ruff-specific-rules-ruf
"S102", # https://docs.astral.sh/ruff/rules/#flake8-bandit-s
"S110", # https://docs.astral.sh/ruff/rules/#flake8-bandit-s
"S112", # https://docs.astral.sh/ruff/rules/#flake8-bandit-s
"SIM", # https://docs.astral.sh/ruff/rules/#flake8-simplify-sim
"T100", # https://docs.astral.sh/ruff/rules/#flake8-debugger-t10
"T20", # https://docs.astral.sh/ruff/rules/#flake8-print-t20
"TC", # https://docs.astral.sh/ruff/rules/#flake8-type-checking-tc
"TID", # https://docs.astral.sh/ruff/rules/#flake8-tidy-imports-tid
"TRY002", # https://docs.astral.sh/ruff/rules/#tryceratops-try
"TRY004", # https://docs.astral.sh/ruff/rules/#tryceratops-try
"TRY201", # https://docs.astral.sh/ruff/rules/#tryceratops-try
"TRY203", # https://docs.astral.sh/ruff/rules/#tryceratops-try
"TRY401", # https://docs.astral.sh/ruff/rules/#tryceratops-try
"UP", # https://docs.astral.sh/ruff/rules/#pyupgrade-up
"W", # https://docs.astral.sh/ruff/rules/#pycodestyle-e-w
"YTT", # https://docs.astral.sh/ruff/rules/#flake8-2020-ytt
] ]
ignore = [ ignore = [
"DJ001", "DJ001",
@@ -305,7 +224,6 @@ per-file-ignores."*/migrations/*.py" = [
] ]
# Testing # Testing
per-file-ignores."*/tests/*.py" = [ per-file-ignores."*/tests/*.py" = [
"DTZ",
"E501", "E501",
"SIM117", "SIM117",
] ]
+17 -14
View File
@@ -27,7 +27,7 @@ from documents.models import DocumentType
from documents.models import PaperlessTask from documents.models import PaperlessTask
from documents.models import StoragePath from documents.models import StoragePath
from documents.models import Tag from documents.models import Tag
from documents.permissions import set_permissions_for_object from documents.permissions import set_permissions_for_objects
from documents.plugins.helpers import DocumentsStatusManager from documents.plugins.helpers import DocumentsStatusManager
from documents.tasks import bulk_update_documents from documents.tasks import bulk_update_documents
from documents.tasks import consume_file from documents.tasks import consume_file
@@ -430,10 +430,13 @@ def set_permissions(
else: else:
qs.update(owner=owner) qs.update(owner=owner)
for doc in qs:
set_permissions_for_object(permissions=set_permissions, object=doc, merge=merge)
affected_docs = list(qs.values_list("pk", flat=True)) affected_docs = list(qs.values_list("pk", flat=True))
set_permissions_for_objects(
permissions=set_permissions,
model=Document,
pks=affected_docs,
merge=merge,
)
bulk_update_documents.apply_async( bulk_update_documents.apply_async(
kwargs={"document_ids": affected_docs}, kwargs={"document_ids": affected_docs},
@@ -507,8 +510,8 @@ def rotate(
logger.info( logger.info(
f"Queued new rotated version for document {pair.root_doc.id} by {degrees} degrees", f"Queued new rotated version for document {pair.root_doc.id} by {degrees} degrees",
) )
except Exception: except Exception as e:
logger.exception(f"Error rotating document {pair.root_doc.id}") logger.exception(f"Error rotating document {pair.root_doc.id}: {e}")
return "OK" return "OK"
@@ -554,9 +557,9 @@ def merge(
affected_docs.append(doc.id) affected_docs.append(doc.id)
if handoff_asn is None and doc.archive_serial_number is not None: if handoff_asn is None and doc.archive_serial_number is not None:
handoff_asn = doc.archive_serial_number handoff_asn = doc.archive_serial_number
except Exception: except Exception as e:
logger.exception( logger.exception(
f"Error merging document {doc.id}, it will not be included in the merge", f"Error merging document {doc.id}, it will not be included in the merge: {e}",
) )
if len(affected_docs) == 0: if len(affected_docs) == 0:
logger.warning("No documents were merged") logger.warning("No documents were merged")
@@ -805,8 +808,8 @@ def split(
else: else:
group(consume_tasks).delay() group(consume_tasks).delay()
except Exception: except Exception as e:
logger.exception(f"Error splitting document {doc.id}") logger.exception(f"Error splitting document {doc.id}: {e}")
return "OK" return "OK"
@@ -858,8 +861,8 @@ def delete_pages(
logger.info( logger.info(
f"Queued new version for document {pair.root_doc.id} after deleting pages {pages}", f"Queued new version for document {pair.root_doc.id} after deleting pages {pages}",
) )
except Exception: except Exception as e:
logger.exception(f"Error deleting pages from document {pair.root_doc.id}") logger.exception(f"Error deleting pages from document {pair.root_doc.id}: {e}")
return "OK" return "OK"
@@ -986,7 +989,7 @@ def edit_pdf(
group(consume_tasks).delay() group(consume_tasks).delay()
except Exception as e: except Exception as e:
logger.exception(f"Error editing document {pair.root_doc.id}") logger.exception(f"Error editing document {pair.root_doc.id}: {e}")
raise ValueError( raise ValueError(
f"An error occurred while editing the document: {e}", f"An error occurred while editing the document: {e}",
) from e ) from e
@@ -1097,7 +1100,7 @@ def remove_password(
except Exception as e: except Exception as e:
logger.exception( logger.exception(
f"Error removing password from document {pair.root_doc.id}", f"Error removing password from document {pair.root_doc.id}: {e}",
) )
raise ValueError( raise ValueError(
f"An error occurred while removing the password: {e}", f"An error occurred while removing the password: {e}",
+7 -7
View File
@@ -69,8 +69,8 @@ def load_classifier(*, raise_exception: bool = False) -> DocumentClassifier | No
Path(settings.MODEL_FILE).unlink() Path(settings.MODEL_FILE).unlink()
classifier = None classifier = None
if raise_exception: if raise_exception:
raise raise e
except ClassifierModelCorruptError: except ClassifierModelCorruptError as e:
# there's something wrong with the model file. # there's something wrong with the model file.
logger.exception( logger.exception(
"Unrecoverable error while loading document " "Unrecoverable error while loading document "
@@ -79,17 +79,17 @@ def load_classifier(*, raise_exception: bool = False) -> DocumentClassifier | No
Path(settings.MODEL_FILE).unlink() Path(settings.MODEL_FILE).unlink()
classifier = None classifier = None
if raise_exception: if raise_exception:
raise raise e
except OSError: except OSError as e:
logger.exception("IO error while loading document classification model") logger.exception("IO error while loading document classification model")
classifier = None classifier = None
if raise_exception: if raise_exception:
raise raise e
except Exception: # pragma: no cover except Exception as e: # pragma: no cover
logger.exception("Unknown error while loading document classification model") logger.exception("Unknown error while loading document classification model")
classifier = None classifier = None
if raise_exception: if raise_exception:
raise raise e
return classifier return classifier
+6 -4
View File
@@ -216,7 +216,7 @@ class ConsumerPluginMixin:
current_progress, current_progress,
max_progress, max_progress,
document_id=document_id, document_id=document_id,
owner_id=self.metadata.owner_id or None, owner_id=self.metadata.owner_id if self.metadata.owner_id else None,
users_can_view=(self.metadata.view_users or []) users_can_view=(self.metadata.view_users or [])
+ (self.metadata.change_users or []), + (self.metadata.change_users or []),
groups_can_view=(self.metadata.view_groups or []) groups_can_view=(self.metadata.view_groups or [])
@@ -674,7 +674,9 @@ class ConsumerPlugin(
document=document, document=document,
logging_group=self.logging_group, logging_group=self.logging_group,
classifier=classifier, classifier=classifier,
original_file=self.unmodified_original or self.working_copy, original_file=self.unmodified_original
if self.unmodified_original
else self.working_copy,
) )
# After everything is in the database, copy the files into # After everything is in the database, copy the files into
@@ -847,7 +849,7 @@ class ConsumerPlugin(
else: else:
stats = Path(self.input_doc.original_file).stat() stats = Path(self.input_doc.original_file).stat()
create_date = timezone.make_aware( create_date = timezone.make_aware(
datetime.datetime.fromtimestamp(stats.st_mtime), # noqa: DTZ006 - make_aware() requires a naive datetime datetime.datetime.fromtimestamp(stats.st_mtime),
) )
self.log.debug(f"Creation date from st_mtime: {create_date}") self.log.debug(f"Creation date from st_mtime: {create_date}")
@@ -961,7 +963,7 @@ class ConsumerPlugin(
try: try:
copy_basic_file_stats(source, target) copy_basic_file_stats(source, target)
except Exception: # pragma: no cover except Exception: # pragma: no cover
self.log.debug("Unable to copy file stats from %s to %s", source, target) pass
class ConsumerPreflightPlugin( class ConsumerPreflightPlugin(
+2 -4
View File
@@ -78,9 +78,7 @@ class CollatePlugin(NoCleanupPluginMixin, NoSetupPluginMixin, ConsumeTaskPlugin)
stats = staging.stat() stats = staging.stat()
# if the file is older than the timeout, we don't consider # if the file is older than the timeout, we don't consider
# it valid # it valid
if ( if (dt.datetime.now().timestamp() - stats.st_mtime) > TIMEOUT_SECONDS:
dt.datetime.now(tz=dt.UTC).timestamp() - stats.st_mtime
) > TIMEOUT_SECONDS:
logger.warning("Outdated double sided staging file exists, deleting it") logger.warning("Outdated double sided staging file exists, deleting it")
staging.unlink() staging.unlink()
else: else:
@@ -136,7 +134,7 @@ class CollatePlugin(NoCleanupPluginMixin, NoSetupPluginMixin, ConsumeTaskPlugin)
shutil.move(pdf_file, staging) shutil.move(pdf_file, staging)
# update access to modification time so we know if the file # update access to modification time so we know if the file
# is outdated when another file gets uploaded # is outdated when another file gets uploaded
timestamp = dt.datetime.now(tz=dt.UTC).timestamp() timestamp = dt.datetime.now().timestamp()
os.utime(staging, (timestamp, timestamp)) os.utime(staging, (timestamp, timestamp))
logger.info( logger.info(
"Got scan with odd numbered pages of double-sided scan, moved it to %s", "Got scan with odd numbered pages of double-sided scan, moved it to %s",
+1 -1
View File
@@ -734,7 +734,7 @@ class CustomFieldQueryParser:
) )
# Check if any of the requested IDs are missing. # Check if any of the requested IDs are missing.
missing_ids = set(value) - {link.document_id for link in links} missing_ids = set(value) - set(link.document_id for link in links)
if missing_ids: if missing_ids:
# The result should be an empty set in this case. # The result should be an empty set in this case.
return Q(id__in=[]) return Q(id__in=[])
@@ -631,25 +631,23 @@ class Command(BaseCommand):
): ):
# Process each change # Process each change
for change_type, path in changes: for change_type, path in changes:
resolved_path = Path(path).resolve() path = Path(path).resolve()
if change_type == Change.deleted: if change_type == Change.deleted:
# Consumed (or otherwise removed); a later file # Consumed (or otherwise removed); a later file
# reusing this name must not be skipped as # reusing this name must not be skipped as
# already-queued. # already-queued.
queued.discard(resolved_path) queued.discard(path)
if not resolved_path.is_file(): if not path.is_file():
continue continue
if resolved_path in queued: if path in queued:
# Already queued and awaiting consumption; a stray # Already queued and awaiting consumption; a stray
# event (NAS metadata touch, AV scan, etc.) while # event (NAS metadata touch, AV scan, etc.) while
# the file sits on disk mid-consumption must not # the file sits on disk mid-consumption must not
# cause it to be queued a second time (GH #13511). # cause it to be queued a second time (GH #13511).
logger.debug( logger.debug(f"Ignoring event for queued file: {path}")
f"Ignoring event for queued file: {resolved_path}",
)
continue continue
logger.debug(f"Event: {change_type.name} for {resolved_path}") logger.debug(f"Event: {change_type.name} for {path}")
tracker.track(resolved_path, change_type) tracker.track(path, change_type)
# Check for stable files # Check for stable files
for stable_path in tracker.get_stable_files(): for stable_path in tracker.get_stable_files():
+1 -7
View File
@@ -30,10 +30,6 @@ if TYPE_CHECKING:
logger = logging.getLogger("paperless.matching") logger = logging.getLogger("paperless.matching")
class UnsupportedWorkflowTriggerTypeError(Exception):
pass
def log_reason( def log_reason(
matching_model: MatchingModel | WorkflowTrigger, matching_model: MatchingModel | WorkflowTrigger,
document: Document, document: Document,
@@ -695,9 +691,7 @@ def document_matches_workflow(
) )
else: else:
# New trigger types need to be explicitly checked above # New trigger types need to be explicitly checked above
raise UnsupportedWorkflowTriggerTypeError( raise Exception(f"Trigger type {trigger_type} not yet supported")
f"Trigger type {trigger_type} not yet supported",
)
if trigger_matched: if trigger_matched:
logger.info(f"Document matched {trigger} from {workflow}") logger.info(f"Document matched {trigger} from {workflow}")
+1 -1
View File
@@ -377,7 +377,7 @@ class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-
from documents.versioning import versions_newest_first from documents.versioning import versions_newest_first
if hasattr(self, "effective_content"): if hasattr(self, "effective_content"):
return self.effective_content return getattr(self, "effective_content")
if self.root_document_id is not None or self.pk is None: if self.root_document_id is not None or self.pk is None:
return self.content return self.content
+1 -1
View File
@@ -41,7 +41,7 @@ def get_default_file_extension(mime_type: str) -> str:
return supported[mime_type] return supported[mime_type]
ext = mimetypes.guess_extension(mime_type) ext = mimetypes.guess_extension(mime_type)
return ext or "" return ext if ext else ""
def is_file_ext_supported(ext: str) -> bool: def is_file_ext_supported(ext: str) -> bool:
+178
View File
@@ -173,6 +173,184 @@ def set_permissions_for_object(
) )
def _resolve_permissions(codenames: set[str], ctype: ContentType) -> list[Permission]:
"""
Resolves `codenames` to Permission rows, raising like the single-object
assign_perm() this bulk path replaces does (via a `.get()` internally)
if any codename doesn't exist -- e.g. a client-supplied action name that
was never validated (BulkEditObjectsSerializer._validate_permissions
calls validate_set_permissions() only for its side-effecting id checks
and discards the filtered dict it returns, so an unrecognized action key
reaches this function as-is). A plain `.filter()` with no existence
check would otherwise silently build zero rows and no-op instead of
reporting the bad input.
"""
permission_objs = list(
Permission.objects.filter(content_type=ctype, codename__in=codenames),
)
missing = codenames - {p.codename for p in permission_objs}
if missing:
raise Permission.DoesNotExist(
f"Permission matching query does not exist for codename(s): "
f"{', '.join(sorted(missing))}",
)
return permission_objs
# Target number of permission rows to build in Python before handing them to
# bulk_create -- keeps peak memory bounded for a large "apply to all" call,
# independent of bulk_create's own batch_size (which only caps the size of
# each INSERT statement, not how many row objects exist in memory at once).
_PERMISSION_ROW_CHUNK_SIZE = 5000
def _apply_bulk_permission_entry(
*,
perm_model: type[UserObjectPermission] | type[GroupObjectPermission],
identity_model: type[User] | type[Group],
identity_field: str,
ids: list[int],
codename: str,
permission_objs: list[Permission],
ctype: ContentType,
object_pks: list[str],
merge: bool,
) -> None:
# Only the ids are needed to build permission rows (via `<field>_id=`),
# so avoid fetching full User/Group rows for identities that may not
# even end up being granted anything new.
add_ids = set(
identity_model.objects.filter(id__in=ids).values_list("id", flat=True),
)
if not merge:
existing_ids = set(
perm_model.objects.filter(
content_type=ctype,
object_pk__in=object_pks,
permission__codename=codename,
)
.values_list(f"{identity_field}_id", flat=True)
.distinct(),
)
remove_ids = existing_ids - add_ids
if remove_ids:
perm_model.objects.filter(
content_type=ctype,
object_pk__in=object_pks,
permission__codename=codename,
**{f"{identity_field}_id__in": remove_ids},
).delete()
if not add_ids:
return
rows_per_pk = len(permission_objs) * len(add_ids)
pks_per_chunk = max(1, _PERMISSION_ROW_CHUNK_SIZE // rows_per_pk)
for start in range(0, len(object_pks), pks_per_chunk):
pk_chunk = object_pks[start : start + pks_per_chunk]
rows = [
perm_model(
content_type=ctype,
object_pk=pk,
permission=permission_obj,
**{f"{identity_field}_id": identity_id},
)
for permission_obj in permission_objs
for pk in pk_chunk
for identity_id in add_ids
]
# ignore_conflicts skips only rows that already exist as an exact
# (identity, permission, object) match -- the same de-dup the
# underlying (user|group, permission, object_pk) unique constraint
# already enforces for the single-object assign_perm() this
# replaces, so it doesn't change what counts as "already granted".
# batch_size caps how many rows go into a single INSERT so a huge
# chunk doesn't build one enormous statement.
perm_model.objects.bulk_create(rows, ignore_conflicts=True, batch_size=1000)
def set_permissions_for_objects(
permissions: dict,
model: type[Model],
pks: QuerySet | list,
*,
merge: bool = False,
) -> None:
"""
Bulk equivalent of set_permissions_for_object: applies the same
permission changes to every object identified by `pks` at once.
Takes a model + pks (rather than model instances) deliberately -- the
permission rows built below only ever need `pk`, `content_type`, and
identity ids, so callers shouldn't have to fetch full rows (with every
other field) just to hand them to this function.
Deliberately does not use guardian's queryset/list-aware assign_perm:
passing a list as the object routes to bulk_assign_perm, which skips
creating a direct permission row for anyone who already has the
permission via ANY group membership (it checks
ObjectPermissionChecker.has_perm, which is group-inheritance-aware) --
unlike the single-object assign_perm this replaces, which always
ensures a direct row via get_or_create regardless of group-derived
access. Losing that guarantee would mean a later revocation of the
group's grant silently strips access an admin explicitly asked to be
direct. Bulk-creating rows straight against the permission models
instead (see _apply_bulk_permission_entry) preserves the original
always-create-a-direct-row semantics while still batching every object
and every identity into one query per action, rather than one query per
(object, user) pair.
"""
object_pks = [str(pk) for pk in pks]
if not object_pks: # pragma: no cover
return
model_name = model.__name__.lower()
ctype = ContentType.objects.get_for_model(model)
for action, entry in permissions.items():
codename = f"{action}_{model_name}"
implied_codenames = {codename}
if action == "change":
# change gives view too
implied_codenames.add(f"view_{model_name}")
# Resolved once per action (not once per users/groups branch) and
# shared between both below -- also where an unrecognized action
# name (see _resolve_permissions) is caught.
permission_objs = (
_resolve_permissions(implied_codenames, ctype)
if "users" in entry or "groups" in entry
else []
)
if "users" in entry:
_apply_bulk_permission_entry(
perm_model=UserObjectPermission,
identity_model=User,
identity_field="user",
ids=entry["users"],
codename=codename,
permission_objs=permission_objs,
ctype=ctype,
object_pks=object_pks,
merge=merge,
)
if "groups" in entry:
_apply_bulk_permission_entry(
perm_model=GroupObjectPermission,
identity_model=Group,
identity_field="group",
ids=entry["groups"],
codename=codename,
permission_objs=permission_objs,
ctype=ctype,
object_pks=object_pks,
merge=merge,
)
def permitted_object_ids( def permitted_object_ids(
user: User | None, user: User | None,
model: type[Model], model: type[Model],
@@ -43,8 +43,8 @@ def _discover_parser_class() -> type[DateParserPluginBase]:
valid_plugins.append(ep) valid_plugins.append(ep)
else: else:
logger.warning(f"Plugin {ep.name} does not subclass DateParser.") logger.warning(f"Plugin {ep.name} does not subclass DateParser.")
except Exception: except Exception as e:
logger.exception(f"Unable to load date parser plugin {ep.name}") logger.exception(f"Unable to load date parser plugin {ep.name}: {e}")
if not valid_plugins: if not valid_plugins:
return RegexDateParserPlugin return RegexDateParserPlugin
+2 -2
View File
@@ -91,8 +91,8 @@ class DateParserPluginBase(ABC):
}, },
locales=self.config.languages, locales=self.config.languages,
) )
except Exception: except Exception as e:
logger.exception(f"Error while parsing date string '{date_string}'") logger.exception(f"Error while parsing date string '{date_string}': {e}")
return None return None
def _filter_date( def _filter_date(
+6 -4
View File
@@ -59,10 +59,11 @@ def safe_regex_match(pattern: str, text: str, *, flags: int = 0):
try: try:
validate_regex_pattern(pattern) validate_regex_pattern(pattern)
compiled = regex.compile(pattern, flags=flags) compiled = regex.compile(pattern, flags=flags)
except (regex.error, ValueError): except (regex.error, ValueError) as exc:
logger.exception( logger.exception(
"Error while processing regular expression %s", "Error while processing regular expression %s: %s",
textwrap.shorten(pattern, width=80, placeholder=""), textwrap.shorten(pattern, width=80, placeholder=""),
exc,
) )
return None return None
@@ -85,10 +86,11 @@ def safe_regex_sub(pattern: str, repl: str, text: str, *, flags: int = 0) -> str
try: try:
validate_regex_pattern(pattern) validate_regex_pattern(pattern)
compiled = regex.compile(pattern, flags=flags) compiled = regex.compile(pattern, flags=flags)
except (regex.error, ValueError): except (regex.error, ValueError) as exc:
logger.exception( logger.exception(
"Error while processing regular expression %s", "Error while processing regular expression %s: %s",
textwrap.shorten(pattern, width=80, placeholder=""), textwrap.shorten(pattern, width=80, placeholder=""),
exc,
) )
return None return None
+2 -2
View File
@@ -1142,7 +1142,7 @@ def get_backend() -> TantivyBackend:
Returns: Returns:
Thread-safe singleton TantivyBackend instance Thread-safe singleton TantivyBackend instance
""" """
global _backend, _backend_path # noqa: PLW0603 - module-level singleton, no class to hold this state global _backend, _backend_path
current_path: Path = settings.INDEX_DIR current_path: Path = settings.INDEX_DIR
@@ -1173,7 +1173,7 @@ def reset_backend() -> None:
Forces creation of a new backend instance on the next get_backend() call. Forces creation of a new backend instance on the next get_backend() call.
Used for test isolation and when switching between different index directories. Used for test isolation and when switching between different index directories.
""" """
global _backend, _backend_path # noqa: PLW0603 - module-level singleton, no class to hold this state global _backend, _backend_path
with _backend_lock: with _backend_lock:
if _backend is not None: if _backend is not None:
+1 -1
View File
@@ -240,7 +240,7 @@ def parse_user_query(
DEFAULT_SEARCH_FIELDS, DEFAULT_SEARCH_FIELDS,
field_boosts=_FIELD_BOOSTS, field_boosts=_FIELD_BOOSTS,
# (prefix=True, distance=1, transposition_cost_one=True) — edit-distance fuzziness # (prefix=True, distance=1, transposition_cost_one=True) — edit-distance fuzziness
fuzzy_fields=dict.fromkeys(DEFAULT_SEARCH_FIELDS, (True, 1, True)), fuzzy_fields={f: (True, 1, True) for f in DEFAULT_SEARCH_FIELDS},
) )
# 0.1 boost keeps fuzzy hits ranked below exact matches (intentional) # 0.1 boost keeps fuzzy hits ranked below exact matches (intentional)
clauses.append((tantivy.Occur.Should, tantivy.Query.boost_query(fuzzy, 0.1))) clauses.append((tantivy.Occur.Should, tantivy.Query.boost_query(fuzzy, 0.1)))
+12 -11
View File
@@ -433,7 +433,7 @@ class OwnedObjectSerializer(
return set() return set()
ctype = ContentType.objects.get_for_model(first_obj) ctype = ContentType.objects.get_for_model(first_obj)
object_pks = [obj.pk for obj in objects] object_pks = list(obj.pk for obj in objects)
pk_type = type(first_obj.pk) pk_type = type(first_obj.pk)
def get_pks_for_permission_type(model): def get_pks_for_permission_type(model):
@@ -727,7 +727,7 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
self.instance.clean() self.instance.clean()
except ValidationError as e: except ValidationError as e:
logger.debug("Tag parent validation failed: %s", e) logger.debug("Tag parent validation failed: %s", e)
raise raise e
finally: finally:
self.instance.tn_parent = original_parent self.instance.tn_parent = original_parent
else: else:
@@ -737,7 +737,7 @@ class TagSerializer(MatchingModelSerializer, OwnedObjectSerializer):
temp.clean() temp.clean()
except ValidationError as e: except ValidationError as e:
logger.debug("Tag parent validation failed: %s", e) logger.debug("Tag parent validation failed: %s", e)
raise raise e
return super().validate(attrs) return super().validate(attrs)
@@ -1147,7 +1147,7 @@ class DocumentSerializer(
def to_representation(self, instance): def to_representation(self, instance):
doc = super().to_representation(instance) doc = super().to_representation(instance)
if "content" in self.fields and hasattr(instance, "effective_content"): if "content" in self.fields and hasattr(instance, "effective_content"):
doc["content"] = instance.effective_content or "" doc["content"] = getattr(instance, "effective_content") or ""
if self.truncate_content and "content" in self.fields: if self.truncate_content and "content" in self.fields:
doc["content"] = doc.get("content")[0:550] doc["content"] = doc.get("content")[0:550]
return doc return doc
@@ -1857,8 +1857,8 @@ class BulkEditSerializer(
if isinstance(custom_fields, dict): if isinstance(custom_fields, dict):
try: try:
ids = [int(i[0]) for i in custom_fields.items()] ids = [int(i[0]) for i in custom_fields.items()]
except Exception: except Exception as e:
logger.exception("Error validating custom fields") logger.exception(f"Error validating custom fields: {e}")
raise serializers.ValidationError( raise serializers.ValidationError(
f"{name} must be a list of integers or a dict of id:value pairs, see the log for details", f"{name} must be a list of integers or a dict of id:value pairs, see the log for details",
) )
@@ -2056,12 +2056,13 @@ class BulkEditSerializer(
for doc in docs: for doc in docs:
if "-" in doc: if "-" in doc:
pages.append( pages.append(
list( [
range( x
for x in range(
int(doc.split("-")[0]), int(doc.split("-")[0]),
int(doc.split("-")[1]) + 1, int(doc.split("-")[1]) + 1,
), )
), ],
) )
else: else:
pages.append([int(doc)]) pages.append([int(doc)])
@@ -2922,7 +2923,7 @@ class ShareLinkBundleSerializer(OwnedObjectSerializer):
return share_link_bundle return share_link_bundle
def get_document_count(self, obj: ShareLinkBundle) -> int: def get_document_count(self, obj: ShareLinkBundle) -> int:
return obj.document_total or obj.documents.count() return getattr(obj, "document_total") or obj.documents.count()
class BulkEditObjectsSerializer(SerializerWithPerms, SetPermissionsMixin): class BulkEditObjectsSerializer(SerializerWithPerms, SetPermissionsMixin):
+4 -5
View File
@@ -636,7 +636,7 @@ def update_filename_and_move_files(
# so this is not the end of the world. # so this is not the end of the world.
# B: if moving the original file failed, nothing has changed # B: if moving the original file failed, nothing has changed
# anyway. # anyway.
logger.exception("Error reverting document changes") pass
# restore old values on the instance # restore old values on the instance
instance.filename = old_filename instance.filename = old_filename
@@ -1101,11 +1101,10 @@ def _extract_input_data(
if v is None or k.startswith("_"): if v is None or k.startswith("_"):
continue continue
if isinstance(v, datetime.date): if isinstance(v, datetime.date):
override_dict[k] = v.isoformat() v = v.isoformat()
elif isinstance(v, Path): elif isinstance(v, Path):
override_dict[k] = str(v) v = str(v)
else: override_dict[k] = v
override_dict[k] = v
if override_dict: if override_dict:
data["overrides"] = override_dict data["overrides"] = override_dict
return data return data
+7 -6
View File
@@ -217,9 +217,9 @@ def consume_file(
overrides.filename or input_doc.original_file.name, overrides.filename or input_doc.original_file.name,
self.request.id, self.request.id,
) as status_mgr, ) as status_mgr,
TemporaryDirectory(dir=settings.SCRATCH_DIR) as tmp_dir_name, TemporaryDirectory(dir=settings.SCRATCH_DIR) as tmp_dir,
): ):
tmp_dir = Path(tmp_dir_name) tmp_dir = Path(tmp_dir)
msg = None msg = None
for plugin_class in plugins: for plugin_class in plugins:
plugin_name = plugin_class.NAME plugin_name = plugin_class.NAME
@@ -261,7 +261,7 @@ def consume_file(
) )
except Exception as e: except Exception as e:
logger.exception(f"{plugin_name} failed") logger.exception(f"{plugin_name} failed: {e}")
status_mgr.send_progress( status_mgr.send_progress(
ProgressStatusOptions.FAILED, ProgressStatusOptions.FAILED,
f"{e}", f"{e}",
@@ -495,8 +495,8 @@ def empty_trash(doc_ids=None) -> None:
content_type=ContentType.objects.get_for_model(Document), content_type=ContentType.objects.get_for_model(Document),
object_id__in=deleted_document_ids, object_id__in=deleted_document_ids,
).delete() ).delete()
except Exception: # pragma: no cover except Exception as e: # pragma: no cover
logger.exception("Error while emptying trash") logger.exception(f"Error while emptying trash: {e}")
finally: finally:
models.signals.post_delete.disconnect( models.signals.post_delete.disconnect(
cleanup_document_deletion, cleanup_document_deletion,
@@ -832,8 +832,9 @@ def build_share_link_bundle(bundle_id: int) -> None:
logger.info("Built share link bundle %s", bundle.pk) logger.info("Built share link bundle %s", bundle.pk)
except Exception as exc: except Exception as exc:
logger.exception( logger.exception(
"Failed to build share link bundle %s", "Failed to build share link bundle %s: %s",
bundle_id, bundle_id,
exc,
) )
bundle.status = ShareLinkBundle.Status.FAILED bundle.status = ShareLinkBundle.Status.FAILED
bundle.last_error = { bundle.last_error = {
-4
View File
@@ -78,10 +78,6 @@ class PlaceholderString(str):
def __ne__(self, other) -> bool: def __ne__(self, other) -> bool:
return not self.__eq__(other) return not self.__eq__(other)
def __hash__(self) -> int:
# Equal to both "-none-" and "none", so hash to a single canonical value
return hash("-none-")
NO_VALUE_PLACEHOLDER = PlaceholderString("-none-") NO_VALUE_PLACEHOLDER = PlaceholderString("-none-")
+3 -3
View File
@@ -138,9 +138,9 @@ def parse_w_workflow_placeholders(
# We're good! # We're good!
return rendered_template return rendered_template
except UndefinedError: except UndefinedError as e:
# The undefined class logs this already for us # The undefined class logs this already for us
raise raise e
except TemplateSyntaxError as e: except TemplateSyntaxError as e:
logger.warning(f"Template syntax error in title generation: {e}") logger.warning(f"Template syntax error in title generation: {e}")
except SecurityError as e: except SecurityError as e:
@@ -150,5 +150,5 @@ def parse_w_workflow_placeholders(
logger.warning( logger.warning(
f"Invalid title format '{text}', workflow not applied: {e}", f"Invalid title format '{text}', workflow not applied: {e}",
) )
raise raise e
return None return None
@@ -296,7 +296,7 @@ class TestRegexDateParser:
# simulate parse failure for malformed input # simulate parse failure for malformed input
if "99/99/9999" in date_string or "bad date" in date_string: if "99/99/9999" in date_string or "bad date" in date_string:
raise Exception("parse failed for malformed date") # noqa: TRY002 - simulates a generic parser failure raise Exception("parse failed for malformed date")
return None return None
@@ -57,13 +57,13 @@ class MultiprocessCommand(PaperlessCommand):
def handle(self, *args, **options): def handle(self, *args, **options):
items = list(range(5)) items = list(range(5))
results = list( results = []
self.process_parallel( for result in self.process_parallel(
_double_value, _double_value,
items, items,
description="Processing...", description="Processing...",
), ):
) results.append(result)
successes = sum(1 for r in results if r.success) successes = sum(1 for r in results if r.success)
self.stdout.write(f"Successes: {successes}") self.stdout.write(f"Successes: {successes}")
@@ -66,7 +66,7 @@ class TestWriteBatchLockRetry:
) )
mock_sleep = mocker.patch( mock_sleep = mocker.patch(
"documents.search._backend.time.sleep", "documents.search._backend.time.sleep",
side_effect=sleep_values.append, side_effect=lambda s: sleep_values.append(s),
) )
# Should not raise — 4th attempt succeeds # Should not raise — 4th attempt succeeds
@@ -111,7 +111,7 @@ class TestWriteBatchLockRetry:
sleep_values: list[float] = [] sleep_values: list[float] = []
mocker.patch( mocker.patch(
"documents.search._backend.time.sleep", "documents.search._backend.time.sleep",
side_effect=sleep_values.append, side_effect=lambda s: sleep_values.append(s),
) )
for _ in range(50): for _ in range(50):
sleep_values.clear() sleep_values.clear()
+2 -2
View File
@@ -1003,8 +1003,8 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
for correspondent in response.data[field]: for correspondent in response.data[field]:
self.assertEqual(correspondent["document_count"], 0) self.assertEqual(correspondent["document_count"], 0)
self.assertCountEqual( self.assertCountEqual(
(c["id"] for c in response.data[field]), map(lambda c: c["id"], response.data[field]),
(c["id"] for c in Entity.objects.values("id")), map(lambda c: c["id"], Entity.objects.values("id")),
) )
def test_api_selection_data(self) -> None: def test_api_selection_data(self) -> None:
+44
View File
@@ -2,10 +2,13 @@ import datetime
import json import json
from unittest import mock from unittest import mock
from django.contrib.auth.models import Group
from django.contrib.auth.models import Permission from django.contrib.auth.models import Permission
from django.contrib.auth.models import User from django.contrib.auth.models import User
from django.test import override_settings from django.test import override_settings
from guardian.shortcuts import assign_perm from guardian.shortcuts import assign_perm
from guardian.shortcuts import get_groups_with_perms
from guardian.shortcuts import get_users_with_perms
from rest_framework import status from rest_framework import status
from rest_framework.test import APITestCase from rest_framework.test import APITestCase
@@ -815,6 +818,47 @@ class TestBulkEditObjects(APITestCase):
self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(StoragePath.objects.count(), 0) self.assertEqual(StoragePath.objects.count(), 0)
def test_bulk_objects_set_permissions_batched_across_object_count(
self,
) -> None:
"""
GIVEN:
- Many tags are being bulk-edited to set permissions at once
WHEN:
- bulk_edit_objects API endpoint is called with set_permissions
operation over a small batch vs. a much larger one
THEN:
- Permissions are applied correctly at both scales
"""
group1 = Group.objects.create(name="perm-group")
permissions = {
"view": {"users": [self.user1.id, self.user2.id], "groups": [group1.id]},
"change": {"users": [self.user1.id], "groups": [group1.id]},
}
def run_with_n_tags(n: int) -> None:
tags = [Tag.objects.create(name=f"perm-tag-{n}-{i}") for i in range(n)]
response = self.client.post(
"/api/bulk_edit_objects/",
json.dumps(
{
"objects": [t.id for t in tags],
"object_type": "tags",
"operation": "set_permissions",
"permissions": permissions,
"merge": False,
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
for tag in tags:
self.assertEqual(get_users_with_perms(tag).count(), 2)
self.assertEqual(get_groups_with_perms(tag).count(), 1)
run_with_n_tags(5)
run_with_n_tags(50)
def test_bulk_objects_delete_all_filtered(self) -> None: def test_bulk_objects_delete_all_filtered(self) -> None:
""" """
GIVEN: GIVEN:
+2 -2
View File
@@ -18,8 +18,8 @@ class MockOpenIDProvider:
def get_brands(self): def get_brands(self):
default_servers = [ default_servers = [
{"id": "yahoo", "name": "Yahoo", "openid_url": "http://me.yahoo.com"}, dict(id="yahoo", name="Yahoo", openid_url="http://me.yahoo.com"),
{"id": "hyves", "name": "Hyves", "openid_url": "http://hyves.nl"}, dict(id="hyves", name="Hyves", openid_url="http://hyves.nl"),
] ]
return default_servers return default_servers
+2 -2
View File
@@ -205,12 +205,12 @@ class TestBarcode(
- Barcode is detected on page 1 (zero indexed) - Barcode is detected on page 1 (zero indexed)
""" """
for test_filename in [ for test_file in [
"patch-code-t-middle-reverse.pdf", "patch-code-t-middle-reverse.pdf",
"patch-code-t-middle-distorted.pdf", "patch-code-t-middle-distorted.pdf",
"patch-code-t-middle-fuzzy.pdf", "patch-code-t-middle-fuzzy.pdf",
]: ]:
test_file = self.BARCODE_SAMPLE_DIR / test_filename test_file = self.BARCODE_SAMPLE_DIR / test_file
with self.get_reader(test_file) as reader: with self.get_reader(test_file) as reader:
reader.detect() reader.detect()
+119 -3
View File
@@ -5,6 +5,7 @@ from unittest import mock
import pikepdf import pikepdf
from django.contrib.auth.models import Group from django.contrib.auth.models import Group
from django.contrib.auth.models import Permission
from django.contrib.auth.models import User from django.contrib.auth.models import User
from django.test import TestCase from django.test import TestCase
from guardian.shortcuts import assign_perm from guardian.shortcuts import assign_perm
@@ -19,6 +20,7 @@ from documents.models import Document
from documents.models import DocumentType from documents.models import DocumentType
from documents.models import StoragePath from documents.models import StoragePath
from documents.models import Tag from documents.models import Tag
from documents.permissions import set_permissions_for_objects
from documents.tests.utils import DirectoriesMixin from documents.tests.utils import DirectoriesMixin
@@ -510,6 +512,120 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
) )
self.assertEqual(groups_with_perms.count(), 2) self.assertEqual(groups_with_perms.count(), 2)
@mock.patch("documents.tasks.bulk_update_documents.apply_async")
def test_set_permissions_batched_across_document_count(
self,
m,
) -> None:
"""
GIVEN:
- Many documents are being bulk-edited to set permissions at once
WHEN:
- set_permissions runs over a small batch vs. a much larger one
THEN:
- Permissions are applied correctly at both scales
"""
permissions = {
"view": {
"users": [self.user1.id, self.user2.id],
"groups": [self.group2.id],
},
"change": {
"users": [self.user1.id],
"groups": [self.group2.id],
},
}
def run_with_n_documents(n: int) -> None:
docs = [
Document.objects.create(checksum=f"perm-{n}-{i}", title=f"perm-{n}-{i}")
for i in range(n)
]
bulk_edit.set_permissions(
[doc.id for doc in docs],
set_permissions=permissions,
owner=self.owner,
merge=False,
)
for doc in docs:
self.assertEqual(get_users_with_perms(doc).count(), 2)
self.assertEqual(get_groups_with_perms(doc).count(), 1)
run_with_n_documents(5)
run_with_n_documents(50)
@mock.patch("documents.tasks.bulk_update_documents.apply_async")
def test_set_permissions_grants_direct_perm_even_if_already_granted_via_group(
self,
m,
) -> None:
"""
GIVEN:
- A user already has view access to a document via group
membership, with no direct grant of their own
WHEN:
- set_permissions explicitly grants that same user direct view
access via bulk_edit
THEN:
- A direct permission grant is created for the user, not skipped
because they already have equivalent access via the group
Regression test: guardian's queryset-aware assign_perm() (routed to
when the target is a list/queryset) skips creating a direct row for
anyone whose ObjectPermissionChecker.has_perm() already returns True
-- which includes group-derived access. The single-object assign_perm
this bulk path replaces has no such check; it always ensures a
direct row via get_or_create. Losing that guarantee would mean
revoking the group's grant later silently strips access that was
supposed to be explicit.
"""
self.doc1.owner = self.user1
self.doc1.save()
self.user1.groups.add(self.group1)
assign_perm("view_document", self.group1, self.doc1)
bulk_edit.set_permissions(
[self.doc1.id],
set_permissions={
"view": {"users": [self.user1.id], "groups": []},
},
merge=True,
)
direct_users = get_users_with_perms(
self.doc1,
only_with_perms_in=["view_document"],
with_group_users=False,
)
self.assertIn(self.user1, direct_users)
def test_set_permissions_for_objects_raises_for_unknown_action(self) -> None:
"""
GIVEN:
- An unrecognized permission action name with users to grant it
to
WHEN:
- set_permissions_for_objects is called
THEN:
- Permission.DoesNotExist is raised, not a silent no-op
Regression test: the endpoint that calls this
(BulkEditObjectPermissionsView) never actually validates action
names against the raw client-supplied permissions dict --
BulkEditObjectsSerializer._validate_permissions calls
validate_set_permissions() only for its side-effecting user/group id
checks and discards the filtered dict it returns -- so a bogus
action key reaches this function as-is. Resolving the Permission via
a bare `.filter()` (which returns empty instead of raising) would
silently drop the grant and report success.
"""
with self.assertRaises(Permission.DoesNotExist):
set_permissions_for_objects(
{"not_a_real_action": {"users": [self.user1.id], "groups": []}},
Document,
[self.doc1.pk],
)
@mock.patch("documents.models.Document.delete") @mock.patch("documents.models.Document.delete")
def test_delete_documents_old_uuid_field(self, m) -> None: def test_delete_documents_old_uuid_field(self, m) -> None:
m.side_effect = Exception("Data too long for column 'transaction_id' at row 1") m.side_effect = Exception("Data too long for column 'transaction_id' at row 1")
@@ -777,7 +893,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
sig.set.return_value.apply_async.side_effect = Exception("boom") sig.set.return_value.apply_async.side_effect = Exception("boom")
mock_consume_file.return_value = sig mock_consume_file.return_value = sig
with self.assertRaises(Exception): # noqa: B017 - mock injects a bare Exception with self.assertRaises(Exception):
bulk_edit.merge(doc_ids, delete_originals=True) bulk_edit.merge(doc_ids, delete_originals=True)
self.doc1.refresh_from_db() self.doc1.refresh_from_db()
@@ -1318,7 +1434,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
sig.apply_async.side_effect = Exception("boom") sig.apply_async.side_effect = Exception("boom")
mock_chord.return_value = sig mock_chord.return_value = sig
with self.assertRaises(Exception): # noqa: B017 - mock injects a bare Exception with self.assertRaises(Exception):
bulk_edit.edit_pdf(doc_ids, operations, delete_original=True) bulk_edit.edit_pdf(doc_ids, operations, delete_original=True)
self.doc2.refresh_from_db() self.doc2.refresh_from_db()
@@ -1430,7 +1546,7 @@ class TestPDFActions(DirectoriesMixin, TestCase):
{"page": 9999}, # invalid page, forces error during PDF load {"page": 9999}, # invalid page, forces error during PDF load
] ]
with self.assertLogs("paperless.bulk_edit", level="ERROR"): with self.assertLogs("paperless.bulk_edit", level="ERROR"):
with self.assertRaises(ValueError): with self.assertRaises(Exception):
bulk_edit.edit_pdf(doc_ids, operations) bulk_edit.edit_pdf(doc_ids, operations)
mock_group.assert_not_called() mock_group.assert_not_called()
mock_consume_file.assert_not_called() mock_consume_file.assert_not_called()
+1 -1
View File
@@ -783,7 +783,7 @@ class TestClassifier(DirectoriesMixin, TestCase):
Path(settings.MODEL_FILE).touch() Path(settings.MODEL_FILE).touch()
mock_load.side_effect = Exception() mock_load.side_effect = Exception()
with self.assertRaises(Exception): # noqa: B017 - mock injects a bare Exception with self.assertRaises(Exception):
load_classifier(raise_exception=True) load_classifier(raise_exception=True)
+2 -2
View File
@@ -137,7 +137,7 @@ class FaultyParser(_BaseNewStyleParser):
class FaultyGenericExceptionParser(_BaseNewStyleParser): class FaultyGenericExceptionParser(_BaseNewStyleParser):
def parse(self, document_path, mime_type, *, produce_archive: bool = True) -> None: def parse(self, document_path, mime_type, *, produce_archive: bool = True) -> None:
raise Exception("Generic exception.") # noqa: TRY002 - deliberately not a ParseError raise Exception("Generic exception.")
def fake_magic_from_file(file, *, mime=False): # NOSONAR def fake_magic_from_file(file, *, mime=False): # NOSONAR
@@ -1333,7 +1333,7 @@ class PreConsumeTestCase(DirectoriesMixin, GetConsumerMixin, TestCase):
script_calls = [ script_calls = [
call call
for call in m.call_args_list for call in m.call_args_list
if call.args and call.args[0] and call.args[0][0] != "pdftotext" if call.args and call.args[0] and call.args[0][0] not in ("pdftotext",)
] ]
self.assertEqual(script_calls, []) self.assertEqual(script_calls, [])
+1 -8
View File
@@ -44,7 +44,6 @@ from documents import tasks
from documents.data_models import ConsumableDocument from documents.data_models import ConsumableDocument
from documents.data_models import DocumentMetadataOverrides from documents.data_models import DocumentMetadataOverrides
from documents.data_models import DocumentSource from documents.data_models import DocumentSource
from documents.matching import UnsupportedWorkflowTriggerTypeError
from documents.matching import document_matches_workflow from documents.matching import document_matches_workflow
from documents.matching import existing_document_matches_workflow from documents.matching import existing_document_matches_workflow
from documents.matching import prefilter_documents_by_workflowtrigger from documents.matching import prefilter_documents_by_workflowtrigger
@@ -2852,13 +2851,7 @@ class TestWorkflows(
doc = Document.objects.create( doc = Document.objects.create(
title="test", title="test",
) )
self.assertRaises( self.assertRaises(Exception, document_matches_workflow, doc, w, 99)
UnsupportedWorkflowTriggerTypeError,
document_matches_workflow,
doc,
w,
99,
)
def test_removal_action_document_updated_workflow(self) -> None: def test_removal_action_document_updated_workflow(self) -> None:
""" """
+5 -9
View File
@@ -21,32 +21,28 @@ def uri_validator(value: str, allowed_schemes: set[str] | None = None) -> None:
parts = urlparse(value) parts = urlparse(value)
if not parts.scheme: if not parts.scheme:
raise ValidationError( raise ValidationError(
_("Unable to parse URI %(value)s, missing scheme"), _(f"Unable to parse URI {value}, missing scheme"),
params={"value": value}, params={"value": value},
) )
elif not parts.netloc and not parts.path: elif not parts.netloc and not parts.path:
raise ValidationError( raise ValidationError(
_("Unable to parse URI %(value)s, missing net location or path"), _(f"Unable to parse URI {value}, missing net location or path"),
params={"value": value}, params={"value": value},
) )
if allowed_schemes and parts.scheme not in allowed_schemes: if allowed_schemes and parts.scheme not in allowed_schemes:
raise ValidationError( raise ValidationError(
_( _(
"URI scheme '%(scheme)s' is not allowed. Allowed schemes: %(allowed_schemes)s", f"URI scheme '{parts.scheme}' is not allowed. Allowed schemes: {', '.join(allowed_schemes)}",
), ),
params={ params={"value": value, "scheme": parts.scheme},
"value": value,
"scheme": parts.scheme,
"allowed_schemes": ", ".join(allowed_schemes),
},
) )
except ValidationError: except ValidationError:
raise raise
except Exception as e: except Exception as e:
raise ValidationError( raise ValidationError(
_("Unable to parse URI %(value)s"), _(f"Unable to parse URI {value}"),
params={"value": value}, params={"value": value},
) from e ) from e
+33 -29
View File
@@ -178,7 +178,7 @@ from documents.permissions import has_perms_owner_aware
from documents.permissions import has_system_status_permission from documents.permissions import has_system_status_permission
from documents.permissions import permitted_document_ids from documents.permissions import permitted_document_ids
from documents.permissions import permitted_object_ids from documents.permissions import permitted_object_ids
from documents.permissions import set_permissions_for_object from documents.permissions import set_permissions_for_objects
from documents.plugins.date_parsing import get_date_parser from documents.plugins.date_parsing import get_date_parser
from documents.schema import generate_object_with_permissions_schema from documents.schema import generate_object_with_permissions_schema
from documents.search import SearchHit from documents.search import SearchHit
@@ -1440,7 +1440,7 @@ class DocumentViewSet(
try: try:
lang = detect(doc.content) lang = detect(doc.content)
except Exception: except Exception:
logger.debug("Unable to detect language for document %s", doc.pk) pass
meta["lang"] = lang meta["lang"] = lang
return Response(meta) return Response(meta)
@@ -1478,12 +1478,13 @@ class DocumentViewSet(
with get_date_parser() as date_parser: with get_date_parser() as date_parser:
gen = date_parser.parse(doc.filename, doc.content) gen = date_parser.parse(doc.filename, doc.content)
dates = sorted( dates = sorted(
set( {
itertools.islice( i
for i in itertools.islice(
gen, gen,
settings.NUMBER_OF_SUGGESTED_DATES, settings.NUMBER_OF_SUGGESTED_DATES,
), )
), },
) )
resp_data = { resp_data = {
@@ -1567,16 +1568,21 @@ class DocumentViewSet(
except ValueError as exc: except ValueError as exc:
logger.exception( logger.exception(
"Invalid AI configuration while generating suggestions for " "Invalid AI configuration while generating suggestions for "
"document %s", "document %s: %s",
doc.pk, doc.pk,
exc,
exc_info=True,
) )
raise ValidationError( raise ValidationError(
{"ai": [_("Invalid AI configuration.")]}, {"ai": [_("Invalid AI configuration.")]},
) from exc ) from exc
except LLMTimeoutError: except LLMTimeoutError as exc:
logger.exception( logger.exception(
"AI backend timed out while generating suggestions for document %s", "AI backend timed out while generating suggestions for "
"document %s: %s",
doc.pk, doc.pk,
exc,
exc_info=True,
) )
return Response( return Response(
{"ai": [_("AI backend request timed out.")]}, {"ai": [_("AI backend request timed out.")]},
@@ -2049,7 +2055,7 @@ class DocumentViewSet(
doc_name, doc_data = serializer.validated_data.get("document") doc_name, doc_data = serializer.validated_data.get("document")
version_label = serializer.validated_data.get("version_label") version_label = serializer.validated_data.get("version_label")
t = int(mktime(datetime.now().timetuple())) # noqa: DTZ005 - mktime() requires a local time tuple t = int(mktime(datetime.now().timetuple()))
settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True) settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True)
@@ -3319,7 +3325,7 @@ class PostDocumentView(GenericAPIView[Any]):
cf = serializer.validated_data.get("custom_fields") cf = serializer.validated_data.get("custom_fields")
from_webui = serializer.validated_data.get("from_webui") from_webui = serializer.validated_data.get("from_webui")
t = int(mktime(datetime.now().timetuple())) # noqa: DTZ005 - mktime() requires a local time tuple t = int(mktime(datetime.now().timetuple()))
settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True) settings.SCRATCH_DIR.mkdir(parents=True, exist_ok=True)
@@ -4129,7 +4135,7 @@ class UiSettingsView(GenericAPIView[Any]):
user_resp["last_name"] = user.last_name user_resp["last_name"] = user.last_name
# strip <app_label>. # strip <app_label>.
roles = (re.sub(r"^\w+.", "", perm) for perm in user.get_all_permissions()) roles = map(lambda perm: re.sub(r"^\w+.", "", perm), user.get_all_permissions())
return Response( return Response(
{ {
"user": user_resp, "user": user_resp,
@@ -4908,12 +4914,12 @@ class BulkEditObjectsView(PassUserMixin):
qs_owner_update.update(owner=owner) qs_owner_update.update(owner=owner)
if "permissions" in serializer.validated_data: if "permissions" in serializer.validated_data:
for obj in qs: set_permissions_for_objects(
set_permissions_for_object( permissions=permissions,
permissions=permissions, model=object_class,
object=obj, pks=qs.values_list("pk", flat=True),
merge=merge, merge=merge,
) )
except Exception as e: except Exception as e:
logger.warning( logger.warning(
@@ -5156,11 +5162,11 @@ class SystemStatusView(PassUserMixin):
f"{m.app}.{m.name}" f"{m.app}.{m.name}"
for m in MigrationRecorder.Migration.objects.all().order_by("id") for m in MigrationRecorder.Migration.objects.all().order_by("id")
] ]
except Exception: # pragma: no cover except Exception as e: # pragma: no cover
applied_migrations = [] applied_migrations = []
db_status = "ERROR" db_status = "ERROR"
logger.exception( logger.exception(
"System status detected a possible problem while connecting to the database", f"System status detected a possible problem while connecting to the database: {e}",
) )
db_error = "Error connecting to database, check logs for more detail." db_error = "Error connecting to database, check logs for more detail."
@@ -5176,10 +5182,10 @@ class SystemStatusView(PassUserMixin):
try: try:
client.ping() client.ping()
redis_status = "OK" redis_status = "OK"
except Exception: except Exception as e:
redis_status = "ERROR" redis_status = "ERROR"
logger.exception( logger.exception(
"System status detected a possible problem while connecting to redis", f"System status detected a possible problem while connecting to redis: {e}",
) )
redis_error = "Error connecting to redis, check logs for more detail." redis_error = "Error connecting to redis, check logs for more detail."
@@ -5209,10 +5215,10 @@ class SystemStatusView(PassUserMixin):
else: else:
celery_active = "WARNING" celery_active = "WARNING"
celery_error = "Celery worker responded unexpectedly." celery_error = "Celery worker responded unexpectedly."
except Exception: except Exception as e:
celery_active = "ERROR" celery_active = "ERROR"
logger.exception( logger.exception(
"System status detected a possible problem while connecting to celery", f"System status detected a possible problem while connecting to celery: {e}",
) )
celery_error = "Error connecting to celery, check logs for more detail." celery_error = "Error connecting to celery, check logs for more detail."
@@ -5227,15 +5233,13 @@ class SystemStatusView(PassUserMixin):
index_dir = settings.INDEX_DIR index_dir = settings.INDEX_DIR
mtimes = [p.stat().st_mtime for p in index_dir.iterdir() if p.is_file()] mtimes = [p.stat().st_mtime for p in index_dir.iterdir() if p.is_file()]
index_last_modified = ( index_last_modified = (
make_aware(datetime.fromtimestamp(max(mtimes))) # noqa: DTZ006 - make_aware() requires a naive datetime make_aware(datetime.fromtimestamp(max(mtimes))) if mtimes else None
if mtimes
else None
) )
except Exception: except Exception as e:
index_status = "ERROR" index_status = "ERROR"
index_error = "Error opening index, check logs for more detail." index_error = "Error opening index, check logs for more detail."
logger.exception( logger.exception(
"System status detected a possible problem while opening the index", f"System status detected a possible problem while opening the index: {e}",
) )
index_last_modified = None index_last_modified = None
+5 -5
View File
@@ -66,7 +66,7 @@ def build_workflow_action_context(
else None else None
) )
filename = document.original_file or "" filename = document.original_file if document.original_file else ""
return { return {
"title": overrides.title "title": overrides.title
if overrides and overrides.title if overrides and overrides.title
@@ -179,9 +179,9 @@ def execute_email_action(
f"Sent {n_messages} notification email(s) to {action.email.to}", f"Sent {n_messages} notification email(s) to {action.email.to}",
extra={"group": logging_group}, extra={"group": logging_group},
) )
except Exception: except Exception as e:
logger.exception( logger.exception(
"Error occurred sending notification email", f"Error occurred sending notification email: {e}",
extra={"group": logging_group}, extra={"group": logging_group},
) )
@@ -265,9 +265,9 @@ def execute_webhook_action(
f"Webhook to {action.webhook.url} queued", f"Webhook to {action.webhook.url} queued",
extra={"group": logging_group}, extra={"group": logging_group},
) )
except Exception: except Exception as e:
logger.exception( logger.exception(
"Error occurred sending webhook", f"Error occurred sending webhook: {e}",
extra={"group": logging_group}, extra={"group": logging_group},
) )
+1 -1
View File
@@ -47,7 +47,7 @@ def resolve_date(dates: list[str]) -> date | None:
""" """
for value in dates: for value in dates:
try: try:
return datetime.strptime(value, "%Y-%m-%d").date() # noqa: DTZ007 - only the calendar date is used, time/tz is discarded return datetime.strptime(value, "%Y-%m-%d").date()
except (TypeError, ValueError): except (TypeError, ValueError):
logger.debug("Ignoring unparsable suggested date %s", value) logger.debug("Ignoring unparsable suggested date %s", value)
return None return None
+1 -1
View File
@@ -70,6 +70,6 @@ def send_webhook(
logger.error( logger.error(
f"Failed attempt sending webhook to {url}: {e}", f"Failed attempt sending webhook to {url}: {e}",
) )
raise raise e
finally: finally:
transport.close() transport.close()
+1 -2
View File
@@ -241,7 +241,7 @@ def check_v3_minimum_upgrade_version(
return [] return []
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
last_applied = max(applied) if applied else "(none)" last_applied = sorted(applied)[-1] if applied else "(none)"
logger.error( logger.error(
"V3 upgrade check failed: last applied documents migration is %r. " "V3 upgrade check failed: last applied documents migration is %r. "
"Expected '1075_workflowaction_order' (v2.20.15). " "Expected '1075_workflowaction_order' (v2.20.15). "
@@ -341,7 +341,6 @@ def get_tesseract_langs():
proc = subprocess.run( proc = subprocess.run(
[shutil.which("tesseract"), "--list-langs"], [shutil.which("tesseract"), "--list-langs"],
capture_output=True, capture_output=True,
check=False,
) )
# Decode bytes to string, split on newlines, trim out the header # Decode bytes to string, split on newlines, trim out the header
+3 -3
View File
@@ -84,7 +84,7 @@ def get_parser_registry() -> ParserRegistry:
ParserRegistry ParserRegistry
The shared registry singleton. The shared registry singleton.
""" """
global _registry, _discovery_complete # noqa: PLW0603 - module-level singleton, no class to hold this state global _registry, _discovery_complete
with _lock: with _lock:
if _registry is None: if _registry is None:
@@ -113,7 +113,7 @@ def init_builtin_parsers() -> None:
------- -------
None None
""" """
global _registry # noqa: PLW0603 - module-level singleton, no class to hold this state global _registry
with _lock: with _lock:
if _registry is None: if _registry is None:
@@ -137,7 +137,7 @@ def reset_parser_registry() -> None:
------- -------
None None
""" """
global _registry, _discovery_complete # noqa: PLW0603 - module-level singleton, no class to hold this state global _registry, _discovery_complete
_registry = None _registry = None
_discovery_complete = False _discovery_complete = False
+2 -2
View File
@@ -76,7 +76,7 @@ class RemoteEngineConfig:
def engine_is_valid(self) -> bool: def engine_is_valid(self) -> bool:
"""Return True when the engine is known and fully configured.""" """Return True when the engine is known and fully configured."""
return ( return (
self.engine == "azureai" self.engine in ("azureai",)
and self.api_key is not None and self.api_key is not None
and not (self.engine == "azureai" and self.endpoint is None) and not (self.engine == "azureai" and self.endpoint is None)
) )
@@ -467,7 +467,7 @@ class RemoteDocumentParser:
return result.content return result.content
except Exception as e: except Exception as e:
logger.exception("Azure AI Vision parsing failed") logger.exception("Azure AI Vision parsing failed: %s", e)
raise ParseError(f"Azure AI Vision parsing failed: {e}") from e raise ParseError(f"Azure AI Vision parsing failed: {e}") from e
finally: finally:
+3 -4
View File
@@ -306,9 +306,8 @@ def extract_pdf_metadata(
for key, value in meta.items(): for key, value in meta.items():
if isinstance(value, list): if isinstance(value, list):
str_value = " ".join(str(e) for e in value) value = " ".join(str(e) for e in value)
else: value = str(value)
str_value = str(value)
try: try:
m = namespace_pattern.match(key) m = namespace_pattern.match(key)
@@ -330,7 +329,7 @@ def extract_pdf_metadata(
namespace=namespace, namespace=namespace,
prefix=meta.REVERSE_NS[namespace], prefix=meta.REVERSE_NS[namespace],
key=key_value, key=key_value,
value=str_value, value=value,
), ),
) )
except Exception as e: except Exception as e:
+9 -14
View File
@@ -294,7 +294,7 @@ if _CHANNELS_BACKEND.startswith("channels_redis."):
############################################################################### ###############################################################################
EMAIL_HOST: Final[str] = os.getenv("PAPERLESS_EMAIL_HOST", "localhost") EMAIL_HOST: Final[str] = os.getenv("PAPERLESS_EMAIL_HOST", "localhost")
EMAIL_PORT: Final[int] = get_int_from_env("PAPERLESS_EMAIL_PORT", 25) EMAIL_PORT: Final[int] = int(os.getenv("PAPERLESS_EMAIL_PORT", 25))
EMAIL_HOST_USER: Final[str] = os.getenv("PAPERLESS_EMAIL_HOST_USER", "") EMAIL_HOST_USER: Final[str] = os.getenv("PAPERLESS_EMAIL_HOST_USER", "")
EMAIL_HOST_PASSWORD: Final[str] = os.getenv("PAPERLESS_EMAIL_HOST_PASSWORD", "") EMAIL_HOST_PASSWORD: Final[str] = os.getenv("PAPERLESS_EMAIL_HOST_PASSWORD", "")
DEFAULT_FROM_EMAIL: Final[str] = os.getenv("PAPERLESS_EMAIL_FROM", EMAIL_HOST_USER) DEFAULT_FROM_EMAIL: Final[str] = os.getenv("PAPERLESS_EMAIL_FROM", EMAIL_HOST_USER)
@@ -381,9 +381,8 @@ ACCOUNT_SESSION_REMEMBER = get_bool_from_env(
"True", "True",
) )
SESSION_EXPIRE_AT_BROWSER_CLOSE = not ACCOUNT_SESSION_REMEMBER SESSION_EXPIRE_AT_BROWSER_CLOSE = not ACCOUNT_SESSION_REMEMBER
SESSION_COOKIE_AGE = get_int_from_env( SESSION_COOKIE_AGE = int(
"PAPERLESS_SESSION_COOKIE_AGE", os.getenv("PAPERLESS_SESSION_COOKIE_AGE", 60 * 60 * 24 * 7 * 3),
60 * 60 * 24 * 7 * 3,
) )
# https://docs.djangoproject.com/en/5.1/ref/settings/#std-setting-SESSION_ENGINE # https://docs.djangoproject.com/en/5.1/ref/settings/#std-setting-SESSION_ENGINE
SESSION_ENGINE = "django.contrib.sessions.backends.cached_db" SESSION_ENGINE = "django.contrib.sessions.backends.cached_db"
@@ -396,6 +395,7 @@ if AUTO_LOGIN_USERNAME:
def _parse_remote_user_settings() -> str: def _parse_remote_user_settings() -> str:
global MIDDLEWARE, AUTHENTICATION_BACKENDS, REST_FRAMEWORK
enable = get_bool_from_env("PAPERLESS_ENABLE_HTTP_REMOTE_USER") enable = get_bool_from_env("PAPERLESS_ENABLE_HTTP_REMOTE_USER")
enable_api = get_bool_from_env("PAPERLESS_ENABLE_HTTP_REMOTE_USER_API") enable_api = get_bool_from_env("PAPERLESS_ENABLE_HTTP_REMOTE_USER_API")
if enable or enable_api: if enable or enable_api:
@@ -454,6 +454,7 @@ if ALLOWED_HOSTS != ["*"]:
def _parse_paperless_url(): def _parse_paperless_url():
global CSRF_TRUSTED_ORIGINS, CORS_ALLOWED_ORIGINS, ALLOWED_HOSTS
url = os.getenv("PAPERLESS_URL") url = os.getenv("PAPERLESS_URL")
if url: if url:
CSRF_TRUSTED_ORIGINS.append(url) CSRF_TRUSTED_ORIGINS.append(url)
@@ -613,8 +614,8 @@ USE_TZ = True
LOGGING_DIR.mkdir(parents=True, exist_ok=True) LOGGING_DIR.mkdir(parents=True, exist_ok=True)
LOGROTATE_MAX_SIZE = get_int_from_env("PAPERLESS_LOGROTATE_MAX_SIZE", 1024 * 1024) LOGROTATE_MAX_SIZE = os.getenv("PAPERLESS_LOGROTATE_MAX_SIZE", 1024 * 1024)
LOGROTATE_MAX_BACKUPS = get_int_from_env("PAPERLESS_LOGROTATE_MAX_BACKUPS", 20) LOGROTATE_MAX_BACKUPS = os.getenv("PAPERLESS_LOGROTATE_MAX_BACKUPS", 20)
LOGGING = { LOGGING = {
"version": 1, "version": 1,
@@ -810,15 +811,9 @@ IGNORABLE_FILES: Final[list[str]] = [
"Thumbs.db", "Thumbs.db",
] ]
CONSUMER_POLLING_INTERVAL = get_float_from_env( CONSUMER_POLLING_INTERVAL = float(os.getenv("PAPERLESS_CONSUMER_POLLING_INTERVAL", 0))
"PAPERLESS_CONSUMER_POLLING_INTERVAL",
0.0,
)
CONSUMER_STABILITY_DELAY = get_float_from_env( CONSUMER_STABILITY_DELAY = float(os.getenv("PAPERLESS_CONSUMER_STABILITY_DELAY", 5))
"PAPERLESS_CONSUMER_STABILITY_DELAY",
5.0,
)
CONSUMER_DELETE_DUPLICATES = get_bool_from_env("PAPERLESS_CONSUMER_DELETE_DUPLICATES") CONSUMER_DELETE_DUPLICATES = get_bool_from_env("PAPERLESS_CONSUMER_DELETE_DUPLICATES")
+1 -1
View File
@@ -111,7 +111,7 @@ def parse_dict_from_str(
return False return False
settings: dict[str, Any] = copy.deepcopy(defaults) if defaults else {} settings: dict[str, Any] = copy.deepcopy(defaults) if defaults else {}
_type_map = type_map or {} _type_map = type_map if type_map else {}
if not env_str: if not env_str:
return settings return settings
@@ -114,17 +114,17 @@ def test_cache_hit_when_enabled() -> None:
assert settings.CACHALOT_TIMEOUT == 1 assert settings.CACHALOT_TIMEOUT == 1
# Read a table to populate the cache # Read a table to populate the cache
list(Tag.objects.values_list("id", flat=True)) list(list(Tag.objects.values_list("id", flat=True)))
# Invalidate the cache then read the database, there should be DB hit # Invalidate the cache then read the database, there should be DB hit
invalidate_db_cache() invalidate_db_cache()
with CaptureQueriesContext(connection) as ctx: with CaptureQueriesContext(connection) as ctx:
list(Tag.objects.values_list("id", flat=True)) list(list(Tag.objects.values_list("id", flat=True)))
assert len(ctx) assert len(ctx)
# Doing the same request again should hit the cache, not the DB # Doing the same request again should hit the cache, not the DB
with CaptureQueriesContext(connection) as ctx: with CaptureQueriesContext(connection) as ctx:
list(Tag.objects.values_list("id", flat=True)) list(list(Tag.objects.values_list("id", flat=True)))
assert not len(ctx) assert not len(ctx)
# Wait the end of TTL # Wait the end of TTL
@@ -133,7 +133,7 @@ def test_cache_hit_when_enabled() -> None:
# Read the DB again. The DB should be hit because the cache has expired # Read the DB again. The DB should be hit because the cache has expired
with CaptureQueriesContext(connection) as ctx: with CaptureQueriesContext(connection) as ctx:
list(Tag.objects.values_list("id", flat=True)) list(list(Tag.objects.values_list("id", flat=True)))
assert len(ctx) assert len(ctx)
# Invalidate the cache at the end of test # Invalidate the cache at the end of test
@@ -149,7 +149,7 @@ def test_cache_is_disabled_by_default() -> None:
# Read the table multiple times: the DB should always be hit without cache # Read the table multiple times: the DB should always be hit without cache
for _ in range(3): for _ in range(3):
with CaptureQueriesContext(connection) as ctx: with CaptureQueriesContext(connection) as ctx:
list(Tag.objects.values_list("id", flat=True)) list(list(Tag.objects.values_list("id", flat=True)))
assert len(ctx) assert len(ctx)
# Invalidate the cache at the end of test # Invalidate the cache at the end of test
+1 -1
View File
@@ -59,7 +59,7 @@ def test_ocr_to_dateparser_languages_exception(
raise RuntimeError("Simulated error") raise RuntimeError("Simulated error")
with caplog.at_level(logging.WARNING): with caplog.at_level(logging.WARNING):
monkeypatch.setattr(utils, "LocaleDataLoader", DummyLoader) monkeypatch.setattr(utils, "LocaleDataLoader", lambda: DummyLoader())
result = utils.ocr_to_dateparser_languages("eng+fra") result = utils.ocr_to_dateparser_languages("eng+fra")
assert result == [] assert result == []
assert ( assert (
+2 -2
View File
@@ -103,8 +103,8 @@ def stream_chat_with_documents(
documents, documents,
output_language=output_language, output_language=output_language,
) )
except Exception: except Exception as e:
logger.exception("Failed to stream document chat response") logger.exception("Failed to stream document chat response: %s", e)
yield CHAT_ERROR_MESSAGE yield CHAT_ERROR_MESSAGE
+1 -1
View File
@@ -167,7 +167,7 @@ def test_get_ai_document_classification_failure(mock_run_llm_query, mock_documen
""" """
mock_run_llm_query.side_effect = Exception("LLM query failed") mock_run_llm_query.side_effect = Exception("LLM query failed")
with pytest.raises(Exception): # noqa: B017 - mock injects a bare Exception with pytest.raises(Exception):
get_ai_document_classification(mock_document) get_ai_document_classification(mock_document)
@@ -21,6 +21,5 @@ class TestLazyAiImports:
capture_output=True, capture_output=True,
text=True, text=True,
cwd=_SRC_DIR, cwd=_SRC_DIR,
check=False,
) )
assert result.returncode == 0, result.stdout + result.stderr assert result.returncode == 0, result.stdout + result.stderr
+6 -5
View File
@@ -7,6 +7,7 @@ import ssl
import tempfile import tempfile
import traceback import traceback
import unicodedata import unicodedata
from datetime import date
from datetime import timedelta from datetime import timedelta
from fnmatch import fnmatch from fnmatch import fnmatch
from pathlib import Path from pathlib import Path
@@ -405,7 +406,7 @@ def make_criterias(rule: MailRule, *, supports_gmail_labels: bool):
Returns criteria to be applied to MailBox.fetch for the given rule. Returns criteria to be applied to MailBox.fetch for the given rule.
""" """
maximum_age = timezone.localdate() - timedelta(days=rule.maximum_age) maximum_age = date.today() - timedelta(days=rule.maximum_age)
criterias = {} criterias = {}
if rule.maximum_age > 0: if rule.maximum_age > 0:
criterias["date_gte"] = maximum_age criterias["date_gte"] = maximum_age
@@ -722,9 +723,9 @@ class MailAccountHandler(LoggingMixin):
f"Rule {rule}: Stopping processing rules due to stop_processing flag", f"Rule {rule}: Stopping processing rules due to stop_processing flag",
) )
break break
except Exception: except Exception as e:
self.log.exception( self.log.exception(
f"Rule {rule}: Error while processing rule", f"Rule {rule}: Error while processing rule: {e}",
) )
except MailError: except MailError:
raise raise
@@ -873,9 +874,9 @@ class MailAccountHandler(LoggingMixin):
total_processed_files += processed_files total_processed_files += processed_files
mails_processed += 1 mails_processed += 1
except Exception: except Exception as e:
self.log.exception( self.log.exception(
f"Rule {rule}: Error while processing mail {message.uid}", f"Rule {rule}: Error while processing mail {message.uid}: {e}",
) )
self.log.debug(f"Rule {rule}: Processed {mails_processed} matching mail(s)") self.log.debug(f"Rule {rule}: Processed {mails_processed} matching mail(s)")
+1 -5
View File
@@ -11,10 +11,6 @@ from imap_tools import MailMessage
from documents.loggers import LoggingMixin from documents.loggers import LoggingMixin
class MailDecryptionError(Exception):
pass
class MailMessagePreprocessor(abc.ABC): class MailMessagePreprocessor(abc.ABC):
""" """
Defines the interface for preprocessors that alter messages before they are handled in MailAccountHandler Defines the interface for preprocessors that alter messages before they are handled in MailAccountHandler
@@ -73,7 +69,7 @@ class MailMessageDecryptor(MailMessagePreprocessor, LoggingMixin):
f"Message decryption failed with status message " f"Message decryption failed with status message "
f"{decrypted_raw_message.status}", f"{decrypted_raw_message.status}",
) )
raise MailDecryptionError( raise Exception(
f"Decryption failed: {decrypted_raw_message.status}, {decrypted_raw_message.stderr}", f"Decryption failed: {decrypted_raw_message.status}, {decrypted_raw_message.stderr}",
) )
self.log.debug("Message decrypted successfully.") self.log.debug("Message decrypted successfully.")
+1 -1
View File
@@ -50,7 +50,7 @@ class ProcessedMailFactory(DjangoModelFactory[ProcessedMail]):
rule = factory.SubFactory(MailRuleFactory) rule = factory.SubFactory(MailRuleFactory)
folder = "INBOX" folder = "INBOX"
uid = factory.Sequence(str) uid = factory.Sequence(lambda n: str(n))
subject = factory.Faker("sentence", nb_words=4) subject = factory.Faker("sentence", nb_words=4)
received = factory.LazyFunction(timezone.now) received = factory.LazyFunction(timezone.now)
processed = factory.LazyFunction(timezone.now) processed = factory.LazyFunction(timezone.now)
+1 -1
View File
@@ -214,7 +214,7 @@ class BogusMailBox(AbstractContextManager):
) )
self.messages = list(filter(lambda m: m.uid not in uid_list, self.messages)) self.messages = list(filter(lambda m: m.uid not in uid_list, self.messages))
else: else:
raise Exception # noqa: TRY002 - test double simulating a generic mailbox failure raise Exception
def fake_magic_from_buffer(buffer, *, mime=False): def fake_magic_from_buffer(buffer, *, mime=False):
@@ -14,7 +14,6 @@ from imap_tools import MailMessage
from paperless_mail.mail import MailAccountHandler from paperless_mail.mail import MailAccountHandler
from paperless_mail.models import MailRule from paperless_mail.models import MailRule
from paperless_mail.preprocessor import MailDecryptionError
from paperless_mail.preprocessor import MailMessageDecryptor from paperless_mail.preprocessor import MailMessageDecryptor
from paperless_mail.tests.factories import MailAccountFactory from paperless_mail.tests.factories import MailAccountFactory
from paperless_mail.tests.test_mail import TestMail from paperless_mail.tests.test_mail import TestMail
@@ -83,9 +82,7 @@ class MessageEncryptor:
armor=True, armor=True,
) )
if not encrypted_data.ok: if not encrypted_data.ok:
raise Exception( # noqa: TRY002 - test fixture setup, not production code raise Exception(f"Encryption failed: {encrypted_data.stderr}")
f"Encryption failed: {encrypted_data.stderr}",
)
encrypted_email_content = encrypted_data.data encrypted_email_content = encrypted_data.data
new_email = MIMEMultipart("encrypted", protocol="application/pgp-encrypted") new_email = MIMEMultipart("encrypted", protocol="application/pgp-encrypted")
@@ -187,11 +184,7 @@ class TestMailMessageGpgDecryptor(TestMail):
EMAIL_GNUPG_HOME=empty_gpg_home, EMAIL_GNUPG_HOME=empty_gpg_home,
): ):
message_decryptor = MailMessageDecryptor() message_decryptor = MailMessageDecryptor()
self.assertRaises( self.assertRaises(Exception, message_decryptor.run, encrypted_message)
MailDecryptionError,
message_decryptor.run,
encrypted_message,
)
finally: finally:
# Clean up the temporary GPG home used only by this test # Clean up the temporary GPG home used only by this test
try: try:
+2 -1
View File
@@ -1,3 +1,4 @@
import datetime
import logging import logging
from datetime import timedelta from datetime import timedelta
from http import HTTPStatus from http import HTTPStatus
@@ -86,7 +87,7 @@ class MailAccountViewSet(PassUserMixin, ModelViewSet[MailAccount]):
@action(methods=["post"], detail=False) @action(methods=["post"], detail=False)
def test(self, request): def test(self, request):
logger = logging.getLogger("paperless_mail") logger = logging.getLogger("paperless_mail")
request.data["name"] = timezone.now().isoformat() request.data["name"] = datetime.datetime.now().isoformat()
serializer = self.get_serializer(data=request.data) serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True) serializer.is_valid(raise_exception=True)
existing_account = None existing_account = None