From 3e56dace7353f0363d49452e2c2e6db988ae55ee Mon Sep 17 00:00:00 2001 From: Philipp Defner Date: Wed, 9 Sep 2026 23:40:24 +0100 Subject: [PATCH 001/258] Enhancement: Improve matching for correspondents, storage path and labels by removing bias + adding minimum match threshold (#12164) --- docs/configuration.md | 9 ++ src/documents/classifier.py | 94 +++++++++++----- src/documents/tests/test_classifier.py | 145 +++++++++++++++++++++++++ src/paperless/settings/__init__.py | 7 ++ 4 files changed, 227 insertions(+), 28 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 37a552f1a..e045af077 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1200,6 +1200,15 @@ still perform some basic text pre-processing before matching. Defaults to true, enabling the feature. +#### [`PAPERLESS_CLASSIFIER_MATCH_THRESHOLD=`](#PAPERLESS_CLASSIFIER_MATCH_THRESHOLD) {#PAPERLESS_CLASSIFIER_MATCH_THRESHOLD} + +: Sets the minimum confidence score (0.0-1.0) required for the automatic +classifier to assign a correspondent, document type, or storage path to a +document. Predictions below this threshold are discarded and the field is +left unassigned, preventing low-confidence guesses from being applied. + + Defaults to 0.6. + #### [`PAPERLESS_DATE_PARSER_LANGUAGES=`](#PAPERLESS_DATE_PARSER_LANGUAGES) {#PAPERLESS_DATE_PARSER_LANGUAGES} : Specifies which language Paperless should use when parsing dates from documents. diff --git a/src/documents/classifier.py b/src/documents/classifier.py index a5c272062..cfed4725c 100644 --- a/src/documents/classifier.py +++ b/src/documents/classifier.py @@ -34,6 +34,27 @@ from paperless.signed_pickle import signed_pickle_loads logger = logging.getLogger("paperless.classifier") + +def _predict_with_threshold(classifier, X, threshold: float) -> int | None: + """ + Return the predicted class id, or None if: + - the prediction is -1 (no match), or + - the winning class probability is below the configured threshold. + + Using predict_proba() instead of predict() lets us apply a minimum-confidence + cutoff so that uncertain predictions are discarded rather than assigned. + """ + probas = classifier.predict_proba(X)[0] + best_idx = int(probas.argmax()) + best_class = int(classifier.classes_[best_idx]) + + if best_class == -1: + return None + if threshold > 0.0 and probas[best_idx] < threshold: + return None + return best_class + + ADVANCED_TEXT_PROCESSING_ENABLED = ( settings.NLTK_LANGUAGE is not None and settings.NLTK_ENABLED ) @@ -102,7 +123,8 @@ class DocumentClassifier: # v8 - Added storage path classifier # v9 - Changed from hashing to time/ids for re-train check # v10 - HMAC-signed model file - FORMAT_VERSION = 10 + # v11 - Use sample_weight for balanced training; predict_proba with threshold + FORMAT_VERSION = 11 HMAC_SIZE = 32 # SHA-256 digest length @@ -324,6 +346,13 @@ class DocumentClassifier: from sklearn.preprocessing import LabelBinarizer from sklearn.preprocessing import MultiLabelBinarizer + # MLPClassifier does not support class_weight directly + # (https://github.com/scikit-learn/scikit-learn/issues/9113), so we use + # compute_sample_weight to balance classes during training and prevent + # over-represented correspondents from dominating predictions. + # https://scikit-learn.org/stable/modules/generated/sklearn.utils.class_weight.compute_sample_weight.html + from sklearn.utils.class_weight import compute_sample_weight + # Step 2: vectorize data logger.debug("Vectorizing data...") notify("Vectorizing document content...") @@ -369,7 +398,7 @@ class DocumentClassifier: self.tags_binarizer = MultiLabelBinarizer() labels_tags_vectorized = self.tags_binarizer.fit_transform(labels_tags) - self.tags_classifier = MLPClassifier(tol=0.01) + self.tags_classifier = MLPClassifier(tol=0.01, random_state=0) self.tags_classifier.fit(data_vectorized, labels_tags_vectorized) else: self.tags_classifier = None @@ -380,8 +409,12 @@ class DocumentClassifier: notify( f"Training correspondent classifier ({num_correspondents} correspondent(s))...", ) - self.correspondent_classifier = MLPClassifier(tol=0.01) - self.correspondent_classifier.fit(data_vectorized, labels_correspondent) + self.correspondent_classifier = MLPClassifier(tol=0.01, random_state=0) + self.correspondent_classifier.fit( + data_vectorized, + labels_correspondent, + sample_weight=compute_sample_weight("balanced", labels_correspondent), + ) else: self.correspondent_classifier = None logger.debug( @@ -393,8 +426,12 @@ class DocumentClassifier: notify( f"Training document type classifier ({num_document_types} type(s))...", ) - self.document_type_classifier = MLPClassifier(tol=0.01) - self.document_type_classifier.fit(data_vectorized, labels_document_type) + self.document_type_classifier = MLPClassifier(tol=0.01, random_state=0) + self.document_type_classifier.fit( + data_vectorized, + labels_document_type, + sample_weight=compute_sample_weight("balanced", labels_document_type), + ) else: self.document_type_classifier = None logger.debug( @@ -406,10 +443,11 @@ class DocumentClassifier: "Training storage paths classifier...", ) notify(f"Training storage path classifier ({num_storage_paths} path(s))...") - self.storage_path_classifier = MLPClassifier(tol=0.01) + self.storage_path_classifier = MLPClassifier(tol=0.01, random_state=0) self.storage_path_classifier.fit( data_vectorized, labels_storage_path, + sample_weight=compute_sample_weight("balanced", labels_storage_path), ) else: self.storage_path_classifier = None @@ -546,24 +584,24 @@ class DocumentClassifier: def predict_correspondent(self, content: str) -> int | None: if self.correspondent_classifier: X = self._vectorize(content) - correspondent_id = self.correspondent_classifier.predict(X) - if correspondent_id != -1: - return correspondent_id - else: - return None - else: - return None + predicted_id = _predict_with_threshold( + self.correspondent_classifier, + X, + settings.CLASSIFIER_MATCH_THRESHOLD, + ) + return predicted_id + return None def predict_document_type(self, content: str) -> int | None: if self.document_type_classifier: X = self._vectorize(content) - document_type_id = self.document_type_classifier.predict(X) - if document_type_id != -1: - return document_type_id - else: - return None - else: - return None + predicted_id = _predict_with_threshold( + self.document_type_classifier, + X, + settings.CLASSIFIER_MATCH_THRESHOLD, + ) + return predicted_id + return None def predict_tags(self, content: str) -> list[int]: from sklearn.utils.multiclass import type_of_target @@ -589,10 +627,10 @@ class DocumentClassifier: def predict_storage_path(self, content: str) -> int | None: if self.storage_path_classifier: X = self._vectorize(content) - storage_path_id = self.storage_path_classifier.predict(X) - if storage_path_id != -1: - return storage_path_id - else: - return None - else: - return None + predicted_id = _predict_with_threshold( + self.storage_path_classifier, + X, + settings.CLASSIFIER_MATCH_THRESHOLD, + ) + return predicted_id + return None diff --git a/src/documents/tests/test_classifier.py b/src/documents/tests/test_classifier.py index 12dfea4d8..0e09ff074 100644 --- a/src/documents/tests/test_classifier.py +++ b/src/documents/tests/test_classifier.py @@ -3,6 +3,7 @@ import warnings from pathlib import Path from unittest import mock +import numpy as np import pytest from django.conf import settings from django.test import TestCase @@ -11,6 +12,7 @@ from django.test import override_settings from documents.classifier import ClassifierModelCorruptError from documents.classifier import DocumentClassifier from documents.classifier import IncompatibleClassifierVersionError +from documents.classifier import _predict_with_threshold from documents.classifier import load_classifier from documents.models import Correspondent from documents.models import Document @@ -625,6 +627,103 @@ class TestClassifier(DirectoriesMixin, TestCase): self.assertEqual(self.classifier.predict_storage_path(doc1.content), sp.pk) self.assertIsNone(self.classifier.predict_storage_path(doc2.content)) + def test_predict_rejects_prediction_below_match_threshold(self) -> None: + """ + GIVEN: + - Classifiers trained against test data with confident predictions + WHEN: + - CLASSIFIER_MATCH_THRESHOLD exceeds the model's confidence + THEN: + - Every predict_* method discards the match in favor of no match + """ + c1 = Correspondent.objects.create( + name="c1", + matching_algorithm=Correspondent.MATCH_AUTO, + ) + dt1 = DocumentType.objects.create( + name="dt1", + matching_algorithm=DocumentType.MATCH_AUTO, + ) + sp1 = StoragePath.objects.create( + name="sp1", + matching_algorithm=StoragePath.MATCH_AUTO, + ) + + doc1 = Document.objects.create( + title="doc1", + content="this is a document from c1", + correspondent=c1, + document_type=dt1, + storage_path=sp1, + checksum="A", + ) + Document.objects.create( + title="doc2", + content="this is a document from no one", + checksum="B", + ) + + self.classifier.train() + + predictors = { + "correspondent": self.classifier.predict_correspondent, + "document_type": self.classifier.predict_document_type, + "storage_path": self.classifier.predict_storage_path, + } + # No real prediction can reach a confidence this high, so this + # isolates the threshold check from the model's actual output. + with override_settings(CLASSIFIER_MATCH_THRESHOLD=0.999999): + for name, predict in predictors.items(): + with self.subTest(field=name): + self.assertIsNone(predict(doc1.content)) + + def test_train_uses_balanced_sample_weight(self) -> None: + """ + GIVEN: + - A training set with correspondents, document types and storage paths + WHEN: + - The classifier is trained + THEN: + - Each MLP classifier is fit with balanced sample weights, so that + over-represented classes don't dominate predictions + """ + c1 = Correspondent.objects.create( + name="c1", + matching_algorithm=Correspondent.MATCH_AUTO, + ) + dt1 = DocumentType.objects.create( + name="dt1", + matching_algorithm=DocumentType.MATCH_AUTO, + ) + sp1 = StoragePath.objects.create( + name="sp1", + matching_algorithm=StoragePath.MATCH_AUTO, + ) + + Document.objects.create( + title="doc1", + content="this is a document from c1", + correspondent=c1, + document_type=dt1, + storage_path=sp1, + checksum="A", + ) + Document.objects.create( + title="doc2", + content="this is a document from no one", + checksum="B", + ) + + with mock.patch( + "sklearn.utils.class_weight.compute_sample_weight", + return_value=None, + ) as mocked_compute_sample_weight: + self.classifier.train() + + self.assertEqual(mocked_compute_sample_weight.call_count, 3) + for call in mocked_compute_sample_weight.call_args_list: + self.assertEqual(call.args[0], "balanced") + def test_one_tag_predict(self) -> None: t1 = Tag.objects.create(name="t1", matching_algorithm=Tag.MATCH_AUTO, pk=12) @@ -810,6 +909,52 @@ class TestClassifier(DirectoriesMixin, TestCase): load_classifier(raise_exception=True) +class _StubProbaClassifier: + """ + A fake scikit-learn classifier exposing just enough of the API for + `_predict_with_threshold`: `classes_` and `predict_proba`. + """ + + def __init__(self, classes: list[int], probabilities: list[float]) -> None: + self.classes_ = np.array(classes) + self._probabilities = np.array([probabilities]) + + def predict_proba(self, X) -> np.ndarray: + return self._probabilities + + +@pytest.mark.parametrize( + ("classes", "probabilities", "threshold", "expected"), + [ + # confident prediction above the threshold is returned + ([-1, 3], [0.1, 0.9], 0.6, 3), + # prediction below the threshold is discarded + ([-1, 3], [0.45, 0.55], 0.6, None), + # boundary: exactly at the threshold is accepted, not discarded + ([-1, 3], [0.4, 0.6], 0.6, 3), + # the winning class is the "no match" pseudo-class, regardless of its + # own confidence + ([-1, 3], [0.99, 0.01], 0.0, None), + # threshold of 0.0 disables the confidence check entirely + ([-1, 3], [0.45, 0.55], 0.0, 3), + ], +) +def test_predict_with_threshold(classes, probabilities, threshold, expected) -> None: + classifier = _StubProbaClassifier(classes, probabilities) + result = _predict_with_threshold(classifier, X=None, threshold=threshold) + assert result == expected + + +def test_classifier_match_threshold_default() -> None: + """ + GIVEN: + - No PAPERLESS_CLASSIFIER_MATCH_THRESHOLD environment variable is set + THEN: + - The classifier match threshold defaults to 0.6 + """ + assert settings.CLASSIFIER_MATCH_THRESHOLD == 0.6 + + def test_preprocess_content() -> None: """ GIVEN: diff --git a/src/paperless/settings/__init__.py b/src/paperless/settings/__init__.py index a0dc0e585..14f2fb9ac 100644 --- a/src/paperless/settings/__init__.py +++ b/src/paperless/settings/__init__.py @@ -96,6 +96,13 @@ MODEL_FILE = get_path_from_env( "PAPERLESS_MODEL_FILE", DATA_DIR / "classification_model.pickle", ) + +# Minimum confidence (0.0-1.0) for the ML classifier to assign a correspondent, +# document type, or storage path. 0.0 disables the threshold. +CLASSIFIER_MATCH_THRESHOLD: Final[float] = get_float_from_env( + "PAPERLESS_CLASSIFIER_MATCH_THRESHOLD", + 0.6, +) LLM_INDEX_DIR = DATA_DIR / "llm_index" LLM_INDEX_LOCK = LLM_INDEX_DIR / "index.lock" # Cross-process read/write lock guarding the LLM index compaction/migration From 1c9681962588a441b2a3139efd426e4520e95aad Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 22:41:48 +0000 Subject: [PATCH 002/258] Auto translate strings --- src/locale/en_US/LC_MESSAGES/django.po | 76 +++++++++++++------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/src/locale/en_US/LC_MESSAGES/django.po b/src/locale/en_US/LC_MESSAGES/django.po index 0a766f170..5327ac8be 100644 --- a/src/locale/en_US/LC_MESSAGES/django.po +++ b/src/locale/en_US/LC_MESSAGES/django.po @@ -2,7 +2,7 @@ msgid "" msgstr "" "Project-Id-Version: paperless-ngx\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2026-09-09 16:03+0000\n" +"POT-Creation-Date: 2026-09-09 22:41+0000\n" "PO-Revision-Date: 2022-02-17 04:17\n" "Last-Translator: \n" "Language-Team: English\n" @@ -2258,151 +2258,151 @@ msgstr "" msgid "paperless application settings" msgstr "" -#: paperless/settings/__init__.py:562 +#: paperless/settings/__init__.py:569 msgid "English (US)" msgstr "" -#: paperless/settings/__init__.py:563 +#: paperless/settings/__init__.py:570 msgid "Arabic" msgstr "" -#: paperless/settings/__init__.py:564 +#: paperless/settings/__init__.py:571 msgid "Afrikaans" msgstr "" -#: paperless/settings/__init__.py:565 +#: paperless/settings/__init__.py:572 msgid "Belarusian" msgstr "" -#: paperless/settings/__init__.py:566 +#: paperless/settings/__init__.py:573 msgid "Bulgarian" msgstr "" -#: paperless/settings/__init__.py:567 +#: paperless/settings/__init__.py:574 msgid "Catalan" msgstr "" -#: paperless/settings/__init__.py:568 +#: paperless/settings/__init__.py:575 msgid "Czech" msgstr "" -#: paperless/settings/__init__.py:569 +#: paperless/settings/__init__.py:576 msgid "Danish" msgstr "" -#: paperless/settings/__init__.py:570 +#: paperless/settings/__init__.py:577 msgid "German" msgstr "" -#: paperless/settings/__init__.py:571 +#: paperless/settings/__init__.py:578 msgid "Greek" msgstr "" -#: paperless/settings/__init__.py:572 +#: paperless/settings/__init__.py:579 msgid "English (GB)" msgstr "" -#: paperless/settings/__init__.py:573 +#: paperless/settings/__init__.py:580 msgid "Spanish" msgstr "" -#: paperless/settings/__init__.py:574 +#: paperless/settings/__init__.py:581 msgid "Persian" msgstr "" -#: paperless/settings/__init__.py:575 +#: paperless/settings/__init__.py:582 msgid "Finnish" msgstr "" -#: paperless/settings/__init__.py:576 +#: paperless/settings/__init__.py:583 msgid "French" msgstr "" -#: paperless/settings/__init__.py:577 +#: paperless/settings/__init__.py:584 msgid "Hungarian" msgstr "" -#: paperless/settings/__init__.py:578 +#: paperless/settings/__init__.py:585 msgid "Indonesian" msgstr "" -#: paperless/settings/__init__.py:579 +#: paperless/settings/__init__.py:586 msgid "Italian" msgstr "" -#: paperless/settings/__init__.py:580 +#: paperless/settings/__init__.py:587 msgid "Japanese" msgstr "" -#: paperless/settings/__init__.py:581 +#: paperless/settings/__init__.py:588 msgid "Korean" msgstr "" -#: paperless/settings/__init__.py:582 +#: paperless/settings/__init__.py:589 msgid "Luxembourgish" msgstr "" -#: paperless/settings/__init__.py:583 +#: paperless/settings/__init__.py:590 msgid "Norwegian" msgstr "" -#: paperless/settings/__init__.py:584 +#: paperless/settings/__init__.py:591 msgid "Dutch" msgstr "" -#: paperless/settings/__init__.py:585 +#: paperless/settings/__init__.py:592 msgid "Polish" msgstr "" -#: paperless/settings/__init__.py:586 +#: paperless/settings/__init__.py:593 msgid "Portuguese (Brazil)" msgstr "" -#: paperless/settings/__init__.py:587 +#: paperless/settings/__init__.py:594 msgid "Portuguese" msgstr "" -#: paperless/settings/__init__.py:588 +#: paperless/settings/__init__.py:595 msgid "Romanian" msgstr "" -#: paperless/settings/__init__.py:589 +#: paperless/settings/__init__.py:596 msgid "Russian" msgstr "" -#: paperless/settings/__init__.py:590 +#: paperless/settings/__init__.py:597 msgid "Slovak" msgstr "" -#: paperless/settings/__init__.py:591 +#: paperless/settings/__init__.py:598 msgid "Slovenian" msgstr "" -#: paperless/settings/__init__.py:592 +#: paperless/settings/__init__.py:599 msgid "Serbian" msgstr "" -#: paperless/settings/__init__.py:593 +#: paperless/settings/__init__.py:600 msgid "Swedish" msgstr "" -#: paperless/settings/__init__.py:594 +#: paperless/settings/__init__.py:601 msgid "Turkish" msgstr "" -#: paperless/settings/__init__.py:595 +#: paperless/settings/__init__.py:602 msgid "Ukrainian" msgstr "" -#: paperless/settings/__init__.py:596 +#: paperless/settings/__init__.py:603 msgid "Vietnamese" msgstr "" -#: paperless/settings/__init__.py:597 +#: paperless/settings/__init__.py:604 msgid "Chinese Simplified" msgstr "" -#: paperless/settings/__init__.py:598 +#: paperless/settings/__init__.py:605 msgid "Chinese Traditional" msgstr "" From 60709b8319fa636b0e24a520fd004785566412a5 Mon Sep 17 00:00:00 2001 From: Trenton H <797416+stumpylog@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:47:07 -0700 Subject: [PATCH 003/258] Performance: cut redundant per-document lookups in bulk `modify_custom_fields` (#13807) * Perf: batch the repeated lookups in modify_custom_fields `modify_custom_fields()` re-resolved the same objects inside its per-document loop: `custom_fields.get(id=field_id)` re-ran a CustomField query for every document, and doc link fields called `Document.objects.get(id=doc_id)` a second time for a document that was already known. Resolve both up front with `in_bulk()` and hand the resolved objects to `update_or_create()` rather than bare ids. Passing the objects also populates the FK cache on the newly created instance, so auditlog's post_save receiver touching `.document`/`.field` no longer costs a reload per row. The document map defers `content`, the one field here that is both large and unused. The symmetrical-link removal pass and `remove_doclink()` get `select_related()` for the same auditlog reason. Measured over 50 documents, sqlite, audit log enabled: before after add 3 string fields 1502 1054 update 1 string field 451 403 add doc link 851 653 remove doc link 604 354 `update_or_create()` is kept as-is. Dropping it for a hand-rolled get-or-construct loop removes a further ~4 statements per row, but those are the SAVEPOINT/RELEASE pairs of its `transaction.atomic()`, and the `select_for_update()` and IntegrityError fallback that go with them. The (document, field) unique constraint depends on that when two bulk edits overlap, and the wall clock did not move to pay for it (331 ms vs 310 ms for the string case above). Also normalises the field ids to int once at the top so the old dict API, whose keys may arrive as strings, indexes the resolved map correctly. The `if custom_field:` branch it replaces was dead: `.get()` raises DoesNotExist, it never returns None. Co-Authored-By: Claude Opus 5 * Fixes the comment --------- Co-authored-by: Claude Opus 5 --- src/documents/bulk_edit.py | 75 ++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/src/documents/bulk_edit.py b/src/documents/bulk_edit.py index 79a79e6cf..363193d48 100644 --- a/src/documents/bulk_edit.py +++ b/src/documents/bulk_edit.py @@ -299,53 +299,55 @@ def modify_custom_fields( ) -> Literal["OK"]: qs = Document.objects.filter(id__in=doc_ids).only("pk") affected_docs = list(qs.values_list("pk", flat=True)) - # Ensure add_custom_fields is a list of tuples, supports old API + # Ensure add_custom_fields is a list of (int, value) tuples, supports old API add_custom_fields = ( - add_custom_fields.items() + [(int(field), value) for field, value in add_custom_fields.items()] if isinstance(add_custom_fields, dict) - else [(field, None) for field in add_custom_fields] + else [(int(field), None) for field in add_custom_fields] ) - custom_fields = CustomField.objects.filter( - id__in=[int(field) for field, _ in add_custom_fields], - ).distinct() + # Resolved once, instead of re-querying the same field for every document + custom_fields_by_id: dict[int, CustomField] = CustomField.objects.in_bulk( + [field_id for field_id, _ in add_custom_fields], + ) + # Passed to update_or_create() below rather than a bare id, so the FK is + # cached on the created instance and auditlog's post_save receiver does + # not reload it per row. Only needed for additions. content is deferred: + # the one field here that is both large and unused. + docs_by_id: dict[int, Document] = ( + Document.objects.defer("content").in_bulk(affected_docs) + if add_custom_fields + else {} + ) for field_id, value in add_custom_fields: + custom_field = custom_fields_by_id[field_id] + value_field = CustomFieldInstance.TYPE_TO_DATA_STORE_NAME_MAP[ + custom_field.data_type + ] + is_doclink = custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK for doc_id in affected_docs: - defaults = {} - custom_field = custom_fields.get(id=field_id) - if custom_field: - value_field = CustomFieldInstance.TYPE_TO_DATA_STORE_NAME_MAP[ - custom_field.data_type - ] - defaults[value_field] = value - if ( - custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK - and value - and doc_id in value - ): - # Prevent self-linking - continue + if is_doclink and value and doc_id in value: + # Prevent self-linking + continue CustomFieldInstance.objects.update_or_create( - document_id=doc_id, - field_id=field_id, - defaults=defaults, + document=docs_by_id[doc_id], + field=custom_field, + defaults={value_field: value}, ) - if custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK: - doc = Document.objects.get(id=doc_id) - reflect_doclinks(doc, custom_field, value) + if is_doclink: + reflect_doclinks(docs_by_id[doc_id], custom_field, value) - # For doc link fields that are being removed, remove symmetrical links + # For doc link fields that are being removed, remove symmetrical links. + # select_related avoids a per-instance reload of the document and field. for doclink_being_removed_instance in CustomFieldInstance.objects.filter( document_id__in=affected_docs, field__id__in=remove_custom_fields, field__data_type=CustomField.FieldDataType.DOCUMENTLINK, value_document_ids__isnull=False, - ): + ).select_related("field", "document"): for target_doc_id in doclink_being_removed_instance.value: remove_doclink( - document=Document.objects.get( - id=doclink_being_removed_instance.document.id, - ), + document=doclink_being_removed_instance.document, field=doclink_being_removed_instance.field, target_doc_id=target_doc_id, ) @@ -1178,10 +1180,13 @@ def remove_doclink( """ Removes a 'symmetrical' link to `document` from the target document's existing custom field instance """ - target_doc_field_instance = CustomFieldInstance.objects.filter( - document_id=target_doc_id, - field=field, - ).first() + # select_related: a signal receiver (auditlog) touches .document/.field on + # the save() below, without this that is a per-call reload query + target_doc_field_instance = ( + CustomFieldInstance.objects.filter(document_id=target_doc_id, field=field) + .select_related("document", "field") + .first() + ) if ( target_doc_field_instance is not None and document.id in target_doc_field_instance.value From 2d646840432397436b7cdfbbd685aca5eb723636 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Wed, 9 Sep 2026 22:00:37 -0700 Subject: [PATCH 004/258] Enhancement: customize sidebar items (#14052) --- .../admin/settings/settings.component.html | 16 ++++ .../admin/settings/settings.component.spec.ts | 47 ++++++++++- .../admin/settings/settings.component.ts | 60 +++++++++++++- .../app-frame/app-frame.component.html | 35 ++++++--- .../app-frame/app-frame.component.spec.ts | 78 ++++++++++++++++++- .../app-frame/app-frame.component.ts | 15 +++- .../common/input/switch/switch.component.html | 10 +-- .../input/switch/switch.component.spec.ts | 10 +++ .../common/input/switch/switch.component.ts | 3 + src-ui/src/app/data/ui-settings.ts | 16 ++++ .../src/app/services/settings.service.spec.ts | 35 ++++++++- src-ui/src/app/services/settings.service.ts | 36 +++++++++ 12 files changed, 339 insertions(+), 22 deletions(-) diff --git a/src-ui/src/app/components/admin/settings/settings.component.html b/src-ui/src/app/components/admin/settings/settings.component.html index 305bb8a96..b520e8218 100644 --- a/src-ui/src/app/components/admin/settings/settings.component.html +++ b/src-ui/src/app/components/admin/settings/settings.component.html @@ -112,6 +112,22 @@ +

Sidebar items to show:

+ @for (option of sidebarItemOptions; track option.id) { +
+ + +
+ } + diff --git a/src-ui/src/app/components/admin/settings/settings.component.spec.ts b/src-ui/src/app/components/admin/settings/settings.component.spec.ts index 7b65281d0..71889e74b 100644 --- a/src-ui/src/app/components/admin/settings/settings.component.spec.ts +++ b/src-ui/src/app/components/admin/settings/settings.component.spec.ts @@ -24,7 +24,7 @@ import { SystemStatus, SystemStatusItemStatus, } from 'src/app/data/system-status' -import { SETTINGS_KEYS } from 'src/app/data/ui-settings' +import { HideableSidebarItemID, SETTINGS_KEYS } from 'src/app/data/ui-settings' import { IfOwnerDirective } from 'src/app/directives/if-owner.directive' import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive' import { PermissionsGuard } from 'src/app/guards/permissions.guard' @@ -209,6 +209,45 @@ describe('SettingsComponent', () => { fixture.detectChanges() } + it('supports configuring sidebar items and canceling changes', () => { + completeSetup() + + component.toggleSidebarItem(HideableSidebarItemID.Workflows, false) + fixture.detectChanges() + + expect(component.settingsForm.value.sidebarHiddenItems).toContain( + HideableSidebarItemID.Workflows + ) + + settingsService.updateSidebarItemVisibility( + HideableSidebarItemID.Mail, + false + ) + + expect(component.settingsForm.value.sidebarHiddenItems).toContain( + HideableSidebarItemID.Mail + ) + + component.reset() + + expect(component.settingsForm.value.sidebarHiddenItems).not.toContain( + HideableSidebarItemID.Workflows + ) + expect(component.settingsForm.value.sidebarHiddenItems).not.toContain( + HideableSidebarItemID.Mail + ) + }) + + it('enables sidebar item controls on general settings until destroyed', () => { + completeSetup() + + expect(settingsService.organizingSidebarItems()).toBe(true) + + component.ngOnDestroy() + + expect(settingsService.organizingSidebarItems()).toBe(false) + }) + it('should support tabbed settings & change URL, prevent navigation if dirty confirmation rejected', async () => { completeSetup() const navigateSpy = jest.spyOn(router, 'navigate') @@ -249,6 +288,7 @@ describe('SettingsComponent', () => { it('should support save local settings updating appearance settings and calling API, show error', () => { completeSetup() + component.toggleSidebarItem(HideableSidebarItemID.Workflows, false) const toastErrorSpy = jest.spyOn(toastService, 'showError') const toastSpy = jest.spyOn(toastService, 'show') const storeSpy = jest.spyOn(settingsService, 'storeSettings') @@ -267,7 +307,10 @@ describe('SettingsComponent', () => { expect(toastErrorSpy).toHaveBeenCalled() expect(storeSpy).toHaveBeenCalled() expect(appearanceSettingsSpy).not.toHaveBeenCalled() - expect(setSpy).toHaveBeenCalledTimes(33) + expect(setSpy).toHaveBeenCalledTimes(34) + expect(setSpy).toHaveBeenCalledWith(SETTINGS_KEYS.SIDEBAR_HIDDEN_ITEMS, [ + HideableSidebarItemID.Workflows, + ]) // succeed storeSpy.mockReturnValueOnce(of(true)) diff --git a/src-ui/src/app/components/admin/settings/settings.component.ts b/src-ui/src/app/components/admin/settings/settings.component.ts index 218f6683c..9076d2a3e 100644 --- a/src-ui/src/app/components/admin/settings/settings.component.ts +++ b/src-ui/src/app/components/admin/settings/settings.component.ts @@ -39,7 +39,12 @@ import { SystemStatus, SystemStatusItemStatus, } from 'src/app/data/system-status' -import { GlobalSearchType, SETTINGS_KEYS } from 'src/app/data/ui-settings' +import { + GlobalSearchType, + HIDEABLE_SIDEBAR_ITEM_IDS, + HideableSidebarItemID, + SETTINGS_KEYS, +} from 'src/app/data/ui-settings' import { User } from 'src/app/data/user' import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive' import { CustomDatePipe } from 'src/app/pipes/custom-date.pipe' @@ -102,6 +107,14 @@ const documentDetailFieldOptions = [ { id: DocumentDetailFieldID.Tags, label: $localize`Tags` }, ] +const sidebarItemLabels: Record = { + [HideableSidebarItemID.Dashboard]: $localize`Dashboard`, + [HideableSidebarItemID.SavedViews]: $localize`Saved Views`, + [HideableSidebarItemID.Workflows]: $localize`Workflows`, + [HideableSidebarItemID.Mail]: $localize`Mail`, + [HideableSidebarItemID.Documentation]: $localize`Documentation`, +} + @Component({ selector: 'pngx-settings', templateUrl: './settings.component.html', @@ -149,6 +162,7 @@ export class SettingsComponent bulkEditApplyOnClose: new FormControl(null), documentListItemPerPage: new FormControl(null), slimSidebarEnabled: new FormControl(null), + sidebarHiddenItems: new FormControl([]), darkModeUseSystem: new FormControl(null), darkModeEnabled: new FormControl(null), darkModeInvertThumbs: new FormControl(null), @@ -186,6 +200,7 @@ export class SettingsComponent store: BehaviorSubject storeSub: Subscription + sidebarItemsSub: Subscription isDirty$: Observable isDirty: boolean = false unsubscribeNotifier: Subject = new Subject() @@ -203,6 +218,10 @@ export class SettingsComponent public readonly PdfEditorEditMode = PdfEditorEditMode public readonly documentDetailFieldOptions = documentDetailFieldOptions + public readonly sidebarItemOptions = HIDEABLE_SIDEBAR_ITEM_IDS.map((id) => ({ + id, + label: sidebarItemLabels[id], + })) get systemStatusHasErrors(): boolean { const status = this.systemStatus() @@ -230,6 +249,10 @@ export class SettingsComponent constructor() { super() + this.sidebarItemsSub = + this.settings.sidebarHiddenItemsEditingChanged.subscribe((hiddenItems) => + this.settingsForm.controls.sidebarHiddenItems.setValue(hiddenItems) + ) this.settings.settingsSaved.subscribe(() => { if (!this.savePending) this.initialize() this.savedViewsService.maybeRefreshDocumentCounts() @@ -279,14 +302,21 @@ export class SettingsComponent this.activatedRoute.paramMap.subscribe((paramMap) => { const section = paramMap.get('section') + let navID = SettingsNavIDs.General if (section) { const navIDKey: string = Object.keys(SettingsNavIDs).find( (navID) => navID.toLowerCase() == section ) if (navIDKey) { - this.activeNavID.set(SettingsNavIDs[navIDKey]) + navID = SettingsNavIDs[navIDKey] } } + this.activeNavID.set(navID) + this.settings.sidebarHiddenItemsEditing.set( + navID === SettingsNavIDs.General + ? [...this.settingsForm.controls.sidebarHiddenItems.value] + : null + ) }) } @@ -310,6 +340,7 @@ export class SettingsComponent SETTINGS_KEYS.DOCUMENT_LIST_SIZE ), slimSidebarEnabled: this.settings.get(SETTINGS_KEYS.SLIM_SIDEBAR), + sidebarHiddenItems: this.settings.get(SETTINGS_KEYS.SIDEBAR_HIDDEN_ITEMS), darkModeUseSystem: this.settings.get(SETTINGS_KEYS.DARK_MODE_USE_SYSTEM), darkModeEnabled: this.settings.get(SETTINGS_KEYS.DARK_MODE_ENABLED), darkModeInvertThumbs: this.settings.get( @@ -436,6 +467,12 @@ export class SettingsComponent this.settingsForm.patchValue(currentFormValue) } + if (this.settings.organizingSidebarItems()) { + this.settings.sidebarHiddenItemsEditing.set([ + ...this.settingsForm.controls.sidebarHiddenItems.value, + ]) + } + if (this.canViewSystemStatus) { this.systemStatusService.get().subscribe((status) => { this.systemStatus.set(status) @@ -444,8 +481,18 @@ export class SettingsComponent } ngOnDestroy() { + this.settings.sidebarHiddenItemsEditing.set(null) if (this.isDirty) this.settings.updateAppearanceSettings() // in case user changed appearance but didn't save this.storeSub && this.storeSub.unsubscribe() + this.sidebarItemsSub.unsubscribe() + } + + isSidebarItemShown(item: HideableSidebarItemID): boolean { + return !(this.settingsForm.value.sidebarHiddenItems || []).includes(item) + } + + toggleSidebarItem(item: HideableSidebarItemID, checked: boolean): void { + this.settings.updateSidebarItemVisibility(item, checked) } public saveSettings() { @@ -473,6 +520,10 @@ export class SettingsComponent SETTINGS_KEYS.SLIM_SIDEBAR, this.settingsForm.value.slimSidebarEnabled ) + this.settings.set( + SETTINGS_KEYS.SIDEBAR_HIDDEN_ITEMS, + this.settingsForm.value.sidebarHiddenItems + ) this.settings.set( SETTINGS_KEYS.DARK_MODE_USE_SYSTEM, this.settingsForm.value.darkModeUseSystem @@ -632,6 +683,11 @@ export class SettingsComponent reset() { this.settingsForm.patchValue(this.store.getValue()) + if (this.settings.organizingSidebarItems()) { + this.settings.sidebarHiddenItemsEditing.set([ + ...this.settingsForm.controls.sidebarHiddenItems.value, + ]) + } } clearThemeColor() { diff --git a/src-ui/src/app/components/app-frame/app-frame.component.html b/src-ui/src/app/components/app-frame/app-frame.component.html index 6faddbb8a..a8dd12d34 100644 --- a/src-ui/src/app/components/app-frame/app-frame.component.html +++ b/src-ui/src/app/components/app-frame/app-frame.component.html @@ -86,12 +86,15 @@ }