diff --git a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.html b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.html index 492b2a4e4..a5e8897f7 100644 --- a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.html +++ b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.html @@ -10,7 +10,7 @@ } - @if (textFilterTarget === 'asn') { + @if (textFilterTarget === 'asn' || textFilterTarget === 'duplicates') { { expect(component.textFilterTarget).toEqual('mime-type') // TEXT_FILTER_TARGET_MIME_TYPE }) + it('should ingest filter rules for documents with duplicates', () => { + component.filterRules = [ + { + rule_type: FILTER_HAS_DUPLICATES, + value: 'true', + }, + ] + fixture.detectChanges() + + expect(component.textFilterTarget).toEqual('duplicates') + expect(component.textFilterModifier).toEqual('has-duplicates') + expect(component.textFilterInputDisabled).toBeTruthy() + }) + + it('should ingest filter rules for documents without duplicates', () => { + component.filterRules = [ + { + rule_type: FILTER_HAS_DUPLICATES, + value: 'false', + }, + ] + + expect(component.textFilterTarget).toEqual('duplicates') + expect(component.textFilterModifier).toEqual('does-not-have-duplicates') + expect(component.filterRules).toEqual([ + { + rule_type: FILTER_HAS_DUPLICATES, + value: 'false', + }, + ]) + }) + it('should ingest text filter rules for fulltext query', () => { expect(component.textFilter).toEqual(null) component.filterRules = [ @@ -1390,6 +1423,33 @@ describe('FilterEditorComponent', () => { ]) }) + it('should convert duplicate target input to the correct filter rule', () => { + const textFieldTargetDropdown = fixture.debugElement.queryAll( + By.directive(NgbDropdownItem) + )[5] + textFieldTargetDropdown.triggerEventHandler('click') + fixture.detectChanges() + + expect(component.textFilterTarget).toEqual('duplicates') + expect(component.filterRules).toEqual([ + { + rule_type: FILTER_HAS_DUPLICATES, + value: 'true', + }, + ]) + + const textFieldModifierSelect = fixture.debugElement.query(By.css('select')) + textFieldModifierSelect.nativeElement.value = 'does-not-have-duplicates' + textFieldModifierSelect.nativeElement.dispatchEvent(new Event('change')) + fixture.detectChanges() + expect(component.filterRules).toEqual([ + { + rule_type: FILTER_HAS_DUPLICATES, + value: 'false', + }, + ]) + }) + it('should convert user input to correct filter rules on full text query', () => { component.textFilterInput.nativeElement.value = 'foo' component.textFilterInput.nativeElement.dispatchEvent(new Event('input')) @@ -2178,6 +2238,22 @@ describe('FilterEditorComponent', () => { ] expect(component.generateFilterName()).toEqual('Without any tag') + component.filterRules = [ + { + rule_type: FILTER_HAS_DUPLICATES, + value: 'true', + }, + ] + expect(component.generateFilterName()).toEqual('With duplicates') + + component.filterRules = [ + { + rule_type: FILTER_HAS_DUPLICATES, + value: 'false', + }, + ] + expect(component.generateFilterName()).toEqual('Without duplicates') + component.filterRules = [ { rule_type: FILTER_CUSTOM_FIELDS_QUERY, diff --git a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts index 16bf3cccb..ed08cee1d 100644 --- a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts +++ b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts @@ -65,6 +65,7 @@ import { FILTER_HAS_CUSTOM_FIELDS_ALL, FILTER_HAS_CUSTOM_FIELDS_ANY, FILTER_HAS_DOCUMENT_TYPE_ANY, + FILTER_HAS_DUPLICATES, FILTER_HAS_STORAGE_PATH_ANY, FILTER_HAS_TAGS_ALL, FILTER_HAS_TAGS_ANY, @@ -129,12 +130,15 @@ const TEXT_FILTER_TARGET_FULLTEXT_QUERY = 'fulltext-query' const TEXT_FILTER_TARGET_FULLTEXT_MORELIKE = 'fulltext-morelike' const TEXT_FILTER_TARGET_CUSTOM_FIELDS = 'custom-fields' const TEXT_FILTER_TARGET_MIME_TYPE = 'mime-type' +const TEXT_FILTER_TARGET_DUPLICATES = 'duplicates' const TEXT_FILTER_MODIFIER_EQUALS = 'equals' const TEXT_FILTER_MODIFIER_NULL = 'is null' const TEXT_FILTER_MODIFIER_NOTNULL = 'not null' const TEXT_FILTER_MODIFIER_GT = 'greater' const TEXT_FILTER_MODIFIER_LT = 'less' +const TEXT_FILTER_MODIFIER_HAS_DUPLICATES = 'has-duplicates' +const TEXT_FILTER_MODIFIER_DOES_NOT_HAVE_DUPLICATES = 'does-not-have-duplicates' const RELATIVE_DATE_QUERY_REGEXP_CREATED = /created:[\["]([^\]]+)[\]"]/g const RELATIVE_DATE_QUERY_REGEXP_ADDED = /added:[\["]([^\]]+)[\]"]/g @@ -205,6 +209,7 @@ const DEFAULT_TEXT_FILTER_TARGET_OPTIONS = [ id: TEXT_FILTER_TARGET_FULLTEXT_QUERY, name: $localize`Advanced search`, }, + { id: TEXT_FILTER_TARGET_DUPLICATES, name: $localize`Duplicates` }, ] const DEPRECATED_CUSTOM_FIELDS_TEXT_FILTER_TARGET_OPTION = { @@ -241,6 +246,17 @@ const DEFAULT_TEXT_FILTER_MODIFIER_OPTIONS = [ }, ] +const DUPLICATES_FILTER_MODIFIER_OPTIONS = [ + { + id: TEXT_FILTER_MODIFIER_HAS_DUPLICATES, + label: $localize`exist`, + }, + { + id: TEXT_FILTER_MODIFIER_DOES_NOT_HAVE_DUPLICATES, + label: $localize`do not exist`, + }, +] + @Component({ selector: 'pngx-filter-editor', templateUrl: './filter-editor.component.html', @@ -320,6 +336,12 @@ export class FilterEditorComponent if (rule.value == 'false') { return $localize`Without any tag` } + break + + case FILTER_HAS_DUPLICATES: + return rule.value == 'false' + ? $localize`Without duplicates` + : $localize`With duplicates` case FILTER_CUSTOM_FIELDS_QUERY: return $localize`Custom fields query` @@ -390,7 +412,9 @@ export class FilterEditorComponent public textFilterModifier: string get textFilterModifiers() { - return DEFAULT_TEXT_FILTER_MODIFIER_OPTIONS + return this.textFilterTarget === TEXT_FILTER_TARGET_DUPLICATES + ? DUPLICATES_FILTER_MODIFIER_OPTIONS + : DEFAULT_TEXT_FILTER_MODIFIER_OPTIONS } get textFilterModifierIsNull(): boolean { @@ -399,6 +423,13 @@ export class FilterEditorComponent ) } + get textFilterInputDisabled(): boolean { + return ( + this.textFilterModifierIsNull || + this.textFilterTarget === TEXT_FILTER_TARGET_DUPLICATES + ) + } + tagSelectionModel = new FilterableDropdownSelectionModel(true) correspondentSelectionModel = new FilterableDropdownSelectionModel() documentTypeSelectionModel = new FilterableDropdownSelectionModel() @@ -444,6 +475,7 @@ export class FilterEditorComponent this.customFieldQueriesModel.clear(false) this._textFilter = null this._moreLikeId = null + this.textFilterTarget = TEXT_FILTER_TARGET_TITLE_CONTENT this.dateAddedTo = null this.dateAddedFrom = null this.dateCreatedTo = null @@ -477,6 +509,13 @@ export class FilterEditorComponent this.textFilterTarget = TEXT_FILTER_TARGET_MIME_TYPE this._textFilter = rule.value break + case FILTER_HAS_DUPLICATES: + this.textFilterTarget = TEXT_FILTER_TARGET_DUPLICATES + this.textFilterModifier = + rule.value == 'false' || rule.value == '0' + ? TEXT_FILTER_MODIFIER_DOES_NOT_HAVE_DUPLICATES + : TEXT_FILTER_MODIFIER_HAS_DUPLICATES + break case FILTER_FULLTEXT_QUERY: let allQueryArgs = rule.value.split(',') let textQueryArgs = [] @@ -800,6 +839,14 @@ export class FilterEditorComponent value: this._textFilter.trim(), }) } + if (this.textFilterTarget == TEXT_FILTER_TARGET_DUPLICATES) { + filterRules.push({ + rule_type: FILTER_HAS_DUPLICATES, + value: ( + this.textFilterModifier == TEXT_FILTER_MODIFIER_HAS_DUPLICATES + ).toString(), + }) + } if (this._textFilter && this.textFilterTarget == TEXT_FILTER_TARGET_TITLE) { filterRules.push({ rule_type: FILTER_SIMPLE_TITLE, @@ -1163,7 +1210,7 @@ export class FilterEditorComponent } get textFilter() { - return this.textFilterModifierIsNull ? '' : this._textFilter + return this.textFilterInputDisabled ? '' : this._textFilter } set textFilter(value) { @@ -1363,12 +1410,24 @@ export class FilterEditorComponent this._textFilter = '' } this.textFilterTarget = target + if (target == TEXT_FILTER_TARGET_DUPLICATES) { + this._textFilter = '' + this.textFilterModifier = TEXT_FILTER_MODIFIER_HAS_DUPLICATES + } else if ( + [ + TEXT_FILTER_MODIFIER_HAS_DUPLICATES, + TEXT_FILTER_MODIFIER_DOES_NOT_HAVE_DUPLICATES, + ].includes(this.textFilterModifier) + ) { + this.textFilterModifier = TEXT_FILTER_MODIFIER_EQUALS + } this.textFilterInput.nativeElement.focus() this.updateRules() } textFilterModifierChange() { if ( + this.textFilterTarget == TEXT_FILTER_TARGET_DUPLICATES || this.textFilterModifierIsNull || ([ TEXT_FILTER_MODIFIER_EQUALS, diff --git a/src-ui/src/app/data/filter-rule-type.ts b/src-ui/src/app/data/filter-rule-type.ts index 6330eb44c..5feaf3867 100644 --- a/src-ui/src/app/data/filter-rule-type.ts +++ b/src-ui/src/app/data/filter-rule-type.ts @@ -49,6 +49,7 @@ export const FILTER_MODIFIED_AFTER = 16 export const FILTER_TITLE_CONTENT = 19 // Deprecated in favor of Tantivy-backed `text` filtervar. Keep for now for existing saved views export const FILTER_SIMPLE_TITLE = 48 export const FILTER_SIMPLE_TEXT = 49 +export const FILTER_HAS_DUPLICATES = 50 export const FILTER_FULLTEXT_QUERY = 20 export const FILTER_FULLTEXT_MORELIKE = 21 @@ -382,6 +383,13 @@ export const FILTER_RULE_TYPES: FilterRuleType[] = [ datatype: 'string', multi: false, }, + { + id: FILTER_HAS_DUPLICATES, + filtervar: 'has_duplicates', + datatype: 'boolean', + multi: false, + default: true, + }, ] export interface FilterRuleType { diff --git a/src-ui/src/app/utils/query-params.spec.ts b/src-ui/src/app/utils/query-params.spec.ts index 7fd8f6808..7137ed53b 100644 --- a/src-ui/src/app/utils/query-params.spec.ts +++ b/src-ui/src/app/utils/query-params.spec.ts @@ -7,6 +7,7 @@ import { FILTER_HAS_ANY_TAG, FILTER_HAS_CUSTOM_FIELDS_ALL, FILTER_HAS_CUSTOM_FIELDS_ANY, + FILTER_HAS_DUPLICATES, FILTER_HAS_TAGS_ALL, FILTER_SIMPLE_TEXT, FILTER_SIMPLE_TITLE, @@ -132,6 +133,16 @@ describe('QueryParams Utils', () => { is_tagged: 0, }) + params = queryParamsFromFilterRules([ + { + rule_type: FILTER_HAS_DUPLICATES, + value: 'false', + }, + ]) + expect(params).toEqual({ + has_duplicates: 0, + }) + params = queryParamsFromFilterRules([ { rule_type: FILTER_TITLE_CONTENT, @@ -247,6 +258,18 @@ describe('QueryParams Utils', () => { }, ]) + rules = filterRulesFromQueryParams( + convertToParamMap({ + has_duplicates: 'true', + }) + ) + expect(rules).toEqual([ + { + rule_type: FILTER_HAS_DUPLICATES, + value: 'true', + }, + ]) + rules = filterRulesFromQueryParams( convertToParamMap({ correspondent__isnull: '1', diff --git a/src/documents/filters.py b/src/documents/filters.py index 40ae8978b..4c2aa1ecf 100644 --- a/src/documents/filters.py +++ b/src/documents/filters.py @@ -25,6 +25,7 @@ from django.db.models import Sum from django.db.models import Value from django.db.models import When from django.db.models.functions import Cast +from django.db.models.functions import NullIf from django.utils.translation import gettext_lazy as _ from django_filters import DateFilter from django_filters.rest_framework import BooleanFilter @@ -50,6 +51,7 @@ from documents.models import ShareLink from documents.models import ShareLinkBundle from documents.models import StoragePath from documents.models import Tag +from documents.permissions import permitted_document_ids from documents.permissions import permitted_object_ids if TYPE_CHECKING: @@ -793,6 +795,12 @@ class CustomFieldQueryFilter(Filter): class DocumentFilterSet(FilterSet): + has_duplicates = BooleanFilter(method="filter_has_duplicates") + + def __init__(self, *args: Any, user: Any = None, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self._user = user + is_tagged = BooleanFilter( label="Is tagged", field_name="tags", @@ -852,6 +860,38 @@ class DocumentFilterSet(FilterSet): mime_type = MimeTypeFilter() + def filter_has_duplicates(self, queryset, name, value): + if value is None: + return queryset + + user = ( + self._user + if self._user is not None + else getattr(self.request, "user", None) + ) + queryset = queryset.alias( + nonempty_archive_checksum=NullIf("archive_checksum", Value("")), + ) + + visible_root_documents = Document.global_objects.filter( + root_document__isnull=True, + pk__in=permitted_document_ids( + user, + include_deleted=True, + ), + ).exclude(pk=OuterRef("pk")) + # see serialisers._get_viewable_duplicates(). + matching_duplicates = visible_root_documents.filter( + Q(checksum=OuterRef("checksum")) + | Q(checksum=OuterRef("nonempty_archive_checksum")) + | Q(archive_checksum=OuterRef("checksum")) + | Q(archive_checksum=OuterRef("nonempty_archive_checksum")), + ) + + return queryset.alias( + has_visible_duplicates=Exists(matching_duplicates), + ).filter(has_visible_duplicates=value) + # Backwards compatibility created__date__gt = DateFilter(field_name="created", lookup_expr="gt") created__date__gte = DateFilter(field_name="created", lookup_expr="gte") diff --git a/src/documents/migrations/0026_alter_document_archive_checksum_and_more.py b/src/documents/migrations/0026_alter_document_archive_checksum_and_more.py new file mode 100644 index 000000000..ea08e437c --- /dev/null +++ b/src/documents/migrations/0026_alter_document_archive_checksum_and_more.py @@ -0,0 +1,86 @@ +# Generated by Django 5.2.16 on 2026-09-05 16:29 + +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + dependencies = [ + ("documents", "0025_workflowaction_apply_ai_suggestions"), + ] + + operations = [ + migrations.AlterField( + model_name="document", + name="archive_checksum", + field=models.CharField( + blank=True, + db_index=True, + editable=False, + help_text="The checksum of the archived document.", + max_length=64, + null=True, + verbose_name="archive checksum", + ), + ), + migrations.AlterField( + model_name="savedviewfilterrule", + name="rule_type", + field=models.PositiveSmallIntegerField( + choices=[ + (0, "title contains"), + (1, "content contains"), + (2, "ASN is"), + (3, "correspondent is"), + (4, "document type is"), + (5, "is in inbox"), + (6, "has tag"), + (7, "has any tag"), + (8, "created before"), + (9, "created after"), + (10, "created year is"), + (11, "created month is"), + (12, "created day is"), + (13, "added before"), + (14, "added after"), + (15, "modified before"), + (16, "modified after"), + (17, "does not have tag"), + (18, "does not have ASN"), + (19, "title or content contains"), + (20, "fulltext query"), + (21, "more like this"), + (22, "has tags in"), + (23, "ASN greater than"), + (24, "ASN less than"), + (25, "storage path is"), + (26, "has correspondent in"), + (27, "does not have correspondent in"), + (28, "has document type in"), + (29, "does not have document type in"), + (30, "has storage path in"), + (31, "does not have storage path in"), + (32, "owner is"), + (33, "has owner in"), + (34, "does not have owner"), + (35, "does not have owner in"), + (36, "has custom field value"), + (37, "is shared by me"), + (38, "has custom fields"), + (39, "has custom field in"), + (40, "does not have custom field in"), + (41, "does not have custom field"), + (42, "custom fields query"), + (43, "created to"), + (44, "created from"), + (45, "added to"), + (46, "added from"), + (47, "mime type is"), + (48, "simple title search"), + (49, "simple text search"), + (50, "has duplicates"), + ], + verbose_name="rule type", + ), + ), + ] diff --git a/src/documents/models.py b/src/documents/models.py index 8dc3090f2..0cc4decda 100644 --- a/src/documents/models.py +++ b/src/documents/models.py @@ -227,6 +227,7 @@ class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager- editable=False, blank=True, null=True, + db_index=True, help_text=_("The checksum of the archived document."), ) @@ -706,6 +707,7 @@ class SavedViewFilterRule(models.Model): (47, _("mime type is")), (48, _("simple title search")), (49, _("simple text search")), + (50, _("has duplicates")), ] saved_view = models.ForeignKey( diff --git a/src/documents/tests/test_api_bulk_edit.py b/src/documents/tests/test_api_bulk_edit.py index cf447501c..2f34b2425 100644 --- a/src/documents/tests/test_api_bulk_edit.py +++ b/src/documents/tests/test_api_bulk_edit.py @@ -717,6 +717,44 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): self.assertEqual(args[0], [self.doc2.id]) self.assertEqual(kwargs["storage_path"], self.sp1.id) + @mock.patch("documents.serialisers.bulk_edit.set_storage_path") + def test_api_bulk_edit_with_all_true_resolves_owned_duplicates(self, m) -> None: + self.setup_mock(m, "set_storage_path") + user = User.objects.create_user(username="duplicate-owner") + user.user_permissions.add( + Permission.objects.get(codename="change_document"), + ) + first_duplicate = Document.objects.create( + checksum="owned-duplicate", + title="First duplicate", + owner=user, + ) + second_duplicate = Document.objects.create( + checksum="owned-duplicate", + title="Second duplicate", + owner=user, + ) + self.client.force_authenticate(user=user) + + response = self.client.post( + "/api/documents/bulk_edit/", + json.dumps( + { + "all": True, + "filters": {"has_duplicates": True}, + "method": "set_storage_path", + "parameters": {"storage_path": self.sp1.id}, + }, + ), + content_type="application/json", + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + m.assert_called_once() + args, kwargs = m.call_args + self.assertCountEqual(args[0], [first_duplicate.id, second_duplicate.id]) + self.assertEqual(kwargs["storage_path"], self.sp1.id) + @mock.patch("documents.search.get_backend") @mock.patch("documents.serialisers.bulk_edit.set_storage_path") def test_api_bulk_edit_with_all_true_resolves_documents_from_search_filters( diff --git a/src/documents/tests/test_api_documents.py b/src/documents/tests/test_api_documents.py index b67b87772..2aceb3fc9 100644 --- a/src/documents/tests/test_api_documents.py +++ b/src/documents/tests/test_api_documents.py @@ -981,6 +981,128 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase): self.assertEqual(len(results), 1) self.assertEqual(results[0]["id"], doc.id) + def test_has_duplicates_filter(self) -> None: + original_match = Document.objects.create( + title="original match", + checksum="same-original", + ) + second_original_match = Document.objects.create( + title="second original match", + checksum="same-original", + ) + archive_match = Document.objects.create( + title="archive match", + checksum="archive-source", + archive_checksum="same-archive", + ) + original_to_archive_match = Document.objects.create( + title="original to archive match", + checksum="same-archive", + ) + first_archive_match = Document.objects.create( + title="first archive match", + checksum="first-archive-source", + archive_checksum="same-archive-only", + ) + second_archive_match = Document.objects.create( + title="second archive match", + checksum="second-archive-source", + archive_checksum="same-archive-only", + ) + first_empty_archive = Document.objects.create( + title="first empty archive", + checksum="first-empty-archive", + archive_checksum="", + ) + second_empty_archive = Document.objects.create( + title="second empty archive", + checksum="second-empty-archive", + archive_checksum="", + ) + unique = Document.objects.create(title="unique", checksum="unique") + version_root = Document.objects.create( + title="version root", + checksum="version-root", + ) + Document.objects.create( + title="version", + checksum=unique.checksum, + root_document=version_root, + version_index=1, + ) + trash_match = Document.objects.create( + title="trash match", + checksum="trash-match", + ) + trashed_duplicate = Document.objects.create( + title="trashed duplicate", + checksum="trash-match", + ) + trashed_duplicate.delete() + + response = self.client.get("/api/documents/?has_duplicates=true") + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertCountEqual( + [document["id"] for document in response.data["results"]], + [ + original_match.id, + second_original_match.id, + archive_match.id, + original_to_archive_match.id, + first_archive_match.id, + second_archive_match.id, + trash_match.id, + ], + ) + + response = self.client.get("/api/documents/?has_duplicates=false") + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertCountEqual( + [document["id"] for document in response.data["results"]], + [ + unique.id, + version_root.id, + first_empty_archive.id, + second_empty_archive.id, + ], + ) + + response = self.client.get(f"/api/documents/{first_empty_archive.id}/") + self.assertEqual(response.data["duplicate_documents"], []) + + def test_has_duplicates_filter_respects_document_permissions(self) -> None: + owner = User.objects.create_user(username="duplicate-owner") + requester = User.objects.create_user(username="duplicate-requester") + requester.user_permissions.add( + Permission.objects.get(codename="view_document"), + ) + visible_document = Document.objects.create( + title="visible document", + checksum="permission-match", + owner=requester, + ) + hidden_duplicate = Document.objects.create( + title="hidden duplicate", + checksum="permission-match", + owner=owner, + ) + self.client.force_authenticate(user=requester) + + response = self.client.get("/api/documents/?has_duplicates=true") + self.assertNotIn( + visible_document.id, + [document["id"] for document in response.data["results"]], + ) + + assign_perm("view_document", requester, hidden_duplicate) + response = self.client.get("/api/documents/?has_duplicates=true") + self.assertIn( + visible_document.id, + [document["id"] for document in response.data["results"]], + ) + def test_custom_fields_icontains_filter_no_duplicates(self) -> None: """ GIVEN: diff --git a/src/documents/views.py b/src/documents/views.py index fadd160c4..0d256115a 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -2816,6 +2816,7 @@ class DocumentSelectionMixin: filtered_documents = DocumentFilterSet( data=orm_filters, queryset=permitted_documents, + user=user, ).qs.distinct() # tantivy-filtered docs (if search params provided) search_filtered_ids = self._get_search_document_ids(