Compare commits

...
Author SHA1 Message Date
shamoon 8af982d85f Exclude empty checksums (Codex) 2026-09-05 10:56:33 -07:00
shamoon b8c9215a21 Thread the user through for bulk edit all 2026-09-05 10:39:52 -07:00
shamoon 4ab91dfea4 Add to the filter editor 2026-09-05 09:45:47 -07:00
shamoon 28f33763cc Add the duplicates filter to frontend query-params 2026-09-05 09:41:13 -07:00
shamoon ae507afda3 Backend tests for the filter 2026-09-05 09:36:09 -07:00
shamoon 512c52e129 Add the filter, migration 2026-09-05 09:34:13 -07:00
11 changed files with 459 additions and 4 deletions
@@ -10,7 +10,7 @@
}
</div>
</div>
@if (textFilterTarget === 'asn') {
@if (textFilterTarget === 'asn' || textFilterTarget === 'duplicates') {
<select class="form-select flex-grow-0 w-auto" [(ngModel)]="textFilterModifier" (change)="textFilterModifierChange()">
@for (m of textFilterModifiers; track m) {
<option ngbDropdownItem [value]="m.id">{{m.label}}</option>
@@ -23,7 +23,7 @@
</button>
}
<input #textFilterInput class="form-control form-control-sm" type="text"
[disabled]="textFilterModifierIsNull"
[disabled]="textFilterInputDisabled"
[(ngModel)]="textFilter"
(keydown)="textFilterKeydown($event)"
[ngbTypeahead]="searchAutoComplete"
@@ -53,6 +53,7 @@ import {
FILTER_HAS_CUSTOM_FIELDS_ALL,
FILTER_HAS_CUSTOM_FIELDS_ANY,
FILTER_HAS_DOCUMENT_TYPE_ANY,
FILTER_HAS_DUPLICATES,
FILTER_HAS_STORAGE_PATH_ANY,
FILTER_HAS_TAGS_ALL,
FILTER_HAS_TAGS_ANY,
@@ -427,6 +428,38 @@ describe('FilterEditorComponent', () => {
expect(component.textFilterTarget).toEqual('mime-type') // TEXT_FILTER_TARGET_MIME_TYPE
})
it('should ingest filter rules for documents with duplicates', () => {
component.filterRules = [
{
rule_type: FILTER_HAS_DUPLICATES,
value: 'true',
},
]
fixture.detectChanges()
expect(component.textFilterTarget).toEqual('duplicates')
expect(component.textFilterModifier).toEqual('has-duplicates')
expect(component.textFilterInputDisabled).toBeTruthy()
})
it('should ingest filter rules for documents without duplicates', () => {
component.filterRules = [
{
rule_type: FILTER_HAS_DUPLICATES,
value: 'false',
},
]
expect(component.textFilterTarget).toEqual('duplicates')
expect(component.textFilterModifier).toEqual('does-not-have-duplicates')
expect(component.filterRules).toEqual([
{
rule_type: FILTER_HAS_DUPLICATES,
value: 'false',
},
])
})
it('should ingest text filter rules for fulltext query', () => {
expect(component.textFilter).toEqual(null)
component.filterRules = [
@@ -1390,6 +1423,33 @@ describe('FilterEditorComponent', () => {
])
})
it('should convert duplicate target input to the correct filter rule', () => {
const textFieldTargetDropdown = fixture.debugElement.queryAll(
By.directive(NgbDropdownItem)
)[5]
textFieldTargetDropdown.triggerEventHandler('click')
fixture.detectChanges()
expect(component.textFilterTarget).toEqual('duplicates')
expect(component.filterRules).toEqual([
{
rule_type: FILTER_HAS_DUPLICATES,
value: 'true',
},
])
const textFieldModifierSelect = fixture.debugElement.query(By.css('select'))
textFieldModifierSelect.nativeElement.value = 'does-not-have-duplicates'
textFieldModifierSelect.nativeElement.dispatchEvent(new Event('change'))
fixture.detectChanges()
expect(component.filterRules).toEqual([
{
rule_type: FILTER_HAS_DUPLICATES,
value: 'false',
},
])
})
it('should convert user input to correct filter rules on full text query', () => {
component.textFilterInput.nativeElement.value = 'foo'
component.textFilterInput.nativeElement.dispatchEvent(new Event('input'))
@@ -2178,6 +2238,22 @@ describe('FilterEditorComponent', () => {
]
expect(component.generateFilterName()).toEqual('Without any tag')
component.filterRules = [
{
rule_type: FILTER_HAS_DUPLICATES,
value: 'true',
},
]
expect(component.generateFilterName()).toEqual('With duplicates')
component.filterRules = [
{
rule_type: FILTER_HAS_DUPLICATES,
value: 'false',
},
]
expect(component.generateFilterName()).toEqual('Without duplicates')
component.filterRules = [
{
rule_type: FILTER_CUSTOM_FIELDS_QUERY,
@@ -65,6 +65,7 @@ import {
FILTER_HAS_CUSTOM_FIELDS_ALL,
FILTER_HAS_CUSTOM_FIELDS_ANY,
FILTER_HAS_DOCUMENT_TYPE_ANY,
FILTER_HAS_DUPLICATES,
FILTER_HAS_STORAGE_PATH_ANY,
FILTER_HAS_TAGS_ALL,
FILTER_HAS_TAGS_ANY,
@@ -129,12 +130,15 @@ const TEXT_FILTER_TARGET_FULLTEXT_QUERY = 'fulltext-query'
const TEXT_FILTER_TARGET_FULLTEXT_MORELIKE = 'fulltext-morelike'
const TEXT_FILTER_TARGET_CUSTOM_FIELDS = 'custom-fields'
const TEXT_FILTER_TARGET_MIME_TYPE = 'mime-type'
const TEXT_FILTER_TARGET_DUPLICATES = 'duplicates'
const TEXT_FILTER_MODIFIER_EQUALS = 'equals'
const TEXT_FILTER_MODIFIER_NULL = 'is null'
const TEXT_FILTER_MODIFIER_NOTNULL = 'not null'
const TEXT_FILTER_MODIFIER_GT = 'greater'
const TEXT_FILTER_MODIFIER_LT = 'less'
const TEXT_FILTER_MODIFIER_HAS_DUPLICATES = 'has-duplicates'
const TEXT_FILTER_MODIFIER_DOES_NOT_HAVE_DUPLICATES = 'does-not-have-duplicates'
const RELATIVE_DATE_QUERY_REGEXP_CREATED = /created:[\["]([^\]]+)[\]"]/g
const RELATIVE_DATE_QUERY_REGEXP_ADDED = /added:[\["]([^\]]+)[\]"]/g
@@ -205,6 +209,7 @@ const DEFAULT_TEXT_FILTER_TARGET_OPTIONS = [
id: TEXT_FILTER_TARGET_FULLTEXT_QUERY,
name: $localize`Advanced search`,
},
{ id: TEXT_FILTER_TARGET_DUPLICATES, name: $localize`Duplicates` },
]
const DEPRECATED_CUSTOM_FIELDS_TEXT_FILTER_TARGET_OPTION = {
@@ -241,6 +246,17 @@ const DEFAULT_TEXT_FILTER_MODIFIER_OPTIONS = [
},
]
const DUPLICATES_FILTER_MODIFIER_OPTIONS = [
{
id: TEXT_FILTER_MODIFIER_HAS_DUPLICATES,
label: $localize`exist`,
},
{
id: TEXT_FILTER_MODIFIER_DOES_NOT_HAVE_DUPLICATES,
label: $localize`do not exist`,
},
]
@Component({
selector: 'pngx-filter-editor',
templateUrl: './filter-editor.component.html',
@@ -320,6 +336,12 @@ export class FilterEditorComponent
if (rule.value == 'false') {
return $localize`Without any tag`
}
break
case FILTER_HAS_DUPLICATES:
return rule.value == 'false'
? $localize`Without duplicates`
: $localize`With duplicates`
case FILTER_CUSTOM_FIELDS_QUERY:
return $localize`Custom fields query`
@@ -390,7 +412,9 @@ export class FilterEditorComponent
public textFilterModifier: string
get textFilterModifiers() {
return DEFAULT_TEXT_FILTER_MODIFIER_OPTIONS
return this.textFilterTarget === TEXT_FILTER_TARGET_DUPLICATES
? DUPLICATES_FILTER_MODIFIER_OPTIONS
: DEFAULT_TEXT_FILTER_MODIFIER_OPTIONS
}
get textFilterModifierIsNull(): boolean {
@@ -399,6 +423,13 @@ export class FilterEditorComponent
)
}
get textFilterInputDisabled(): boolean {
return (
this.textFilterModifierIsNull ||
this.textFilterTarget === TEXT_FILTER_TARGET_DUPLICATES
)
}
tagSelectionModel = new FilterableDropdownSelectionModel(true)
correspondentSelectionModel = new FilterableDropdownSelectionModel()
documentTypeSelectionModel = new FilterableDropdownSelectionModel()
@@ -444,6 +475,7 @@ export class FilterEditorComponent
this.customFieldQueriesModel.clear(false)
this._textFilter = null
this._moreLikeId = null
this.textFilterTarget = TEXT_FILTER_TARGET_TITLE_CONTENT
this.dateAddedTo = null
this.dateAddedFrom = null
this.dateCreatedTo = null
@@ -477,6 +509,13 @@ export class FilterEditorComponent
this.textFilterTarget = TEXT_FILTER_TARGET_MIME_TYPE
this._textFilter = rule.value
break
case FILTER_HAS_DUPLICATES:
this.textFilterTarget = TEXT_FILTER_TARGET_DUPLICATES
this.textFilterModifier =
rule.value == 'false' || rule.value == '0'
? TEXT_FILTER_MODIFIER_DOES_NOT_HAVE_DUPLICATES
: TEXT_FILTER_MODIFIER_HAS_DUPLICATES
break
case FILTER_FULLTEXT_QUERY:
let allQueryArgs = rule.value.split(',')
let textQueryArgs = []
@@ -800,6 +839,14 @@ export class FilterEditorComponent
value: this._textFilter.trim(),
})
}
if (this.textFilterTarget == TEXT_FILTER_TARGET_DUPLICATES) {
filterRules.push({
rule_type: FILTER_HAS_DUPLICATES,
value: (
this.textFilterModifier == TEXT_FILTER_MODIFIER_HAS_DUPLICATES
).toString(),
})
}
if (this._textFilter && this.textFilterTarget == TEXT_FILTER_TARGET_TITLE) {
filterRules.push({
rule_type: FILTER_SIMPLE_TITLE,
@@ -1163,7 +1210,7 @@ export class FilterEditorComponent
}
get textFilter() {
return this.textFilterModifierIsNull ? '' : this._textFilter
return this.textFilterInputDisabled ? '' : this._textFilter
}
set textFilter(value) {
@@ -1363,12 +1410,24 @@ export class FilterEditorComponent
this._textFilter = ''
}
this.textFilterTarget = target
if (target == TEXT_FILTER_TARGET_DUPLICATES) {
this._textFilter = ''
this.textFilterModifier = TEXT_FILTER_MODIFIER_HAS_DUPLICATES
} else if (
[
TEXT_FILTER_MODIFIER_HAS_DUPLICATES,
TEXT_FILTER_MODIFIER_DOES_NOT_HAVE_DUPLICATES,
].includes(this.textFilterModifier)
) {
this.textFilterModifier = TEXT_FILTER_MODIFIER_EQUALS
}
this.textFilterInput.nativeElement.focus()
this.updateRules()
}
textFilterModifierChange() {
if (
this.textFilterTarget == TEXT_FILTER_TARGET_DUPLICATES ||
this.textFilterModifierIsNull ||
([
TEXT_FILTER_MODIFIER_EQUALS,
+8
View File
@@ -49,6 +49,7 @@ export const FILTER_MODIFIED_AFTER = 16
export const FILTER_TITLE_CONTENT = 19 // Deprecated in favor of Tantivy-backed `text` filtervar. Keep for now for existing saved views
export const FILTER_SIMPLE_TITLE = 48
export const FILTER_SIMPLE_TEXT = 49
export const FILTER_HAS_DUPLICATES = 50
export const FILTER_FULLTEXT_QUERY = 20
export const FILTER_FULLTEXT_MORELIKE = 21
@@ -382,6 +383,13 @@ export const FILTER_RULE_TYPES: FilterRuleType[] = [
datatype: 'string',
multi: false,
},
{
id: FILTER_HAS_DUPLICATES,
filtervar: 'has_duplicates',
datatype: 'boolean',
multi: false,
default: true,
},
]
export interface FilterRuleType {
+23
View File
@@ -7,6 +7,7 @@ import {
FILTER_HAS_ANY_TAG,
FILTER_HAS_CUSTOM_FIELDS_ALL,
FILTER_HAS_CUSTOM_FIELDS_ANY,
FILTER_HAS_DUPLICATES,
FILTER_HAS_TAGS_ALL,
FILTER_SIMPLE_TEXT,
FILTER_SIMPLE_TITLE,
@@ -132,6 +133,16 @@ describe('QueryParams Utils', () => {
is_tagged: 0,
})
params = queryParamsFromFilterRules([
{
rule_type: FILTER_HAS_DUPLICATES,
value: 'false',
},
])
expect(params).toEqual({
has_duplicates: 0,
})
params = queryParamsFromFilterRules([
{
rule_type: FILTER_TITLE_CONTENT,
@@ -247,6 +258,18 @@ describe('QueryParams Utils', () => {
},
])
rules = filterRulesFromQueryParams(
convertToParamMap({
has_duplicates: 'true',
})
)
expect(rules).toEqual([
{
rule_type: FILTER_HAS_DUPLICATES,
value: 'true',
},
])
rules = filterRulesFromQueryParams(
convertToParamMap({
correspondent__isnull: '1',
+40
View File
@@ -25,6 +25,7 @@ from django.db.models import Sum
from django.db.models import Value
from django.db.models import When
from django.db.models.functions import Cast
from django.db.models.functions import NullIf
from django.utils.translation import gettext_lazy as _
from django_filters import DateFilter
from django_filters.rest_framework import BooleanFilter
@@ -50,6 +51,7 @@ from documents.models import ShareLink
from documents.models import ShareLinkBundle
from documents.models import StoragePath
from documents.models import Tag
from documents.permissions import permitted_document_ids
from documents.permissions import permitted_object_ids
if TYPE_CHECKING:
@@ -793,6 +795,12 @@ class CustomFieldQueryFilter(Filter):
class DocumentFilterSet(FilterSet):
has_duplicates = BooleanFilter(method="filter_has_duplicates")
def __init__(self, *args: Any, user: Any = None, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)
self._user = user
is_tagged = BooleanFilter(
label="Is tagged",
field_name="tags",
@@ -852,6 +860,38 @@ class DocumentFilterSet(FilterSet):
mime_type = MimeTypeFilter()
def filter_has_duplicates(self, queryset, name, value):
if value is None:
return queryset
user = (
self._user
if self._user is not None
else getattr(self.request, "user", None)
)
queryset = queryset.alias(
nonempty_archive_checksum=NullIf("archive_checksum", Value("")),
)
visible_root_documents = Document.global_objects.filter(
root_document__isnull=True,
pk__in=permitted_document_ids(
user,
include_deleted=True,
),
).exclude(pk=OuterRef("pk"))
# see serialisers._get_viewable_duplicates().
matching_duplicates = visible_root_documents.filter(
Q(checksum=OuterRef("checksum"))
| Q(checksum=OuterRef("nonempty_archive_checksum"))
| Q(archive_checksum=OuterRef("checksum"))
| Q(archive_checksum=OuterRef("nonempty_archive_checksum")),
)
return queryset.alias(
has_visible_duplicates=Exists(matching_duplicates),
).filter(has_visible_duplicates=value)
# Backwards compatibility
created__date__gt = DateFilter(field_name="created", lookup_expr="gt")
created__date__gte = DateFilter(field_name="created", lookup_expr="gte")
@@ -0,0 +1,86 @@
# Generated by Django 5.2.16 on 2026-09-05 16:29
from django.db import migrations
from django.db import models
class Migration(migrations.Migration):
dependencies = [
("documents", "0025_workflowaction_apply_ai_suggestions"),
]
operations = [
migrations.AlterField(
model_name="document",
name="archive_checksum",
field=models.CharField(
blank=True,
db_index=True,
editable=False,
help_text="The checksum of the archived document.",
max_length=64,
null=True,
verbose_name="archive checksum",
),
),
migrations.AlterField(
model_name="savedviewfilterrule",
name="rule_type",
field=models.PositiveSmallIntegerField(
choices=[
(0, "title contains"),
(1, "content contains"),
(2, "ASN is"),
(3, "correspondent is"),
(4, "document type is"),
(5, "is in inbox"),
(6, "has tag"),
(7, "has any tag"),
(8, "created before"),
(9, "created after"),
(10, "created year is"),
(11, "created month is"),
(12, "created day is"),
(13, "added before"),
(14, "added after"),
(15, "modified before"),
(16, "modified after"),
(17, "does not have tag"),
(18, "does not have ASN"),
(19, "title or content contains"),
(20, "fulltext query"),
(21, "more like this"),
(22, "has tags in"),
(23, "ASN greater than"),
(24, "ASN less than"),
(25, "storage path is"),
(26, "has correspondent in"),
(27, "does not have correspondent in"),
(28, "has document type in"),
(29, "does not have document type in"),
(30, "has storage path in"),
(31, "does not have storage path in"),
(32, "owner is"),
(33, "has owner in"),
(34, "does not have owner"),
(35, "does not have owner in"),
(36, "has custom field value"),
(37, "is shared by me"),
(38, "has custom fields"),
(39, "has custom field in"),
(40, "does not have custom field in"),
(41, "does not have custom field"),
(42, "custom fields query"),
(43, "created to"),
(44, "created from"),
(45, "added to"),
(46, "added from"),
(47, "mime type is"),
(48, "simple title search"),
(49, "simple text search"),
(50, "has duplicates"),
],
verbose_name="rule type",
),
),
]
+2
View File
@@ -227,6 +227,7 @@ class Document(SoftDeleteModel, ModelWithOwner): # type: ignore[django-manager-
editable=False,
blank=True,
null=True,
db_index=True,
help_text=_("The checksum of the archived document."),
)
@@ -706,6 +707,7 @@ class SavedViewFilterRule(models.Model):
(47, _("mime type is")),
(48, _("simple title search")),
(49, _("simple text search")),
(50, _("has duplicates")),
]
saved_view = models.ForeignKey(
+38
View File
@@ -717,6 +717,44 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
self.assertEqual(args[0], [self.doc2.id])
self.assertEqual(kwargs["storage_path"], self.sp1.id)
@mock.patch("documents.serialisers.bulk_edit.set_storage_path")
def test_api_bulk_edit_with_all_true_resolves_owned_duplicates(self, m) -> None:
self.setup_mock(m, "set_storage_path")
user = User.objects.create_user(username="duplicate-owner")
user.user_permissions.add(
Permission.objects.get(codename="change_document"),
)
first_duplicate = Document.objects.create(
checksum="owned-duplicate",
title="First duplicate",
owner=user,
)
second_duplicate = Document.objects.create(
checksum="owned-duplicate",
title="Second duplicate",
owner=user,
)
self.client.force_authenticate(user=user)
response = self.client.post(
"/api/documents/bulk_edit/",
json.dumps(
{
"all": True,
"filters": {"has_duplicates": True},
"method": "set_storage_path",
"parameters": {"storage_path": self.sp1.id},
},
),
content_type="application/json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
m.assert_called_once()
args, kwargs = m.call_args
self.assertCountEqual(args[0], [first_duplicate.id, second_duplicate.id])
self.assertEqual(kwargs["storage_path"], self.sp1.id)
@mock.patch("documents.search.get_backend")
@mock.patch("documents.serialisers.bulk_edit.set_storage_path")
def test_api_bulk_edit_with_all_true_resolves_documents_from_search_filters(
+122
View File
@@ -981,6 +981,128 @@ class TestDocumentApi(DirectoriesMixin, ConsumeTaskMixin, APITestCase):
self.assertEqual(len(results), 1)
self.assertEqual(results[0]["id"], doc.id)
def test_has_duplicates_filter(self) -> None:
original_match = Document.objects.create(
title="original match",
checksum="same-original",
)
second_original_match = Document.objects.create(
title="second original match",
checksum="same-original",
)
archive_match = Document.objects.create(
title="archive match",
checksum="archive-source",
archive_checksum="same-archive",
)
original_to_archive_match = Document.objects.create(
title="original to archive match",
checksum="same-archive",
)
first_archive_match = Document.objects.create(
title="first archive match",
checksum="first-archive-source",
archive_checksum="same-archive-only",
)
second_archive_match = Document.objects.create(
title="second archive match",
checksum="second-archive-source",
archive_checksum="same-archive-only",
)
first_empty_archive = Document.objects.create(
title="first empty archive",
checksum="first-empty-archive",
archive_checksum="",
)
second_empty_archive = Document.objects.create(
title="second empty archive",
checksum="second-empty-archive",
archive_checksum="",
)
unique = Document.objects.create(title="unique", checksum="unique")
version_root = Document.objects.create(
title="version root",
checksum="version-root",
)
Document.objects.create(
title="version",
checksum=unique.checksum,
root_document=version_root,
version_index=1,
)
trash_match = Document.objects.create(
title="trash match",
checksum="trash-match",
)
trashed_duplicate = Document.objects.create(
title="trashed duplicate",
checksum="trash-match",
)
trashed_duplicate.delete()
response = self.client.get("/api/documents/?has_duplicates=true")
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertCountEqual(
[document["id"] for document in response.data["results"]],
[
original_match.id,
second_original_match.id,
archive_match.id,
original_to_archive_match.id,
first_archive_match.id,
second_archive_match.id,
trash_match.id,
],
)
response = self.client.get("/api/documents/?has_duplicates=false")
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertCountEqual(
[document["id"] for document in response.data["results"]],
[
unique.id,
version_root.id,
first_empty_archive.id,
second_empty_archive.id,
],
)
response = self.client.get(f"/api/documents/{first_empty_archive.id}/")
self.assertEqual(response.data["duplicate_documents"], [])
def test_has_duplicates_filter_respects_document_permissions(self) -> None:
owner = User.objects.create_user(username="duplicate-owner")
requester = User.objects.create_user(username="duplicate-requester")
requester.user_permissions.add(
Permission.objects.get(codename="view_document"),
)
visible_document = Document.objects.create(
title="visible document",
checksum="permission-match",
owner=requester,
)
hidden_duplicate = Document.objects.create(
title="hidden duplicate",
checksum="permission-match",
owner=owner,
)
self.client.force_authenticate(user=requester)
response = self.client.get("/api/documents/?has_duplicates=true")
self.assertNotIn(
visible_document.id,
[document["id"] for document in response.data["results"]],
)
assign_perm("view_document", requester, hidden_duplicate)
response = self.client.get("/api/documents/?has_duplicates=true")
self.assertIn(
visible_document.id,
[document["id"] for document in response.data["results"]],
)
def test_custom_fields_icontains_filter_no_duplicates(self) -> None:
"""
GIVEN:
+1
View File
@@ -2815,6 +2815,7 @@ class DocumentSelectionMixin:
filtered_documents = DocumentFilterSet(
data=orm_filters,
queryset=permitted_documents,
user=user,
).qs.distinct()
# tantivy-filtered docs (if search params provided)
search_filtered_ids = self._get_search_document_ids(