Compare commits

...
11 changed files with 329 additions and 30 deletions
+71 -22
View File
@@ -5973,7 +5973,7 @@
<source>Open <x id="PH" equiv-text="this.title"/> filter</source> <source>Open <x id="PH" equiv-text="this.title"/> filter</source>
<context-group purpose="location"> <context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/filterable-dropdown/filterable-dropdown.component.ts</context> <context context-type="sourcefile">src/app/components/common/filterable-dropdown/filterable-dropdown.component.ts</context>
<context context-type="linenumber">828</context> <context context-type="linenumber">831</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="7005745151564974365" datatype="html"> <trans-unit id="7005745151564974365" datatype="html">
@@ -6382,27 +6382,6 @@
<context context-type="linenumber">94</context> <context context-type="linenumber">94</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="5947558132119506443" datatype="html">
<source>My documents</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/permissions-filter-dropdown/permissions-filter-dropdown.component.html</context>
<context context-type="linenumber">25,26</context>
</context-group>
</trans-unit>
<trans-unit id="231920238966427751" datatype="html">
<source>Shared with me</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/permissions-filter-dropdown/permissions-filter-dropdown.component.html</context>
<context context-type="linenumber">35,36</context>
</context-group>
</trans-unit>
<trans-unit id="175385209536581523" datatype="html">
<source>Shared by me</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/permissions-filter-dropdown/permissions-filter-dropdown.component.html</context>
<context context-type="linenumber">45,46</context>
</context-group>
</trans-unit>
<trans-unit id="5151074932731293042" datatype="html"> <trans-unit id="5151074932731293042" datatype="html">
<source>Unowned</source> <source>Unowned</source>
<context-group purpose="location"> <context-group purpose="location">
@@ -6417,6 +6396,76 @@
<context context-type="linenumber">85</context> <context context-type="linenumber">85</context>
</context-group> </context-group>
</trans-unit> </trans-unit>
<trans-unit id="5947558132119506443" datatype="html">
<source>My documents</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/permissions-filter-dropdown/permissions-filter-dropdown.component.ts</context>
<context context-type="linenumber">101</context>
</context-group>
</trans-unit>
<trans-unit id="1930869169119109336" datatype="html">
<source>Owned by <x id="PH" equiv-text="username"/></source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/permissions-filter-dropdown/permissions-filter-dropdown.component.ts</context>
<context context-type="linenumber">106</context>
</context-group>
</trans-unit>
<trans-unit id="5339682692608120628" datatype="html">
<source>Owned by another user</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/permissions-filter-dropdown/permissions-filter-dropdown.component.ts</context>
<context context-type="linenumber">107</context>
</context-group>
</trans-unit>
<trans-unit id="231920238966427751" datatype="html">
<source>Shared with me</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/permissions-filter-dropdown/permissions-filter-dropdown.component.ts</context>
<context context-type="linenumber">117</context>
</context-group>
</trans-unit>
<trans-unit id="1894556100995563325" datatype="html">
<source>Not owned by <x id="PH" equiv-text="usernames.join(&apos;, &apos;)"/></source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/permissions-filter-dropdown/permissions-filter-dropdown.component.ts</context>
<context context-type="linenumber">124</context>
</context-group>
</trans-unit>
<trans-unit id="4647949080250052038" datatype="html">
<source>Not owned by another user</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/permissions-filter-dropdown/permissions-filter-dropdown.component.ts</context>
<context context-type="linenumber">127</context>
</context-group>
</trans-unit>
<trans-unit id="8858352775080403297" datatype="html">
<source>Not owned by selected users</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/permissions-filter-dropdown/permissions-filter-dropdown.component.ts</context>
<context context-type="linenumber">128</context>
</context-group>
</trans-unit>
<trans-unit id="175385209536581523" datatype="html">
<source>Shared by me</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/permissions-filter-dropdown/permissions-filter-dropdown.component.ts</context>
<context context-type="linenumber">136</context>
</context-group>
</trans-unit>
<trans-unit id="5140574576358170412" datatype="html">
<source>Shared by <x id="PH" equiv-text="username"/></source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/permissions-filter-dropdown/permissions-filter-dropdown.component.ts</context>
<context context-type="linenumber">141</context>
</context-group>
</trans-unit>
<trans-unit id="391557549689505150" datatype="html">
<source>Shared by another user</source>
<context-group purpose="location">
<context context-type="sourcefile">src/app/components/common/permissions-filter-dropdown/permissions-filter-dropdown.component.ts</context>
<context context-type="linenumber">142</context>
</context-group>
</trans-unit>
<trans-unit id="941924371433275463" datatype="html"> <trans-unit id="941924371433275463" datatype="html">
<source>Global permissions define what areas of the app and API endpoints users can access.</source> <source>Global permissions define what areas of the app and API endpoints users can access.</source>
<context-group purpose="location"> <context-group purpose="location">
@@ -1,4 +1,4 @@
<div ngbDropdown #fieldDropdown="ngbDropdown" (openChange)="onOpenClose($event)" [popperOptions]="popperOptions"> <div ngbDropdown #fieldDropdown="ngbDropdown" (openChange)="onOpenClose($event)" placement="bottom-end" [popperOptions]="popperOptions">
<button type="button" class="btn btn-sm btn-outline-primary" id="customFieldsDropdown" [disabled]="disabled" ngbDropdownToggle> <button type="button" class="btn btn-sm btn-outline-primary" id="customFieldsDropdown" [disabled]="disabled" ngbDropdownToggle>
<i-bs name="ui-radios"></i-bs><div class="d-none d-lg-inline ms-1"><ng-container i18n>Custom Fields</ng-container></div> <i-bs name="ui-radios"></i-bs><div class="d-none d-lg-inline ms-1"><ng-container i18n>Custom Fields</ng-container></div>
</button> </button>
@@ -6,5 +6,16 @@
&.show { &.show {
margin-left: -245px !important; margin-left: -245px !important;
} }
.list-group {
max-height: min(50dvh, 20rem);
overflow-y: auto;
}
.list-group-item:first-child {
position: sticky;
top: 0;
z-index: 1;
}
} }
} }
@@ -49,7 +49,7 @@
</cdk-virtual-scroll-viewport> </cdk-virtual-scroll-viewport>
} }
@if (editing) { @if (editing) {
@if (filteredItems.length === 0 && createRef !== undefined) { @if (filteredItems.length === 0 && createRef !== undefined && filterText?.length > 0) {
<button class="list-group-item list-group-item-action bg-light" (click)="createClicked()" [disabled]="disabled"> <button class="list-group-item list-group-item-action bg-light" (click)="createClicked()" [disabled]="disabled">
<small class="ms-2"><ng-container i18n>Create</ng-container> "{{filterText}}"</small> <small class="ms-2"><ng-container i18n>Create</ng-container> "{{filterText}}"</small>
<i-bs width="1.5em" height="1em" name="plus"></i-bs> <i-bs width="1.5em" height="1em" name="plus"></i-bs>
@@ -62,7 +62,7 @@
</button> </button>
} }
} }
@if (extraButtonTitle) { @if (extraButtonTitle && (showExtraButtonIfEmpty || filteredItems?.length > 0)) {
<button class="list-group-item list-group-item-action bg-light d-flex align-items-center" (click)="extraButtonClicked($event)" [disabled]="disabled"> <button class="list-group-item list-group-item-action bg-light d-flex align-items-center" (click)="extraButtonClicked($event)" [disabled]="disabled">
<small class="ms-2 fw-bold">{{extraButtonTitle}}</small> <small class="ms-2 fw-bold">{{extraButtonTitle}}</small>
<i-bs width="1.5em" height="1em" name="arrow-right"></i-bs> <i-bs width="1.5em" height="1em" name="arrow-right"></i-bs>
@@ -911,6 +911,25 @@ describe('FilterableDropdownComponent & FilterableDropdownSelectionModel', () =>
expect(createSpy).toHaveBeenCalled() expect(createSpy).toHaveBeenCalled()
}) })
it('should only show create when a non-empty filter has no matches', () => {
component.selectionModel.items = []
component.icon = 'tag-fill'
component.editing = true
component.createRef = jest.fn()
fixture.detectChanges()
expect(fixture.nativeElement.textContent).not.toContain('Create')
component.listFilterEnter()
expect(component.createRef).not.toHaveBeenCalled()
const filterInput: HTMLInputElement =
fixture.nativeElement.querySelector('input[type="text"]')
filterInput.value = 'FooBar'
filterInput.dispatchEvent(new Event('input'))
fixture.detectChanges()
expect(fixture.nativeElement.textContent).toContain('Create "FooBar"')
})
it('should exclude item and trigger change event', () => { it('should exclude item and trigger change event', () => {
const id = 1 const id = 1
const state = ToggleableItemState.Selected const state = ToggleableItemState.Selected
@@ -970,4 +989,18 @@ describe('FilterableDropdownComponent & FilterableDropdownSelectionModel', () =>
expect(extraButtonClicked).toBeTruthy() expect(extraButtonClicked).toBeTruthy()
expect(applied).toBeFalsy() expect(applied).toBeFalsy()
}) })
it('should only show the extra button for an empty result when enabled', () => {
component.selectionModel.items = items
component.icon = 'tag-fill'
component.extraButtonTitle = 'Extra'
component.filterText = 'FooBar'
fixture.detectChanges()
expect(fixture.nativeElement.textContent).not.toContain('Extra')
fixture.componentRef.setInput('showExtraButtonIfEmpty', true)
fixture.detectChanges()
expect(fixture.nativeElement.textContent).toContain('Extra')
})
}) })
@@ -774,6 +774,9 @@ export class FilterableDropdownComponent
@Input() @Input()
extraButtonTitle: string extraButtonTitle: string
@Input()
showExtraButtonIfEmpty: boolean = false
creating: boolean = false creating: boolean = false
@Output() @Output()
@@ -892,7 +895,11 @@ export class FilterableDropdownComponent
this.dropdown.close() this.dropdown.close()
} }
}, 200) }, 200)
} else if (filtered.length == 0 && this.createRef) { } else if (
filtered.length == 0 &&
this.createRef &&
this.filterText?.length > 0
) {
this.createClicked() this.createClicked()
} }
} }
@@ -22,7 +22,7 @@
} }
</div> </div>
<div class="me-1"> <div class="me-1">
<small i18n>My documents</small> <small>{{ownerFilterLabel}}</small>
</div> </div>
</button> </button>
<button class="list-group-item list-group-item-action d-flex align-items-center p-2 border-top-0 border-start-0 border-end-0 border-bottom" role="menuitem" (click)="setFilter(OwnerFilterType.NOT_SELF)" [disabled]="disabled"> <button class="list-group-item list-group-item-action d-flex align-items-center p-2 border-top-0 border-start-0 border-end-0 border-bottom" role="menuitem" (click)="setFilter(OwnerFilterType.NOT_SELF)" [disabled]="disabled">
@@ -32,7 +32,7 @@
} }
</div> </div>
<div class="me-1"> <div class="me-1">
<small i18n>Shared with me</small> <small>{{ownerExclusionFilterLabel}}</small>
</div> </div>
</button> </button>
<button class="list-group-item list-group-item-action d-flex align-items-center p-2 border-top-0 border-start-0 border-end-0 border-bottom" role="menuitem" (click)="setFilter(OwnerFilterType.SHARED_BY_ME)" [disabled]="disabled"> <button class="list-group-item list-group-item-action d-flex align-items-center p-2 border-top-0 border-start-0 border-end-0 border-bottom" role="menuitem" (click)="setFilter(OwnerFilterType.SHARED_BY_ME)" [disabled]="disabled">
@@ -42,7 +42,7 @@
} }
</div> </div>
<div class="me-1"> <div class="me-1">
<small i18n>Shared by me</small> <small>{{sharedByFilterLabel}}</small>
</div> </div>
</button> </button>
<button class="list-group-item list-group-item-action d-flex align-items-center p-2 border-top-0 border-start-0 border-end-0 border-bottom" role="menuitem" (click)="setFilter(OwnerFilterType.UNOWNED)" [disabled]="disabled"> <button class="list-group-item list-group-item-action d-flex align-items-center p-2 border-top-0 border-start-0 border-end-0 border-bottom" role="menuitem" (click)="setFilter(OwnerFilterType.UNOWNED)" [disabled]="disabled">
@@ -94,6 +94,58 @@ describe('PermissionsFilterDropdownComponent', () => {
expect(component.isActive).toBeTruthy() expect(component.isActive).toBeTruthy()
}) })
it('should describe concrete user filters honestly', () => {
component.selectionModel.ownerFilter = OwnerFilterType.SELF
component.selectionModel.userID = 1
expect(component.ownerFilterLabel).toEqual('Owned by user1')
component.selectionModel.ownerFilter = OwnerFilterType.NOT_SELF
component.selectionModel.excludeUsers = [1]
expect(component.ownerExclusionFilterLabel).toEqual('Not owned by user1')
component.selectionModel.ownerFilter = OwnerFilterType.SHARED_BY_ME
component.selectionModel.userID = 1
expect(component.sharedByFilterLabel).toEqual('Shared by user1')
})
it('should describe concrete filters when usernames are unavailable', () => {
component.selectionModel.ownerFilter = OwnerFilterType.SELF
component.selectionModel.userID = 99
expect(component.ownerFilterLabel).toEqual('Owned by another user')
component.selectionModel.ownerFilter = OwnerFilterType.NOT_SELF
component.selectionModel.excludeUsers = [99]
expect(component.ownerExclusionFilterLabel).toEqual(
'Not owned by another user'
)
component.selectionModel.excludeUsers = [98, 99]
expect(component.ownerExclusionFilterLabel).toEqual(
'Not owned by selected users'
)
component.selectionModel.ownerFilter = OwnerFilterType.SHARED_BY_ME
component.selectionModel.userID = 99
expect(component.sharedByFilterLabel).toEqual('Shared by another user')
})
it('should retain relative labels for filters bound to the current user', () => {
component.selectionModel.userID = currentUserID
expect(component.ownerFilterLabel).toEqual('My documents')
expect(component.sharedByFilterLabel).toEqual('Shared by me')
component.selectionModel.excludeUsers = [currentUserID]
expect(component.ownerExclusionFilterLabel).toEqual('Shared with me')
})
it('should retain relative labels for inactive filter choices', () => {
component.selectionModel.ownerFilter = OwnerFilterType.NONE
expect(component.ownerFilterLabel).toEqual('My documents')
expect(component.ownerExclusionFilterLabel).toEqual('Shared with me')
expect(component.sharedByFilterLabel).toEqual('Shared by me')
})
it('should support reset', () => { it('should support reset', () => {
component.setFilter(OwnerFilterType.OTHERS) component.setFilter(OwnerFilterType.OTHERS)
expect(component.selectionModel.ownerFilter).not.toEqual( expect(component.selectionModel.ownerFilter).not.toEqual(
@@ -93,6 +93,55 @@ export class PermissionsFilterDropdownComponent extends ComponentWithPermissions
) )
} }
get ownerFilterLabel(): string {
if (
this.selectionModel?.ownerFilter !== OwnerFilterType.SELF ||
this.selectionModel?.userID === this.settingsService.currentUser()?.id
) {
return $localize`My documents`
}
const username = this.getUsername(this.selectionModel?.userID)
return username
? $localize`Owned by ${username}`
: $localize`Owned by another user`
}
get ownerExclusionFilterLabel(): string {
const excludedUsers = this.selectionModel?.excludeUsers ?? []
if (
this.selectionModel?.ownerFilter !== OwnerFilterType.NOT_SELF ||
(excludedUsers.length === 1 &&
excludedUsers[0] === this.settingsService.currentUser()?.id)
) {
return $localize`Shared with me`
}
const usernames = excludedUsers
.map((id) => this.getUsername(id))
.filter(Boolean)
if (usernames.length === excludedUsers.length && usernames.length > 0) {
return $localize`Not owned by ${usernames.join(', ')}`
}
return excludedUsers.length === 1
? $localize`Not owned by another user`
: $localize`Not owned by selected users`
}
get sharedByFilterLabel(): string {
if (
this.selectionModel?.ownerFilter !== OwnerFilterType.SHARED_BY_ME ||
this.selectionModel?.userID === this.settingsService.currentUser()?.id
) {
return $localize`Shared by me`
}
const username = this.getUsername(this.selectionModel?.userID)
return username
? $localize`Shared by ${username}`
: $localize`Shared by another user`
}
constructor() { constructor() {
const userService = inject(UserService) const userService = inject(UserService)
@@ -164,4 +213,8 @@ export class PermissionsFilterDropdownComponent extends ComponentWithPermissions
} }
this.onChange() this.onChange()
} }
private getUsername(userID: number): string {
return this.users().find((user) => user.id === userID)?.username
}
} }
+21 -1
View File
@@ -223,7 +223,27 @@ class WriteBatch:
) )
time.sleep(sleep_s) time.sleep(sleep_s)
self._raw_writer = self._backend._index.writer() # Open a fresh Index (and thus a fresh Tantivy ManagedDirectory)
# for the write, rather than reusing the process-local cached
# index. ManagedDirectory loads its GC bookkeeping (.managed.json)
# once, at construction, and never re-reads it; paperless runs
# several long-lived processes (Granian workers, Celery workers)
# that take turns writing under the file lock above. A cached,
# long-lived writer index would carry a stale managed-files view
# and, on commit, overwrite .managed.json with that stale view -
# permanently losing track of segment files other processes
# registered in the meantime, so they can never be garbage
# collected. Reopening fresh here always picks up the current
# on-disk state. The long-lived self._backend._index is used for
# reads only and is reloaded (not reopened) after commit below.
write_index = tantivy.Index(
build_schema(),
path=str(self._backend._path),
)
register_tokenizers(write_index, settings.SEARCH_LANGUAGE)
self._raw_writer = write_index.writer()
else:
self._raw_writer = self._backend._index.writer()
return self return self
def __exit__(self, exc_type, exc_val, exc_tb): def __exit__(self, exc_type, exc_val, exc_tb):
@@ -1,3 +1,6 @@
import json
from pathlib import Path
import pytest import pytest
from django.contrib.auth.models import Group from django.contrib.auth.models import Group
from django.contrib.auth.models import User from django.contrib.auth.models import User
@@ -21,6 +24,17 @@ from documents.tests.factories import UserFactory
pytestmark = [pytest.mark.search, pytest.mark.django_db] pytestmark = [pytest.mark.search, pytest.mark.django_db]
# Extensions of actual Tantivy segment data files, as opposed to its own
# bookkeeping files (meta.json, .managed.json, lock files).
_SEGMENT_FILE_EXTENSIONS = (
".fast",
".fieldnorm",
".idx",
".pos",
".store",
".term",
)
class TestWriteBatch: class TestWriteBatch:
"""Test WriteBatch context manager functionality.""" """Test WriteBatch context manager functionality."""
@@ -1014,3 +1028,63 @@ class TestHighlightHits:
hits = backend.highlight_hits("quick", [doc.pk]) hits = backend.highlight_hits("quick", [doc.pk])
assert len(hits) == 0 assert len(hits) == 0
class TestIndexDirectoryGarbageCollection:
"""Regression tests for Tantivy segment files leaking on disk when
multiple long-lived worker processes (Granian/Celery) take turns writing
to the same on-disk index (issue #13679)."""
def test_no_permanently_orphaned_segment_files_across_worker_processes(
self,
tmp_path: Path,
) -> None:
"""Simulate two long-lived worker processes, each with its own
process-local ``TantivyBackend``/``Index`` opened once at process
start, alternating turns as the writer -- exactly how paperless runs
in production (several Granian + Celery worker processes).
Every segment file physically present on disk must still be tracked
in Tantivy's ``.managed.json`` bookkeeping; otherwise it can never be
garbage collected by anyone again and the index directory grows
without bound.
"""
index_dir = tmp_path / "index"
index_dir.mkdir()
worker_a = TantivyBackend(path=index_dir)
worker_a.open()
worker_b = TantivyBackend(path=index_dir)
worker_b.open()
workers = [worker_a, worker_b]
docs = [
DocumentFactory.create(checksum=f"GC{i}", title=f"gc doc {i}")
for i in range(5)
]
try:
# Alternate writers across many commits, repeatedly upserting the
# same documents so segments accumulate and get superseded,
# forcing the delete+add upsert pattern and eventual merges.
for i in range(30):
worker = workers[i % len(workers)]
doc = docs[i % len(docs)]
worker.add_or_update(doc)
finally:
worker_a.close()
worker_b.close()
managed_path = index_dir / ".managed.json"
managed = set(json.loads(managed_path.read_text()))
on_disk = {
p.name
for p in index_dir.iterdir()
if p.is_file() and p.suffix in _SEGMENT_FILE_EXTENSIONS
}
orphans = on_disk - managed
assert not orphans, (
"Segment files present on disk but absent from Tantivy's "
f".managed.json bookkeeping (permanently un-collectible): {orphans}"
)