From 14aaa42777d674187cd7cfb8e3efb1b1707225ac Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:17:22 -0700 Subject: [PATCH] Fix: support combined text/query searches (#14443) --- .../filter-editor.component.spec.ts | 56 ++++++++++++++--- .../filter-editor/filter-editor.component.ts | 17 ----- src/documents/search/_backend.py | 9 +++ src/documents/tests/test_api_bulk_edit.py | 3 +- src/documents/tests/test_api_search.py | 62 ++++++++++++++++++- src/documents/views.py | 56 ++++++++++------- 6 files changed, 152 insertions(+), 51 deletions(-) diff --git a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.spec.ts b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.spec.ts index 9bf88ae69..d7fc23959 100644 --- a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.spec.ts +++ b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.spec.ts @@ -484,6 +484,22 @@ describe('FilterEditorComponent', () => { expect(component.textFilter).toBeNull() }) + it('should ingest title filter rules alongside a fulltext query with date created', () => { + component.filterRules = [ + { + rule_type: FILTER_SIMPLE_TITLE, + value: 'foo', + }, + { + rule_type: FILTER_FULLTEXT_QUERY, + value: 'created:[-1 week to now]', + }, + ] + expect(component.dateCreatedRelativeDate).toEqual(1) // RELATIVE_DATE_QUERYSTRINGS['-1 week to now'] + expect(component.textFilter).toEqual('foo') + expect(component.textFilterTarget).toEqual('title') // TEXT_FILTER_TARGET_TITLE + }) + it('should ingest text filter rules for fulltext query that include date added', () => { expect(component.dateAddedRelativeDate).toBeNull() component.filterRules = [ @@ -1845,7 +1861,7 @@ describe('FilterEditorComponent', () => { ]) }) - it('should carry over text filtering on date created with relative date', () => { + it('should keep text filtering separate from date created relative date', () => { component.textFilter = 'foo' const dateCreatedDropdown = fixture.debugElement.queryAll( By.directive(DatesDropdownComponent) @@ -1855,14 +1871,18 @@ describe('FilterEditorComponent', () => { fixture.detectChanges() tick(400) expect(component.filterRules).toEqual([ + { + rule_type: FILTER_SIMPLE_TEXT, + value: 'foo', + }, { rule_type: FILTER_FULLTEXT_QUERY, - value: 'foo,created:[-1 week to now]', + value: 'created:[-1 week to now]', }, ]) }) - it('should carry over text filtering once with created and added relative dates', () => { + it('should keep text filtering separate from created and added relative dates', () => { component.textFilter = 'foo' const datesDropdown = fixture.debugElement.query( By.directive(DatesDropdownComponent) @@ -1873,14 +1893,18 @@ describe('FilterEditorComponent', () => { fixture.detectChanges() tick(400) expect(component.filterRules).toEqual([ + { + rule_type: FILTER_SIMPLE_TEXT, + value: 'foo', + }, { rule_type: FILTER_FULLTEXT_QUERY, - value: 'foo,created:[-1 week to now],added:[-1 month to now]', + value: 'created:[-1 week to now],added:[-1 month to now]', }, ]) }) - it('should convert legacy title filters into full text query when adding a created relative date', () => { + it('should keep legacy title filters as a title search when adding a created relative date', () => { component.filterRules = [ { rule_type: FILTER_TITLE, @@ -1895,14 +1919,18 @@ describe('FilterEditorComponent', () => { fixture.detectChanges() tick(400) expect(component.filterRules).toEqual([ + { + rule_type: FILTER_SIMPLE_TITLE, + value: 'foo', + }, { rule_type: FILTER_FULLTEXT_QUERY, - value: 'foo,created:[-1 week to now]', + value: 'created:[-1 week to now]', }, ]) }) - it('should convert simple title filters into full text query when adding a created relative date', () => { + it('should keep simple title filters as a title search when adding a created relative date', () => { component.filterRules = [ { rule_type: FILTER_SIMPLE_TITLE, @@ -1917,9 +1945,13 @@ describe('FilterEditorComponent', () => { fixture.detectChanges() tick(400) expect(component.filterRules).toEqual([ + { + rule_type: FILTER_SIMPLE_TITLE, + value: 'foo', + }, { rule_type: FILTER_FULLTEXT_QUERY, - value: 'foo,created:[-1 week to now]', + value: 'created:[-1 week to now]', }, ]) }) @@ -2005,7 +2037,7 @@ describe('FilterEditorComponent', () => { ]) }) - it('should carry over text filtering on date added with relative date', () => { + it('should keep text filtering separate from date added relative date', () => { component.textFilter = 'foo' const datesDropdown = fixture.debugElement.query( By.directive(DatesDropdownComponent) @@ -2015,9 +2047,13 @@ describe('FilterEditorComponent', () => { fixture.detectChanges() tick(400) expect(component.filterRules).toEqual([ + { + rule_type: FILTER_SIMPLE_TEXT, + value: 'foo', + }, { rule_type: FILTER_FULLTEXT_QUERY, - value: 'foo,added:[-1 week to now]', + value: 'added:[-1 week to now]', }, ]) }) diff --git a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts index 7da3f6373..1aabd7b64 100644 --- a/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts +++ b/src-ui/src/app/components/document-list/filter-editor/filter-editor.component.ts @@ -1083,23 +1083,6 @@ export class FilterEditorComponent (fr) => fr.rule_type == FILTER_FULLTEXT_QUERY ) - // if had a title / content search and added a relative date we need to carry it over... - if ( - !existingRule && - this._textFilter?.length > 0 && - (this.textFilterTarget == TEXT_FILTER_TARGET_TITLE_CONTENT || - this.textFilterTarget == TEXT_FILTER_TARGET_TITLE) - ) { - existingRule = filterRules.find( - (fr) => - fr.rule_type == FILTER_TITLE_CONTENT || - fr.rule_type == FILTER_SIMPLE_TEXT || - fr.rule_type == FILTER_TITLE || - fr.rule_type == FILTER_SIMPLE_TITLE - ) - existingRule.rule_type = FILTER_FULLTEXT_QUERY - } - let queryArgs = existingRule?.value.split(',') ?? [] if (this.dateCreatedRelativeDate !== null) { const rd = RELATIVE_DATE_QUERYSTRINGS.find( diff --git a/src/documents/search/_backend.py b/src/documents/search/_backend.py index aaa4ed395..2227df0f5 100644 --- a/src/documents/search/_backend.py +++ b/src/documents/search/_backend.py @@ -926,6 +926,7 @@ class TantivyBackend: sort_field: str | None = None, sort_reverse: bool = False, search_mode: SearchMode = SearchMode.QUERY, + filter_query: str | None = None, limit: int | None = None, ) -> list[int]: """ @@ -941,6 +942,7 @@ class TantivyBackend: sort_field: Field to sort by (None for relevance ranking) sort_reverse: Whether to reverse the sort order search_mode: Query parsing mode (QUERY, TEXT, or TITLE) + filter_query: Optional advanced query results must also match limit: Maximum number of IDs to return (None = all matching docs) Returns: @@ -948,6 +950,13 @@ class TantivyBackend: """ self._ensure_open() user_query = self._parse_query(query, search_mode) + if filter_query is not None: + user_query = user_query.and_must_match( + tantivy.Query.const_score_query( + self._parse_query(filter_query, SearchMode.QUERY), + 0.0, # zero so doesn't affect relevance + ), + ) final_query = self._apply_permission_filter(user_query, user) searcher = self._index.searcher() diff --git a/src/documents/tests/test_api_bulk_edit.py b/src/documents/tests/test_api_bulk_edit.py index 9c7ba3db8..208657564 100644 --- a/src/documents/tests/test_api_bulk_edit.py +++ b/src/documents/tests/test_api_bulk_edit.py @@ -835,6 +835,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): for filters in ( {"text": "new doc 2017-03-16"}, {"title_search": "apple"}, + {"title_search": "apple", "query": "created:[-1 year to now]"}, ): with self.subTest(filters=filters): get_backend.return_value.search_ids.return_value = [self.doc2.id] @@ -886,7 +887,7 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase): "all": True, "filters": { "text": "B", - "query": "c1", + "title_search": "c1", }, "method": "set_storage_path", "parameters": {"storage_path": self.sp1.id}, diff --git a/src/documents/tests/test_api_search.py b/src/documents/tests/test_api_search.py index 0b7e4961b..d7a07f5ed 100644 --- a/src/documents/tests/test_api_search.py +++ b/src/documents/tests/test_api_search.py @@ -245,8 +245,68 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase): self.assertEqual(response.data["count"], 1) self.assertEqual(response.data["results"][0]["id"], title_match.id) + def test_simple_search_combined_with_query_filter(self) -> None: + """ + GIVEN: + - A document with the term in its title, created this year + - A document with the term only in its content, created this year + - A document with the term in its title, created years ago + WHEN: + - A title or text search is combined with a relative created + date query, as the filter editor sends it + THEN: + - The simple search keeps its own field scope and the query only + narrows the results (issue #14436) + """ + this_year = timezone.localdate().replace(month=1, day=1) + title_match = Document.objects.create( + title="foo report", + content="nothing relevant", + checksum="F1", + created=this_year, + pk=21, + ) + content_match = Document.objects.create( + title="something else", + content="foo appears here", + checksum="F2", + created=this_year, + pk=22, + ) + old_title_match = Document.objects.create( + title="foo archive", + content="nothing relevant", + checksum="F3", + created=datetime.date(2001, 1, 1), + pk=23, + ) + backend = get_backend() + for doc in (title_match, content_match, old_title_match): + backend.add_or_update(doc) + + query = "created:[-1 year to now]" + response = self.client.get( + "/api/documents/", + {"title_search": "foo", "query": query}, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual( + [r["id"] for r in response.data["results"]], + [title_match.id], + ) + + response = self.client.get( + "/api/documents/", + {"text": "foo", "query": query}, + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertCountEqual( + [r["id"] for r in response.data["results"]], + [title_match.id, content_match.id], + ) + def test_search_rejects_multiple_search_modes(self) -> None: - response = self.client.get("/api/documents/?text=bank&query=bank") + response = self.client.get("/api/documents/?text=bank&title_search=bank") self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) self.assertEqual( response.data["detail"], diff --git a/src/documents/views.py b/src/documents/views.py index 06b63b663..30d790a07 100644 --- a/src/documents/views.py +++ b/src/documents/views.py @@ -289,6 +289,7 @@ logger = logging.getLogger("paperless.api") # clauses efficiently, so this threshold mainly protects SQLite users. _TANTIVY_INTERSECT_THRESHOLD = 5_000 _TANTIVY_SEARCH_PARAM_NAMES = ("text", "title_search", "query", "more_like_id") +_TANTIVY_COMBINABLE_SEARCH_PARAMS = ({"text", "query"}, {"title_search", "query"}) # whoosh-compat's fieldname tagger (used only for SearchMode.QUERY, via the # whoosh grammar in parse_user_query) is O(n^2) in plain word characters: @@ -308,6 +309,18 @@ _TANTIVY_SEARCH_PARAM_NAMES = ("text", "title_search", "query", "more_like_id") _MAX_QUERY_LENGTH: Final[int] = 4096 +def _validate_tantivy_search_params(active: list[str]) -> None: + # may also have a filter query alongside a simple search e.g. relative dates + if len(active) > 1 and set(active) not in _TANTIVY_COMBINABLE_SEARCH_PARAMS: + raise ValidationError( + { + "detail": _( + "Specify only one of text, title_search, query, or more_like_id.", + ), + }, + ) + + def _get_tantivy_query_and_mode(params): from documents.search import QueryTooLongError from documents.search import SearchMode @@ -321,9 +334,15 @@ def _get_tantivy_query_and_mode(params): else: return None # pragma: no cover - if len(raw) > _MAX_QUERY_LENGTH: - raise QueryTooLongError(len(raw), _MAX_QUERY_LENGTH) - return raw, mode + filter_query = ( + str(params["query"]) + if mode is not SearchMode.QUERY and "query" in params + else None + ) + for value in (raw, filter_query): + if value is not None and len(value) > _MAX_QUERY_LENGTH: + raise QueryTooLongError(len(value), _MAX_QUERY_LENGTH) + return raw, mode, filter_query def _get_more_like_id(query_params: dict[str, Any], user: User | None) -> int: @@ -2566,15 +2585,9 @@ class UnifiedSearchViewSet(DocumentViewSet): def parse_search_params() -> SearchParams: """Extract query string, search mode, and ordering from request.""" - active = self._get_active_search_params(request) - if len(active) > 1: - raise ValidationError( - { - "detail": _( - "Specify only one of text, title_search, query, or more_like_id.", - ), - }, - ) + _validate_tantivy_search_params( + self._get_active_search_params(request), + ) ordering_param = request.query_params.get("ordering", "") sort_reverse = ordering_param.startswith("-") @@ -2636,7 +2649,9 @@ class UnifiedSearchViewSet(DocumentViewSet): filtered_qs: QuerySet[Document], ) -> SearchResultPage: """Handle text/title/query search: IDs, ORM intersection, page highlights.""" - query_str, search_mode = _get_tantivy_query_and_mode(request.query_params) + query_str, search_mode, filter_query = _get_tantivy_query_and_mode( + request.query_params, + ) # "score" is not a real Tantivy sort field — it means relevance order, # which is Tantivy's default when no sort field is specified. @@ -2649,6 +2664,7 @@ class UnifiedSearchViewSet(DocumentViewSet): ), sort_reverse=sort_reverse, search_mode=search_mode, + filter_query=filter_query, ) ordered_ids = intersect_and_order( all_ids, @@ -2883,14 +2899,7 @@ class DocumentSelectionMixin: ] if not search_filters: return None - if len(search_filters) > 1: - raise ValidationError( - { - "detail": _( - "Specify only one of text, title_search, query, or more_like_id.", - ), - }, - ) + _validate_tantivy_search_params(search_filters) from documents.search import SearchQueryError from documents.search import get_backend @@ -2909,11 +2918,14 @@ class DocumentSelectionMixin: user=search_user, ) else: - query_str, search_mode = _get_tantivy_query_and_mode(filters) + query_str, search_mode, filter_query = _get_tantivy_query_and_mode( + filters, + ) search_ids = backend.search_ids( query_str, user=search_user, search_mode=search_mode, + filter_query=filter_query, ) except SearchQueryError as e: # Same user-fixable-query mapping as the search list endpoint: