Performance: optimize dropdown filtering

docs/migration-v3.md

@@ -326,3 +326,11 @@ behind a reverse proxy may need to set
 [`PAPERLESS_TRUSTED_PROXIES`](configuration.md#PAPERLESS_TRUSTED_PROXIES),
 [`PAPERLESS_ALLAUTH_TRUSTED_CLIENT_IP_HEADER`](configuration.md#PAPERLESS_ALLAUTH_TRUSTED_CLIENT_IP_HEADER),
 or both, to avoid `403 Forbidden` errors on login.
+
+## Database Migrations
+
+Some integer fields have been changed to smaller types to reduce database size. If you have any `MailRule` records with a `maximum_age` greater than 32767, they will be clamped to 32767 during the migration to avoid errors during migration.
+
+### Action Required
+
+No user action is required. The migration will automatically clamp any `MailRule.maximum_age` values greater than 32767 to 32767 during the migration process.

src-ui/src/app/components/common/input/tags/tags.component.ts

@@ -61,12 +61,15 @@ export class TagsComponent implements OnInit, ControlValueAccessor {
   writeValue(newValue: number[]): void {
     this.value = newValue
   }
+
   registerOnChange(fn: any): void {
     this.onChange = fn
   }
+
   registerOnTouched(fn: any): void {
     this.onTouched = fn
   }
+
   setDisabledState?(isDisabled: boolean): void {
     this.disabled = isDisabled
   }
@@ -74,6 +77,7 @@ export class TagsComponent implements OnInit, ControlValueAccessor {
   ngOnInit(): void {
     this.tagService.listAll().subscribe((result) => {
       this.tags = result.results
+      this.tagSearchTextCache.clear()
     })
   }
 
@@ -113,15 +117,12 @@ export class TagsComponent implements OnInit, ControlValueAccessor {
 
   tags: Tag[] = []
 
+  private readonly tagSearchTextCache = new Map<number, string>()
+
   public createTagRef: (name) => void
 
   public searchFn = (term: string, tag: Tag): boolean =>
-    matchesSearchText(
-      [this.getParentChain(tag?.id).map((parent) => parent.name), tag?.name]
-        .flat()
-        .join(' '),
-      term
-    )
+    matchesSearchText(this.getTagSearchText(tag), term)
 
   getTag(id: number) {
     if (this.tags) {
@@ -152,8 +153,8 @@ export class TagsComponent implements OnInit, ControlValueAccessor {
 
   private removeChildren(tagIDs: number[], tag: Tag) {
     if (tag.children?.length) {
-      const childIDs = tag.children.map((child) => child.id)
-      tagIDs = tagIDs.filter((id) => !childIDs.includes(id))
+      const childIDs = new Set(tag.children.map((child) => child.id))
+      tagIDs = tagIDs.filter((id) => !childIDs.has(id))
       for (const child of tag.children) {
         tagIDs = this.removeChildren(tagIDs, child)
       }
@@ -188,6 +189,7 @@ export class TagsComponent implements OnInit, ControlValueAccessor {
         tap((newTag) => {
           this.tagService.listAll().subscribe((tags) => {
             this.tags = tags.results
+            this.tagSearchTextCache.clear()
             add && this.addTag(newTag.id)
           })
         })
@@ -238,4 +240,22 @@ export class TagsComponent implements OnInit, ControlValueAccessor {
     }
     return chain
   }
+
+  private getTagSearchText(tag: Tag): string {
+    if (!tag) {
+      return ''
+    }
+
+    const cachedSearchText = this.tagSearchTextCache.get(tag.id)
+    if (cachedSearchText !== undefined) {
+      return cachedSearchText
+    }
+
+    const searchText = [
+      ...this.getParentChain(tag.id).map((parent) => parent.name),
+      tag.name,
+    ].join(' ')
+    this.tagSearchTextCache.set(tag.id, searchText)
+    return searchText
+  }
 }

src-ui/src/app/pipes/filter.pipe.spec.ts

@@ -46,4 +46,23 @@ describe('FilterPipe', () => {
     itemsReturned = pipe.transform(items, 'slug', 'slug')
     expect(itemsReturned).toEqual([items[0]])
   })
+
+  it('should filter matchingmodel items by normalized independent terms', () => {
+    const pipe = new FilterPipe()
+    const items: MatchingModel[] = [
+      {
+        id: 1,
+        name: 'Financ\u00e9 Taxes 2026',
+        slug: 'finance-taxes',
+      },
+      {
+        id: 2,
+        name: 'Receipt Archive 2026',
+        slug: 'receipt-archive',
+      },
+    ]
+
+    expect(pipe.transform(items, 'finance 26')).toEqual([items[0]])
+    expect(pipe.transform(items, 'finance receipt')).toEqual([])
+  })
 })

src-ui/src/app/utils/text-search.ts

@@ -8,8 +8,39 @@ export type SearchTextValue =
   | null
   | undefined
 
+const normalizedStringCache = new Map<string, string>()
+const searchTermsCache = new Map<string, string[]>()
+
+function normalizeString(value: string): string {
+  const cachedValue = normalizedStringCache.get(value)
+  if (cachedValue !== undefined) {
+    return cachedValue
+  }
+
+  const normalizedValue = normalizeSync(value).toLocaleLowerCase()
+  normalizedStringCache.set(value, normalizedValue)
+  return normalizedValue
+}
+
 export function normalizeSearchText(value: SearchTextValue): string {
-  return normalizeSync(String(value ?? '')).toLocaleLowerCase()
+  return normalizeString(String(value ?? ''))
+}
+
+function getSearchTerms(searchText: SearchTextValue): string[] {
+  const normalizedSearchText = normalizeSearchText(searchText).trim()
+
+  if (!normalizedSearchText) {
+    return []
+  }
+
+  const cachedTerms = searchTermsCache.get(normalizedSearchText)
+  if (cachedTerms !== undefined) {
+    return cachedTerms
+  }
+
+  const searchTerms = normalizedSearchText.split(/\s+/)
+  searchTermsCache.set(normalizedSearchText, searchTerms)
+  return searchTerms
 }
 
 export function matchesSearchText(
@@ -17,7 +48,7 @@ export function matchesSearchText(
   searchText: SearchTextValue
 ): boolean {
   const normalizedValue = normalizeSearchText(value)
-  const searchTerms = normalizeSearchText(searchText).trim().split(/\s+/)
+  const searchTerms = getSearchTerms(searchText)
 
   return searchTerms.every((term) => normalizedValue.includes(term))
 }

src/documents/search/_tokenizer.py

@@ -48,6 +48,9 @@ _LANGUAGE_MAP: dict[str, str] = {
 }
 
 SUPPORTED_LANGUAGES: frozenset[str] = frozenset(_LANGUAGE_MAP)
+# Document.title is max_length=128, so use 129 as the limit for
+# Tantivy's remove_long filter
+_TOKEN_REMOVE_LONG_LIMIT: Final[int] = 129
 
 
 def register_tokenizers(index: tantivy.Index, language: str | None) -> None:
@@ -77,10 +80,10 @@ def register_tokenizers(index: tantivy.Index, language: str | None) -> None:
 
 
 def _paperless_text(language: str | None) -> tantivy.TextAnalyzer:
-    """Main full-text tokenizer for content, title, etc: simple -> remove_long(65) -> lowercase -> ascii_fold [-> stemmer]"""
+    """Main full-text tokenizer for content, title, etc: simple -> remove_long(129) -> lowercase -> ascii_fold [-> stemmer]"""
     builder = (
         tantivy.TextAnalyzerBuilder(tantivy.Tokenizer.simple())
-        .filter(tantivy.Filter.remove_long(65))
+        .filter(tantivy.Filter.remove_long(_TOKEN_REMOVE_LONG_LIMIT))
         .filter(tantivy.Filter.lowercase())
         .filter(tantivy.Filter.ascii_fold())
     )
@@ -119,12 +122,12 @@ def _bigram_analyzer() -> tantivy.TextAnalyzer:
 
 
 def _simple_search_analyzer() -> tantivy.TextAnalyzer:
-    """Tokenizer for simple substring search fields: non-whitespace chunks -> remove_long(65) -> lowercase -> ascii_fold."""
+    """Tokenizer for simple substring search fields: non-whitespace chunks -> remove_long(129) -> lowercase -> ascii_fold."""
     return (
         tantivy.TextAnalyzerBuilder(
             tantivy.Tokenizer.regex(r"\S+"),
         )
-        .filter(tantivy.Filter.remove_long(65))
+        .filter(tantivy.Filter.remove_long(_TOKEN_REMOVE_LONG_LIMIT))
         .filter(tantivy.Filter.lowercase())
         .filter(tantivy.Filter.ascii_fold())
         .build()

src/documents/search/_translate.py

@@ -163,7 +163,12 @@ def scan(query: str) -> list[Token]:
             i += 1
             continue
         token, i = matched
-        _flush(buf, tokens)
+        if buf and buf[-1] == ",":
+            buf.pop()
+            _flush(buf, tokens)
+            tokens.append(Comma())
+        else:
+            _flush(buf, tokens)
         tokens.append(token)
         i = _maybe_comma(query, i, tokens)
     _flush(buf, tokens)

src/documents/tests/search/test_backend.py

@@ -261,6 +261,36 @@ class TestSearch:
             == 1
         )
 
+    @pytest.mark.parametrize(
+        ("search_mode", "query"),
+        [
+            pytest.param(SearchMode.TITLE, "12345", id="title_search"),
+            pytest.param(SearchMode.TEXT, "12345", id="text_search"),
+            pytest.param(SearchMode.QUERY, None, id="query_title_exact"),
+        ],
+    )
+    def test_search_modes_match_model_limit_title_tokens(
+        self,
+        backend: TantivyBackend,
+        search_mode: SearchMode,
+        query: str | None,
+    ) -> None:
+        """Search must keep filename-like title tokens up to the model limit."""
+        long_title = "1234567890" * 12 + "12345678"
+        doc = Document.objects.create(
+            title=long_title,
+            content="ordinary content",
+            checksum="TXT12",
+            pk=18,
+        )
+        backend.add_or_update(doc)
+
+        assert backend.search_ids(
+            query or f"title:{long_title}",
+            user=None,
+            search_mode=search_mode,
+        ) == [doc.pk]
+
     @pytest.mark.parametrize(
         ("mode", "title", "content", "hits", "misses"),
         [

src/documents/tests/search/test_tokenizer.py

@@ -99,6 +99,25 @@ class TestTokenizers:
         )
         assert simple_search_index.searcher().search(q, limit=5).count == 1
 
+    def test_simple_search_analyzer_supports_model_limit_token_substrings(
+        self,
+        simple_search_index: tantivy.Index,
+    ) -> None:
+        """Simple substring search keeps tokens up to Document.title's model limit."""
+        long_token = "abcdefghij" * 12 + "abcdefgh"
+        writer = simple_search_index.writer()
+        doc = tantivy.Document()
+        doc.add_text("simple_content", long_token)
+        writer.add_document(doc)
+        writer.commit()
+        simple_search_index.reload()
+        q = tantivy.Query.regex_query(
+            simple_search_index.schema,
+            "simple_content",
+            ".*cdefg.*",
+        )
+        assert simple_search_index.searcher().search(q, limit=5).count == 1
+
     def test_unsupported_language_logs_warning(self, caplog: LogCaptureFixture) -> None:
         """Unsupported language codes should log a warning and disable stemming gracefully."""
         sb = tantivy.SchemaBuilder()

src/documents/tests/search/test_translate.py

@@ -716,6 +716,12 @@ class TestISODatetimeBounds:
             "added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
         )
 
+    def test_translate_query_text_before_comma_separated_date_clause(self) -> None:
+        result = translate_query("schäfersee,created:previous year", UTC)
+        assert result == (
+            "schäfersee AND created:[2025-01-01T00:00:00Z TO 2026-01-01T00:00:00Z]"
+        )
+
     def test_invalid_iso_datetime_raises(self) -> None:
         # A token with "T" that is not valid ISO datetime -> raise.
         with pytest.raises(InvalidDateQuery) as exc_info:

src/documents/views.py

@@ -3769,13 +3769,20 @@ class BulkDownloadView(DocumentSelectionMixin, GenericAPIView[Any]):
             validated_data=serializer.validated_data,
         )
         documents = Document.objects.filter(pk__in=ids)
+        versioned_documents = []
         compression = serializer.validated_data.get("compression")
         content = serializer.validated_data.get("content")
         follow_filename_format = serializer.validated_data.get("follow_formatting")
 
         for document in documents:
-            if not has_perms_owner_aware(request.user, "change_document", document):
+            root_doc = get_root_document(document)
+            if not has_perms_owner_aware(request.user, "view_document", root_doc):
                 return HttpResponseForbidden("Insufficient permissions")
+            versioned_documents.append(
+                get_latest_version_for_root(
+                    root_doc,
+                ),
+            )
 
         if content == "both":
             strategy_class = OriginalAndArchiveStrategy
@@ -3798,7 +3805,7 @@ class BulkDownloadView(DocumentSelectionMixin, GenericAPIView[Any]):
                     zipf,
                     follow_formatting=follow_filename_format,
                 )
-                for document in documents:
+                for document in versioned_documents:
                     strategy.add_document(document)
 
             f = temp_path.open("rb")

src/paperless_mail/migrations/0002_optimize_integer_field_sizes.py

@@ -4,6 +4,12 @@ from django.db import migrations
 from django.db import models
 
 
+def clamp_mailrule_maximum_age(apps, schema_editor):
+    # Clamp the maximum_age field of MailRule because of PositiveIntegerField --> PositiveSmallIntegerField
+    MailRule = apps.get_model("paperless_mail", "MailRule")
+    MailRule.objects.filter(maximum_age__gt=32767).update(maximum_age=32767)
+
+
 class Migration(migrations.Migration):
     dependencies = [
         ("paperless_mail", "0001_squashed"),
@@ -112,6 +118,10 @@ class Migration(migrations.Migration):
                 verbose_name="consumption scope",
             ),
         ),
+        migrations.RunPython(
+            clamp_mailrule_maximum_age,
+            migrations.RunPython.noop,
+        ),
         migrations.AlterField(
             model_name="mailrule",
             name="maximum_age",