Fix: consolidate and extend unicode NFC normalization for filenames and matching

Consolidates the scattered unicodedata.normalize("NFC", ...) calls introduced
by my earlier filename/path normalization work into a single
documents.utils.normalize_unicode() helper, and closes several gaps where
NFD-normalized filenames could still slip through unnormalized:

- Document.get_public_filename() now normalizes, fixing exported filenames
  built from an NFD title/correspondent name (the default, non-format export
  path was not covered by the earlier fix).
- DocumentViewSet.update_version() now normalizes the uploaded filename,
  matching PostDocumentView's existing behavior.
- ConsumerPlugin normalizes self.filename once at consumption time, covering
  the title fallback and Document.original_filename for every document
  source (consume folder, mail, API, barcode splits).
- Workflow trigger and mail rule filename/path matching (documents/matching.py,
  paperless_mail/mail.py) now normalize the document/attachment side before
  comparing, so an NFD filename matches an NFC-typed filter pattern instead of
  silently failing to match.
- WorkflowTriggerSerializer and MailRuleSerializer normalize filter_filename/
  filter_path and the attachment include/exclude patterns once at write time,
  so the read side isn't re-normalizing an already-canonical value on every
  match.

docker/rootfs/etc/s6-overlay/s6-rc.d/init-compile-bytecode/run

@@ -1,58 +0,0 @@
-#!/command/with-contenv /usr/bin/bash
-# shellcheck shell=bash
-declare -r log_prefix="[init-compile-bytecode]"
-
-# PYTHONDONTWRITEBYTECODE=1 is set for the whole container. This unit compiles a
-# scoped set of libraries anyway, to speed up startup without bloating image size.
-
-# Handle the people using a read only file system
-if [[ "${S6_READ_ONLY_ROOT}" == "1" ]]; then
-	echo "${log_prefix} S6_READ_ONLY_ROOT=1, skipping (nothing to write bytecode to)"
-	exit 0
-fi
-
-# When running as a non-root user, site-packages is still root-owned and unwritable,
-# so this step would just fail loudly on every container start. Skip it.
-if [[ -n "${USER_IS_NON_ROOT}" ]]; then
-	echo "${log_prefix} USER_IS_NON_ROOT is set, skipping (site-packages is not writable)"
-	exit 0
-fi
-
-declare -r site_packages="$(python3 -c 'import site; print(site.getsitepackages()[0])')"
-
-# Deliberately scoped to packages that paperless.settings/paperless/__init__.py import
-# unconditionally on every manage.py invocation (Django itself, the always-loaded
-# INSTALLED_APPS, and celery). This is NOT "compile everything" - the optional AI stack
-# (torch, llama-index, sentence-transformers, ...) is intentionally excluded since it is
-# lazy-imported and large.
-declare -a scope=(
-	"${PAPERLESS_SRC_DIR}"
-	"${site_packages}/django"
-	"${site_packages}/celery"
-	"${site_packages}/kombu"
-	"${site_packages}/rest_framework"
-	"${site_packages}/django_filters"
-	"${site_packages}/whitenoise"
-	"${site_packages}/corsheaders"
-	"${site_packages}/django_extensions"
-	"${site_packages}/guardian"
-	"${site_packages}/allauth"
-	"${site_packages}/drf_spectacular"
-	"${site_packages}/drf_spectacular_sidecar"
-	"${site_packages}/treenode"
-	"${site_packages}/compression_middleware"
-)
-
-declare -a existing_scope=()
-for path in "${scope[@]}"; do
-	[[ -d "${path}" ]] && existing_scope+=("${path}")
-done
-
-echo "${log_prefix} Compiling bytecode for: ${existing_scope[*]}"
-declare -r start_seconds=${SECONDS}
-
-if ! PYTHONDONTWRITEBYTECODE= python3 -m compileall -q "${existing_scope[@]}"; then
-	echo "${log_prefix} WARNING: compileall reported errors (read-only filesystem or unwritable site-packages?); continuing without a bytecode cache"
-fi
-
-echo "${log_prefix} Done in $((SECONDS - start_seconds))s"

docker/rootfs/etc/s6-overlay/s6-rc.d/init-compile-bytecode/type

@@ -1 +0,0 @@
-oneshot

docker/rootfs/etc/s6-overlay/s6-rc.d/init-compile-bytecode/up

@@ -1 +0,0 @@
-/etc/s6-overlay/s6-rc.d/init-compile-bytecode/run

docs/configuration.md

@@ -1200,15 +1200,6 @@ still perform some basic text pre-processing before matching.
 
     Defaults to true, enabling the feature.
 
-#### [`PAPERLESS_CLASSIFIER_MATCH_THRESHOLD=<float>`](#PAPERLESS_CLASSIFIER_MATCH_THRESHOLD) {#PAPERLESS_CLASSIFIER_MATCH_THRESHOLD}
-
-: Sets the minimum confidence score (0.0-1.0) required for the automatic
-classifier to assign a correspondent, document type, or storage path to a
-document. Predictions below this threshold are discarded and the field is
-left unassigned, preventing low-confidence guesses from being applied.
-
-    Defaults to 0.6.
-
 #### [`PAPERLESS_DATE_PARSER_LANGUAGES=<lang>`](#PAPERLESS_DATE_PARSER_LANGUAGES) {#PAPERLESS_DATE_PARSER_LANGUAGES}
 
 : Specifies which language Paperless should use when parsing dates from documents.

pyproject.toml

@@ -77,7 +77,6 @@ dependencies = [
   "torch~=2.13.0",
   "watchfiles>=1.2",
   "whitenoise~=6.11",
-  "whoosh-compat[tantivy]==0.1",
   "zxing-cpp~=3.1.0",
 ]
 [project.optional-dependencies]

src-ui/src/app/components/admin/settings/settings.component.html

@@ -112,22 +112,6 @@
 
                 <pngx-input-check i18n-title title="Use 'slim' sidebar (icons only)" formControlName="slimSidebarEnabled"></pngx-input-check>
 
-                <p class="mb-2 mt-3" i18n>Sidebar items to show:</p>
-                @for (option of sidebarItemOptions; track option.id) {
-                  <div class="form-check">
-                    <input
-                      class="form-check-input"
-                      type="checkbox"
-                      [id]="'sidebar-item-setting-' + option.id"
-                      [checked]="isSidebarItemShown(option.id)"
-                      (change)="toggleSidebarItem(option.id, $event.target.checked)"
-                    />
-                    <label class="form-check-label" [for]="'sidebar-item-setting-' + option.id">
-                      {{ option.label }}
-                    </label>
-                  </div>
-                }
-
               </div>
             </div>
 

src-ui/src/app/components/admin/settings/settings.component.spec.ts

@@ -24,7 +24,7 @@ import {
   SystemStatus,
   SystemStatusItemStatus,
 } from 'src/app/data/system-status'
-import { HideableSidebarItemID, SETTINGS_KEYS } from 'src/app/data/ui-settings'
+import { SETTINGS_KEYS } from 'src/app/data/ui-settings'
 import { IfOwnerDirective } from 'src/app/directives/if-owner.directive'
 import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive'
 import { PermissionsGuard } from 'src/app/guards/permissions.guard'
@@ -209,45 +209,6 @@ describe('SettingsComponent', () => {
     fixture.detectChanges()
   }
 
-  it('supports configuring sidebar items and canceling changes', () => {
-    completeSetup()
-
-    component.toggleSidebarItem(HideableSidebarItemID.Workflows, false)
-    fixture.detectChanges()
-
-    expect(component.settingsForm.value.sidebarHiddenItems).toContain(
-      HideableSidebarItemID.Workflows
-    )
-
-    settingsService.updateSidebarItemVisibility(
-      HideableSidebarItemID.Mail,
-      false
-    )
-
-    expect(component.settingsForm.value.sidebarHiddenItems).toContain(
-      HideableSidebarItemID.Mail
-    )
-
-    component.reset()
-
-    expect(component.settingsForm.value.sidebarHiddenItems).not.toContain(
-      HideableSidebarItemID.Workflows
-    )
-    expect(component.settingsForm.value.sidebarHiddenItems).not.toContain(
-      HideableSidebarItemID.Mail
-    )
-  })
-
-  it('enables sidebar item controls on general settings until destroyed', () => {
-    completeSetup()
-
-    expect(settingsService.organizingSidebarItems()).toBe(true)
-
-    component.ngOnDestroy()
-
-    expect(settingsService.organizingSidebarItems()).toBe(false)
-  })
-
   it('should support tabbed settings & change URL, prevent navigation if dirty confirmation rejected', async () => {
     completeSetup()
     const navigateSpy = jest.spyOn(router, 'navigate')
@@ -288,7 +249,6 @@ describe('SettingsComponent', () => {
 
   it('should support save local settings updating appearance settings and calling API, show error', () => {
     completeSetup()
-    component.toggleSidebarItem(HideableSidebarItemID.Workflows, false)
     const toastErrorSpy = jest.spyOn(toastService, 'showError')
     const toastSpy = jest.spyOn(toastService, 'show')
     const storeSpy = jest.spyOn(settingsService, 'storeSettings')
@@ -307,10 +267,7 @@ describe('SettingsComponent', () => {
     expect(toastErrorSpy).toHaveBeenCalled()
     expect(storeSpy).toHaveBeenCalled()
     expect(appearanceSettingsSpy).not.toHaveBeenCalled()
-    expect(setSpy).toHaveBeenCalledTimes(34)
+    expect(setSpy).toHaveBeenCalledTimes(33)
-    expect(setSpy).toHaveBeenCalledWith(SETTINGS_KEYS.SIDEBAR_HIDDEN_ITEMS, [
-      HideableSidebarItemID.Workflows,
-    ])
 
     // succeed
     storeSpy.mockReturnValueOnce(of(true))

src-ui/src/app/components/admin/settings/settings.component.ts

@@ -39,12 +39,7 @@ import {
   SystemStatus,
   SystemStatusItemStatus,
 } from 'src/app/data/system-status'
-import {
+import { GlobalSearchType, SETTINGS_KEYS } from 'src/app/data/ui-settings'
-  GlobalSearchType,
-  HIDEABLE_SIDEBAR_ITEM_IDS,
-  HideableSidebarItemID,
-  SETTINGS_KEYS,
-} from 'src/app/data/ui-settings'
 import { User } from 'src/app/data/user'
 import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive'
 import { CustomDatePipe } from 'src/app/pipes/custom-date.pipe'
@@ -107,14 +102,6 @@ const documentDetailFieldOptions = [
   { id: DocumentDetailFieldID.Tags, label: $localize`Tags` },
 ]
 
-const sidebarItemLabels: Record<HideableSidebarItemID, string> = {
-  [HideableSidebarItemID.Dashboard]: $localize`Dashboard`,
-  [HideableSidebarItemID.SavedViews]: $localize`Saved Views`,
-  [HideableSidebarItemID.Workflows]: $localize`Workflows`,
-  [HideableSidebarItemID.Mail]: $localize`Mail`,
-  [HideableSidebarItemID.Documentation]: $localize`Documentation`,
-}
-
 @Component({
   selector: 'pngx-settings',
   templateUrl: './settings.component.html',
@@ -162,7 +149,6 @@ export class SettingsComponent
     bulkEditApplyOnClose: new FormControl(null),
     documentListItemPerPage: new FormControl(null),
     slimSidebarEnabled: new FormControl(null),
-    sidebarHiddenItems: new FormControl<HideableSidebarItemID[]>([]),
     darkModeUseSystem: new FormControl(null),
     darkModeEnabled: new FormControl(null),
     darkModeInvertThumbs: new FormControl(null),
@@ -200,7 +186,6 @@ export class SettingsComponent
 
   store: BehaviorSubject<any>
   storeSub: Subscription
-  sidebarItemsSub: Subscription
   isDirty$: Observable<boolean>
   isDirty: boolean = false
   unsubscribeNotifier: Subject<any> = new Subject()
@@ -218,10 +203,6 @@ export class SettingsComponent
   public readonly PdfEditorEditMode = PdfEditorEditMode
 
   public readonly documentDetailFieldOptions = documentDetailFieldOptions
-  public readonly sidebarItemOptions = HIDEABLE_SIDEBAR_ITEM_IDS.map((id) => ({
-    id,
-    label: sidebarItemLabels[id],
-  }))
 
   get systemStatusHasErrors(): boolean {
     const status = this.systemStatus()
@@ -249,10 +230,6 @@ export class SettingsComponent
 
   constructor() {
     super()
-    this.sidebarItemsSub =
-      this.settings.sidebarHiddenItemsEditingChanged.subscribe((hiddenItems) =>
-        this.settingsForm.controls.sidebarHiddenItems.setValue(hiddenItems)
-      )
     this.settings.settingsSaved.subscribe(() => {
       if (!this.savePending) this.initialize()
       this.savedViewsService.maybeRefreshDocumentCounts()
@@ -302,21 +279,14 @@ export class SettingsComponent
 
     this.activatedRoute.paramMap.subscribe((paramMap) => {
       const section = paramMap.get('section')
-      let navID = SettingsNavIDs.General
       if (section) {
         const navIDKey: string = Object.keys(SettingsNavIDs).find(
           (navID) => navID.toLowerCase() == section
         )
         if (navIDKey) {
-          navID = SettingsNavIDs[navIDKey]
+          this.activeNavID.set(SettingsNavIDs[navIDKey])
         }
       }
-      this.activeNavID.set(navID)
-      this.settings.sidebarHiddenItemsEditing.set(
-        navID === SettingsNavIDs.General
-          ? [...this.settingsForm.controls.sidebarHiddenItems.value]
-          : null
-      )
     })
   }
 
@@ -340,7 +310,6 @@ export class SettingsComponent
         SETTINGS_KEYS.DOCUMENT_LIST_SIZE
       ),
       slimSidebarEnabled: this.settings.get(SETTINGS_KEYS.SLIM_SIDEBAR),
-      sidebarHiddenItems: this.settings.get(SETTINGS_KEYS.SIDEBAR_HIDDEN_ITEMS),
       darkModeUseSystem: this.settings.get(SETTINGS_KEYS.DARK_MODE_USE_SYSTEM),
       darkModeEnabled: this.settings.get(SETTINGS_KEYS.DARK_MODE_ENABLED),
       darkModeInvertThumbs: this.settings.get(
@@ -467,12 +436,6 @@ export class SettingsComponent
       this.settingsForm.patchValue(currentFormValue)
     }
 
-    if (this.settings.organizingSidebarItems()) {
-      this.settings.sidebarHiddenItemsEditing.set([
-        ...this.settingsForm.controls.sidebarHiddenItems.value,
-      ])
-    }
-
     if (this.canViewSystemStatus) {
       this.systemStatusService.get().subscribe((status) => {
         this.systemStatus.set(status)
@@ -481,18 +444,8 @@ export class SettingsComponent
   }
 
   ngOnDestroy() {
-    this.settings.sidebarHiddenItemsEditing.set(null)
     if (this.isDirty) this.settings.updateAppearanceSettings() // in case user changed appearance but didn't save
     this.storeSub && this.storeSub.unsubscribe()
-    this.sidebarItemsSub.unsubscribe()
-  }
-
-  isSidebarItemShown(item: HideableSidebarItemID): boolean {
-    return !(this.settingsForm.value.sidebarHiddenItems || []).includes(item)
-  }
-
-  toggleSidebarItem(item: HideableSidebarItemID, checked: boolean): void {
-    this.settings.updateSidebarItemVisibility(item, checked)
   }
 
   public saveSettings() {
@@ -520,10 +473,6 @@ export class SettingsComponent
       SETTINGS_KEYS.SLIM_SIDEBAR,
       this.settingsForm.value.slimSidebarEnabled
     )
-    this.settings.set(
-      SETTINGS_KEYS.SIDEBAR_HIDDEN_ITEMS,
-      this.settingsForm.value.sidebarHiddenItems
-    )
     this.settings.set(
       SETTINGS_KEYS.DARK_MODE_USE_SYSTEM,
       this.settingsForm.value.darkModeUseSystem
@@ -683,11 +632,6 @@ export class SettingsComponent
 
   reset() {
     this.settingsForm.patchValue(this.store.getValue())
-    if (this.settings.organizingSidebarItems()) {
-      this.settings.sidebarHiddenItemsEditing.set([
-        ...this.settingsForm.controls.sidebarHiddenItems.value,
-      ])
-    }
   }
 
   clearThemeColor() {

src-ui/src/app/components/app-frame/app-frame.component.html

@@ -86,15 +86,12 @@
       }
       <div class="sidebar-sticky pt-3 pb-1 d-flex flex-column justify-space-around">
         <ul class="nav flex-column">
-          <li class="nav-item app-link position-relative" [class.d-none]="settingsService.sidebarItemIsHidden(HideableSidebarItemID.Dashboard) && !settingsService.organizingSidebarItems()">
+          <li class="nav-item app-link">
-            <a class="nav-link" [class.opacity-50]="settingsService.sidebarItemIsHidden(HideableSidebarItemID.Dashboard)" [class.pe-5]="settingsService.organizingSidebarItems() && !slimSidebarEnabled && !slimSidebarAnimating()" routerLink="dashboard" routerLinkActive="active" (click)="closeMenu()"
+            <a class="nav-link" routerLink="dashboard" routerLinkActive="active" (click)="closeMenu()"
               ngbPopover="Dashboard" i18n-ngbPopover [disablePopover]="!slimSidebarPopoversEnabled" placement="end"
               container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
               <i-bs class="me-2" name="house"></i-bs><span class="nav-link-label"><ng-container i18n>Dashboard</ng-container></span>
             </a>
-            @if (settingsService.organizingSidebarItems()) {
-              <pngx-input-switch class="position-absolute top-50 end-0 translate-middle-y me-1" [class.d-none]="slimSidebarEnabled || slimSidebarAnimating()" [compact]="true" title="Dashboard" i18n-title [ngModel]="!settingsService.sidebarItemIsHidden(HideableSidebarItemID.Dashboard)" (ngModelChange)="toggleSidebarItem(HideableSidebarItemID.Dashboard, $event)"></pngx-input-switch>
-            }
           </li>
           <li class="nav-item app-link" *pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.Document }">
             <a class="nav-link" routerLink="documents" routerLinkActive="active"
@@ -240,38 +237,29 @@
                 </div>
               </li>
             }
-            <li class="nav-item app-link position-relative" [class.d-none]="settingsService.sidebarItemIsHidden(HideableSidebarItemID.SavedViews) && !settingsService.organizingSidebarItems()" *pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.SavedView }">
+            <li class="nav-item app-link" *pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.SavedView }">
-              <a class="nav-link" [class.opacity-50]="settingsService.sidebarItemIsHidden(HideableSidebarItemID.SavedViews)" [class.pe-5]="settingsService.organizingSidebarItems() && !slimSidebarEnabled && !slimSidebarAnimating()" routerLink="savedviews" routerLinkActive="active" (click)="closeMenu()"
+              <a class="nav-link" routerLink="savedviews" routerLinkActive="active" (click)="closeMenu()"
                 ngbPopover="Saved Views" i18n-ngbPopover [disablePopover]="!slimSidebarPopoversEnabled" placement="end"
                 container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
                 <i-bs class="me-2" name="window-stack"></i-bs><span class="nav-link-label"><ng-container i18n>Saved Views</ng-container></span>
               </a>
-              @if (settingsService.organizingSidebarItems()) {
-                <pngx-input-switch class="position-absolute top-50 end-0 translate-middle-y me-1" [class.d-none]="slimSidebarEnabled || slimSidebarAnimating()" [compact]="true" title="Saved Views" i18n-title [ngModel]="!settingsService.sidebarItemIsHidden(HideableSidebarItemID.SavedViews)" (ngModelChange)="toggleSidebarItem(HideableSidebarItemID.SavedViews, $event)"></pngx-input-switch>
-              }
             </li>
-            <li class="nav-item app-link position-relative" [class.d-none]="settingsService.sidebarItemIsHidden(HideableSidebarItemID.Workflows) && !settingsService.organizingSidebarItems()"
+            <li class="nav-item app-link"
               *pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.Workflow }"
               tourAnchor="tour.workflows">
-              <a class="nav-link" [class.opacity-50]="settingsService.sidebarItemIsHidden(HideableSidebarItemID.Workflows)" [class.pe-5]="settingsService.organizingSidebarItems() && !slimSidebarEnabled && !slimSidebarAnimating()" routerLink="workflows" routerLinkActive="active" (click)="closeMenu()"
+              <a class="nav-link" routerLink="workflows" routerLinkActive="active" (click)="closeMenu()"
                 ngbPopover="Workflows" i18n-ngbPopover [disablePopover]="!slimSidebarPopoversEnabled" placement="end"
                 container="body" triggers="mouseenter:mouseleave" popoverClass="popover-slim">
                 <i-bs class="me-2" name="boxes"></i-bs><span class="nav-link-label"><ng-container i18n>Workflows</ng-container></span>
               </a>
-              @if (settingsService.organizingSidebarItems()) {
-                <pngx-input-switch class="position-absolute top-50 end-0 translate-middle-y me-1" [class.d-none]="slimSidebarEnabled || slimSidebarAnimating()" [compact]="true" title="Workflows" i18n-title [ngModel]="!settingsService.sidebarItemIsHidden(HideableSidebarItemID.Workflows)" (ngModelChange)="toggleSidebarItem(HideableSidebarItemID.Workflows, $event)"></pngx-input-switch>
-              }
             </li>
-            <li class="nav-item app-link position-relative" [class.d-none]="settingsService.sidebarItemIsHidden(HideableSidebarItemID.Mail) && !settingsService.organizingSidebarItems()" *pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.MailAccount }"
+            <li class="nav-item app-link" *pngxIfPermissions="{ action: PermissionAction.View, type: PermissionType.MailAccount }"
               tourAnchor="tour.mail">
-              <a class="nav-link" [class.opacity-50]="settingsService.sidebarItemIsHidden(HideableSidebarItemID.Mail)" [class.pe-5]="settingsService.organizingSidebarItems() && !slimSidebarEnabled && !slimSidebarAnimating()" routerLink="mail" routerLinkActive="active" (click)="closeMenu()" ngbPopover="Mail"
+              <a class="nav-link" routerLink="mail" routerLinkActive="active" (click)="closeMenu()" ngbPopover="Mail"
                 i18n-ngbPopover [disablePopover]="!slimSidebarPopoversEnabled" placement="end" container="body"
                 triggers="mouseenter:mouseleave" popoverClass="popover-slim">
                 <i-bs class="me-2" name="envelope"></i-bs><span class="nav-link-label"><ng-container i18n>Mail</ng-container></span>
               </a>
-              @if (settingsService.organizingSidebarItems()) {
-                <pngx-input-switch class="position-absolute top-50 end-0 translate-middle-y me-1" [class.d-none]="slimSidebarEnabled || slimSidebarAnimating()" [compact]="true" title="Mail" i18n-title [ngModel]="!settingsService.sidebarItemIsHidden(HideableSidebarItemID.Mail)" (ngModelChange)="toggleSidebarItem(HideableSidebarItemID.Mail, $event)"></pngx-input-switch>
-              }
             </li>
             <li class="nav-item app-link" *pngxIfPermissions="{ action: PermissionAction.Delete, type: PermissionType.Document }">
               <a class="nav-link" routerLink="trash" routerLinkActive="active" (click)="closeMenu()" ngbPopover="Trash"
@@ -334,16 +322,13 @@
                 </a>
               </li>
             }
-            <li class="nav-item mt-2 position-relative" [class.d-none]="settingsService.sidebarItemIsHidden(HideableSidebarItemID.Documentation) && !settingsService.organizingSidebarItems()" tourAnchor="tour.outro">
+            <li class="nav-item mt-2" tourAnchor="tour.outro">
-              <a class="text-muted small d-flex align-items-center flex-wrap text-decoration-none nav-anchor" [class.opacity-50]="settingsService.sidebarItemIsHidden(HideableSidebarItemID.Documentation)" [class.pe-5]="settingsService.organizingSidebarItems() && !slimSidebarEnabled && !slimSidebarAnimating()"
+              <a class="text-muted small d-flex align-items-center flex-wrap text-decoration-none nav-anchor"
                 target="_blank" rel="noopener noreferrer" href="https://docs.paperless-ngx.com" ngbPopover="Documentation"
                 i18n-ngbPopover [disablePopover]="!slimSidebarPopoversEnabled" placement="end" container="body"
                 triggers="mouseenter:mouseleave" popoverClass="popover-slim">
                 <i-bs class="d-flex me-2" name="question-circle"></i-bs><span><ng-container i18n>Documentation</ng-container></span>
               </a>
-              @if (settingsService.organizingSidebarItems()) {
-                <pngx-input-switch class="position-absolute top-50 end-0 translate-middle-y me-1" [class.d-none]="slimSidebarEnabled || slimSidebarAnimating()" [compact]="true" title="Documentation" i18n-title [ngModel]="!settingsService.sidebarItemIsHidden(HideableSidebarItemID.Documentation)" (ngModelChange)="toggleSidebarItem(HideableSidebarItemID.Documentation, $event)"></pngx-input-switch>
-              }
             </li>
             <li class="nav-item" [class.visually-hidden]="slimSidebarEnabled">
               <div class="text-muted small d-flex align-items-center flex-wrap nav-label">

src-ui/src/app/components/app-frame/app-frame.component.spec.ts

@@ -15,7 +15,7 @@ import { provideUiTour } from 'ngx-ui-tour-ng-bootstrap'
 import { of, throwError } from 'rxjs'
 import { routes } from 'src/app/app-routing.module'
 import { SavedView } from 'src/app/data/saved-view'
-import { HideableSidebarItemID, SETTINGS_KEYS } from 'src/app/data/ui-settings'
+import { SETTINGS_KEYS } from 'src/app/data/ui-settings'
 import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive'
 import { PermissionsGuard } from 'src/app/guards/permissions.guard'
 import {
@@ -287,82 +287,6 @@ describe('AppFrameComponent', () => {
     jest.useRealTimers()
   })
 
-  it('should hide configured sidebar items', () => {
-    settingsService.set(SETTINGS_KEYS.SIDEBAR_HIDDEN_ITEMS, [
-      HideableSidebarItemID.Dashboard,
-      HideableSidebarItemID.Workflows,
-    ])
-    fixture.detectChanges()
-
-    expect(
-      fixture.nativeElement.querySelector('[routerLink="dashboard"]')
-        .parentElement.classList
-    ).toContain('d-none')
-    expect(
-      fixture.nativeElement.querySelector('[routerLink="workflows"]')
-        .parentElement.classList
-    ).toContain('d-none')
-    expect(
-      fixture.nativeElement.querySelector('[routerLink="mail"]').parentElement
-        .classList
-    ).not.toContain('d-none')
-  })
-
-  it('should show hidden items and visibility switches while customizing', () => {
-    settingsService.set(SETTINGS_KEYS.SIDEBAR_HIDDEN_ITEMS, [
-      HideableSidebarItemID.Dashboard,
-    ])
-    settingsService.sidebarHiddenItemsEditing.set([
-      HideableSidebarItemID.Dashboard,
-    ])
-    fixture.detectChanges()
-
-    expect(
-      fixture.nativeElement.querySelectorAll('pngx-input-switch').length
-    ).toBe(5)
-    expect(
-      fixture.nativeElement.querySelector('[routerLink="dashboard"]')
-        .parentElement.classList
-    ).not.toContain('d-none')
-    expect(
-      fixture.nativeElement.querySelector('[routerLink="dashboard"]').classList
-    ).toContain('opacity-50')
-
-    settingsService.set(SETTINGS_KEYS.SLIM_SIDEBAR, true)
-    fixture.detectChanges()
-
-    expect(
-      Array.from(
-        fixture.nativeElement.querySelectorAll('pngx-input-switch')
-      ).every((toggle: HTMLElement) => toggle.classList.contains('d-none'))
-    ).toBe(true)
-    expect(
-      fixture.nativeElement.querySelector('[routerLink="dashboard"]').classList
-    ).not.toContain('pe-5')
-
-    settingsService.set(SETTINGS_KEYS.SLIM_SIDEBAR, false)
-    component.slimSidebarAnimating.set(true)
-    fixture.detectChanges()
-
-    expect(
-      Array.from(
-        fixture.nativeElement.querySelectorAll('pngx-input-switch')
-      ).every((toggle: HTMLElement) => toggle.classList.contains('d-none'))
-    ).toBe(true)
-
-    component.slimSidebarAnimating.set(false)
-    fixture.detectChanges()
-
-    expect(
-      Array.from(
-        fixture.nativeElement.querySelectorAll('pngx-input-switch')
-      ).every((toggle: HTMLElement) => !toggle.classList.contains('d-none'))
-    ).toBe(true)
-    expect(
-      fixture.nativeElement.querySelector('[routerLink="dashboard"]').classList
-    ).toContain('pe-5')
-  })
-
   it('should show error on toggle slim sidebar if store settings fails', () => {
     jest.spyOn(console, 'warn').mockImplementation(() => {})
     const toastSpy = jest.spyOn(toastService, 'showError')

src-ui/src/app/components/app-frame/app-frame.component.ts

@@ -7,7 +7,6 @@ import {
 } from '@angular/cdk/drag-drop'
 import { NgClass } from '@angular/common'
 import { Component, HostListener, inject, OnInit, signal } from '@angular/core'
-import { FormsModule } from '@angular/forms'
 import { ActivatedRoute, Router, RouterModule } from '@angular/router'
 import {
   NgbCollapseModule,
@@ -22,11 +21,7 @@ import { Observable } from 'rxjs'
 import { first } from 'rxjs/operators'
 import { Document } from 'src/app/data/document'
 import { SavedView } from 'src/app/data/saved-view'
-import {
+import { CollapsibleSection, SETTINGS_KEYS } from 'src/app/data/ui-settings'
-  CollapsibleSection,
-  HideableSidebarItemID,
-  SETTINGS_KEYS,
-} from 'src/app/data/ui-settings'
 import { IfPermissionsDirective } from 'src/app/directives/if-permissions.directive'
 import { ComponentCanDeactivate } from 'src/app/guards/dirty-doc.guard'
 import { DocumentTitlePipe } from 'src/app/pipes/document-title.pipe'
@@ -53,7 +48,6 @@ import { ChatComponent } from '../chat/chat/chat.component'
 import { BrandMarkComponent } from '../common/logo/brand-mark/brand-mark.component'
 import { LogoComponent } from '../common/logo/logo.component'
 import { ProfileEditDialogComponent } from '../common/profile-edit-dialog/profile-edit-dialog.component'
-import { SwitchComponent } from '../common/input/switch/switch.component'
 import { DocumentDetailComponent } from '../document-detail/document-detail.component'
 import { ComponentWithPermissions } from '../with-permissions/with-permissions.component'
 import { GlobalSearchComponent } from './global-search/global-search.component'
@@ -82,8 +76,6 @@ const SCROLL_THRESHOLD = 16
     NgxBootstrapIconsModule,
     DragDropModule,
     TourNgBootstrap,
-    FormsModule,
-    SwitchComponent,
   ],
 })
 export class AppFrameComponent
@@ -106,7 +98,6 @@ export class AppFrameComponent
   readonly isMenuCollapsed = signal(true)
   readonly slimSidebarAnimating = signal(false)
   readonly mobileSearchHidden = signal(false)
-  readonly HideableSidebarItemID = HideableSidebarItemID
   private readonly versionSetting = this.settingsService.getSignal<string>(
     SETTINGS_KEYS.VERSION
   )
@@ -204,10 +195,6 @@ export class AppFrameComponent
     }, 200) // slightly longer than css animation for slim sidebar
   }
 
-  toggleSidebarItem(item: HideableSidebarItemID, visible: boolean): void {
-    this.settingsService.updateSidebarItemVisibility(item, visible)
-  }
-
   toggleAttributesSections(event?: Event): void {
     event?.preventDefault()
     event?.stopPropagation()

src-ui/src/app/components/common/input/switch/switch.component.html

@@ -1,6 +1,6 @@
-<div [class.mb-3]="!compact">
+<div class="mb-3">
-  <div [class.row]="!compact">
+  <div class="row">
-    @if (!horizontal && !compact) {
+    @if (!horizontal) {
       <div class="d-flex align-items-center position-relative hidden-button-container col-md-3">
         <label class="form-label" [for]="inputId" [ngbTooltip]="showUnsetNote && isUnset ? tipContent: null" placement="end">
           {{title}}
@@ -17,8 +17,8 @@
     }
     <div [ngClass]="{'align-items-center': horizontal, 'd-flex': horizontal}">
       <div class="form-check form-switch">
-        <input #inputField type="checkbox" class="form-check-input" [id]="inputId" [(ngModel)]="value" [ngModelOptions]="{standalone: true}" (change)="onChange(value)" (blur)="onTouched()" [disabled]="disabled" [attr.aria-label]="compact ? title : null">
+        <input #inputField type="checkbox" class="form-check-input" [id]="inputId" [(ngModel)]="value" [ngModelOptions]="{standalone: true}" (change)="onChange(value)" (blur)="onTouched()" [disabled]="disabled">
-        @if (horizontal && !compact) {
+        @if (horizontal) {
           <label class="form-check-label" [class.text-muted]="showUnsetNote && isUnset" [for]="inputId" [ngbTooltip]="showUnsetNote && isUnset ? tipContent: null" placement="end">
             {{title}}
             @if (showUnsetNote && isUnset) {

src-ui/src/app/components/common/input/switch/switch.component.spec.ts

@@ -48,14 +48,4 @@ describe('SwitchComponent', () => {
     component.value = undefined
     expect(component.isUnset).toBeTruthy()
   })
-
-  it('should support a compact layout', () => {
-    component.compact = true
-    component.title = 'Test switch'
-    fixture.detectChanges()
-
-    expect(fixture.nativeElement.querySelector('.mb-3')).toBeNull()
-    expect(fixture.nativeElement.querySelector('.row')).toBeNull()
-    expect(input.getAttribute('aria-label')).toEqual('Test switch')
-  })
 })

src-ui/src/app/components/common/input/switch/switch.component.ts

@@ -25,9 +25,6 @@ export class SwitchComponent extends AbstractInputComponent<boolean> {
   @Input()
   showUnsetNote: boolean = false
 
-  @Input()
-  compact: boolean = false
-
   constructor() {
     super()
   }

src-ui/src/app/data/ui-settings.ts

@@ -24,16 +24,6 @@ export enum CollapsibleSection {
   ATTRIBUTES = 'attributes',
 }
 
-export enum HideableSidebarItemID {
-  Dashboard = 'dashboard',
-  SavedViews = 'saved_views',
-  Workflows = 'workflows',
-  Mail = 'mail',
-  Documentation = 'documentation',
-}
-
-export const HIDEABLE_SIDEBAR_ITEM_IDS = Object.values(HideableSidebarItemID)
-
 export const PAPERLESS_GREEN_HEX = '#17541f'
 
 export const SETTINGS_KEYS = {
@@ -66,7 +56,6 @@ export const SETTINGS_KEYS = {
   NOTES_ENABLED: 'general-settings:notes-enabled',
   AUDITLOG_ENABLED: 'general-settings:auditlog-enabled',
   SLIM_SIDEBAR: 'general-settings:slim-sidebar',
-  SIDEBAR_HIDDEN_ITEMS: 'general-settings:sidebar:hidden-items',
   ATTRIBUTES_SECTIONS_COLLAPSED:
     'general-settings:attributes-sections-collapsed',
   UPDATE_CHECKING_ENABLED: 'general-settings:update-checking:enabled',
@@ -138,11 +127,6 @@ export const SETTINGS: UiSetting[] = [
     type: 'boolean',
     default: false,
   },
-  {
-    key: SETTINGS_KEYS.SIDEBAR_HIDDEN_ITEMS,
-    type: 'array',
-    default: [],
-  },
   {
     key: SETTINGS_KEYS.ATTRIBUTES_SECTIONS_COLLAPSED,
     type: 'array',

src-ui/src/app/services/settings.service.spec.ts

@@ -14,11 +14,7 @@ import { CustomFieldDataType } from '../data/custom-field'
 import { DEFAULT_DISPLAY_FIELDS, DisplayField } from '../data/document'
 import { SavedView } from '../data/saved-view'
 import { RemoteOCRModeConfig } from '../data/paperless-config'
-import {
+import { SETTINGS_KEYS, UiSettings } from '../data/ui-settings'
-  HideableSidebarItemID,
-  SETTINGS_KEYS,
-  UiSettings,
-} from '../data/ui-settings'
 import { PermissionsService } from './permissions.service'
 import { CustomFieldsService } from './rest/custom-fields.service'
 import { SettingsService } from './settings.service'
@@ -234,35 +230,6 @@ describe('SettingsService', () => {
     expect(notesEnabled()).toBeFalsy()
   })
 
-  it('updates sidebar item visibility', () => {
-    httpTestingController
-      .expectOne(`${environment.apiBaseUrl}ui_settings/`)
-      .flush(ui_settings)
-
-    expect(
-      settingsService.sidebarItemIsHidden(HideableSidebarItemID.Workflows)
-    ).toBe(false)
-
-    settingsService.updateSidebarItemVisibility(
-      HideableSidebarItemID.Workflows,
-      false
-    )
-
-    expect(
-      settingsService.sidebarItemIsHidden(HideableSidebarItemID.Workflows)
-    ).toBe(true)
-    expect(settingsService.get(SETTINGS_KEYS.SIDEBAR_HIDDEN_ITEMS)).toEqual([])
-
-    settingsService.updateSidebarItemVisibility(
-      HideableSidebarItemID.Workflows,
-      true
-    )
-
-    expect(
-      settingsService.sidebarItemIsHidden(HideableSidebarItemID.Workflows)
-    ).toBe(false)
-  })
-
   it('updates setting signals when settings are reinitialized', () => {
     let req = httpTestingController.expectOne(
       `${environment.apiBaseUrl}ui_settings/`

src-ui/src/app/services/settings.service.ts

@@ -24,7 +24,6 @@ import { DEFAULT_DISPLAY_FIELDS, DisplayField } from '../data/document'
 import { RemoteOCRModeConfig } from '../data/paperless-config'
 import { SavedView } from '../data/saved-view'
 import {
-  HideableSidebarItemID,
   PAPERLESS_GREEN_HEX,
   SETTINGS,
   SETTINGS_KEYS,
@@ -314,18 +313,6 @@ export class SettingsService {
   readonly globalDropzoneEnabled = signal(true)
   readonly globalDropzoneActive = signal(false)
   readonly organizingSidebarSavedViews = signal(false)
-  readonly sidebarHiddenItemsEditing = signal<HideableSidebarItemID[] | null>(
-    null
-  )
-  readonly organizingSidebarItems = computed(
-    () => this.sidebarHiddenItemsEditing() !== null
-  )
-  readonly sidebarHiddenItemsEditingChanged = new EventEmitter<
-    HideableSidebarItemID[]
-  >()
-  readonly hiddenSidebarItems = this.getSignal<HideableSidebarItemID[]>(
-    SETTINGS_KEYS.SIDEBAR_HIDDEN_ITEMS
-  )
 
   readonly allDisplayFields = signal<Array<{ id: DisplayField; name: string }>>(
     DEFAULT_DISPLAY_FIELDS
@@ -762,29 +749,6 @@ export class SettingsService {
     return this.storeSettings()
   }
 
-  sidebarItemIsHidden(item: HideableSidebarItemID): boolean {
-    return (
-      this.sidebarHiddenItemsEditing() ?? this.hiddenSidebarItems()
-    ).includes(item)
-  }
-
-  updateSidebarItemVisibility(
-    item: HideableSidebarItemID,
-    visible: boolean
-  ): void {
-    const hiddenItems = new Set(
-      this.sidebarHiddenItemsEditing() ?? this.hiddenSidebarItems()
-    )
-    if (visible) {
-      hiddenItems.delete(item)
-    } else {
-      hiddenItems.add(item)
-    }
-    const updatedHiddenItems = [...hiddenItems]
-    this.sidebarHiddenItemsEditing.set(updatedHiddenItems)
-    this.sidebarHiddenItemsEditingChanged.emit(updatedHiddenItems)
-  }
-
   updateSavedViewsVisibility(
     dashboardVisibleViewIds: number[],
     sidebarVisibleViewIds: number[]

src/documents/apps.py

@@ -28,7 +28,6 @@ class DocumentsConfig(AppConfig):
         document_consumption_finished.connect(set_storage_path)
         document_consumption_finished.connect(add_to_index)
         document_consumption_finished.connect(run_workflows_added)
-        document_consumption_finished.connect(add_to_index)
         document_consumption_finished.connect(add_or_update_document_in_llm_index)
         document_updated.connect(run_workflows_updated)
         document_updated.connect(send_websocket_document_updated)

src/documents/bulk_edit.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import logging
 import tempfile
-import uuid
 from pathlib import Path
 from typing import TYPE_CHECKING
 from typing import Literal
@@ -299,55 +298,53 @@ def modify_custom_fields(
 ) -> Literal["OK"]:
     qs = Document.objects.filter(id__in=doc_ids).only("pk")
     affected_docs = list(qs.values_list("pk", flat=True))
-    # Ensure add_custom_fields is a list of (int, value) tuples, supports old API
+    # Ensure add_custom_fields is a list of tuples, supports old API
     add_custom_fields = (
-        [(int(field), value) for field, value in add_custom_fields.items()]
+        add_custom_fields.items()
         if isinstance(add_custom_fields, dict)
-        else [(int(field), None) for field in add_custom_fields]
+        else [(field, None) for field in add_custom_fields]
     )
 
-    # Resolved once, instead of re-querying the same field for every document
+    custom_fields = CustomField.objects.filter(
-    custom_fields_by_id: dict[int, CustomField] = CustomField.objects.in_bulk(
+        id__in=[int(field) for field, _ in add_custom_fields],
-        [field_id for field_id, _ in add_custom_fields],
+    ).distinct()
-    )
-    # Passed to update_or_create() below rather than a bare id, so the FK is
-    # cached on the created instance and auditlog's post_save receiver does
-    # not reload it per row. Only needed for additions. content is deferred:
-    # the one field here that is both large and unused.
-    docs_by_id: dict[int, Document] = (
-        Document.objects.defer("content").in_bulk(affected_docs)
-        if add_custom_fields
-        else {}
-    )
     for field_id, value in add_custom_fields:
-        custom_field = custom_fields_by_id[field_id]
-        value_field = CustomFieldInstance.TYPE_TO_DATA_STORE_NAME_MAP[
-            custom_field.data_type
-        ]
-        is_doclink = custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK
         for doc_id in affected_docs:
-            if is_doclink and value and doc_id in value:
+            defaults = {}
-                # Prevent self-linking
+            custom_field = custom_fields.get(id=field_id)
-                continue
+            if custom_field:
+                value_field = CustomFieldInstance.TYPE_TO_DATA_STORE_NAME_MAP[
+                    custom_field.data_type
+                ]
+                defaults[value_field] = value
+                if (
+                    custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK
+                    and value
+                    and doc_id in value
+                ):
+                    # Prevent self-linking
+                    continue
             CustomFieldInstance.objects.update_or_create(
-                document=docs_by_id[doc_id],
+                document_id=doc_id,
-                field=custom_field,
+                field_id=field_id,
-                defaults={value_field: value},
+                defaults=defaults,
             )
-            if is_doclink:
+            if custom_field.data_type == CustomField.FieldDataType.DOCUMENTLINK:
-                reflect_doclinks(docs_by_id[doc_id], custom_field, value)
+                doc = Document.objects.get(id=doc_id)
+                reflect_doclinks(doc, custom_field, value)
 
-    # For doc link fields that are being removed, remove symmetrical links.
+    # For doc link fields that are being removed, remove symmetrical links
-    # select_related avoids a per-instance reload of the document and field.
     for doclink_being_removed_instance in CustomFieldInstance.objects.filter(
         document_id__in=affected_docs,
         field__id__in=remove_custom_fields,
         field__data_type=CustomField.FieldDataType.DOCUMENTLINK,
         value_document_ids__isnull=False,
-    ).select_related("field", "document"):
+    ):
         for target_doc_id in doclink_being_removed_instance.value:
             remove_doclink(
-                document=doclink_being_removed_instance.document,
+                document=Document.objects.get(
+                    id=doclink_being_removed_instance.document.id,
+                ),
                 field=doclink_being_removed_instance.field,
                 target_doc_id=target_doc_id,
             )
@@ -382,7 +379,7 @@ def delete(doc_ids: list[int]) -> Literal["OK"]:
         )
         delete_ids = list({*doc_ids, *version_ids})
 
-        Document.objects.filter(id__in=delete_ids).delete(transaction_id=uuid.uuid4())
+        Document.objects.filter(id__in=delete_ids).delete()
 
         from documents.search import get_backend
 
@@ -1180,13 +1177,10 @@ def remove_doclink(
     """
     Removes a 'symmetrical' link to `document` from the target document's existing custom field instance
     """
-    # select_related: a signal receiver (auditlog) touches .document/.field on
+    target_doc_field_instance = CustomFieldInstance.objects.filter(
-    # the save() below, without this that is a per-call reload query
+        document_id=target_doc_id,
-    target_doc_field_instance = (
+        field=field,
-        CustomFieldInstance.objects.filter(document_id=target_doc_id, field=field)
+    ).first()
-        .select_related("document", "field")
-        .first()
-    )
     if (
         target_doc_field_instance is not None
         and document.id in target_doc_field_instance.value

src/documents/classifier.py

@@ -34,27 +34,6 @@ from paperless.signed_pickle import signed_pickle_loads
 
 logger = logging.getLogger("paperless.classifier")
 
-
-def _predict_with_threshold(classifier, X, threshold: float) -> int | None:
-    """
-    Return the predicted class id, or None if:
-    - the prediction is -1 (no match), or
-    - the winning class probability is below the configured threshold.
-
-    Using predict_proba() instead of predict() lets us apply a minimum-confidence
-    cutoff so that uncertain predictions are discarded rather than assigned.
-    """
-    probas = classifier.predict_proba(X)[0]
-    best_idx = int(probas.argmax())
-    best_class = int(classifier.classes_[best_idx])
-
-    if best_class == -1:
-        return None
-    if threshold > 0.0 and probas[best_idx] < threshold:
-        return None
-    return best_class
-
-
 ADVANCED_TEXT_PROCESSING_ENABLED = (
     settings.NLTK_LANGUAGE is not None and settings.NLTK_ENABLED
 )
@@ -123,8 +102,7 @@ class DocumentClassifier:
     # v8 - Added storage path classifier
     # v9 - Changed from hashing to time/ids for re-train check
     # v10 - HMAC-signed model file
-    # v11 - Use sample_weight for balanced training; predict_proba with threshold
+    FORMAT_VERSION = 10
-    FORMAT_VERSION = 11
 
     HMAC_SIZE = 32  # SHA-256 digest length
 
@@ -346,13 +324,6 @@ class DocumentClassifier:
         from sklearn.preprocessing import LabelBinarizer
         from sklearn.preprocessing import MultiLabelBinarizer
 
-        # MLPClassifier does not support class_weight directly
-        # (https://github.com/scikit-learn/scikit-learn/issues/9113), so we use
-        # compute_sample_weight to balance classes during training and prevent
-        # over-represented correspondents from dominating predictions.
-        # https://scikit-learn.org/stable/modules/generated/sklearn.utils.class_weight.compute_sample_weight.html
-        from sklearn.utils.class_weight import compute_sample_weight
-
         # Step 2: vectorize data
         logger.debug("Vectorizing data...")
         notify("Vectorizing document content...")
@@ -398,7 +369,7 @@ class DocumentClassifier:
                 self.tags_binarizer = MultiLabelBinarizer()
                 labels_tags_vectorized = self.tags_binarizer.fit_transform(labels_tags)
 
-            self.tags_classifier = MLPClassifier(tol=0.01, random_state=0)
+            self.tags_classifier = MLPClassifier(tol=0.01)
             self.tags_classifier.fit(data_vectorized, labels_tags_vectorized)
         else:
             self.tags_classifier = None
@@ -409,12 +380,8 @@ class DocumentClassifier:
             notify(
                 f"Training correspondent classifier ({num_correspondents} correspondent(s))...",
             )
-            self.correspondent_classifier = MLPClassifier(tol=0.01, random_state=0)
+            self.correspondent_classifier = MLPClassifier(tol=0.01)
-            self.correspondent_classifier.fit(
+            self.correspondent_classifier.fit(data_vectorized, labels_correspondent)
-                data_vectorized,
-                labels_correspondent,
-                sample_weight=compute_sample_weight("balanced", labels_correspondent),
-            )
         else:
             self.correspondent_classifier = None
             logger.debug(
@@ -426,12 +393,8 @@ class DocumentClassifier:
             notify(
                 f"Training document type classifier ({num_document_types} type(s))...",
             )
-            self.document_type_classifier = MLPClassifier(tol=0.01, random_state=0)
+            self.document_type_classifier = MLPClassifier(tol=0.01)
-            self.document_type_classifier.fit(
+            self.document_type_classifier.fit(data_vectorized, labels_document_type)
-                data_vectorized,
-                labels_document_type,
-                sample_weight=compute_sample_weight("balanced", labels_document_type),
-            )
         else:
             self.document_type_classifier = None
             logger.debug(
@@ -443,11 +406,10 @@ class DocumentClassifier:
                 "Training storage paths classifier...",
             )
             notify(f"Training storage path classifier ({num_storage_paths} path(s))...")
-            self.storage_path_classifier = MLPClassifier(tol=0.01, random_state=0)
+            self.storage_path_classifier = MLPClassifier(tol=0.01)
             self.storage_path_classifier.fit(
                 data_vectorized,
                 labels_storage_path,
-                sample_weight=compute_sample_weight("balanced", labels_storage_path),
             )
         else:
             self.storage_path_classifier = None
@@ -584,24 +546,24 @@ class DocumentClassifier:
     def predict_correspondent(self, content: str) -> int | None:
         if self.correspondent_classifier:
             X = self._vectorize(content)
-            predicted_id = _predict_with_threshold(
+            correspondent_id = self.correspondent_classifier.predict(X)
-                self.correspondent_classifier,
+            if correspondent_id != -1:
-                X,
+                return correspondent_id
-                settings.CLASSIFIER_MATCH_THRESHOLD,
+            else:
-            )
+                return None
-            return predicted_id
+        else:
-        return None
+            return None
 
     def predict_document_type(self, content: str) -> int | None:
         if self.document_type_classifier:
             X = self._vectorize(content)
-            predicted_id = _predict_with_threshold(
+            document_type_id = self.document_type_classifier.predict(X)
-                self.document_type_classifier,
+            if document_type_id != -1:
-                X,
+                return document_type_id
-                settings.CLASSIFIER_MATCH_THRESHOLD,
+            else:
-            )
+                return None
-            return predicted_id
+        else:
-        return None
+            return None
 
     def predict_tags(self, content: str) -> list[int]:
         from sklearn.utils.multiclass import type_of_target
@@ -627,10 +589,10 @@ class DocumentClassifier:
     def predict_storage_path(self, content: str) -> int | None:
         if self.storage_path_classifier:
             X = self._vectorize(content)
-            predicted_id = _predict_with_threshold(
+            storage_path_id = self.storage_path_classifier.predict(X)
-                self.storage_path_classifier,
+            if storage_path_id != -1:
-                X,
+                return storage_path_id
-                settings.CLASSIFIER_MATCH_THRESHOLD,
+            else:
-            )
+                return None
-            return predicted_id
+        else:
-        return None
+            return None

src/documents/consumer.py

@@ -52,6 +52,7 @@ from documents.templating.workflows import parse_w_workflow_placeholders
 from documents.utils import compute_checksum
 from documents.utils import copy_basic_file_stats
 from documents.utils import copy_file_with_basic_stats
+from documents.utils import normalize_unicode
 from documents.utils import run_subprocess
 from paperless.config import OcrConfig
 from paperless.config import RemoteOCRConfig
@@ -201,7 +202,9 @@ class ConsumerPluginMixin:
 
         self.renew_logging_group()
 
-        self.filename = self.metadata.filename or self.input_doc.original_file.name
+        self.filename = normalize_unicode(
+            self.metadata.filename or self.input_doc.original_file.name,
+        )
 
     def _send_progress(
         self,

src/documents/management/commands/document_consumer.py

@@ -156,15 +156,6 @@ class FileStabilityTracker:
                     logger.debug(f"File disappeared during stability check: {path}")
                 continue
 
-            # Stable, but empty: some scanners create a zero byte placeholder
-            # and only write the page some time later. Consuming it now can
-            # only fail  so drop it and let the writer's next event
-            # (or the periodic rescan) bring it back once it has content
-            if not tracked.last_size:
-                to_remove.append(path)
-                logger.debug("Ignoring stable but empty file: %s", path)
-                continue
-
             # File is stable, we can return it
             to_yield.append(path)
             logger.info(f"File is stable: {path}")

src/documents/matching.py

@@ -21,6 +21,7 @@ from documents.models import Workflow
 from documents.models import WorkflowTrigger
 from documents.permissions import permitted_object_ids
 from documents.regex import safe_regex_search
+from documents.utils import normalize_unicode
 
 if TYPE_CHECKING:
     from django.db.models import QuerySet
@@ -311,11 +312,12 @@ def consumable_document_matches_workflow(
         trigger_matched = False
 
     # Document filename vs trigger filename
+    document_filename = normalize_unicode(document.original_file.name)
     if (
         trigger.filter_filename is not None
         and len(trigger.filter_filename) > 0
         and not fnmatch(
-            document.original_file.name.lower(),
+            document_filename.lower(),
             trigger.filter_filename.lower(),
         )
     ):
@@ -328,10 +330,12 @@ def consumable_document_matches_workflow(
     # Document path vs trigger path
 
     # Use the original_path if set, else us the original_file
-    match_against = (
+    match_against = normalize_unicode(
-        document.original_path
+        str(
-        if document.original_path is not None
+            document.original_path
-        else document.original_file
+            if document.original_path is not None
+            else document.original_file,
+        ),
     )
 
     if (
@@ -536,7 +540,7 @@ def existing_document_matches_workflow(
         and len(trigger.filter_filename) > 0
         and document.original_filename is not None
         and not fnmatch(
-            document.original_filename.lower(),
+            normalize_unicode(document.original_filename).lower(),
             trigger.filter_filename.lower(),
         )
     ):

src/documents/models.py

@@ -1,5 +1,4 @@
 import datetime
-import uuid
 from pathlib import Path
 from typing import Final
 
@@ -28,6 +27,7 @@ from django_softdelete.models import SoftDeleteModel
 
 from documents.data_models import DocumentSource
 from documents.parsers import get_default_file_extension
+from documents.utils import normalize_unicode
 
 
 class ModelWithOwner(models.Model):
@@ -468,7 +468,7 @@ class Document(SoftDeleteModel, ModelWithOwner):  # type: ignore[django-manager-
         context_document = (
             self.root_document if self.root_document_id is not None else self
         )
-        result = str(context_document)
+        result = normalize_unicode(str(context_document))
 
         if counter:
             result += f"_{counter:02}"
@@ -515,20 +515,13 @@ class Document(SoftDeleteModel, ModelWithOwner):  # type: ignore[django-manager-
     def delete(
         self,
         *args,
-        transaction_id=None,
         **kwargs,
     ):
-        # Versions must share the root's transaction ID so they are restored
+        # If deleting a root document, move all its versions to trash as well.
-        # together by django-softdelete.
-        if transaction_id is None:
-            transaction_id = uuid.uuid4()
         if self.root_document_id is None:
-            Document.objects.filter(root_document=self).delete(
+            Document.objects.filter(root_document=self).delete()
-                transaction_id=transaction_id,
-            )
         return super().delete(
             *args,
-            transaction_id=transaction_id,
             **kwargs,
         )
 

src/documents/search/__init__.py

@@ -6,20 +6,13 @@ from documents.search._backend import TantivyRelevanceList
 from documents.search._backend import WriteBatch
 from documents.search._backend import get_backend
 from documents.search._backend import reset_backend
-from documents.search._errors import InvalidDateQuery
-from documents.search._errors import InvalidNumberQuery
-from documents.search._errors import MultipleSearchQueryErrors
-from documents.search._errors import QueryTooLongError
-from documents.search._errors import SearchQueryError
-from documents.search._errors import search_query_error_messages
 from documents.search._schema import needs_rebuild
 from documents.search._schema import wipe_index
+from documents.search._translate import InvalidDateQuery
+from documents.search._translate import SearchQueryError
 
 __all__ = [
     "InvalidDateQuery",
-    "InvalidNumberQuery",
-    "MultipleSearchQueryErrors",
-    "QueryTooLongError",
     "SearchHit",
     "SearchIndexLockError",
     "SearchMode",
@@ -30,6 +23,5 @@ __all__ = [
     "get_backend",
     "needs_rebuild",
     "reset_backend",
-    "search_query_error_messages",
     "wipe_index",
 ]

src/documents/search/_backend.py

@@ -22,6 +22,7 @@ import tantivy
 from django.conf import settings
 from django.utils.timezone import get_current_timezone
 
+from documents.search._query import build_permission_filter
 from documents.search._query import extract_cjk_text
 from documents.search._query import parse_simple_text_highlight_query
 from documents.search._query import parse_simple_text_query
@@ -39,7 +40,6 @@ from documents.utils import QuerySetStream
 from documents.utils import identity
 
 if TYPE_CHECKING:
-    from collections.abc import Iterable
     from collections.abc import Iterator
     from collections.abc import Sequence
     from pathlib import Path
@@ -325,47 +325,6 @@ class WriteBatch:
             self._writer.add_document(doc)
 
 
-def build_permission_filter(
-    schema: tantivy.Schema,
-    user: AbstractUser,
-    viewer_group_ids: Iterable[int] = (),
-) -> tantivy.Query:
-    """
-    Build a query filter for user document permissions.
-
-    Creates a query that matches only documents visible to the specified user
-    according to paperless-ngx permission rules:
-    - Public documents (no owner) are visible to all users
-    - Private documents are visible to their owner
-    - Documents explicitly shared with the user are visible
-    - Documents shared with one of the user's current groups are visible
-
-    Args:
-        schema: Tantivy schema for field validation
-        user: User to check permissions for
-        viewer_group_ids: Current group memberships for the user
-
-    Returns:
-        Tantivy query that filters results to visible documents
-    """
-    owner_any = tantivy.Query.exists_query("owner_id")
-    no_owner = tantivy.Query.boolean_query(
-        [
-            (tantivy.Occur.Must, tantivy.Query.all_query()),
-            (tantivy.Occur.MustNot, owner_any),
-        ],
-    )
-    owned = tantivy.Query.term_query(schema, "owner_id", user.pk)
-    shared = tantivy.Query.term_query(schema, "viewer_id", user.pk)
-    group_shared = [
-        tantivy.Query.term_query(schema, "viewer_group_id", group_id)
-        for group_id in viewer_group_ids
-    ]
-    return tantivy.Query.disjunction_max_query(
-        [no_owner, owned, shared, *group_shared],
-    )
-
-
 class TantivyBackend:
     """
     Tantivy search backend with explicit lifecycle management.
@@ -539,6 +498,7 @@ class TantivyBackend:
             doc.add_text("correspondent_sort", document.correspondent.name)
             if cjk_corr := extract_cjk_text(document.correspondent.name):
                 doc.add_text("bigram_correspondent", cjk_corr)
+            doc.add_unsigned("correspondent_id", document.correspondent_id)
 
         # Document type
         if document.document_type:
@@ -546,10 +506,12 @@ class TantivyBackend:
             doc.add_text("type_sort", document.document_type.name)
             if cjk_type := extract_cjk_text(document.document_type.name):
                 doc.add_text("bigram_document_type", cjk_type)
+            doc.add_unsigned("document_type_id", document.document_type_id)
 
         # Storage path
         if document.storage_path:
             doc.add_text("storage_path", document.storage_path.name)
+            doc.add_unsigned("storage_path_id", document.storage_path_id)
 
         # Tags — collect names for autocomplete in the same pass
         tag_names: list[str] = []
@@ -557,13 +519,12 @@ class TantivyBackend:
             doc.add_text("tag", tag.name)
             if cjk_tag := extract_cjk_text(tag.name):
                 doc.add_text("bigram_tag", cjk_tag)
+            doc.add_unsigned("tag_id", tag.pk)
             tag_names.append(tag.name)
 
         # Notes — JSON for structured queries (notes.user:alice, notes.note:text).
         # notes_text is a plain-text companion for snippet/highlight generation;
-        # tantivy's SnippetGenerator does not support JSON fields. It is not in
+        # tantivy's SnippetGenerator does not support JSON fields.
-        # _DEFAULT_SEARCH_FIELDS, so an unqualified query never searches it: a
-        # note matches through the JSON field or not at all.
         num_notes = 0
         note_texts: list[str] = []
         for note in document.notes.all():
@@ -579,9 +540,8 @@ class TantivyBackend:
         if note_texts:
             doc.add_text("notes_text", " ".join(note_texts))
 
-        # Custom fields: JSON for structured queries (custom_fields.name:x,
+        # Custom fields — JSON for structured queries (custom_fields.name:x, custom_fields.value:y),
-        # custom_fields.value:y). There is no companion text field here, unlike
+        # companion text field for default full-text search.
-        # notes: custom field values are reachable only through the JSON field.
         for cfi in document.custom_fields.all():
             search_value = cfi.value_for_search
             # Skip fields where there is no value yet
@@ -748,17 +708,7 @@ class TantivyBackend:
         user_query = self._parse_query(query, search_mode)
         highlight_query = user_query
         if search_mode is SearchMode.TEXT:
-            try:
+            highlight_query = parse_simple_text_highlight_query(self._index, query)
-                highlight_query = parse_simple_text_highlight_query(
-                    self._index,
-                    query,
-                )
-            except ValueError:
-                logger.debug(
-                    "Skipping simple text highlight query: token string is not "
-                    "valid tantivy query syntax: %r",
-                    query,
-                )
 
         # For notes_text snippet generation, we need a query that targets the
         # notes_text field directly. user_query may contain JSON-field terms

src/documents/search/_dates.py

@@ -0,0 +1,171 @@
+from __future__ import annotations
+
+from datetime import UTC
+from datetime import date
+from datetime import datetime
+from datetime import timedelta
+from typing import TYPE_CHECKING
+from typing import Final
+
+from dateutil.relativedelta import relativedelta
+
+if TYPE_CHECKING:
+    from datetime import tzinfo
+
+_DATE_ONLY_FIELDS = frozenset({"created"})
+
+_TODAY: Final[str] = "today"
+_YESTERDAY: Final[str] = "yesterday"
+_PREVIOUS_WEEK: Final[str] = "previous week"
+_THIS_MONTH: Final[str] = "this month"
+_PREVIOUS_MONTH: Final[str] = "previous month"
+_THIS_YEAR: Final[str] = "this year"
+_PREVIOUS_YEAR: Final[str] = "previous year"
+_PREVIOUS_QUARTER: Final[str] = "previous quarter"
+
+_DATE_KEYWORDS = frozenset(
+    {
+        _TODAY,
+        _YESTERDAY,
+        _PREVIOUS_WEEK,
+        _THIS_MONTH,
+        _PREVIOUS_MONTH,
+        _THIS_YEAR,
+        _PREVIOUS_YEAR,
+        _PREVIOUS_QUARTER,
+    },
+)
+
+
+def _fmt(dt: datetime) -> str:
+    """Format a datetime as an ISO 8601 UTC string for use in Tantivy range queries."""
+    return dt.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def _iso_range(lo: datetime, hi: datetime) -> str:
+    """
+    Format a half-open ``[lo TO hi)`` range in ISO 8601 for Tantivy query syntax.
+
+    ``hi`` is always the exclusive ceiling of a computed period (the start of
+    the *next* day/week/month/quarter/year), so the closing bracket must be
+    the Tantivy exclusive-range brace ``}`` rather than ``]`` — otherwise the
+    first instant of the following period (e.g. the 1st of next month) is
+    incorrectly included in the match.
+    """
+    return f"[{_fmt(lo)} TO {_fmt(hi)}}}"
+
+
+def _quarter_start(d: date) -> date:
+    """Return the first day of the calendar quarter containing ``d``."""
+    return date(d.year, ((d.month - 1) // 3) * 3 + 1, 1)
+
+
+def _midnight(d: date, tz: tzinfo) -> datetime:
+    """Convert a calendar date at local-timezone midnight to a UTC datetime."""
+    return datetime(d.year, d.month, d.day, tzinfo=tz).astimezone(UTC)
+
+
+def _keyword_bounds(keyword: str, tz: tzinfo) -> tuple[date, date]:
+    """
+    Map a relative date keyword to ``(start, exclusive_end)`` calendar dates.
+
+    ``tz`` only determines what "today" is; the caller decides how the returned
+    dates become UTC datetime boundaries (date-only vs. local-midnight offset).
+    """
+    today = datetime.now(tz).date()
+    if keyword == _TODAY:
+        return today, today + timedelta(days=1)
+    if keyword == _YESTERDAY:
+        return today - timedelta(days=1), today
+    if keyword == _PREVIOUS_WEEK:
+        this_monday = today - timedelta(days=today.weekday())
+        return this_monday - timedelta(weeks=1), this_monday
+    if keyword == _THIS_MONTH:
+        first = today.replace(day=1)
+        return first, first + relativedelta(months=1)
+    if keyword == _PREVIOUS_MONTH:
+        this_first = today.replace(day=1)
+        return this_first - relativedelta(months=1), this_first
+    if keyword == _THIS_YEAR:
+        return date(today.year, 1, 1), date(today.year + 1, 1, 1)
+    if keyword == _PREVIOUS_YEAR:
+        return date(today.year - 1, 1, 1), date(today.year, 1, 1)
+    if keyword == _PREVIOUS_QUARTER:
+        this_quarter = _quarter_start(today)
+        return this_quarter - relativedelta(months=3), this_quarter
+    raise ValueError(f"Unknown keyword: {keyword}")
+
+
+def _date_only_range(keyword: str, tz: tzinfo) -> str:
+    """
+    For `created` (DateField): use the local calendar date, converted to
+    midnight UTC boundaries. No offset arithmetic — date only.
+    """
+    start, end = _keyword_bounds(keyword, tz)
+    lo = datetime(start.year, start.month, start.day, tzinfo=UTC)
+    hi = datetime(end.year, end.month, end.day, tzinfo=UTC)
+    return _iso_range(lo, hi)
+
+
+def _datetime_range(keyword: str, tz: tzinfo) -> str:
+    """
+    For `added` / `modified` (DateTimeField, stored as UTC): convert local day
+    boundaries to UTC — full offset arithmetic required.
+    """
+    start, end = _keyword_bounds(keyword, tz)
+    return _iso_range(_midnight(start, tz), _midnight(end, tz))
+
+
+def _precision_bounds(digits: str) -> tuple[date, date] | None:
+    """
+    Map a 4/6/8-digit date token to (start, exclusive_end) calendar dates.
+
+    YYYY -> whole year, YYYYMM -> whole month, YYYYMMDD -> single day.
+    Returns None for any unparsable or out-of-range value (e.g. month 23),
+    so callers can emit a no-match clause instead of erroring (Whoosh parity).
+    """
+    try:
+        if len(digits) == 4:
+            year = int(digits)
+            return date(year, 1, 1), date(year + 1, 1, 1)
+        if len(digits) == 6:
+            year, month = int(digits[:4]), int(digits[4:6])
+            start = date(year, month, 1)
+            end = date(year + 1, 1, 1) if month == 12 else date(year, month + 1, 1)
+            return start, end
+        if len(digits) == 8:
+            start = date(int(digits[:4]), int(digits[4:6]), int(digits[6:8]))
+            return start, start + timedelta(days=1)
+    except ValueError:
+        return None
+    return None
+
+
+def _utc_bounds_for_field(
+    field: str,
+    start: date,
+    end: date,
+    tz: tzinfo,
+) -> tuple[datetime, datetime]:
+    """
+    Convert calendar-date bounds to UTC datetimes per the field's storage type.
+
+    For DateField (``created``) the bounds are UTC midnight (no offset). For
+    DateTimeField (``added``/``modified``) the bounds are local-tz midnight
+    converted to UTC, matching how each field is indexed.
+    """
+    if field in _DATE_ONLY_FIELDS:
+        return (
+            datetime(start.year, start.month, start.day, tzinfo=UTC),
+            datetime(end.year, end.month, end.day, tzinfo=UTC),
+        )
+    return (
+        datetime(start.year, start.month, start.day, tzinfo=tz).astimezone(UTC),
+        datetime(end.year, end.month, end.day, tzinfo=tz).astimezone(UTC),
+    )
+
+
+def _field_range_from_dates(field: str, start: date, end: date, tz: tzinfo) -> str:
+    """Build a Tantivy ``field:[lo TO hi]`` ISO range from calendar-date bounds."""
+    lo, hi = _utc_bounds_for_field(field, start, end, tz)
+    return f"{field}:{_iso_range(lo, hi)}"

src/documents/search/_errors.py

@@ -1,71 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from collections.abc import Sequence
-
-
-class SearchQueryError(ValueError):
-    """
-    Base for user-fixable search query errors.
-
-    Carries a message safe to surface to the user (no internal details). The
-    view layer catches this and returns an HTTP 400, so any future subclass
-    gets the same treatment.
-    """
-
-
-class InvalidDateQuery(SearchQueryError):
-    """Raised when a date field value or range bound cannot be parsed."""
-
-    def __init__(self, field: str | None, value: str | None) -> None:
-        self.field = field
-        self.value = value
-        super().__init__(f"Invalid date value {value!r} for field {field!r}.")
-
-
-class InvalidNumberQuery(SearchQueryError):
-    """Raised when a numeric field value or range bound cannot be parsed."""
-
-    def __init__(self, field: str | None, value: str | None) -> None:
-        self.field = field
-        self.value = value
-        super().__init__(f"Invalid numeric value {value!r} for field {field!r}.")
-
-
-class QueryTooLongError(SearchQueryError):
-    """Raised when a query string exceeds the maximum allowed length.
-
-    whoosh-compat's fieldname tagger is O(n^2) in plain word characters, so an
-    unbounded query is a CPU-exhaustion vector against a single request
-    handler. This is a hard boundary, not a validation nicety.
-    """
-
-    def __init__(self, length: int, limit: int) -> None:
-        self.length = length
-        self.limit = limit
-        super().__init__(
-            f"The search query is too long ({length} characters). "
-            f"The maximum allowed length is {limit} characters.",
-        )
-
-
-class MultipleSearchQueryErrors(SearchQueryError):
-    """Aggregates every user-fixable error from one parse, not just the first."""
-
-    def __init__(self, errors: Sequence[SearchQueryError]) -> None:
-        self.errors = tuple(errors)
-        super().__init__("; ".join(str(e) for e in self.errors))
-
-
-def search_query_error_messages(e: SearchQueryError) -> list[str]:
-    """The user-facing message list for a SearchQueryError.
-
-    Every offending value's message, not just the first, so the user can
-    fix them all in one round-trip. Shared by every view that maps
-    SearchQueryError to an HTTP 400.
-    """
-    if isinstance(e, MultipleSearchQueryErrors):
-        return [str(sub) for sub in e.errors]
-    return [str(e)]

src/documents/search/_fields.py

@@ -1,42 +0,0 @@
-from __future__ import annotations
-
-from whoosh_compat import FieldKind
-from whoosh_compat import FieldSpec
-from whoosh_compat import SubpathSpec
-
-# Internal-only schema fields with no query-syntax meaning of their own
-# (sort shadow fields, bigram CJK fields, simple_title/simple_content,
-# autocomplete_word, notes_text) are NOT represented here, they are
-# declared in _schema.py's field_descriptors().
-#
-# analyzer/pattern_normalizer are deliberately left at FieldSpec's default
-# (None): they're language-specific and only meaningful to whoosh-compat's
-# parser, so _registry.py attaches them per-language via dataclasses.replace()
-# rather than PUBLIC_FIELDS declaring them itself. _schema.py only reads
-# name/kind/fast and never sees the analyzer at all.
-PUBLIC_FIELDS: tuple[FieldSpec, ...] = (
-    FieldSpec("title", FieldKind.TEXT),
-    FieldSpec("content", FieldKind.TEXT),
-    FieldSpec("correspondent", FieldKind.TEXT),
-    FieldSpec("document_type", FieldKind.TEXT, aliases=("type",)),
-    FieldSpec("storage_path", FieldKind.TEXT, aliases=("path",)),
-    FieldSpec("original_filename", FieldKind.TEXT),
-    FieldSpec("tag", FieldKind.TEXT, comma_values=True),
-    FieldSpec("checksum", FieldKind.KEYWORD),
-    FieldSpec("asn", FieldKind.U64, fast=True),
-    FieldSpec("page_count", FieldKind.U64, fast=True),
-    FieldSpec("num_notes", FieldKind.U64, fast=True),
-    FieldSpec("created", FieldKind.DATE, date_only=True, fast=True),
-    FieldSpec("modified", FieldKind.DATETIME, fast=True),
-    FieldSpec("added", FieldKind.DATETIME, fast=True),
-    FieldSpec(
-        "notes",
-        FieldKind.JSON,
-        subpaths={"user": SubpathSpec(), "note": SubpathSpec(default=True)},
-    ),
-    FieldSpec(
-        "custom_fields",
-        FieldKind.JSON,
-        subpaths={"name": SubpathSpec(), "value": SubpathSpec(default=True)},
-    ),
-)

src/documents/search/_query.py

@@ -6,30 +6,22 @@ from typing import Final
 
 import regex
 import tantivy
-import whoosh_compat as wc
 from django.conf import settings
-from whoosh_compat.emitters.tantivy_ import emit as tantivy_emit
-from whoosh_compat.errors import Cause
-from whoosh_compat.errors import Diagnostic
-from whoosh_compat.errors import DiagnosticKind
-from whoosh_compat.errors import QueryError
 
-from documents.search._errors import InvalidDateQuery
-from documents.search._errors import InvalidNumberQuery
-from documents.search._errors import MultipleSearchQueryErrors
-from documents.search._errors import SearchQueryError
-from documents.search._registry import get_field_registry
 from documents.search._tokenizer import simple_search_tokens
+from documents.search._translate import SearchQueryError
+from documents.search._translate import translate_query
 
 if TYPE_CHECKING:
+    from collections.abc import Iterable
     from datetime import tzinfo
 
+    from django.contrib.auth.base_user import AbstractBaseUser
+
 logger = logging.getLogger("paperless.search")
 
-# Maximum seconds any single regex substitution over user-supplied query text
+# Maximum seconds any single regex substitution may run.
-# may run. The one remaining use is a character class, which cannot backtrack,
+# Prevents ReDoS on adversarial user-supplied query strings.
-# so the bound is an upper limit on that substitution's cost, not the ReDoS
-# guard it was originally written as.
 _REGEX_TIMEOUT: Final[float] = 1.0
 
 # Matches CJK/Hangul characters so queries can be routed to bigram fields.
@@ -37,64 +29,6 @@ _REGEX_TIMEOUT: Final[float] = 1.0
 _CJK_RE: Final = regex.compile(r"[\p{Han}\p{Hiragana}\p{Katakana}\p{Hangul}]+")
 
 
-def _user_facing_emit_message(d: Diagnostic) -> str:
-    """A user-safe message for an emit-time QueryError's Diagnostic.
-
-    Built from the Diagnostic's structured fields (kind, field), never from
-    d.message: whoosh-compat documents that as developer/log output with no
-    stability guarantee, and PATTERN_TOO_COMPLEX embeds the raw backend
-    error text in it.
-    """
-    field = str(d.field) if d.field is not None else None
-    if d.kind is DiagnosticKind.EXISTS_REQUIRES_FAST:
-        return f"Existence searches (field:*) are not supported for field {field!r}."
-    if d.kind is DiagnosticKind.TEXT_RANGE:
-        return f"Range searches are not supported for field {field!r}."
-    if d.kind is DiagnosticKind.PATTERN_TOO_COMPLEX:
-        return f"The wildcard pattern for field {field!r} is too complex."
-    if d.kind is DiagnosticKind.SCHEMA_FIELD_MISSING:
-        return f"Field {field!r} is not available in the search index."
-    logger.warning("Unmapped emit diagnostic %s: %s", d.kind, d.message)
-    return "The search query could not be executed."
-
-
-def _map_emit_error(e: QueryError) -> SearchQueryError:
-    """Route an emit-time QueryError by its Diagnostic's Cause.
-
-    INVALID_INPUT/UNSUPPORTED are user-input errors, exactly like a parse
-    diagnostic, and map to a 400. INTERNAL means a defect in whoosh-compat
-    or in our own AST handling, never the user's query, so the QueryError is
-    re-raised rather than converted, reaching the generic 500 handler instead
-    of blaming the query. MISCONFIGURED is deliberately both: the registry and
-    the index schema disagree, which only an operator can fix, so it is logged
-    as an error, but a request is still waiting and the query cannot run
-    either way, so it also returns a 400.
-
-    EXISTS_REQUIRES_FAST is the one MISCONFIGURED kind that is not a
-    disagreement. whoosh-compat derives it from the registry's own FieldSpec
-    (kind plus fast) without ever consulting the index schema, so it fires
-    whenever a non-fast field of a kind that cannot answer "exists" is asked
-    to: for us that is only the JSON fields, which field_descriptors() builds
-    non-fast on purpose. "notes:*" and the five other spellings of it are
-    ordinary user error that no operator action can clear, so they get the
-    400 without the alert.
-    """
-    d = e.diagnostic
-    if d.cause is Cause.INTERNAL:
-        raise e
-    if (
-        d.cause is Cause.MISCONFIGURED
-        and d.kind is not DiagnosticKind.EXISTS_REQUIRES_FAST
-    ):
-        logger.error(
-            "Search index misconfiguration for field %s (%s): %s",
-            d.field,
-            d.kind.name,
-            d.message,
-        )
-    return SearchQueryError(_user_facing_emit_message(d))
-
-
 def _has_cjk(text: str) -> bool:
     """Return True if text contains any CJK characters."""
     return bool(_CJK_RE.search(text))
@@ -103,36 +37,14 @@ def _has_cjk(text: str) -> bool:
 def extract_cjk_text(text: str) -> str:
     """Join the CJK runs in ``text`` for indexing into bigram (char-ngram) fields.
 
-    Mirrors the query side, which extracts the CJK runs of whatever it is
+    Mirrors the query side (``_build_cjk_query``): only CJK runs are ever searched
-    about to search for (the raw string in simple modes, the parsed query's
+    against the bigram fields, so only CJK runs are worth indexing there. Latin
-    free-text tokens in query mode): only CJK runs are ever searched against
+    text fed to a character-bigram field is never matched and only bloats the
-    the bigram fields, so only CJK runs are worth indexing there. Latin text
-    fed to a character-bigram field is never matched and only bloats the
     index and slows indexing/merge. Returns "" when there is no CJK text.
     """
     return " ".join(_CJK_RE.findall(text))
 
 
-def _parse_cjk_text(
-    index: tantivy.Index,
-    cjk_text: str,
-    fields: list[str],
-) -> tantivy.Query | None:
-    """Parse a plain CJK run string against ``fields``, or None if it won't parse."""
-    try:
-        return index.parse_query(cjk_text, fields)
-    except Exception:
-        # Broad on purpose, unlike _try_parse_fuzzy_query's narrower
-        # ValueError: cjk_text isn't filtered to a guaranteed-safe token
-        # set the way the fuzzy blend's word string is, so the exact
-        # failure mode tantivy could raise here isn't pinned down.
-        logger.debug(
-            "Skipping CJK search clause: could not parse CJK text: %r",
-            cjk_text,
-        )
-        return None
-
-
 def _build_cjk_query(
     index: tantivy.Index,
     raw_query: str,
@@ -140,259 +52,91 @@ def _build_cjk_query(
 ) -> tantivy.Query | None:
     """Build a bigram-field query from the CJK runs in ``raw_query``.
 
-    For the simple (TEXT/TITLE) modes, whose input is plain text and carries
+    Only the CJK character runs are extracted and parsed; ASCII field prefixes,
-    no query grammar to respect. Only the CJK character runs are extracted, so
+    boolean operators and date keywords are discarded. This keeps the CJK clause
-    a stray ``field:`` prefix or ``-``/``+`` in the input can neither leak
+    plain-text and consistent across query/simple modes (no leaked ``field:``
-    field semantics nor fail the parse, and no Latin token reaches the
+    semantics, no parse failures from spaced ``-``/``+``), and avoids feeding
-    character-bigram matcher (where it would produce spurious matches against
+    Latin tokens into the character-bigram matcher (which would produce spurious
-    unrelated Latin text). Returns None when there is no CJK text or the parse
+    matches against unrelated Latin text). Returns None when there is no CJK
-    fails.
+    text or the parse fails.
     """
-    cjk_text = extract_cjk_text(raw_query)
+    cjk_text = " ".join(_CJK_RE.findall(raw_query))
     if not cjk_text:
         return None
-    return _parse_cjk_text(index, cjk_text, fields)
-
-
-def _build_ast_cjk_query(
-    index: tantivy.Index,
-    ast: wc.ast.Node,
-    registry: wc.FieldRegistry,
-) -> tantivy.Query | None:
-    """Build the bigram clause of a QUERY-mode search from the parsed AST.
-
-    Same discipline as the fuzzy clause (see _try_parse_fuzzy_query): the CJK
-    runs come from whoosh_compat's ``free_text_tokens`` over the parsed tree,
-    never from the raw query string, so a term the user negated or restricted
-    to a field outside the default search fields contributes nothing, instead
-    of resurfacing as a top-level clause matching every bigram field.
-
-    ``free_text_tokens`` reports no field of its own, so the tokens are
-    collected one default field at a time: a bare term, which the parser has
-    already copied onto every default field, is therefore searched across
-    every bigram field, while ``title:東京`` reaches ``bigram_title`` alone.
-    Fields whose CJK text is identical (the bare-term case) share a single
-    parse over all of their bigram fields at once.
-
-    Raw (``analyzed=False``) tokens are used because the bigram fields have
-    their own character-ngram analyzer: the default fields' word analyzers
-    have no useful say over a CJK run, and running them first would only
-    risk dropping it (remove_long) before the run is ever extracted.
-    Returns None when the query has no CJK free text.
-    """
-    fields_by_text: dict[str, list[str]] = {}
-    for field, bigram_field in _CJK_BIGRAM_FIELDS.items():
-        tokens = wc.free_text_tokens(
-            ast,
-            registry=registry,
-            fields=[field],
-            analyzed=False,
-        )
-        cjk_text = extract_cjk_text(" ".join(tokens))
-        if cjk_text:
-            fields_by_text.setdefault(cjk_text, []).append(bigram_field)
-
-    clauses: list[tuple[tantivy.Occur, tantivy.Query]] = [
-        (tantivy.Occur.Should, query)
-        for cjk_text, bigram_fields in fields_by_text.items()
-        if (query := _parse_cjk_text(index, cjk_text, bigram_fields)) is not None
-    ]
-    return _any_of(clauses) if clauses else None
-
-
-# A joined fuzzy word string must stay plain words: it goes back through
-# tantivy's own query parser, and the raw query text the clause collects
-# routinely carries characters that parser reads as grammar (a colon, a
-# bracket, a quote, a leading -). Each token is cut into its word runs and
-# only those are kept, so no field syntax, pattern, range or grouping can
-# reach the parser. Cutting rather than dropping the whole token is what
-# keeps ordinary hyphenated, dotted and quoted input ("COVID-19",
-# "hello@example.com", "tax reports") contributing to the clause at all.
-_WORD_RUN_RE = regex.compile(r"\w+")
-
-# The one piece of tantivy grammar that survives the cut: its boolean
-# keywords are themselves word runs. Only these exact spellings are
-# grammar there ("And"/"and" are ordinary terms), so lowercasing exactly
-# these turns them back into the ordinary terms the field analyzer used to
-# make of them, before the clause switched to raw text. Left alone, a
-# quoted phrase would silently restructure the clause ("tax AND reports"
-# becoming a conjunction) or fail to parse and drop it entirely
-# ("tax AND", or "IN" anywhere).
-#
-# Only these words are touched: tantivy lowercases query terms with the
-# field's own analyzer, and doing it ourselves first is not always the
-# same operation (Python folds a final sigma to a different letter than
-# tantivy does, and turns Turkish 'İ' into a sequence tantivy then splits
-# in two), which would search for terms the index does not contain.
-_TANTIVY_KEYWORDS: Final[frozenset[str]] = frozenset({"AND", "OR", "NOT", "IN"})
-
-
-def _try_parse_fuzzy_query(
-    index: tantivy.Index,
-    ast: wc.ast.Node,
-    registry: wc.FieldRegistry,
-) -> tantivy.Query | None:
-    """Build the fuzzy blend clause from the parsed query's free-text
-    words, or None if it has none.
-
-    The clause is built by handing tantivy's own query parser a plain
-    word string (there's no clean AST-level fuzzy equivalent to
-    whoosh-compat's parse tree, and fuzzy matching was always an
-    approximate, secondary, 0.1-boosted clause). The words come from
-    whoosh_compat's ``free_text_tokens`` over the already-parsed AST,
-    never from the raw query string: raw whoosh grammar (date keywords,
-    ``[2005 to 2009]`` ranges, bracket-class wildcards) is not tantivy
-    syntax, and feeding it here used to knock the fuzzy clause out for
-    the whole query the moment any such construct appeared alongside a
-    typo'd word. The helper also keeps excluded terms out: a ``NOT``'d
-    word must not resurface through the fuzzy clause.
-
-    Chosen trade-off: a term explicitly fielded on one of the default
-    search fields (``correspondent:acme``) contributes its text to the
-    word string UNFIELDED, so the fuzzy clause searches it across all
-    default fields rather than just the one the user named. That is
-    recall-only widening on a secondary 0.1-boosted clause the score
-    threshold already disciplines, accepted in exchange for never feeding
-    field syntax to tantivy's parser. What the word string guarantees is
-    exactly that: no field prefix, pattern, range, grouping or quoting
-    survives, and the boolean keywords that do survive (they are word
-    runs) are lowercased into ordinary terms; see _TANTIVY_KEYWORDS.
-
-    The words are the query's RAW text, not the analyzer's output
-    (``analyzed=False``), because ``index.parse_query`` analyzes whatever
-    it is given and analysis is not idempotent: ``universities`` stems to
-    ``univers``, and handing that back stems it again to ``univ``, a term
-    the index does not contain. ``prefix=True`` hid this as over-broad
-    matching (``univ`` also prefixes ``unicycle``) rather than as no
-    matches at all. Raw text is untokenized, which is why it is cut into
-    word runs above rather than taken whole.
-
-    The ValueError guard stays as insurance (the word string is plain
-    tokens, so tantivy accepting it is expected, not assumed): on a parse
-    failure the fuzzy clause is skipped and the exact/CJK clauses stand,
-    rather than the whole query failing.
-    """
-    tokens = wc.free_text_tokens(
-        ast,
-        registry=registry,
-        fields=_DEFAULT_SEARCH_FIELDS,
-        analyzed=False,
-    )
-    words = list(
-        dict.fromkeys(
-            word.lower() if word in _TANTIVY_KEYWORDS else word
-            for token in tokens
-            for word in _WORD_RUN_RE.findall(token)
-        ),
-    )
-    if not words:
-        return None
-    fuzzy_text = " ".join(words)
     try:
-        return index.parse_query(
+        return index.parse_query(cjk_text, fields)
-            fuzzy_text,
+    except Exception:
-            _DEFAULT_SEARCH_FIELDS,
-            field_boosts=_FIELD_BOOSTS,
-            fuzzy_fields={f: (True, 1, True) for f in _DEFAULT_SEARCH_FIELDS},
-        )
-    except ValueError:
-        logger.debug(
-            "Skipping fuzzy search clause: token string is not valid "
-            "tantivy query syntax: %r",
-            fuzzy_text,
-        )
         return None
 
 
-_DEFAULT_SEARCH_FIELDS: Final[list[str]] = [
+def build_permission_filter(
+    schema: tantivy.Schema,
+    user: AbstractBaseUser,
+    viewer_group_ids: Iterable[int] = (),
+) -> tantivy.Query:
+    """
+    Build a query filter for user document permissions.
+
+    Creates a query that matches only documents visible to the specified user
+    according to paperless-ngx permission rules:
+    - Public documents (no owner) are visible to all users
+    - Private documents are visible to their owner
+    - Documents explicitly shared with the user are visible
+    - Documents shared with one of the user's current groups are visible
+
+    Args:
+        schema: Tantivy schema for field validation
+        user: User to check permissions for
+        viewer_group_ids: Current group memberships for the user
+
+    Returns:
+        Tantivy query that filters results to visible documents
+    """
+    owner_any = tantivy.Query.exists_query("owner_id")
+    no_owner = tantivy.Query.boolean_query(
+        [
+            (tantivy.Occur.Must, tantivy.Query.all_query()),
+            (tantivy.Occur.MustNot, owner_any),
+        ],
+    )
+    owned = tantivy.Query.term_query(schema, "owner_id", user.pk)
+    shared = tantivy.Query.term_query(schema, "viewer_id", user.pk)
+    group_shared = [
+        tantivy.Query.term_query(schema, "viewer_group_id", group_id)
+        for group_id in viewer_group_ids
+    ]
+    return tantivy.Query.disjunction_max_query(
+        [no_owner, owned, shared, *group_shared],
+    )
+
+
+DEFAULT_SEARCH_FIELDS = [
     "title",
     "content",
     "correspondent",
     "document_type",
     "tag",
 ]
-_SIMPLE_SEARCH_FIELDS: Final[list[str]] = ["simple_title", "simple_content"]
+SIMPLE_SEARCH_FIELDS = ["simple_title", "simple_content"]
-_TITLE_SEARCH_FIELDS: Final[list[str]] = ["simple_title"]
+TITLE_SEARCH_FIELDS = ["simple_title"]
-# The bigram (character-ngram) companion of each default search field.
+_CJK_ALL_FIELDS: Final[list[str]] = [
-_CJK_BIGRAM_FIELDS: Final[dict[str, str]] = {
+    "bigram_content",
-    field: f"bigram_{field}" for field in _DEFAULT_SEARCH_FIELDS
+    "bigram_title",
-}
+    "bigram_correspondent",
+    "bigram_document_type",
+    "bigram_tag",
+]
 _CJK_CONTENT_FIELDS: Final[list[str]] = ["bigram_content"]
 _CJK_TITLE_FIELDS: Final[list[str]] = ["bigram_title"]
 _FIELD_BOOSTS = {"title": 2.0}
 _SIMPLE_FIELD_BOOSTS = {"simple_title": 2.0}
 
 
-class _ConjunctiveNegations(wc.ast.Visitor[tuple["wc.ast.Node", ...]]):
+def _simple_query_tokens(raw_query: str) -> list[str]:
-    """Collect the subtrees an AST excludes from every document it matches.
+    # Tokenize and fold via the same analyzer used to index simple_title /
-
+    # simple_content, so query terms fold identically to the indexed terms
-    A negation reached through ``And``/``AndNot``/``Require`` (and through
+    # (single source of truth for ASCII folding).
-    the required half of an ``AndMaybe``) constrains the whole query, so it
+    return simple_search_tokens(raw_query)
-    can be re-stated above the blend. ``Or`` is deliberately not descended
-    into: in ``invoice OR NOT secret`` the negation is one branch's own
-    condition, and hoisting it would throw away documents the other branch
-    matches. Nor is a collected subtree descended into, since a negation
-    inside a negation is not an exclusion.
-
-    Node types with no negation to contribute (every leaf, ``Or``) fall
-    through to ``generic_visit``.
-    """
-
-    def generic_visit(self, node: wc.ast.Node) -> tuple[wc.ast.Node, ...]:
-        return ()
-
-    def visit_not(self, node: wc.ast.Not) -> tuple[wc.ast.Node, ...]:
-        return (node.child,)
-
-    def visit_andnot(self, node: wc.ast.AndNot) -> tuple[wc.ast.Node, ...]:
-        return (*self.visit(node.positive), node.negative)
-
-    def visit_and(self, node: wc.ast.And) -> tuple[wc.ast.Node, ...]:
-        return tuple(
-            negation for child in node.children for negation in self.visit(child)
-        )
-
-    def visit_boosted(self, node: wc.ast.Boosted) -> tuple[wc.ast.Node, ...]:
-        return self.visit(node.child)
-
-    def visit_andmaybe(self, node: wc.ast.AndMaybe) -> tuple[wc.ast.Node, ...]:
-        return self.visit(node.required)
-
-    def visit_require(self, node: wc.ast.Require) -> tuple[wc.ast.Node, ...]:
-        return (*self.visit(node.scored), *self.visit(node.filter_only))
-
-
-def _negation_clauses(
-    index: tantivy.Index,
-    ast: wc.ast.Node,
-    registry: wc.FieldRegistry,
-) -> list[tuple[tantivy.Occur, tantivy.Query]]:
-    """MustNot clauses for everything ``ast`` excludes conjunctively.
-
-    Each excluded subtree is emitted as its own positive query and attached
-    with ``MustNot``, rather than emitting a negative query and hoping
-    tantivy accepts a bare one.
-    """
-    try:
-        return [
-            (
-                tantivy.Occur.MustNot,
-                tantivy_emit(negation, index=index, registry=registry),
-            )
-            for negation in _ConjunctiveNegations().visit(ast)
-        ]
-    except QueryError as e:
-        raise _map_emit_error(e) from e
-
-
-def _any_of(clauses: list[tuple[tantivy.Occur, tantivy.Query]]) -> tantivy.Query:
-    """Collapse a clause list: none -> empty, one -> itself (no wasted
-    single-clause boolean_query wrapping), many -> boolean_query(clauses)."""
-    if not clauses:
-        return tantivy.Query.empty_query()
-    if len(clauses) == 1:
-        return clauses[0][1]
-    return tantivy.Query.boolean_query(clauses)
 
 
 def _build_simple_token_query(
@@ -424,7 +168,9 @@ def _build_simple_token_query(
             query = tantivy.Query.boost_query(query, boost)
         field_queries.append((tantivy.Occur.Should, query))
 
-    return _any_of(field_queries)
+    if len(field_queries) == 1:
+        return field_queries[0][1]
+    return tantivy.Query.boolean_query(field_queries)
 
 
 def parse_user_query(
@@ -433,53 +179,52 @@ def parse_user_query(
     tz: tzinfo,
 ) -> tantivy.Query:
     """
-    Parse user query through whoosh-compat, then blend in fuzzy/CJK clauses.
+    Parse user query through the complete preprocessing pipeline.
 
-    1. wc.parse() against the shared FieldRegistry (whoosh grammar -> AST).
+    Transforms the raw user query through multiple stages:
-       Bare notes:/custom_fields: prefixes resolve to their default subpath
+    1. Date keyword rewriting (today → ISO 8601 ranges)
-       (notes.note:/custom_fields.value:) directly in the registry, via
+    2. Query normalization (comma expansion, whitespace cleanup)
-       each JSON field's SubpathSpec(default=True).
+    3. Tantivy parsing with field boosts
-    2. Any diagnostics (bad dates/numbers) map to SearchQueryError subclasses
+    4. Optional fuzzy query blending (if ADVANCED_FUZZY_SEARCH_THRESHOLD set)
-       and raise, the view returns HTTP 400 with every offending field
+
-       listed, not just the first.
+    Args:
-    3. emit() turns the AST into a tantivy.Query directly (no string
+        index: Tantivy index with registered tokenizers
-       round-trip). A QueryError is routed by its Diagnostic's Cause
+        raw_query: Original user query string
-       (_map_emit_error): a construct that parses but can't execute against
+        tz: Timezone for date boundary calculations
-       tantivy (e.g. a text-field range) is a 400, a registry/schema
+
-       mismatch is logged and a 400, and an INTERNAL defect is re-raised.
+    Returns:
-    4. Optional fuzzy blend (ADVANCED_FUZZY_SEARCH_THRESHOLD) builds a
+        Parsed Tantivy query ready for execution
-       plain word string from the parsed AST's free-text tokens
+
-       (whoosh_compat.free_text_tokens) and feeds THAT to
+    Note:
-       index.parse_query, never raw_query, whose whoosh grammar (date
+        When ADVANCED_FUZZY_SEARCH_THRESHOLD is configured, adds a low-priority
-       keywords, bracket-class wildcards, etc.) tantivy's parser rejects,
+        fuzzy query as a Should clause (0.1 boost) to catch approximate matches
-       which used to silently knock the fuzzy clause out of any mixed
+        while keeping exact matches ranked higher. The threshold value is applied
-       query (see _try_parse_fuzzy_query).
+        as a post-search score filter, not during query construction.
-    5. Optional CJK bigram clause, built from the same parsed AST for the
-       same reason (see _build_ast_cjk_query): a CJK term the query negated
-       or fielded must not resurface through it.
-    6. When any optional clause was added, the query's conjunctive
-       exclusions are restated as MustNot above the blend
-       (_negation_clauses): a clause built from positive terms cannot
-       express them, and as a bare Should it would undo them.
     """
-    registry = get_field_registry(settings.SEARCH_LANGUAGE)
-    result = wc.parse(
-        raw_query,
-        registry=registry,
-        default_fields=_DEFAULT_SEARCH_FIELDS,
-        field_boosts=_FIELD_BOOSTS,
-        tz=tz,
-    )
-    if result.diagnostics:
-        raise _diagnostics_to_error(result.diagnostics)
 
     try:
-        exact = tantivy_emit(result.ast, index=index, registry=registry)
+        query_str = translate_query(raw_query, tz)
-    except QueryError as e:
+    except SearchQueryError:
-        raise _map_emit_error(e) from e
+        # Intentional, user-fixable error (e.g. an unparsable date). Propagate so
+        # the view can return a 400 with a helpful message rather than falling
+        # back to the raw (still-invalid) query.
+        raise
+    except Exception:  # pragma: no cover - defensive
+        logger.warning("Query translation failed; using raw query", exc_info=True)
+        query_str = raw_query
 
+    exact = index.parse_query(
+        query_str,
+        DEFAULT_SEARCH_FIELDS,
+        field_boosts=_FIELD_BOOSTS,
+    )
+
+    # The standard analyzer keeps a whitespace-free CJK run as a single token,
+    # so substring queries can't match content/title (and long runs are dropped
+    # by remove_long). Route CJK queries to the bigram fields, whose ngram
+    # tokenizer indexes overlapping 2-grams for substring matching.
     cjk_query = (
-        _build_ast_cjk_query(index, result.ast, registry)
+        _build_cjk_query(index, raw_query, _CJK_ALL_FIELDS)
         if _has_cjk(raw_query)
         else None
     )
@@ -490,73 +235,22 @@ def parse_user_query(
 
     threshold = settings.ADVANCED_FUZZY_SEARCH_THRESHOLD
     if threshold is not None:
-        fuzzy = _try_parse_fuzzy_query(index, result.ast, registry)
+        fuzzy = index.parse_query(
-        if fuzzy is not None:
+            query_str,
-            clauses.append(
+            DEFAULT_SEARCH_FIELDS,
-                (tantivy.Occur.Should, tantivy.Query.boost_query(fuzzy, 0.1)),
+            field_boosts=_FIELD_BOOSTS,
-            )
+            # (prefix=True, distance=1, transposition_cost_one=True) — edit-distance fuzziness
+            fuzzy_fields={f: (True, 1, True) for f in DEFAULT_SEARCH_FIELDS},
+        )
+        # 0.1 boost keeps fuzzy hits ranked below exact matches (intentional)
+        clauses.append((tantivy.Occur.Should, tantivy.Query.boost_query(fuzzy, 0.1)))
 
     if cjk_query is not None:
         clauses.append((tantivy.Occur.Should, cjk_query))
 
     if len(clauses) == 1:
         return exact
-    # The fuzzy and CJK clauses are built from positive terms only, so as
+    return tantivy.Query.boolean_query(clauses)
-    # plain Shoulds beside the exact clause they re-admit exactly the
-    # documents the query excluded. Restate the exclusions once, above the
-    # whole blend. Redundant against the exact clause, which already
-    # carries them, but idempotently so.
-    negations = _negation_clauses(index, result.ast, registry)
-    if not negations:
-        return _any_of(clauses)
-    return tantivy.Query.boolean_query(
-        [(tantivy.Occur.Must, _any_of(clauses)), *negations],
-    )
-
-
-# The three whoosh-compat kinds for a wildcard on a field that cannot
-# carry one. d.field_kind supplies the discriminator, so naming the field's
-# type needs no second trip through the registry.
-_PATTERN_ON_KINDS: Final = frozenset(
-    {
-        DiagnosticKind.PATTERN_ON_NUMERIC,
-        DiagnosticKind.PATTERN_ON_BOOLEAN_EXISTS,
-        DiagnosticKind.PATTERN_ON_SUBPATH,
-    },
-)
-
-
-def _diagnostics_to_error(diagnostics: tuple[Diagnostic, ...]) -> SearchQueryError:
-    errors = [_single_diagnostic_to_error(d) for d in diagnostics]
-    return errors[0] if len(errors) == 1 else MultipleSearchQueryErrors(errors)
-
-
-def _single_diagnostic_to_error(d: Diagnostic) -> SearchQueryError:
-    # d.field is a FieldRef, not a str: str(d.field) gives the canonical
-    # dotted name (an aliased query, e.g. type:, reports document_type).
-    field_name = str(d.field) if d.field is not None else None
-    if d.kind is DiagnosticKind.BAD_DATE:
-        return InvalidDateQuery(field_name, d.raw_value)
-    if d.kind is DiagnosticKind.BAD_NUMBER:
-        return InvalidNumberQuery(field_name, d.raw_value)
-    if d.kind is DiagnosticKind.TOO_DEEP:
-        return SearchQueryError("The search query is nested too deeply.")
-    if d.kind in _PATTERN_ON_KINDS:
-        kind_label = f" ({d.field_kind.name.lower()})" if d.field_kind else ""
-        return SearchQueryError(
-            f"Wildcard patterns are not supported for field "
-            f"{field_name!r}{kind_label}.",
-        )
-    if d.kind is DiagnosticKind.SINGLE_CHAR_BRACKET_RANGE:
-        field_label = f" for field {field_name!r}" if field_name else ""
-        return SearchQueryError(
-            f"{d.raw_value!r} looks like a bracket range{field_label}, but "
-            "'[' is not a wildcard character on its own. Combine it with a "
-            "wildcard, e.g. a trailing '*', or double-quote the value to "
-            "search it as literal text.",
-        )
-    logger.warning("Unmapped parse diagnostic %s: %s", d.kind, d.message)
-    return SearchQueryError("The search query could not be executed.")
 
 
 def parse_simple_query(
@@ -574,7 +268,7 @@ def parse_simple_query(
     CJK substrings the simple analyzer can't (long whitespace-free runs are
     dropped by remove_long).
     """
-    tokens = simple_search_tokens(raw_query)
+    tokens = _simple_query_tokens(raw_query)
 
     clauses: list[tuple[tantivy.Occur, tantivy.Query]] = []
     if tokens:
@@ -597,14 +291,23 @@ def parse_simple_query(
             )
             for token in tokens
         ]
-        clauses.append((tantivy.Occur.Should, _any_of(token_queries)))
+        simple_query = (
+            token_queries[0][1]
+            if len(token_queries) == 1
+            else tantivy.Query.boolean_query(token_queries)
+        )
+        clauses.append((tantivy.Occur.Should, simple_query))
 
     if cjk_fields and _has_cjk(raw_query):
         cjk_q = _build_cjk_query(index, raw_query, cjk_fields)
         if cjk_q is not None:
             clauses.append((tantivy.Occur.Should, cjk_q))
 
-    return _any_of(clauses)
+    if not clauses:
+        return tantivy.Query.empty_query()
+    if len(clauses) == 1:
+        return clauses[0][1]
+    return tantivy.Query.boolean_query(clauses)
 
 
 def parse_simple_text_highlight_query(
@@ -619,21 +322,13 @@ def parse_simple_text_highlight_query(
 
     # Strip Tantivy operator chars before tokenizing: this is a plain-text
     # highlight query, not a structured boolean query, so +/- are separators.
-    tokens = simple_search_tokens(
+    tokens = _simple_query_tokens(
         regex.sub(r"[-+]", " ", raw_query, timeout=_REGEX_TIMEOUT),
     )
     if not tokens:
         return tantivy.Query.empty_query()
 
-    # Quote each token as its own phrase, escaping backslashes and embedded
+    return index.parse_query(" ".join(tokens), ["content"])
-    # quotes. simple search tokens can carry arbitrary Tantivy syntax
-    # characters (`"`, `:`, `(`, `[`, `/`, ...) that the query-string parser
-    # would otherwise interpret as query grammar rather than literal text.
-    quoted_tokens = [
-        '"' + token.replace("\\", "\\\\").replace('"', '\\"') + '"' for token in tokens
-    ]
-
-    return index.parse_query(" ".join(quoted_tokens), ["content"])
 
 
 def parse_simple_text_query(
@@ -647,7 +342,7 @@ def parse_simple_text_query(
     return parse_simple_query(
         index,
         raw_query,
-        _SIMPLE_SEARCH_FIELDS,
+        SIMPLE_SEARCH_FIELDS,
         cjk_fields=_CJK_CONTENT_FIELDS,
     )
 
@@ -663,6 +358,6 @@ def parse_simple_title_query(
     return parse_simple_query(
         index,
         raw_query,
-        _TITLE_SEARCH_FIELDS,
+        TITLE_SEARCH_FIELDS,
         cjk_fields=_CJK_TITLE_FIELDS,
     )

src/documents/search/_registry.py

@@ -1,91 +0,0 @@
-from __future__ import annotations
-
-import dataclasses
-from typing import TYPE_CHECKING
-
-from whoosh_compat import FieldKind
-from whoosh_compat import FieldRegistry
-
-from documents.search._fields import PUBLIC_FIELDS
-from documents.search._tokenizer import ascii_fold
-from documents.search._tokenizer import paperless_text_analyzer
-from documents.search._tokenizer import stem_pattern_text
-
-if TYPE_CHECKING:
-    from whoosh_compat import PatternNormalizer
-
-_registry_cache: dict[str | None, FieldRegistry] = {}
-
-
-def _identity_analyzer(text: str) -> list[str]:
-    """Analyzer for KEYWORD fields indexed with the raw tokenizer (no splitting)."""
-    return [text]
-
-
-def _fold_normalizer(text: str) -> str:
-    """Wildcard/regex literal-run normalizer for fields indexed without stemming."""
-    return ascii_fold(text.lower())
-
-
-def _make_pattern_normalizer(language: str | None) -> PatternNormalizer:
-    """Build the wildcard/regex literal-run normalizer for a search language."""
-
-    def _pattern_normalizer(text: str) -> tuple[str, ...]:
-        """Normalize a literal run into the forms a term may match.
-
-        TEXT index terms go through lowercase -> ascii_fold -> stem, so a
-        pattern that skips stemming can never match one: "invoice*" would look
-        for a term starting with "invoice" while the index holds "invoic". The
-        run is therefore offered stemmed as well. KEYWORD fields are indexed
-        raw and get _fold_normalizer instead, so their patterns stay literal.
-
-        Both forms are returned, as alternatives, because neither is a prefix
-        of the other in general: English stemming substitutes as well as
-        truncates ("copy" -> "copi"), so the stem alone loses the compounds
-        the typed run reaches ("copyright") while the typed run alone loses
-        the inflections the stem reaches ("copies"). whoosh-compat ORs the
-        alternatives per literal run and deduplicates them, so a run the
-        stemmer leaves alone costs exactly the one branch it did before.
-
-        Inside a bracket class the emitter calls this once per character and
-        uses the answer only if it is a single one-character form; two forms
-        there leave the character as typed. A stemmer does not change a lone
-        character, so the two forms deduplicate to one and the class body is
-        folded as before.
-        """
-        folded = ascii_fold(text.lower())
-        stemmed = stem_pattern_text(folded, language)
-        return (folded, stemmed)
-
-    return _pattern_normalizer
-
-
-def get_field_registry(language: str | None) -> FieldRegistry:
-    """Build (or return the cached) FieldRegistry for the given search language.
-
-    Cached keyed by language, rebuilt on the same trigger register_tokenizers()
-    uses (settings.SEARCH_LANGUAGE change). A fresh call with a new language
-    builds and caches a new registry rather than mutating the old one.
-    """
-    if language in _registry_cache:
-        return _registry_cache[language]
-
-    text_analyzer = paperless_text_analyzer(language).analyze
-    pattern_normalizer = _make_pattern_normalizer(language)
-
-    specs = [
-        dataclasses.replace(
-            field,
-            analyzer=_identity_analyzer
-            if field.kind is FieldKind.KEYWORD
-            else text_analyzer,
-            pattern_normalizer=_fold_normalizer
-            if field.kind is FieldKind.KEYWORD
-            else pattern_normalizer,
-        )
-        for field in PUBLIC_FIELDS
-    ]
-
-    registry = FieldRegistry(specs)
-    _registry_cache[language] = registry
-    return registry

src/documents/search/_schema.py

@@ -1,19 +1,14 @@
 from __future__ import annotations
 
-import hashlib
 import json
 import logging
 import shutil
 from typing import TYPE_CHECKING
 from typing import Final
-from typing import NamedTuple
 from typing import cast
 
 import tantivy
 from django.conf import settings
-from whoosh_compat import FieldKind
-
-from documents.search._fields import PUBLIC_FIELDS
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -21,185 +16,7 @@ if TYPE_CHECKING:
 logger = logging.getLogger("paperless.search")
 
 # v1 - Initial tantivy schema format
-# v2 - build_schema() derived from PUBLIC_FIELDS, changing the field declaration
+SCHEMA_VERSION: Final[int] = 1
-#      order, and the write-only correspondent/document_type/storage_path/tag id
-#      columns dropped. tantivy compares schemas by ordered field list, so an
-#      index built by v1 rejects every write against the v2 schema.
-SCHEMA_VERSION: Final[int] = 2
-
-
-class FieldDescriptor(NamedTuple):
-    """One tantivy field, in declaration order.
-
-    The descriptor vocabulary is paperless', not tantivy-py's: it is both the
-    input to the SchemaBuilder and the input to schema_fingerprint(), so the
-    persisted fingerprint cannot move under a tantivy-py upgrade.
-    """
-
-    name: str
-    kind: str
-    stored: bool
-    indexed: bool
-    fast: bool
-    tokenizer: str | None
-
-
-# (schema kind, tokenizer) for the FieldKind -> FieldDescriptor mapping that
-# doesn't need special-casing. JSON is handled separately below since it can
-# emit a second, synthetic descriptor.
-_KIND_TABLE: Final[dict[FieldKind, tuple[str, str | None]]] = {
-    FieldKind.TEXT: ("text", "paperless_text"),
-    FieldKind.KEYWORD: ("text", "raw"),
-    FieldKind.U64: ("u64", None),
-    FieldKind.DATE: ("date", None),
-    FieldKind.DATETIME: ("date", None),
-}
-# Kinds whose fast-field flag follows FieldSpec.fast rather than always False.
-_FAST_FROM_FIELD: Final[frozenset[FieldKind]] = frozenset(
-    {FieldKind.U64, FieldKind.DATE, FieldKind.DATETIME},
-)
-
-
-def _public_field_descriptors() -> list[FieldDescriptor]:
-    """Descriptors for the query-visible fields declared in PUBLIC_FIELDS."""
-    descriptors: list[FieldDescriptor] = []
-    for field in PUBLIC_FIELDS:
-        if field.kind is FieldKind.JSON:
-            descriptors.append(
-                FieldDescriptor(
-                    field.name,
-                    "json",
-                    stored=True,
-                    indexed=True,
-                    fast=False,
-                    tokenizer="paperless_text",
-                ),
-            )
-            if field.name == "notes":
-                # Plain-text companion for snippet generation: tantivy's
-                # SnippetGenerator does not support JSON fields. Schema-only,
-                # no query-syntax meaning, not in PUBLIC_FIELDS.
-                descriptors.append(
-                    FieldDescriptor(
-                        "notes_text",
-                        "text",
-                        stored=True,
-                        indexed=True,
-                        fast=False,
-                        tokenizer="paperless_text",
-                    ),
-                )
-            continue
-        schema_kind, tokenizer = _KIND_TABLE[field.kind]
-        descriptors.append(
-            FieldDescriptor(
-                field.name,
-                schema_kind,
-                stored=True,
-                indexed=True,
-                fast=field.fast if field.kind in _FAST_FROM_FIELD else False,
-                tokenizer=tokenizer,
-            ),
-        )
-    return descriptors
-
-
-def field_descriptors() -> list[FieldDescriptor]:
-    """Every field of the document index, in the order tantivy declares them.
-
-    tantivy compares schemas by *ordered* field list, so the order here is
-    part of the on-disk contract: schema_fingerprint() hashes it and
-    needs_rebuild() acts on the result.
-    """
-    return [
-        FieldDescriptor(
-            "id",
-            "u64",
-            stored=True,
-            indexed=True,
-            fast=True,
-            tokenizer=None,
-        ),
-        *_public_field_descriptors(),
-        # Shadow sort fields - fast, not stored
-        *(
-            FieldDescriptor(
-                name,
-                "text",
-                stored=False,
-                indexed=True,
-                fast=True,
-                tokenizer="simple_analyzer",
-            )
-            for name in ("title_sort", "correspondent_sort", "type_sort")
-        ),
-        # CJK support - not stored, indexed only
-        *(
-            FieldDescriptor(
-                name,
-                "text",
-                stored=False,
-                indexed=True,
-                fast=False,
-                tokenizer="bigram_analyzer",
-            )
-            for name in (
-                "bigram_content",
-                "bigram_title",
-                "bigram_correspondent",
-                "bigram_document_type",
-                "bigram_tag",
-            )
-        ),
-        # Simple substring search support for title/content - not stored,
-        # indexed only
-        *(
-            FieldDescriptor(
-                name,
-                "text",
-                stored=False,
-                indexed=True,
-                fast=False,
-                tokenizer="simple_search_analyzer",
-            )
-            for name in ("simple_title", "simple_content")
-        ),
-        # Autocomplete prefix scan via terms_with_prefix, which walks the
-        # field's term dictionary - so the field must be indexed (term dict),
-        # not stored. The stored value is never read back, so storing it only
-        # wastes space.
-        FieldDescriptor(
-            "autocomplete_word",
-            "text",
-            stored=False,
-            indexed=True,
-            fast=False,
-            tokenizer="raw",
-        ),
-        # Permission filter columns, read by build_permission_filter.
-        *(
-            FieldDescriptor(
-                name,
-                "u64",
-                stored=False,
-                indexed=True,
-                fast=True,
-                tokenizer=None,
-            )
-            for name in ("owner_id", "viewer_id", "viewer_group_id")
-        ),
-    ]
-
-
-def schema_fingerprint() -> str:
-    """Hash of the field descriptors, stamped into .index_settings.json.
-
-    Changes whenever a field is added, removed, retyped, re-optioned or
-    reordered, so an index built from a different schema shape is detected
-    even when SCHEMA_VERSION was not bumped.
-    """
-    payload = json.dumps([list(descriptor) for descriptor in field_descriptors()])
-    return hashlib.blake2b(payload.encode()).hexdigest()
 
 
 def build_schema() -> tantivy.Schema:
@@ -215,37 +32,85 @@ def build_schema() -> tantivy.Schema:
     """
     sb = tantivy.SchemaBuilder()
 
-    for descriptor in field_descriptors():
+    sb.add_unsigned_field("id", stored=True, indexed=True, fast=True)
-        if descriptor.kind == "text":
+    sb.add_text_field("checksum", stored=True, tokenizer_name="raw")
-            sb.add_text_field(
+
-                descriptor.name,
+    for field in (
-                stored=descriptor.stored,
+        "title",
-                fast=descriptor.fast,
+        "correspondent",
-                tokenizer_name=cast("str", descriptor.tokenizer),
+        "document_type",
-            )
+        "storage_path",
-        elif descriptor.kind == "json":
+        "original_filename",
-            sb.add_json_field(
+        "content",
-                descriptor.name,
+    ):
-                stored=descriptor.stored,
+        sb.add_text_field(field, stored=True, tokenizer_name="paperless_text")
-                fast=descriptor.fast,
+
-                tokenizer_name=cast("str", descriptor.tokenizer),
+    # Shadow sort fields - fast, not stored/indexed
-            )
+    for field in ("title_sort", "correspondent_sort", "type_sort"):
-        elif descriptor.kind == "u64":
+        sb.add_text_field(
-            sb.add_unsigned_field(
+            field,
-                descriptor.name,
+            stored=False,
-                stored=descriptor.stored,
+            tokenizer_name="simple_analyzer",
-                indexed=descriptor.indexed,
+            fast=True,
-                fast=descriptor.fast,
+        )
-            )
+
-        elif descriptor.kind == "date":
+    # CJK support - not stored, indexed only
-            sb.add_date_field(
+    sb.add_text_field("bigram_content", stored=False, tokenizer_name="bigram_analyzer")
-                descriptor.name,
+    sb.add_text_field("bigram_title", stored=False, tokenizer_name="bigram_analyzer")
-                stored=descriptor.stored,
+    sb.add_text_field(
-                indexed=descriptor.indexed,
+        "bigram_correspondent",
-                fast=descriptor.fast,
+        stored=False,
-            )
+        tokenizer_name="bigram_analyzer",
-        else:
+    )
-            raise ValueError(f"Unknown schema field kind: {descriptor.kind}")
+    sb.add_text_field(
+        "bigram_document_type",
+        stored=False,
+        tokenizer_name="bigram_analyzer",
+    )
+    sb.add_text_field("bigram_tag", stored=False, tokenizer_name="bigram_analyzer")
+
+    # Simple substring search support for title/content - not stored, indexed only
+    sb.add_text_field(
+        "simple_title",
+        stored=False,
+        tokenizer_name="simple_search_analyzer",
+    )
+    sb.add_text_field(
+        "simple_content",
+        stored=False,
+        tokenizer_name="simple_search_analyzer",
+    )
+
+    # Autocomplete prefix scan via terms_with_prefix, which walks the field's
+    # term dictionary - so the field must be indexed (term dict), not stored.
+    # The stored value is never read back, so storing it only wastes space.
+    sb.add_text_field("autocomplete_word", stored=False, tokenizer_name="raw")
+
+    sb.add_text_field("tag", stored=True, tokenizer_name="paperless_text")
+
+    # JSON fields — structured queries: notes.user:alice, custom_fields.name:invoice
+    sb.add_json_field("notes", stored=True, tokenizer_name="paperless_text")
+    # Plain-text companion for notes — tantivy's SnippetGenerator does not support
+    # JSON fields, so highlights require a text field with the same content.
+    sb.add_text_field("notes_text", stored=True, tokenizer_name="paperless_text")
+    sb.add_json_field("custom_fields", stored=True, tokenizer_name="paperless_text")
+
+    for field in (
+        "correspondent_id",
+        "document_type_id",
+        "storage_path_id",
+        "tag_id",
+        "owner_id",
+        "viewer_id",
+        "viewer_group_id",
+    ):
+        sb.add_unsigned_field(field, stored=False, indexed=True, fast=True)
+
+    for field in ("created", "modified", "added"):
+        sb.add_date_field(field, stored=True, indexed=True, fast=True)
+
+    for field in ("asn", "page_count", "num_notes"):
+        sb.add_unsigned_field(field, stored=True, indexed=True, fast=True)
 
     return sb.build()
 
@@ -254,9 +119,9 @@ def needs_rebuild(index_dir: Path) -> bool:
     """
     Check if the search index needs rebuilding.
 
-    Reads .index_settings.json to compare the stored schema version, search
+    Reads .index_settings.json to compare the stored schema version and
-    language and schema fingerprint against the current configuration. Returns
+    search language against the current configuration. Returns True if the
-    True if the file is missing, unparsable, or any value mismatches.
+    file is missing, unparsable, or either value mismatches.
 
     Args:
         index_dir: Path to the search index directory
@@ -275,9 +140,6 @@ def needs_rebuild(index_dir: Path) -> bool:
         if "language" not in data or data["language"] != settings.SEARCH_LANGUAGE:
             logger.info("Search index language changed - rebuilding.")
             return True
-        if data.get("schema_fingerprint") != schema_fingerprint():
-            logger.info("Search index schema fingerprint mismatch - rebuilding.")
-            return True
     except ValueError:
         return True
     return False
@@ -308,7 +170,6 @@ def _write_sentinels(index_dir: Path) -> None:
             {
                 "schema_version": SCHEMA_VERSION,
                 "language": settings.SEARCH_LANGUAGE,
-                "schema_fingerprint": schema_fingerprint(),
             },
         ),
     )

src/documents/search/_tokenizer.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import logging
-from functools import cache
 from typing import Final
 
 import tantivy
@@ -72,7 +71,7 @@ def register_tokenizers(index: tantivy.Index, language: str | None) -> None:
         use fast=True and Tantivy requires fast-field tokenizers to exist
         even for documents that omit those fields.
     """
-    index.register_tokenizer("paperless_text", paperless_text_analyzer(language))
+    index.register_tokenizer("paperless_text", _paperless_text(language))
     index.register_tokenizer("simple_analyzer", _simple_analyzer())
     index.register_tokenizer("bigram_analyzer", _bigram_analyzer())
     index.register_tokenizer("simple_search_analyzer", _simple_search_analyzer())
@@ -80,7 +79,7 @@ def register_tokenizers(index: tantivy.Index, language: str | None) -> None:
     index.register_fast_field_tokenizer("simple_analyzer", _simple_analyzer())
 
 
-def paperless_text_analyzer(language: str | None) -> tantivy.TextAnalyzer:
+def _paperless_text(language: str | None) -> tantivy.TextAnalyzer:
     """Main full-text tokenizer for content, title, etc: simple -> remove_long(129) -> lowercase -> ascii_fold [-> stemmer]"""
     builder = (
         tantivy.TextAnalyzerBuilder(tantivy.Tokenizer.simple())
@@ -101,54 +100,6 @@ def paperless_text_analyzer(language: str | None) -> tantivy.TextAnalyzer:
     return builder.build()
 
 
-@cache
-def _pattern_stemmer(language: str | None) -> tantivy.TextAnalyzer | None:
-    """The stemming tail of paperless_text_analyzer, over a whole literal run.
-
-    Same language gate and same Snowball stemmer paperless_text_analyzer
-    applies at index time, so query patterns follow SEARCH_LANGUAGE. Returns
-    None when that gate disables stemming; paperless_text_analyzer already
-    warns about an unsupported language, so this stays quiet.
-
-    The raw tokenizer keeps the run whole (a wildcard literal is a fragment,
-    not necessarily a word), and remove_long is kept so an over-long run is
-    treated the same way the index treats it.
-    """
-    if not language:
-        return None
-    tantivy_lang = _LANGUAGE_MAP.get(language.lower())
-    if tantivy_lang is None:
-        return None
-    return (
-        tantivy.TextAnalyzerBuilder(tantivy.Tokenizer.raw())
-        .filter(tantivy.Filter.remove_long(_TOKEN_REMOVE_LONG_LIMIT))
-        .filter(tantivy.Filter.stemmer(tantivy_lang))
-        .build()
-    )
-
-
-def stem_pattern_text(text: str, language: str | None) -> str:
-    """Stem an already lowercased/ascii-folded run the way index terms are.
-
-    Returns text unchanged when stemming is disabled for language, and also
-    when the stem step does not yield exactly one token: remove_long drops a run
-    past the length limit, leaving no stem to substitute. Falling back to the
-    text as typed is the safe direction for a pattern prefix, since it can only
-    be as narrow as it was before stemming was considered.
-
-    The raw tokenizer emits one token whatever the input and the stemmer is
-    1-to-1, so only the zero-token case can fire today; the guard covers both
-    counts so a tokenizer change cannot turn this into an IndexError.
-    """
-    analyzer = _pattern_stemmer(language)
-    if analyzer is None:
-        return text
-    tokens = analyzer.analyze(text)
-    if len(tokens) != 1:
-        return text
-    return tokens[0]
-
-
 def _simple_analyzer() -> tantivy.TextAnalyzer:
     """Tokenizer for shadow sort fields (title_sort, correspondent_sort, type_sort): simple -> lowercase -> ascii_fold."""
     return (

src/documents/search/_translate.py

@@ -0,0 +1,610 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import UTC
+from datetime import datetime
+from datetime import timedelta
+from typing import TYPE_CHECKING
+from typing import TypeAlias
+
+import regex
+from dateutil.relativedelta import relativedelta
+
+from documents.search._dates import _DATE_KEYWORDS
+from documents.search._dates import _DATE_ONLY_FIELDS
+from documents.search._dates import _date_only_range
+from documents.search._dates import _datetime_range
+from documents.search._dates import _field_range_from_dates
+from documents.search._dates import _fmt
+from documents.search._dates import _precision_bounds
+from documents.search._dates import _utc_bounds_for_field
+
+# Compiled regex that matches any known multi-word (or single-word) date keyword
+# at the start of a match position, longest alternatives first so "previous week"
+# wins over a hypothetical shorter "previous".
+_KEYWORD_VALUE_RE = regex.compile(
+    "|".join(sorted((regex.escape(k) for k in _DATE_KEYWORDS), key=len, reverse=True)),
+    regex.IGNORECASE,
+)
+
+if TYPE_CHECKING:
+    from datetime import tzinfo
+
+# TODO: this module translates date queries into Tantivy *string* syntax, which
+# forces a workaround for something Tantivy's string parser cannot express on
+# date fields: open-ended ranges use far-past/far-future string sentinels
+# (OPEN_LO/OPEN_HI). These can be replaced with a real tantivy.Query object
+# (Query.range_query(..., None) for open bounds) once tantivy-py accepts Python
+# datetimes in range_query/term_query on Date fields. That support exists on
+# tantivy-py master (PRs #655 + #666) but postdates the pinned 0.26.0 wheel, so
+# it is blocked only on a published release > 0.26.0 and a dependency bump.
+# (Unparsable dates now raise InvalidDateQuery -> HTTP 400 rather than using a
+# no-match string sentinel.)
+
+# Fields that store exact, non-analyzed comma-joined tokens in the index and so
+# need explicit comma->AND expansion (Whoosh KEYWORD(commas=True) set).
+MULTI_VALUE_FIELDS = frozenset({"tag", "tag_id", "viewer_id"})
+
+# Date fields whose values/ranges get rewritten to RFC3339 Tantivy ranges.
+DATE_FIELDS = frozenset({"created", "modified", "added"})
+
+# Field aliases: Whoosh (v2) field names that were renamed in the Tantivy schema.
+# Preserved here so v2 queries using the old names continue to work without 400
+# errors instead of silently failing. Applied by _render to non-date field tokens.
+FIELD_ALIASES: dict[str, str] = {
+    "type": "document_type",
+    "type_id": "document_type_id",
+    "path": "storage_path",
+    "path_id": "storage_path_id",
+}
+
+# Known schema fields: a comma immediately followed by ``<known>:`` is a clause
+# separator. Restricting to known fields prevents URL-like ``http:`` misfires.
+KNOWN_FIELDS = frozenset(
+    {
+        "title",
+        "content",
+        "correspondent",
+        "document_type",
+        "type",  # v2 alias -> document_type
+        "storage_path",
+        "path",  # v2 alias -> storage_path
+        "tag",
+        "tag_id",
+        "correspondent_id",
+        "document_type_id",
+        "type_id",  # v2 alias -> document_type_id
+        "storage_path_id",
+        "path_id",  # v2 alias -> storage_path_id
+        "owner_id",
+        "viewer_id",
+        "asn",
+        "page_count",
+        "num_notes",
+        "created",
+        "modified",
+        "added",
+        "original_filename",
+        "checksum",
+        "notes",
+        "custom_fields",
+    },
+)
+
+_FIELD_RE = regex.compile(r"(?P<field>\w+):")
+
+# Matches the TO separator inside a range bracket. Handles three forms:
+#   middle:   "lo TO hi"   (either lo or hi may be empty)
+#   trailing: "lo TO"      (open upper bound)
+#   leading:  "TO hi"      (open lower bound)
+# Bounds MAY contain internal spaces (e.g. "-7 days"), so we use .*? / .+?
+# and split on the whitespace-delimited " TO " / " to " separator.
+_RANGE_RE = regex.compile(
+    r"^\s*(?P<lo>.*?)\s+[Tt][Oo]\s+(?P<hi>.+?)\s*$"
+    r"|"
+    r"^\s*(?P<lo2>.+?)\s+[Tt][Oo]\s*$"
+    r"|"
+    r"^\s*[Tt][Oo]\s+(?P<hi2>.+?)\s*$",
+)
+
+
+@dataclass(frozen=True, slots=True)
+class FieldValue:
+    field: str
+    value: str
+
+
+# Produced by the comma-resolution pass (not by scan()).
+@dataclass(frozen=True, slots=True)
+class FieldValueList:
+    field: str
+    values: tuple[str, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class FieldRange:
+    field: str
+    open: str
+    lo: str
+    hi: str
+    close: str
+
+
+# Produced by the comma-resolution pass (not by scan()).
+@dataclass(frozen=True, slots=True)
+class Comma:
+    pass
+
+
+@dataclass(frozen=True, slots=True)
+class Passthrough:
+    raw: str
+
+
+Token: TypeAlias = FieldValue | FieldValueList | FieldRange | Comma | Passthrough
+
+_CLOSE: dict[str, str] = {"[": "]", "{": "}"}
+
+
+def scan(query: str) -> list[Token]:
+    """
+    Tokenize a raw query into date/comma-aware tokens, leaving everything else
+    as verbatim ``Passthrough`` runs. Non-recursive: finds the first matching
+    close bracket/quote. Nested brackets are not valid Tantivy range syntax and
+    pass through verbatim on mismatch.
+    """
+    tokens: list[Token] = []
+    buf: list[str] = []  # accumulates passthrough chars
+    i, n = 0, len(query)
+    while i < n:
+        matched = _match_field_token(query, i)
+        if matched is None:
+            buf.append(query[i])
+            i += 1
+            continue
+        token, i = matched
+        if buf and buf[-1] == ",":
+            buf.pop()
+            _flush(buf, tokens)
+            tokens.append(Comma())
+        else:
+            _flush(buf, tokens)
+        tokens.append(token)
+        i = _maybe_comma(query, i, tokens)
+    _flush(buf, tokens)
+    return tokens
+
+
+def _flush(buf: list[str], tokens: list[Token]) -> None:
+    """Emit any accumulated passthrough characters as a single token."""
+    if buf:
+        tokens.append(Passthrough("".join(buf)))
+        buf.clear()
+
+
+def _at_word_boundary(query: str, i: int) -> bool:
+    """A field token may begin only at the start or after a non-word character."""
+    return i == 0 or not (query[i - 1].isalnum() or query[i - 1] == "_")
+
+
+def _match_field_token(query: str, i: int) -> tuple[Token, int] | None:
+    """
+    If a known ``field:`` token starts at ``i``, consume it and return
+    ``(token, end_index)``; otherwise return None so the caller treats the
+    character as passthrough. Handles both ``field:[range]`` and ``field:value``,
+    and returns None when the range/value cannot be consumed.
+    """
+    m = _FIELD_RE.match(query, i)
+    if m is None or m.group("field") not in KNOWN_FIELDS:
+        return None
+    if not _at_word_boundary(query, i):
+        return None
+    field = m.group("field")
+    j = m.end()
+    if j < len(query) and query[j] in "[{":
+        return _consume_range(query, j, field)
+    consumed = _consume_field_value(query, field, j)
+    if consumed is None:
+        return None
+    value, end = consumed
+    return FieldValue(field, value), end
+
+
+def _consume_field_value(query: str, field: str, start: int) -> tuple[str, int] | None:
+    """
+    Consume a field value starting at ``start``: a multi-word date keyword phrase
+    (date fields only), or a bare/quoted value, then absorb any comma-joined
+    continuation that is not a clause separator. ``resolve_commas`` later splits a
+    multi-value field's joined value into a ``FieldValueList``; for other fields
+    the comma stays literal.
+    """
+    n = len(query)
+    consumed = None
+    if field in DATE_FIELDS:
+        km = _KEYWORD_VALUE_RE.match(query, start)
+        if km is not None and (km.end() >= n or query[km.end()] in " \t),"):
+            consumed = (km.group(0), km.end())
+    if consumed is None:
+        consumed = _consume_value(query, start)
+    if consumed is None:
+        return None
+    value, k = consumed
+    while k < n and query[k] == ",":
+        if _looks_like_known_field(query, k + 1):
+            break  # clause separator: left for _maybe_comma to emit a Comma()
+        more = _consume_value(query, k + 1)
+        if more is None:
+            break
+        value = f"{value},{more[0]}"
+        k = more[1]
+    return value, k
+
+
+def _consume_range(
+    query: str,
+    start: int,
+    field: str,
+) -> tuple[FieldRange, int] | None:
+    """Consume ``[lo TO hi]`` / ``{lo TO hi}`` from ``start`` (the bracket)."""
+    open_br = query[start]
+    close_br = _CLOSE[open_br]
+    end = query.find(close_br, start + 1)
+    if end == -1:
+        return None
+    inner = query[start + 1 : end]
+    m = _RANGE_RE.match(inner)
+    if m is not None:
+        if m.group("lo") is not None or m.group("hi") is not None:
+            # Middle form: "lo TO hi" (either may be empty string)
+            lo = (m.group("lo") or "").strip()
+            hi = (m.group("hi") or "").strip()
+        elif m.group("lo2") is not None:
+            # Trailing form: "lo TO"
+            lo = m.group("lo2").strip()
+            hi = ""
+        else:
+            # Leading form: "TO hi"
+            lo = ""
+            hi = (m.group("hi2") or "").strip()
+    else:
+        lo, hi = inner.strip(), ""
+    return FieldRange(field, open_br, lo, hi, close_br), end + 1
+
+
+def _consume_value(query: str, start: int) -> tuple[str, int] | None:
+    """Consume a bare or quoted field value from ``start``, stopping at comma."""
+    n = len(query)
+    if start >= n or query[start] in " \t":
+        return None
+    if query[start] in "\"'":
+        quote = query[start]
+        end = query.find(quote, start + 1)
+        if end == -1:
+            return None
+        return query[start : end + 1], end + 1
+    j = start
+    while j < n and query[j] not in " \t),":
+        j += 1
+    return query[start:j], j
+
+
+def _looks_like_known_field(query: str, pos: int) -> bool:
+    """True if a known ``field:`` token starts at ``pos``."""
+    m = _FIELD_RE.match(query, pos)
+    return bool(m and m.group("field") in KNOWN_FIELDS)
+
+
+def _maybe_comma(query: str, i: int, tokens: list) -> int:
+    """If a clause-separator comma follows at ``i``, emit ``Comma()`` and advance."""
+    if i < len(query) and query[i] == "," and _looks_like_known_field(query, i + 1):
+        tokens.append(Comma())
+        return i + 1
+    return i
+
+
+def resolve_commas(tokens: list) -> list:
+    """
+    Collapse value-list commas into ``FieldValueList`` and keep clause-separator
+    commas as ``Comma``. (Clause-sep commas are already emitted by ``scan`` via
+    the value-stop logic; this pass folds value-lists.)
+    """
+    out: list = []
+    for tok in tokens:
+        if (
+            isinstance(tok, FieldValue)
+            and tok.field in MULTI_VALUE_FIELDS
+            and "," in tok.value
+        ):
+            values = tuple(v for v in tok.value.split(",") if v)
+            out.append(FieldValueList(tok.field, values))
+        else:
+            out.append(tok)
+    return out
+
+
+class SearchQueryError(ValueError):
+    """
+    Base for user-fixable search query errors.
+
+    Carries a message safe to surface to the user (no internal details). The view
+    layer catches this and returns an HTTP 400, so any future subclass (unknown
+    field, malformed range, wrapped parser errors) gets the same treatment.
+    """
+
+
+class InvalidDateQuery(SearchQueryError):
+    """Raised when a date field value or range bound cannot be parsed."""
+
+    def __init__(self, field: str, value: str) -> None:
+        self.field = field
+        self.value = value
+        super().__init__(f"Invalid date value {value!r} for field {field!r}.")
+
+
+_DIGITS_RE = regex.compile(r"^\d{4}(?:\d{2}){0,2}$")
+_ISO_RE = regex.compile(r"^\d{4}(?:-\d{2}(?:-\d{2})?)?$")
+
+
+def translate_scalar(field: str, value: str, tz: tzinfo) -> str:
+    """Translate a bare date-field value to a Tantivy range string."""
+    bare = value.strip("\"'").lower()
+    if bare in _DATE_KEYWORDS:
+        if field in _DATE_ONLY_FIELDS:
+            return f"{field}:{_date_only_range(bare, tz)}"
+        return f"{field}:{_datetime_range(bare, tz)}"
+    digits = value.replace("-", "")
+    if _DIGITS_RE.match(value) or _ISO_RE.match(value):
+        bounds = _precision_bounds(digits)
+        if bounds is None:
+            raise InvalidDateQuery(field, value)
+        return _field_range_from_dates(field, bounds[0], bounds[1], tz)
+    if regex.fullmatch(r"\d{14}", value):
+        try:
+            dt = datetime(
+                int(value[0:4]),
+                int(value[4:6]),
+                int(value[6:8]),
+                int(value[8:10]),
+                int(value[10:12]),
+                int(value[12:14]),
+                tzinfo=UTC,
+            )
+        except ValueError:
+            raise InvalidDateQuery(field, value) from None
+        iso = _fmt(dt)
+        return f"{field}:[{iso} TO {iso}]"
+    # Unrecognized shape -> tell the user their date is malformed rather than
+    # silently matching nothing or emitting invalid Tantivy syntax.
+    raise InvalidDateQuery(field, value)
+
+
+# Open-bound sentinels for date ranges. These far-past/far-future strings allow
+# open-ended ranges to be expressed as Tantivy string queries until tantivy-py
+# exposes Query.range_query(..., None) on Date fields (see module TODO).
+OPEN_LO = "0001-01-01T00:00:00Z"
+OPEN_HI = "9999-12-31T23:59:59Z"
+
+
+# Matches compact now-offset tokens like now-7d, now+1h, now-30m.
+_NOW_COMPACT_RE = regex.compile(
+    r"^now(?P<sign>[+-])(?P<n>\d+)(?P<unit>[dhm])$",
+    regex.IGNORECASE,
+)
+
+# Matches "±N <unit>" Whoosh-style offsets (e.g. -7 days, -1 week, +3 hours).
+# Whoosh's own date parser (qparser.dateparse.PlusMinus) additionally accepted
+# abbreviated unit spellings (e.g. "yrs", "yr", "y", "mos", "wks", "hrs", "mins",
+# "secs"); saved views/searches created under the old Whoosh backend can still
+# contain those tokens (e.g. "-999yrs"), so they are accepted here too and
+# normalized to a canonical unit via _UNIT_ALIASES below.
+_NOW_SPACED_RE = regex.compile(
+    r"^(?P<sign>[+-])(?P<n>\d+)\s*"
+    r"(?P<unit>years|year|yrs|yr|ys|y"
+    r"|months|month|mons|mon|mos|mo"
+    r"|weeks|week|wks|wk|ws|w"
+    r"|days|day|dys|dy|ds|d"
+    r"|hours|hour|hrs|hr|hs|h"
+    r"|minutes|minute|mins|min|ms|m"
+    r"|seconds|second|secs|sec|s)$",
+    regex.IGNORECASE,
+)
+
+# Maps every accepted unit spelling (including Whoosh-era abbreviations) to the
+# canonical unit name used as a key into the delta map in _resolve_relative_bound.
+_UNIT_ALIASES: dict[str, str] = {
+    alias: canonical
+    for canonical, aliases in {
+        "year": ("years", "year", "yrs", "yr", "ys", "y"),
+        "month": ("months", "month", "mons", "mon", "mos", "mo"),
+        "week": ("weeks", "week", "wks", "wk", "ws", "w"),
+        "day": ("days", "day", "dys", "dy", "ds", "d"),
+        "hour": ("hours", "hour", "hrs", "hr", "hs", "h"),
+        "minute": ("minutes", "minute", "mins", "min", "ms", "m"),
+        "second": ("seconds", "second", "secs", "sec", "s"),
+    }.items()
+    for alias in aliases
+}
+
+
+def _resolve_relative_bound(token: str) -> datetime | None:
+    """
+    Resolve a relative bound token to an exact UTC instant, or return None.
+
+    Supported forms:
+      - ``now``            -> current UTC instant
+      - ``now+/-<n>d/h/m`` -> now +/- timedelta (d=days, h=hours, m=minutes)
+      - ``±N <unit>``     -> now +/- delta; month/year use relativedelta;
+                              unit also accepts Whoosh-era abbreviations
+                              (e.g. "yrs", "mos", "wks", "hrs", "mins", "secs")
+    """
+    stripped = token.strip()
+    low = stripped.lower()
+    now = datetime.now(UTC)
+
+    if low == "now":
+        return now
+
+    m = _NOW_COMPACT_RE.match(stripped)
+    if m:
+        sign = 1 if m.group("sign") == "+" else -1
+        n = int(m.group("n"))
+        unit = m.group("unit").lower()
+        delta = (
+            sign
+            * {
+                "d": timedelta(days=n),
+                "h": timedelta(hours=n),
+                "m": timedelta(minutes=n),
+            }[unit]
+        )
+        return now + delta
+
+    m = _NOW_SPACED_RE.match(stripped)
+    if m:
+        sign = 1 if m.group("sign") == "+" else -1
+        n = int(m.group("n"))
+        unit = _UNIT_ALIASES[m.group("unit").lower()]
+        delta_map: dict[str, timedelta | relativedelta] = {
+            "second": timedelta(seconds=n),
+            "minute": timedelta(minutes=n),
+            "hour": timedelta(hours=n),
+            "day": timedelta(days=n),
+            "week": timedelta(weeks=n),
+            "month": relativedelta(months=n),
+            "year": relativedelta(years=n),
+        }
+        return now - delta_map[unit] if sign == -1 else now + delta_map[unit]
+
+    return None
+
+
+def _bound_datetimes(
+    field: str,
+    token: str,
+    tz: tzinfo,
+) -> tuple[datetime, datetime] | None:
+    """
+    Return (floor_dt, ceil_dt) UTC datetimes for a single range bound token, or
+    None if the token is unparsable. ``now`` and relative offsets resolve to the
+    current instant (floor == ceil == that instant; no day-flooring).
+    """
+    token = token.strip()
+
+    # Try relative/now forms first (before stripping hyphens which would mangle them).
+    rel = _resolve_relative_bound(token)
+    if rel is not None:
+        return rel, rel
+
+    # Full ISO datetime token (contains "T"): parse directly and return an exact
+    # instant (floor == ceil). Python 3.11+ datetime.fromisoformat accepts trailing Z.
+    if "T" in token:
+        try:
+            dt = datetime.fromisoformat(token)
+            # Ensure timezone-aware UTC result.
+            dt = dt.replace(tzinfo=UTC) if dt.tzinfo is None else dt.astimezone(UTC)
+            return dt, dt
+        except ValueError:
+            return None
+
+    digits = token.replace("-", "")
+    bounds = _precision_bounds(digits)
+    if bounds is None:
+        return None
+    start, end = bounds
+    return _utc_bounds_for_field(field, start, end, tz)
+
+
+def _render(tok: Token, tz: tzinfo) -> str:
+    """Render a single token back to a Tantivy query string fragment."""
+    if isinstance(tok, Passthrough):
+        return tok.raw
+    if isinstance(tok, Comma):
+        return " AND "
+    if isinstance(tok, FieldValueList):
+        field = FIELD_ALIASES.get(tok.field, tok.field)
+        return " AND ".join(f"{field}:{v}" for v in tok.values)
+    if isinstance(tok, FieldValue):
+        field = FIELD_ALIASES.get(tok.field, tok.field)
+        if field in DATE_FIELDS:
+            return translate_scalar(field, tok.value, tz)
+        return f"{field}:{tok.value}"
+    if isinstance(tok, FieldRange):
+        field = FIELD_ALIASES.get(tok.field, tok.field)
+        if field in DATE_FIELDS:
+            return translate_range(field, tok.lo, tok.hi, tz)
+        return f"{field}:{tok.open}{tok.lo} TO {tok.hi}{tok.close}"
+    return ""  # pragma: no cover
+
+
+# Post-render operator normalization patterns: collapse repeated whitespace and
+# strip spaced/trailing Tantivy boolean operators that would otherwise be invalid.
+_MULTI_SPACE_RE = regex.compile(r" {2,}")
+_TRAILING_OP_RE = regex.compile(r"\s+[-+]+\s*$")
+_SPACED_OP_RE = regex.compile(r"\s+[-+]\s+")
+
+
+def _normalize_operators(text: str) -> str:
+    """
+    Collapse multiple spaces, strip trailing dangling operators, and replace
+    spaced operators (`` - `` / `` + ``) with a single space.
+
+    Applied only to Passthrough fragments (the rendered output is scanned for
+    operator artifacts outside bracketed ranges) via a post-render pass on the
+    full rendered string. This preserves date ranges (``[... TO ...]``) verbatim
+    while cleaning natural-language separators in the surrounding text.
+    """
+    text = _MULTI_SPACE_RE.sub(" ", text)
+    text = _TRAILING_OP_RE.sub("", text).strip()
+    text = _SPACED_OP_RE.sub(" ", text).strip()
+    return text
+
+
+def translate_query(raw: str, tz: tzinfo) -> str:
+    """Translate a raw Whoosh-style query into Tantivy-compatible syntax."""
+    tokens = resolve_commas(scan(raw))
+    rendered = "".join(_render(t, tz) for t in tokens)
+    return _normalize_operators(rendered)
+
+
+def translate_range(field: str, lo: str, hi: str, tz: tzinfo) -> str:
+    """Translate a date-field ``[lo TO hi]`` range to a Tantivy ISO range string.
+
+    Handles partial-date bounds (YYYY, YYYYMM, YYYYMMDD, ISO dash variants),
+    open bounds (empty string -> OPEN_LO/OPEN_HI), ``now``, and reversed ranges
+    (swaps tokens before computing floor/ceil so the span is always correct).
+    """
+    lo_s = lo.strip()
+    hi_s = hi.strip()
+
+    # Parse both bounds to (floor, ceil) pairs when present.
+    lo_pair: tuple[datetime, datetime] | None = None
+    hi_pair: tuple[datetime, datetime] | None = None
+
+    if lo_s:
+        lo_pair = _bound_datetimes(field, lo_s, tz)
+        if lo_pair is None:
+            raise InvalidDateQuery(field, lo_s)
+    if hi_s:
+        hi_pair = _bound_datetimes(field, hi_s, tz)
+        if hi_pair is None:
+            raise InvalidDateQuery(field, hi_s)
+
+    # Detect a reversed range: only swap when BOTH bounds are present.
+    if lo_pair is not None and hi_pair is not None and lo_pair[0] > hi_pair[0]:
+        lo_pair, hi_pair = hi_pair, lo_pair
+
+    lo_iso = _fmt(lo_pair[0]) if lo_pair is not None else OPEN_LO
+
+    # A bound resolves to (floor, ceil) where floor == ceil for an exact instant
+    # (a full ISO datetime, "now", or a "+/-N unit" offset) and floor != ceil for
+    # a coarser period token (year/month/day precision). Only the latter needs a
+    # half-open close: its ceil is the start of the *next* period and must be
+    # excluded, or that instant (e.g. the 1st of next month) wrongly matches.
+    if hi_pair is not None:
+        hi_iso = _fmt(hi_pair[1])
+        hi_close = "]" if hi_pair[0] == hi_pair[1] else "}"
+    else:
+        hi_iso = OPEN_HI
+        hi_close = "]"
+
+    return f"{field}:[{lo_iso} TO {hi_iso}{hi_close}"

src/documents/serialisers.py

@@ -87,6 +87,7 @@ from documents.regex import validate_regex_pattern
 from documents.templating.filepath import validate_filepath_template_and_render
 from documents.templating.utils import convert_format_str_to_template_format
 from documents.templating.workflows import validate_workflow_template
+from documents.utils import normalize_unicode
 from documents.validators import uri_validator
 from documents.validators import url_validator
 from documents.versioning import sort_versions_newest_first
@@ -1247,31 +1248,30 @@ class DocumentSerializer(
 
             validated_data["tags"] = list(final_tags)
         if validated_data.get("remove_inbox_tags"):
-            current_tag_ids = {t.pk for t in instance.tags.all()}
+            tag_ids_being_added = (
-            tags = (
+                [
-                validated_data["tags"]
+                    tag.id
+                    for tag in validated_data["tags"]
+                    if tag not in instance.tags.all()
+                ]
                 if "tags" in validated_data
-                else list(instance.tags.all())
+                else []
             )
-
+            inbox_tags_not_being_added = Tag.objects.filter(is_inbox_tag=True).exclude(
-            # Tags newly added in this update, plus their ancestors, are kept
+                id__in=tag_ids_being_added,
-            keep_ids: set[int] = set()
+            )
-            for tag in tags:
+            if "tags" in validated_data:
-                if tag.pk not in current_tag_ids:
+                validated_data["tags"] = [
-                    keep_ids.add(tag.pk)
+                    tag
-                    keep_ids.update(int(pk) for pk in tag.get_ancestors_pks())
+                    for tag in validated_data["tags"]
-
+                    if tag not in inbox_tags_not_being_added
-            # Remove inbox tags and their descendants, except those being kept
+                ]
-            remove_ids: set[int] = set()
+            else:
-            for inbox_tag in (
+                validated_data["tags"] = [
-                Tag.objects.filter(is_inbox_tag=True)
+                    tag
-                .exclude(pk__in=keep_ids)
+                    for tag in instance.tags.all()
-                .only("pk", "tn_descendants_pks")
+                    if tag not in inbox_tags_not_being_added
-            ):
+                ]
-                remove_ids.add(inbox_tag.pk)
-                remove_ids.update(int(pk) for pk in inbox_tag.get_descendants_pks())
-
-            validated_data["tags"] = [t for t in tags if t.pk not in remove_ids]
 
         if settings.AUDIT_LOG_ENABLED:
             with set_actor(self.user):
@@ -3121,6 +3121,13 @@ class WorkflowTriggerSerializer(serializers.ModelSerializer[WorkflowTrigger]):
         ):
             attrs["filter_path"] = None
 
+        # Normalize once at write time, since these are matched against many
+        # documents but edited rarely
+        if attrs.get("filter_filename") is not None:
+            attrs["filter_filename"] = normalize_unicode(attrs["filter_filename"])
+        if attrs.get("filter_path") is not None:
+            attrs["filter_path"] = normalize_unicode(attrs["filter_path"])
+
         if (
             "filter_custom_field_query" in attrs
             and attrs["filter_custom_field_query"] is not None

src/documents/templating/filepath.py

@@ -1,7 +1,6 @@
 import logging
 import os
 import re
-import unicodedata
 from collections.abc import Iterable
 from pathlib import PurePath
 
@@ -26,6 +25,7 @@ from documents.templating.environment import _template_environment
 from documents.templating.filters import format_datetime
 from documents.templating.filters import get_cf_value
 from documents.templating.filters import localize_date
+from documents.utils import normalize_unicode
 
 logger = logging.getLogger("paperless.templating")
 
@@ -42,7 +42,7 @@ class FilePathTemplate(Template):
             3. Removing extra spaces before and after forward slashes
             4. Preserving spaces in other parts of the path
             """
-            value = unicodedata.normalize("NFC", value)
+            value = normalize_unicode(value)
             value = value.replace("\n", "").replace("\r", "")
             value = re.sub(r"\s*/\s*", "/", value)
 
@@ -184,17 +184,17 @@ def get_basic_metadata_context(
     """
     return {
         "title": pathvalidate.sanitize_filename(
-            unicodedata.normalize("NFC", document.title),
+            normalize_unicode(document.title),
             replacement_text="-",
         ),
         "correspondent": pathvalidate.sanitize_filename(
-            unicodedata.normalize("NFC", document.correspondent.name),
+            normalize_unicode(document.correspondent.name),
             replacement_text="-",
         )
         if document.correspondent
         else no_value_default,
         "document_type": pathvalidate.sanitize_filename(
-            unicodedata.normalize("NFC", document.document_type.name),
+            normalize_unicode(document.document_type.name),
             replacement_text="-",
         )
         if document.document_type
@@ -205,8 +205,7 @@ def get_basic_metadata_context(
         "owner_username": document.owner.username
         if document.owner
         else no_value_default,
-        "original_name": unicodedata.normalize(
+        "original_name": normalize_unicode(
-            "NFC",
             PurePath(document.original_filename).with_suffix("").name,
         )
         if document.original_filename
@@ -275,12 +274,12 @@ def get_tags_context(tags: Iterable[Tag]) -> dict[str, str | list[str]]:
     return {
         "tag_list": pathvalidate.sanitize_filename(
             ",".join(
-                sorted(unicodedata.normalize("NFC", tag.name) for tag in tags),
+                sorted(normalize_unicode(tag.name) for tag in tags),
             ),
             replacement_text="-",
         ),
         # Assumed to be ordered, but a template could loop through to find what they want
-        "tag_name_list": [unicodedata.normalize("NFC", x.name) for x in tags],
+        "tag_name_list": [normalize_unicode(x.name) for x in tags],
     }
 
 
@@ -307,7 +306,7 @@ def get_custom_fields_context(
             CustomField.FieldDataType.LONG_TEXT,
         }:
             value = pathvalidate.sanitize_filename(
-                unicodedata.normalize("NFC", field_instance.value),
+                normalize_unicode(field_instance.value),
                 replacement_text="-",
             )
         elif (
@@ -316,8 +315,7 @@ def get_custom_fields_context(
         ):
             options = field_instance.field.extra_data["select_options"]
             value = pathvalidate.sanitize_filename(
-                unicodedata.normalize(
+                normalize_unicode(
-                    "NFC",
                     next(
                         option["label"]
                         for option in options
@@ -330,7 +328,7 @@ def get_custom_fields_context(
             value = field_instance.value
         field_data["custom_fields"][
             pathvalidate.sanitize_filename(
-                unicodedata.normalize("NFC", field_instance.field.name),
+                normalize_unicode(field_instance.field.name),
                 replacement_text="-",
             )
         ] = {

src/documents/tests/search/conftest.py

@@ -1,11 +1,15 @@
 from __future__ import annotations
 
+import tempfile
 from typing import TYPE_CHECKING
 
 import pytest
+import tantivy
 
 from documents.search._backend import TantivyBackend
 from documents.search._backend import reset_backend
+from documents.search._schema import build_schema
+from documents.search._tokenizer import register_tokenizers
 
 if TYPE_CHECKING:
     from collections.abc import Generator
@@ -31,3 +35,11 @@ def backend() -> Generator[TantivyBackend, None, None]:
     finally:
         b.close()
         reset_backend()
+
+
+@pytest.fixture(scope="module")
+def index() -> tantivy.Index:
+    """A real Tantivy index for parse-acceptance tests (module scope for speed)."""
+    idx = tantivy.Index(build_schema(), path=tempfile.mkdtemp())
+    register_tokenizers(idx, "english")
+    return idx

src/documents/tests/search/test_acceptance.py

@@ -1,541 +0,0 @@
-"""Result-level acceptance corpus: real documents indexed via build_schema(),
-real queries run through parse_user_query(), matched-document-ID sets
-asserted, not intermediate ASTs or query strings. This is paperless-ngx's
-analogue of whoosh-compat's own tests/emitter/test_acceptance_e2e.py.
-
-Supersedes test_query.py's TestParseUserQuery result-level cases.
-"""
-
-from __future__ import annotations
-
-from datetime import UTC
-from datetime import datetime
-from typing import TYPE_CHECKING
-
-import pytest
-import time_machine
-from django.contrib.auth.models import User
-
-from documents.models import CustomField
-from documents.models import CustomFieldInstance
-from documents.models import Document
-from documents.models import DocumentType
-from documents.models import Note
-from documents.models import StoragePath
-from documents.search._query import parse_user_query
-
-if TYPE_CHECKING:
-    from documents.search._backend import TantivyBackend
-
-pytestmark = [pytest.mark.search, pytest.mark.django_db]
-
-FROZEN_NOW = datetime(2026, 6, 15, 12, 0, tzinfo=UTC)
-
-
-def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
-    return set(backend.search_ids(query, user=None))
-
-
-def _index(backend: TantivyBackend, **kwargs: object) -> Document:
-    """Create a Document and index it in one step, for the common case
-    where nothing needs to happen between the two (no related Note/
-    CustomFieldInstance to attach first)."""
-    doc = Document.objects.create(**kwargs)
-    backend.add_or_update(doc)
-    return doc
-
-
-@pytest.fixture
-def indexed_documents(backend: TantivyBackend) -> dict[str, int]:
-    """Index a small fixture set, return {label: doc_id} for corpus queries."""
-    docs = {
-        "invoice_2020": _index(
-            backend,
-            title="Invoice 2020",
-            content="invoice total due",
-            checksum="acc-invoice-2020",
-            archive_serial_number=100,
-        ),
-        "invoice_2021": _index(
-            backend,
-            title="Invoice 2021",
-            content="invoice total due",
-            checksum="acc-invoice-2021",
-            archive_serial_number=101,
-        ),
-        "invoice_2023": _index(
-            backend,
-            title="Invoice 2023",
-            content="invoice total due",
-            checksum="acc-invoice-2023",
-            archive_serial_number=102,
-        ),
-        "receipt_2022": _index(
-            backend,
-            title="Receipt 2022",
-            content="receipt total due",
-            checksum="acc-receipt-2022",
-            archive_serial_number=103,
-        ),
-    }
-    return {label: doc.pk for label, doc in docs.items()}
-
-
-class TestIssue13568BracketWildcard:
-    """paperless-ngx#13568: title:202[0-3]* must keep its character class,
-    not fold to a prefix query that silently drops it."""
-
-    def test_bracket_class_wildcard_matches_only_in_range_years(
-        self,
-        backend: TantivyBackend,
-        indexed_documents: dict[str, int],
-    ) -> None:
-        """
-        GIVEN:
-            - Four indexed documents titled Invoice 2020/2021/2023 and
-              Receipt 2022
-        WHEN:
-            - "title:202[0-1]*" is searched ([0-1], not [0-3], is
-              deliberate: the fixture's trailing digits are 0/1/2/3, so a
-              [0-3] class would match all four and pass even if the
-              character class were silently dropped and folded to an
-              unconstrained "202*" prefix; [0-1] partitions the fixture
-              into a genuine in-range/out-of-range split)
-        THEN:
-            - Only the 2020 and 2021 documents match, proving the bracket
-              character class survived (issue #13568's original bug)
-        """
-        matched = _matched_ids(backend, "title:202[0-1]*")
-        expected = {
-            indexed_documents["invoice_2020"],
-            indexed_documents["invoice_2021"],
-        }
-        assert matched == expected, (
-            "title:202[0-1]* must match 2020/2021 titles and exclude 2022/2023 "
-            "- if this matches everything, the wildcard's character class was "
-            "silently dropped (issue #13568's original bug)"
-        )
-
-
-class TestFieldBoosts:
-    def test_title_boost_ranks_title_match_above_content_only_match(
-        self,
-        backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - One document whose title contains the query word and another
-              whose content (not title) contains it
-        WHEN:
-            - The query word is searched unfielded
-        THEN:
-            - The title match ranks first, proving our title field boost
-              actually affects ranking
-        """
-        title_match = _index(
-            backend,
-            title="urgent",
-            content="nothing else relevant",
-            checksum="acc-boost-title",
-        )
-        _index(
-            backend,
-            title="nothing",
-            content="urgent matter here",
-            checksum="acc-boost-content",
-        )
-        query = parse_user_query(backend._index, "urgent", UTC)
-        searcher = backend._index.searcher()
-        results = searcher.search(query, limit=10)
-        ranked_ids = [
-            searcher.doc(addr).to_dict()["id"][0] for _score, addr in results.hits
-        ]
-        assert ranked_ids[0] == title_match.pk
-
-
-class TestJsonSubpaths:
-    def test_notes_user_matches_document_with_that_note_author(
-        self,
-        backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - A document with a Note authored by "alice" and a second,
-              unrelated document with no note
-        WHEN:
-            - "notes.user:alice" is searched
-        THEN:
-            - Only the document with alice's note matches
-        """
-        alice = User.objects.create_user(username="alice")
-        doc_with_note = Document.objects.create(
-            title="Has note",
-            content="x",
-            checksum="acc-note-with",
-        )
-        Note.objects.create(document=doc_with_note, user=alice, note="reminder")
-        backend.add_or_update(doc_with_note)
-        _index(backend, title="No note", content="x", checksum="acc-note-without")
-        matched = _matched_ids(backend, "notes.user:alice")
-        assert matched == {doc_with_note.pk}
-
-    def test_custom_fields_name_and_value_combine(
-        self,
-        backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - A document with a "Contract Number" custom field valued
-              "policy", and a second document with a differently-named
-              custom field also valued "policy"
-        WHEN:
-            - 'custom_fields.name:"Contract Number" custom_fields.value:policy'
-              is searched
-        THEN:
-            - Only the document whose field name AND value both match is
-              returned
-        """
-        field = CustomField.objects.create(
-            name="Contract Number",
-            data_type=CustomField.FieldDataType.STRING,
-        )
-        other_field = CustomField.objects.create(
-            name="Other Field",
-            data_type=CustomField.FieldDataType.STRING,
-        )
-        matching = Document.objects.create(
-            title="Matching",
-            content="x",
-            checksum="acc-cf-matching",
-        )
-        CustomFieldInstance.objects.create(
-            document=matching,
-            field=field,
-            value_text="policy",
-        )
-        backend.add_or_update(matching)
-        non_matching = Document.objects.create(
-            title="Non-matching",
-            content="x",
-            checksum="acc-cf-nonmatching",
-        )
-        CustomFieldInstance.objects.create(
-            document=non_matching,
-            field=other_field,
-            value_text="policy",
-        )
-        backend.add_or_update(non_matching)
-        matched = _matched_ids(
-            backend,
-            'custom_fields.name:"Contract Number" custom_fields.value:policy',
-        )
-        assert matched == {matching.pk}
-
-
-class TestUnregisteredIdFieldFoldsToLiteralText:
-    """tag_id, owner_id, etc. are intentionally excluded from the
-    FieldRegistry - always internal index columns, never meant to be
-    query-addressable. Prove an unregistered field folds to a literal
-    text search that matches nothing, rather than erroring."""
-
-    def test_tag_id_query_matches_nothing(
-        self,
-        backend: TantivyBackend,
-        indexed_documents: dict[str, int],
-    ) -> None:
-        """
-        GIVEN:
-            - A real indexed corpus and "tag_id", a field intentionally
-              excluded from the FieldRegistry (an internal index column,
-              never meant to be query-addressable)
-        WHEN:
-            - "tag_id:5" is searched
-        THEN:
-            - It folds to a literal text search and matches nothing,
-              rather than erroring
-        """
-        matched = _matched_ids(backend, "tag_id:5")
-        assert matched == set()
-
-
-class TestFuzzyBlendSurvivesWhooshGrammar:
-    """A query mixing whoosh-only grammar (a date keyword) with a typo'd
-    free-text word must still fuzzy-match the intended document when
-    ADVANCED_FUZZY_SEARCH_THRESHOLD is enabled. The fuzzy clause is built
-    from the parsed query's free-text tokens (whoosh_compat's
-    free_text_tokens), never from the raw query string, so whoosh grammar
-    that tantivy's own parser rejects cannot knock the fuzzy clause out."""
-
-    def test_typo_fuzzy_matches_alongside_date_keyword(
-        self,
-        backend: TantivyBackend,
-        settings,
-    ) -> None:
-        """
-        GIVEN:
-            - ADVANCED_FUZZY_SEARCH_THRESHOLD enabled, and a document
-              indexed with content "receipt total due"
-        WHEN:
-            - The query blends whoosh-only grammar tantivy's own parser
-              rejects ("added:today") with a one-transposition misspelling
-              of a word in the indexed content
-        THEN:
-            - The document still matches, because the fuzzy clause is
-              built from the parsed query's free-text tokens
-              (whoosh_compat's free_text_tokens), never from the raw
-              query string, so grammar tantivy's parser cannot handle
-              cannot knock the fuzzy clause out
-        """
-        settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5
-        with time_machine.travel(FROZEN_NOW, tick=False):
-            doc = _index(
-                backend,
-                title="Receipt March",
-                content="receipt total due",
-                checksum="fuzzy-blend-1",
-                archive_serial_number=900,
-            )
-            # Sanity: the exact spelling matches through the exact clause.
-            assert doc.pk in _matched_ids(backend, "added:today receipt")
-            # The regression: the misspelling (one transposition) only
-            # matches via the fuzzy clause, and "added:today" is
-            # whoosh-only grammar tantivy's parser rejects, so raw-string
-            # fuzzy parsing skips the clause entirely and this returns
-            # nothing. The typo is deliberate; keep codespell away from it.
-            typo_query = "added:today reciept"  # codespell:ignore reciept
-            assert doc.pk in _matched_ids(backend, typo_query)
-
-    def test_negated_words_do_not_fuzzy_match(
-        self,
-        backend: TantivyBackend,
-        settings,
-    ) -> None:
-        """
-        GIVEN:
-            - ADVANCED_FUZZY_SEARCH_THRESHOLD enabled, and a document
-              containing the NOT'd word ("receipt") but not the positive
-              word ("total"), so nothing matches the exact clause -- the
-              shape a naive fuzzy string built from ALL words (including
-              the NOT'd one) would make this document the sole hit,
-              normalize its score to 1.0, and survive any threshold (a
-              shape with an exact-matching sibling document would NOT
-              discriminate: normalization would rank the resurfaced
-              document far below the exact match and the threshold would
-              cut it even for a naive implementation)
-        WHEN:
-            - "added:today total NOT receipt" is searched
-        THEN:
-            - The document does not match; a term the user excluded must
-              not resurface through the fuzzy clause
-        """
-        settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5
-        with time_machine.travel(FROZEN_NOW, tick=False):
-            _index(
-                backend,
-                title="Receipt Archive",
-                content="receipt archived stack",
-                checksum="fuzzy-blend-2",
-                archive_serial_number=901,
-            )
-            assert _matched_ids(backend, "added:today total NOT receipt") == set()
-
-
-class TestUnquotedDateKeywordPhrases:
-    """The unquoted spelling (added:previous month) is honored natively by
-    whoosh-compat's own grammar for this closed phrase vocabulary, no
-    app-level rewrite is involved. Pins that the historically supported
-    spelling keeps working now that paperless no longer pre-quotes it."""
-
-    @pytest.fixture
-    def period_documents(self, backend: TantivyBackend) -> dict[str, int]:
-        with time_machine.travel(FROZEN_NOW, tick=False):
-            in_may = _index(
-                backend,
-                title="May Doc",
-                content="statement",
-                checksum="kw-may",
-                archive_serial_number=910,
-                added=datetime(2026, 5, 20, 12, 0, tzinfo=UTC),
-            )
-            in_june = _index(
-                backend,
-                title="June Doc",
-                content="statement",
-                checksum="kw-june",
-                archive_serial_number=911,
-                added=datetime(2026, 6, 10, 12, 0, tzinfo=UTC),
-            )
-        return {"in_may": in_may.pk, "in_june": in_june.pk}
-
-    @pytest.mark.parametrize(
-        "query",
-        [
-            pytest.param("added:previous month", id="unquoted"),
-            pytest.param('added:"previous month"', id="quoted"),
-            pytest.param("added:Previous Month", id="unquoted-mixed-case"),
-        ],
-    )
-    def test_unquoted_matches_the_same_documents_as_quoted(
-        self,
-        backend: TantivyBackend,
-        period_documents: dict[str, int],
-        query: str,
-    ) -> None:
-        """
-        GIVEN:
-            - Two documents added in different months, time frozen so
-              only one falls in "previous month"
-        WHEN:
-            - The same date-keyword phrase is spelled unquoted, quoted,
-              and unquoted with mixed case
-        THEN:
-            - All three spellings match the same document; paperless no
-              longer pre-quotes this phrase before parsing, relying on
-              whoosh-compat's own grammar to accept it unquoted natively
-        """
-        with time_machine.travel(FROZEN_NOW, tick=False):
-            assert _matched_ids(backend, query) == {period_documents["in_may"]}
-
-    @pytest.mark.parametrize(
-        "query",
-        [
-            pytest.param("added:this month", id="this-month"),
-            pytest.param("added:this year", id="this-year"),
-            pytest.param("added:previous week", id="previous-week"),
-            pytest.param("added:previous quarter", id="previous-quarter"),
-            pytest.param("added:previous year", id="previous-year"),
-            pytest.param("created:previous month", id="created-field"),
-            pytest.param("modified:previous month", id="modified-field"),
-        ],
-    )
-    def test_every_phrase_and_date_field_parses_without_error(
-        self,
-        backend: TantivyBackend,
-        period_documents: dict[str, int],
-        query: str,
-    ) -> None:
-        """
-        GIVEN:
-            - Our real schema and every date-keyword phrase in the
-              vocabulary, against every date field we expose (added,
-              created, modified)
-        WHEN:
-            - Each combination is searched
-        THEN:
-            - It parses and searches cleanly against our schema (no
-              SearchQueryError, so no HTTP 400); exact window semantics
-              are whoosh-compat's own and are pinned in its own suite
-        """
-        with time_machine.travel(FROZEN_NOW, tick=False):
-            _matched_ids(backend, query)
-
-    def test_text_field_keyword_words_are_ordinary_text(
-        self,
-        backend: TantivyBackend,
-        period_documents: dict[str, int],
-    ) -> None:
-        """
-        GIVEN:
-            - period_documents (indexed by added-date) and a third
-              document whose title literally contains the words
-              "previous month"
-        WHEN:
-            - "title:previous month" is searched
-        THEN:
-            - Only the document whose title contains those words matches;
-              "previous month" after a TEXT field (or unfielded) is
-              ordinary text, not a date phrase, so the date-window
-              documents do not match
-        """
-        with time_machine.travel(FROZEN_NOW, tick=False):
-            wordy = _index(
-                backend,
-                title="Notes from the previous month",
-                content="meeting notes",
-                checksum="kw-text",
-                archive_serial_number=912,
-            )
-            assert _matched_ids(backend, "title:previous month") == {wordy.pk}
-
-
-class TestFieldAliases:
-    """type:/path: are registry aliases for document_type:/storage_path:.
-    The only other alias coverage is parse-shape; these prove resolution
-    end-to-end against a real index."""
-
-    def test_type_alias_and_canonical_name_match_the_same_document(
-        self,
-        backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - A document with document_type "invoice", and a decoy
-              document with no type whose content merely mentions
-              "invoice" (document_type is itself a default search field,
-              so if alias resolution ever broke and "type:invoice"
-              demoted to unfielded text, the token would STILL match the
-              typed document through the field value; the decoy carrying
-              the query word in content is what makes a demoted search
-              distinguishable, since it would then match both documents
-              and fail the exact-set assertion -- the title avoids
-              stemming to "type": English stems Typed -> type)
-        WHEN:
-            - "type:invoice" and "document_type:invoice" are each
-              searched
-        THEN:
-            - Both resolve to the same document, proving the "type" alias
-              and its canonical field name agree end-to-end against a
-              real index
-        """
-        invoice_type = DocumentType.objects.create(name="invoice")
-        typed = _index(
-            backend,
-            title="First",
-            content="quarterly statement",
-            checksum="alias-type-1",
-            document_type=invoice_type,
-        )
-        _index(
-            backend,
-            title="Second",
-            content="invoice mentioned in body",
-            checksum="alias-type-2",
-        )
-        assert _matched_ids(backend, "type:invoice") == {typed.pk}
-        assert _matched_ids(backend, "document_type:invoice") == {typed.pk}
-
-    def test_path_alias_and_canonical_name_match_the_same_document(
-        self,
-        backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - A document stored under storage_path "archive", and a decoy
-              document with no storage_path whose content merely mentions
-              "archive" (storage_path is NOT a default search field
-              today, so a demoted "path:archive" already matches nothing;
-              the content decoy keeps this test discriminating even if
-              storage_path ever joins the defaults)
-        WHEN:
-            - "path:archive" and "storage_path:archive" are each searched
-        THEN:
-            - Both resolve to the same document, proving the "path" alias
-              and its canonical field name agree end-to-end against a
-              real index
-        """
-        archive = StoragePath.objects.create(name="archive", path="archive/{title}")
-        stored = _index(
-            backend,
-            title="Stored",
-            content="quarterly statement",
-            checksum="alias-path-1",
-            storage_path=archive,
-        )
-        _index(
-            backend,
-            title="Loose",
-            content="archive mentioned in body",
-            checksum="alias-path-2",
-        )
-        assert _matched_ids(backend, "path:archive") == {stored.pk}
-        assert _matched_ids(backend, "storage_path:archive") == {stored.pk}

src/documents/tests/search/test_checksum_prefix_queries.py

@@ -1,82 +0,0 @@
-"""``checksum`` wildcard patterns stay literal end to end, once user queries
-route through whoosh-compat.
-
-The registry-level fact (the pattern normalizer folds a KEYWORD pattern
-rather than stemming it) is pinned on its own in
-``test_keyword_pattern_literal.py``. This proves it actually reaches a real
-query: ``checksum:ceded*`` must match only the document whose checksum
-starts with "ceded", not the one whose checksum stems to the same run.
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-import pytest
-
-from documents.models import Document
-
-if TYPE_CHECKING:
-    from documents.search._backend import TantivyBackend
-
-pytestmark = [pytest.mark.search, pytest.mark.django_db]
-
-CEDEF00D = "cedef00ddeadbeef0123456789abcdef01234567"
-CEDEDEAD = "cededeadbeef567801234567" + "89abcdef01234567"
-
-
-class TestChecksumPrefixQueries:
-    @pytest.fixture
-    def indexed(self, backend: TantivyBackend) -> None:
-        for i, checksum in enumerate((CEDEF00D, CEDEDEAD)):
-            doc = Document.objects.create(
-                title=f"Checksum doc {i}",
-                content="invoices for the quarter",
-                checksum=checksum,
-                archive_serial_number=940 + i,
-            )
-            backend.add_or_update(doc)
-
-    def _ids(self, backend: TantivyBackend, query: str) -> set[int]:
-        return set(backend.search_ids(query, user=None))
-
-    def test_prefix_matches_only_the_document_that_starts_with_it(
-        self,
-        backend: TantivyBackend,
-        indexed: None,
-    ) -> None:
-        """
-        GIVEN:
-            - Two documents indexed with checksums that share a stem when
-              run through the English stemmer ("cedef00d..." and
-              "cededead...") but only one literally starts with "ceded"
-        WHEN:
-            - "checksum:ceded*" is searched
-        THEN:
-            - Only the document whose checksum literally starts with
-              "ceded" matches; the pattern normalizer folds a KEYWORD
-              pattern rather than stemming it, so this reaches a real
-              query end to end
-        """
-        matched = self._ids(backend, "checksum:ceded*")
-        expected = Document.objects.get(checksum=CEDEDEAD).pk
-        assert matched == {expected}
-
-    def test_text_prefix_still_reaches_the_stemmed_index(
-        self,
-        backend: TantivyBackend,
-        indexed: None,
-    ) -> None:
-        """
-        GIVEN:
-            - Two documents indexed with content "invoices for the
-              quarter"
-        WHEN:
-            - "invoice*" is searched against the TEXT content field
-        THEN:
-            - Both documents match, confirming the checksum field's
-              literal-pattern behavior is specific to KEYWORD fields and
-              does not affect TEXT field wildcard matching against
-              stemmed terms
-        """
-        assert len(self._ids(backend, "invoice*")) == 2

src/documents/tests/search/test_cjk_clause.py

@@ -1,169 +0,0 @@
-"""The CJK bigram clause blended into QUERY-mode searches.
-
-The clause exists so CJK runs are matchable at all (the default analyzers
-keep a whitespace-free CJK run as one indivisible token), but it must not
-widen the query beyond what the user asked for: a CJK term the query
-excludes, or restricts to one field, must not come back through it.
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-import pytest
-
-from documents.models import Document
-
-if TYPE_CHECKING:
-    from pytest_django.fixtures import SettingsWrapper
-
-    from documents.search._backend import TantivyBackend
-
-pytestmark = [pytest.mark.search, pytest.mark.django_db]
-
-
-def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
-    return set(backend.search_ids(query, user=None))
-
-
-def _index(backend: TantivyBackend, **kwargs: object) -> Document:
-    doc = Document.objects.create(**kwargs)
-    backend.add_or_update(doc)
-    return doc
-
-
-class TestCjkClauseFollowsTheParsedQuery:
-    def test_negated_cjk_term_is_excluded(self, backend: TantivyBackend) -> None:
-        """
-        GIVEN:
-            - Two documents both matching "invoice", one whose content
-              also contains 漢字
-        WHEN:
-            - "invoice NOT 漢字" is searched
-        THEN:
-            - Only the document without 漢字 matches; 'invoice NOT 漢字'
-              must not return the document containing 漢字
-        """
-        with_cjk = _index(
-            backend,
-            title="Invoice A",
-            content="invoice total 漢字",
-            checksum="cjk-neg-1",
-        )
-        without_cjk = _index(
-            backend,
-            title="Invoice B",
-            content="invoice total only",
-            checksum="cjk-neg-2",
-        )
-
-        assert _matched_ids(backend, "invoice") == {with_cjk.pk, without_cjk.pk}
-        assert _matched_ids(backend, "invoice NOT 漢字") == {without_cjk.pk}
-
-    @pytest.mark.parametrize(
-        ("threshold", "expected"),
-        [
-            pytest.param(None, {"titled"}, id="fuzzy_off"),
-            pytest.param(0.0, {"titled", "content_only"}, id="fuzzy_on"),
-        ],
-    )
-    def test_fielded_cjk_term_searches_only_that_field(
-        self,
-        backend: TantivyBackend,
-        settings: SettingsWrapper,
-        threshold: float | None,
-        expected: set[str],
-    ) -> None:
-        """
-        GIVEN:
-            - One document with 東京 in its title, another with 東京 only
-              in its content, and ADVANCED_FUZZY_SEARCH_THRESHOLD either
-              off or on
-        WHEN:
-            - "title:東京" is searched
-        THEN:
-            - With fuzzy off, only the titled document matches: the CJK
-              clause honours the field, so 'title:東京' must not match a
-              document whose 東京 is only in the content. With fuzzy on,
-              the content-only document is also readmitted, because the
-              fuzzy clause contributes every free-text term UNFIELDED by
-              design (see _try_parse_fuzzy_query) on its own
-              0.1-boosted terms -- a documented trade-off, pinned here so
-              it stays deliberate
-        """
-        settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = threshold
-        content_only = _index(
-            backend,
-            title="Tokyo report",
-            content="東京都の人口は約1400万人です",
-            checksum="cjk-field-1",
-        )
-        titled = _index(
-            backend,
-            title="東京都の報告書",
-            content="an english summary",
-            checksum="cjk-field-2",
-        )
-        pks = {"titled": titled.pk, "content_only": content_only.pk}
-
-        assert _matched_ids(backend, "東京") == set(pks.values())
-        assert _matched_ids(backend, "title:東京") == {pks[label] for label in expected}
-
-    def test_cjk_on_a_non_default_field_builds_no_clause(
-        self,
-        backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - A document with 東京 in its content
-        WHEN:
-            - "notes:東京" is searched (a field outside the default
-              search fields)
-        THEN:
-            - Nothing matches; a CJK term restricted to a field outside
-              the default search fields has nothing to contribute to the
-              bigram clause, so it must not fall back to matching 東京 in
-              the content
-        """
-        _index(
-            backend,
-            title="Tokyo report",
-            content="東京都の人口は約1400万人です",
-            checksum="cjk-notes-1",
-        )
-
-        assert _matched_ids(backend, "notes:東京") == set()
-
-    def test_bare_cjk_term_still_matches_every_default_field(
-        self,
-        backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - One document with 重要 in its content, another with 重要 in
-              its title
-        WHEN:
-            - "重要" and "重要 OR report" are each searched unfielded
-        THEN:
-            - Both documents match either way; the clause's reason for
-              existing is that an unfielded CJK run matches wherever it
-              is indexed, and does so alongside a latin term
-        """
-        in_content = _index(
-            backend,
-            title="report",
-            content="本文に重要な情報",
-            checksum="cjk-bare-1",
-        )
-        in_title = _index(
-            backend,
-            title="重要な報告書",
-            content="english only",
-            checksum="cjk-bare-2",
-        )
-
-        assert _matched_ids(backend, "重要") == {in_content.pk, in_title.pk}
-        assert _matched_ids(backend, "重要 OR report") == {
-            in_content.pk,
-            in_title.pk,
-        }

src/documents/tests/search/test_compact_date_forms.py

@@ -1,86 +0,0 @@
-"""Whoosh's compact, separator-free date spelling, resolved end to end.
-
-whoosh-compat owns both widths of this spelling and asserts both forms'
-bounds directly in its own test suite: the 8-digit form as a whole calendar
-day (lower bound, upper bound and exclusivity), and the 14-digit form as a
-single instant. The 14-digit form is kept here as the single representative
-because it is the one that exercises paperless's ``added`` DATETIME fast
-field at full precision: the corpus separates a document at the named
-instant from one on the same calendar day at another hour and one on the
-next day at the same hour, so a query that degrades into a whole-day
-window, or drops the time of day, matches the wrong set rather than passing
-on a corpus that could not tell the difference.
-"""
-
-from __future__ import annotations
-
-from datetime import UTC
-from datetime import datetime
-from typing import TYPE_CHECKING
-
-import pytest
-
-from documents.models import Document
-
-if TYPE_CHECKING:
-    from documents.search._backend import TantivyBackend
-
-pytestmark = [pytest.mark.search, pytest.mark.django_db]
-
-
-def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
-    return set(backend.search_ids(query, user=None))
-
-
-def _index(backend: TantivyBackend, **kwargs: object) -> Document:
-    doc = Document.objects.create(**kwargs)
-    backend.add_or_update(doc)
-    return doc
-
-
-@pytest.fixture
-def docs(backend: TantivyBackend) -> dict[str, int]:
-    return {
-        "instant": _index(
-            backend,
-            title="On the instant",
-            content="x",
-            checksum="compact-date-instant",
-            added=datetime(2005, 3, 4, 15, 30, tzinfo=UTC),
-        ).pk,
-        "same_day": _index(
-            backend,
-            title="Same day, other hour",
-            content="x",
-            checksum="compact-date-same-day",
-            added=datetime(2005, 3, 4, 9, 0, tzinfo=UTC),
-        ).pk,
-        "next_day": _index(
-            backend,
-            title="Next day, same hour",
-            content="x",
-            checksum="compact-date-next-day",
-            added=datetime(2005, 3, 5, 15, 30, tzinfo=UTC),
-        ).pk,
-    }
-
-
-def test_fourteen_digits_is_a_single_instant(
-    backend: TantivyBackend,
-    docs: dict[str, int],
-) -> None:
-    """
-    GIVEN:
-        - Three documents indexed on the ``added`` DATETIME fast field:
-          one at 2005-03-04T15:30:00, one on the same calendar day at a
-          different hour, and one on the next day at the same hour
-    WHEN:
-        - Searching with the 14-digit compact date form
-          ``added:20050304153000``
-    THEN:
-        - Only the document at that exact instant matches; the same-day
-          document is what tells this apart from the 8-digit day-window
-          form, and the next-day document from a form that ignored the
-          time of day altogether
-    """
-    assert _matched_ids(backend, "added:20050304153000") == {docs["instant"]}

src/documents/tests/search/test_date_keyword_phrase_removal.py

@@ -1,91 +0,0 @@
-"""Pins the correctness gained by deleting the pre-parse
-_quote_date_keyword_phrases rewrite.
-
-That rewrite matched date-keyword phrases (e.g. "previous month" after a
-date field) anywhere in the raw query string, including inside an
-unrelated quoted string, and inserted quotes mid-phrase there too. Its
-own docstring gave ``title:"see added:previous month notes"`` as the
-example of what it corrupted. whoosh-compat's grammar accepts the same
-phrase vocabulary unquoted natively (see TestUnquotedDateKeywordPhrases
-in test_acceptance.py), so the rewrite was redundant everywhere it was
-safe and actively wrong everywhere it was not. This is the one case that
-tells the two apart: a literal title phrase that happens to contain
-"added:previous month" as running text.
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-import pytest
-
-from documents.models import Document
-
-if TYPE_CHECKING:
-    from documents.search._backend import TantivyBackend
-
-pytestmark = [pytest.mark.search, pytest.mark.django_db]
-
-
-def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
-    return set(backend.search_ids(query, user=None))
-
-
-def _index(backend: TantivyBackend, **kwargs: object) -> Document:
-    doc = Document.objects.create(**kwargs)
-    backend.add_or_update(doc)
-    return doc
-
-
-class TestQuotedStringContainingDateKeywordText:
-    """A quoted title phrase containing the literal text
-    "added:previous month" as running words must match on that literal
-    text alone, never spill into an unfielded search for "previous" and
-    "month" across the default search fields the way the deleted rewrite
-    would have decomposed it into."""
-
-    def test_matches_only_the_literal_phrase(
-        self,
-        backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - A document whose title literally contains "see
-              added:previous month notes", and a decoy document whose
-              title/content carry the individual fragments the deleted
-              _quote_date_keyword_phrases rewrite would have decomposed
-              the phrase into (the decoy would incorrectly match under
-              the deleted rewrite: its title contains the "see added:"
-              and " notes" fragments the corrupted parse required as
-              title phrases, and its content supplies "previous" and
-              "month" as the decomposed word-match clauses the rewrite
-              turned the middle of the phrase into)
-        WHEN:
-            - 'title:"see added:previous month notes"' is searched
-        THEN:
-            - Only the document with the literal phrase matches; it must
-              never spill into an unfielded search for "previous" and
-              "month" across the default search fields
-        """
-        literal = _index(
-            backend,
-            title="see added:previous month notes",
-            content="quarterly filing",
-            checksum="dkp-literal",
-            archive_serial_number=920,
-        )
-        # Under the deleted rewrite, this decoy would incorrectly match:
-        # its title contains the "see added:" and " notes" fragments the
-        # corrupted parse required as title phrases, and its content
-        # supplies "previous" and "month" as the decomposed word-match
-        # clauses the rewrite turned the middle of the phrase into.
-        decoy = _index(
-            backend,
-            title="see added: quarterly report notes",
-            content="we reviewed the previous statement about month end",
-            checksum="dkp-decoy",
-            archive_serial_number=921,
-        )
-        query = 'title:"see added:previous month notes"'
-        assert _matched_ids(backend, query) == {literal.pk}
-        assert decoy.pk not in _matched_ids(backend, query)

src/documents/tests/search/test_date_keyword_timezone.py

@@ -1,100 +0,0 @@
-"""Date keyword phrases (``today``, etc.) resolved in a non-UTC timezone,
-end to end.
-
-paperless's own ``tz=get_current_timezone()`` plumbing
-(``TantivyBackend._parse_query``) is exercised elsewhere only for
-relative *ranges* (``added:[-1 week to now]``, in
-documents/tests/test_api_search.py). This covers a date *keyword*
-(``today``), whose day boundary depends on the active timezone the same
-way but goes through whoosh-compat's DateParserPlugin resolution instead
-of an explicit range.
-
-Discriminating shape: frozen at 2026-06-15T02:00 UTC, which is
-2026-06-14T22:00 in America/New_York -- still "today" (06-14) there, but
-already "today" (06-15) in UTC. Two documents pin both directions of the
-mistake a hardcoded-UTC bug would make:
-
-- ``in_ny_today`` (added 2026-06-14T20:00 UTC = 2026-06-14T16:00 NY) is
-  inside New York's "today" window and outside a naive UTC-calendar-day
-  window. A ``tz``-ignoring bug would miss it.
-- ``in_utc_calendar_day_only`` (added 2026-06-15T10:00 UTC =
-  2026-06-15T06:00 NY) is inside a naive UTC-calendar-day window but
-  outside New York's actual "today" window. A ``tz``-ignoring bug would
-  wrongly match it.
-"""
-
-from __future__ import annotations
-
-from datetime import UTC
-from datetime import datetime
-from typing import TYPE_CHECKING
-
-import pytest
-import time_machine
-
-from documents.models import Document
-
-if TYPE_CHECKING:
-    from pytest_django.fixtures import SettingsWrapper
-
-    from documents.search._backend import TantivyBackend
-
-pytestmark = [pytest.mark.search, pytest.mark.django_db]
-
-FROZEN_NOW = datetime(2026, 6, 15, 2, 0, tzinfo=UTC)
-
-
-def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
-    return set(backend.search_ids(query, user=None))
-
-
-def _index(backend: TantivyBackend, **kwargs: object) -> Document:
-    doc = Document.objects.create(**kwargs)
-    backend.add_or_update(doc)
-    return doc
-
-
-class TestDateKeywordUsesTheActiveTimezone:
-    def test_today_matches_the_new_york_calendar_day_not_the_utc_one(
-        self,
-        backend: TantivyBackend,
-        settings: SettingsWrapper,
-    ) -> None:
-        """
-        GIVEN:
-            - TIME_ZONE set to America/New_York, time frozen at
-              2026-06-15T02:00 UTC (2026-06-14T22:00 NY -- still "today"
-              there, but already "today" in UTC), and two documents: one
-              added inside New York's "today" window but outside a naive
-              UTC-calendar-day window, the other the reverse (inside a
-              naive UTC-calendar-day window but outside New York's actual
-              "today")
-        WHEN:
-            - "added:today" is searched
-        THEN:
-            - Only the document inside New York's actual "today" window
-              matches, proving our tz=get_current_timezone() plumbing
-              resolves the date keyword in the active timezone rather
-              than a hardcoded UTC calendar day
-        """
-        settings.TIME_ZONE = "America/New_York"
-        with time_machine.travel(FROZEN_NOW, tick=False):
-            in_ny_today = _index(
-                backend,
-                title="NY today",
-                content="x",
-                checksum="tz-keyword-ny-today",
-                added=datetime(2026, 6, 14, 20, 0, tzinfo=UTC),
-            )
-            # Not captured: the exact-set assertion below already proves
-            # this document (inside a naive UTC-calendar-day window, but
-            # outside New York's actual "today") does not match.
-            _index(
-                backend,
-                title="UTC calendar day only",
-                content="x",
-                checksum="tz-keyword-utc-calendar-day-only",
-                added=datetime(2026, 6, 15, 10, 0, tzinfo=UTC),
-            )
-
-            assert _matched_ids(backend, "added:today") == {in_ny_today.pk}

src/documents/tests/search/test_default_search_fields_guard.py

@@ -1,32 +0,0 @@
-"""``_DEFAULT_SEARCH_FIELDS`` must stay a subset of the registered public
-field names.
-
-Nothing enforced this before: a rename in PUBLIC_FIELDS not mirrored in
-``_DEFAULT_SEARCH_FIELDS`` (documents/search/_query.py) would 400 every
-unfielded search at request time, since ``index.parse_query`` and the
-fuzzy/CJK clause builders are handed a field name the schema no longer
-has.
-"""
-
-from __future__ import annotations
-
-from documents.search._fields import PUBLIC_FIELDS
-from documents.search._query import _DEFAULT_SEARCH_FIELDS
-
-
-class TestDefaultSearchFieldsAreRegistered:
-    def test_every_default_search_field_is_a_public_field(self) -> None:
-        """
-        GIVEN:
-            - PUBLIC_FIELDS and _DEFAULT_SEARCH_FIELDS, our own field
-              tables
-        WHEN:
-            - Every name in _DEFAULT_SEARCH_FIELDS is checked against the
-              registered public field names
-        THEN:
-            - Every one is present; a rename in PUBLIC_FIELDS not
-              mirrored here would 400 every unfielded search at request
-              time
-        """
-        public_field_names = {f.name for f in PUBLIC_FIELDS}
-        assert set(_DEFAULT_SEARCH_FIELDS) <= public_field_names

src/documents/tests/search/test_documented_syntax.py

@@ -1,474 +0,0 @@
-"""Pins the search syntax that ``docs/usage.md`` promises users.
-
-Every query here appears verbatim, or as a direct paraphrase, in the
-"Document searches" section of ``docs/usage.md``. Each case indexes real
-documents and asserts on matched document IDs rather than on the parsed
-query, because a query that parses cleanly is not necessarily a query that
-means what the documentation says it means: ``added:now`` parses without a
-single diagnostic and then matches nothing, because it resolves to an
-instant rather than to a span.
-
-The negative cases matter as much as the positive ones. They pin the
-behaviours the docs explicitly warn about, so that if any of them ever
-starts working the warning can be removed deliberately rather than being
-left standing as a lie.
-"""
-
-from __future__ import annotations
-
-from datetime import UTC
-from datetime import datetime
-from typing import TYPE_CHECKING
-
-import pytest
-import time_machine
-
-from documents.models import Document
-from documents.models import Note
-from documents.models import Tag
-from documents.search._errors import InvalidDateQuery
-
-if TYPE_CHECKING:
-    from collections.abc import Generator
-
-    from django.contrib.auth.models import User
-
-    from documents.search._backend import TantivyBackend
-
-pytestmark = [pytest.mark.search, pytest.mark.django_db]
-
-# A Monday, so that "next monday"/"last monday" land a clean week either side.
-FROZEN_NOW = datetime(2026, 6, 15, 12, 0, tzinfo=UTC)
-
-# The checksum used in the docs' `checksum:` example.
-DOC_CHECKSUM = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"
-
-
-def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
-    return set(backend.search_ids(query, user=None))
-
-
-def _index(backend: TantivyBackend, **kwargs: object) -> Document:
-    doc = Document.objects.create(**kwargs)
-    backend.add_or_update(doc)
-    return doc
-
-
-class TestLogicalExpressions:
-    @pytest.fixture
-    def docs(self, backend: TantivyBackend) -> dict[str, int]:
-        return {
-            "secret": _index(
-                backend,
-                title="Invoice one",
-                content="invoice secret contents",
-                checksum="doc-syntax-secret",
-            ).pk,
-            "plain": _index(
-                backend,
-                title="Invoice two",
-                content="invoice ordinary contents",
-                checksum="doc-syntax-plain",
-            ).pk,
-        }
-
-    def test_not_excludes_a_term(
-        self,
-        backend: TantivyBackend,
-        docs: dict[str, int],
-    ) -> None:
-        """
-        GIVEN:
-            - Two indexed documents, one containing "secret" and one not
-        WHEN:
-            - "invoice NOT secret" is searched, as docs/usage.md documents
-        THEN:
-            - Only the document without "secret" matches
-        """
-        assert _matched_ids(backend, "invoice NOT secret") == {docs["plain"]}
-
-    def test_leading_hyphen_requires_the_term_instead_of_excluding_it(
-        self,
-        backend: TantivyBackend,
-        docs: dict[str, int],
-    ) -> None:
-        """
-        GIVEN:
-            - Two indexed documents, one containing "secret" and one not
-        WHEN:
-            - "invoice -secret" is searched (a leading hyphen, not "NOT")
-        THEN:
-            - Only the document containing "secret" matches, because
-              separators are stripped at index time, so "-secret" is
-              indexed as the plain term "secret" and the query becomes an
-              AND rather than an exclusion, exactly as the docs warn
-        """
-        assert _matched_ids(backend, "invoice -secret") == {docs["secret"]}
-
-    def test_or_inside_parentheses_matches_either_branch(
-        self,
-        backend: TantivyBackend,
-        docs: dict[str, int],
-    ) -> None:
-        """
-        GIVEN:
-            - Two indexed documents, one containing "secret" and one
-              containing "ordinary"
-        WHEN:
-            - "invoice AND (secret OR ordinary)" is searched
-        THEN:
-            - Both documents match
-        """
-        matched = _matched_ids(backend, "invoice AND (secret OR ordinary)")
-        assert matched == {docs["secret"], docs["plain"]}
-
-
-class TestPhraseSearch:
-    def test_quoted_phrase_requires_the_words_in_order(
-        self,
-        backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - A document whose content contains "the quick brown fox jumps"
-        WHEN:
-            - A quoted phrase is searched, in order and out of order
-        THEN:
-            - The in-order phrase matches, and the same words reordered do
-              not
-        """
-        doc = _index(
-            backend,
-            title="Phrase",
-            content="the quick brown fox jumps",
-            checksum="doc-syntax-phrase",
-        )
-        assert _matched_ids(backend, '"quick brown fox"') == {doc.pk}
-        assert _matched_ids(backend, '"brown quick fox"') == set()
-
-
-class TestTagCommaList:
-    """``tag:bills,unpaid`` is published syntax (docs/usage.md), so this checks
-    that the documented spelling still returns what the docs promise: only the
-    document carrying every listed tag.
-
-    It is deliberately not proof of paperless's field configuration, and must
-    not be read as such. Removing ``comma_values`` from the ``tag`` FieldSpec
-    leaves this test passing, because paperless's analyzer splits the literal
-    value "bills,unpaid" into the same two tokens the value-list reading
-    produces, so the two readings select the same documents. The registry fact
-    -- that ``tag`` opts in and no other field does -- is observable only at
-    the registry, and is owned by test_registry.py's
-    ``test_tag_is_comma_values``/``test_correspondent_is_not_comma_values``.
-    """
-
-    def test_comma_list_requires_every_listed_tag(
-        self,
-        backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - A document carrying both "bills" and "unpaid" tags, and a
-              second document carrying only "bills" (plus "archived")
-        WHEN:
-            - "tag:bills,unpaid" is searched
-        THEN:
-            - Only the document carrying every listed tag matches, and a
-              single-tag "tag:bills" search still matches both documents
-        """
-        bills = Tag.objects.create(name="bills")
-        unpaid = Tag.objects.create(name="unpaid")
-        archived = Tag.objects.create(name="archived")
-
-        both = Document.objects.create(
-            title="Both tags",
-            content="body",
-            checksum="doc-syntax-tag-both",
-        )
-        both.tags.add(bills, unpaid)
-        backend.add_or_update(both)
-
-        one = Document.objects.create(
-            title="One tag",
-            content="body",
-            checksum="doc-syntax-tag-one",
-        )
-        one.tags.add(bills, archived)
-        backend.add_or_update(one)
-
-        assert _matched_ids(backend, "tag:bills,unpaid") == {both.pk}
-        assert _matched_ids(backend, "tag:bills") == {both.pk, one.pk}
-
-
-class TestArchiveMetadataFields:
-    @pytest.fixture
-    def doc(self, backend: TantivyBackend, admin_user: User) -> Document:
-        doc = Document.objects.create(
-            title="Metadata",
-            content="body",
-            checksum=DOC_CHECKSUM,
-            archive_serial_number=100,
-            page_count=12,
-            original_filename="invoice.pdf",
-        )
-        Note.objects.create(document=doc, user=admin_user, note="a note")
-        backend.add_or_update(doc)
-        return doc
-
-    @pytest.mark.parametrize(
-        "query",
-        [
-            "asn:100",
-            "asn:[50 to 150]",
-            "page_count:12",
-            "page_count:[10 to 20]",
-            "num_notes:1",
-            "num_notes:[1 to 5]",
-            "original_filename:invoice.pdf",
-            f"checksum:{DOC_CHECKSUM}",
-            "checksum:9f86d081*",
-            # A checksum term is stored verbatim, but a checksum *pattern* is
-            # lowercased before it is matched, which the docs now say outright
-            # next to the "only a complete, lowercase checksum matches" rule
-            # that the uppercase term in the negative list below pins.
-            "checksum:9F86D081*",
-        ],
-    )
-    def test_documented_metadata_query_matches(
-        self,
-        backend: TantivyBackend,
-        doc: Document,
-        query: str,
-    ) -> None:
-        """
-        GIVEN:
-            - A document with an ASN, page count, a note, an original
-              filename and a known checksum
-        WHEN:
-            - Every documented metadata-field spelling (exact value,
-              range, and, for checksum, a lowercase prefix pattern
-              regardless of the case the pattern itself is typed in) is
-              searched
-        THEN:
-            - Each one matches the document
-        """
-        assert _matched_ids(backend, query) == {doc.pk}
-
-    @pytest.mark.parametrize(
-        "query",
-        [
-            # The docs say only a complete, lowercase checksum matches.
-            "checksum:9f86d081",
-            f"checksum:{DOC_CHECKSUM.upper()}",
-        ],
-    )
-    def test_partial_or_uppercase_checksum_matches_nothing(
-        self,
-        backend: TantivyBackend,
-        doc: Document,
-        query: str,
-    ) -> None:
-        """
-        GIVEN:
-            - A document with a known, complete, lowercase checksum
-        WHEN:
-            - An exact-value search is run with a partial or uppercase
-              spelling of that checksum
-        THEN:
-            - Nothing matches, as the docs say only a complete, lowercase
-              checksum matches as an exact value
-        """
-        assert _matched_ids(backend, query) == set()
-
-
-class TestDocumentedDateForms:
-    @pytest.fixture(autouse=True)
-    def frozen_now(self) -> Generator[None, None, None]:
-        with time_machine.travel(FROZEN_NOW, tick=False):
-            yield
-
-    @pytest.fixture
-    def dated(self, backend: TantivyBackend) -> dict[str, int]:
-        stamps = {
-            "today": datetime(2026, 6, 15, 9, 0, tzinfo=UTC),
-            "yesterday": datetime(2026, 6, 14, 9, 0, tzinfo=UTC),
-            "tomorrow": datetime(2026, 6, 16, 9, 0, tzinfo=UTC),
-            "next_monday": datetime(2026, 6, 22, 10, 0, tzinfo=UTC),
-            "last_monday": datetime(2026, 6, 8, 10, 0, tzinfo=UTC),
-            "january": datetime(2026, 1, 10, 10, 0, tzinfo=UTC),
-            "old": datetime(2005, 3, 4, 15, 30, tzinfo=UTC),
-        }
-        return {
-            label: _index(
-                backend,
-                title=label,
-                content="dated body",
-                checksum=f"doc-syntax-date-{label}",
-                added=stamp,
-            ).pk
-            for label, stamp in stamps.items()
-        }
-
-    @pytest.mark.parametrize(
-        ("query", "label"),
-        [
-            ("added:today", "today"),
-            ("added:yesterday", "yesterday"),
-            ("added:tomorrow", "tomorrow"),
-            ('added:"next monday"', "next_monday"),
-            ('added:"last monday"', "last_monday"),
-            ("added:january", "january"),
-            ("added:2005-03-04", "old"),
-            ("added:2005-03", "old"),
-            ("added:[2005-01-01 to 2005-12-31]", "old"),
-            ("added:[2005 to 2009]", "old"),
-            # A full timestamp works, but only quoted when it stands alone,
-            # and only unquoted when it is a range bound. The bare standalone
-            # spelling is pinned as a non-match below.
-            ('added:"2005-03-04T15:30:00Z"', "old"),
-            ("added:[2005-03-04T09:00:00Z to 2005-03-04T17:00:00Z]", "old"),
-            # A quoted range bound works when the quotes are single ones; the
-            # double-quoted spelling is pinned as an error below.
-            ("added:['2005-03-04' to 2005-03-05]", "old"),
-        ],
-    )
-    def test_documented_date_form_matches_its_day_or_month(
-        self,
-        backend: TantivyBackend,
-        dated: dict[str, int],
-        query: str,
-        label: str,
-    ) -> None:
-        """
-        GIVEN:
-            - Documents dated today, yesterday, tomorrow, next/last
-              Monday, in January, and on an old fixed date, indexed
-              against a frozen "now" (a Monday)
-        WHEN:
-            - Every documented date-form spelling is searched: relative
-              keywords, quoted multi-word phrases, a bare year-month, an
-              explicit range, a quoted full timestamp standing alone, an
-              unquoted full timestamp as a range bound, and a
-              single-quoted range bound
-        THEN:
-            - Each form matches exactly the document dated on its day or
-              within its month
-        """
-        assert _matched_ids(backend, query) == {dated[label]}
-
-    @pytest.mark.parametrize(
-        "query",
-        [
-            # Zero-width: these resolve to a single instant, not a span, so
-            # nothing in a realistic corpus lands on them. The docs warn
-            # about them rather than presenting them as usable.
-            "added:now",
-            "added:noon",
-            "added:midnight",
-            # Quoting is what rescues the other multi-word date expressions,
-            # so pin that it does not rescue these: the problem is the width
-            # of the resulting range, not the way the value is delimited.
-            # One quoted spelling is enough for that; which keyword sits
-            # inside the quotes is grammar whoosh-compat owns.
-            'added:"now"',
-            # A relative offset, which the warning in the docs names by this
-            # exact spelling. Standing alone it is an instant like the rest of
-            # this list; the same offset used as a range bound is a real
-            # window, pinned by the test below.
-            'added:"-1 week"',
-        ],
-    )
-    def test_forms_the_docs_warn_about_match_nothing(
-        self,
-        backend: TantivyBackend,
-        dated: dict[str, int],
-        query: str,
-    ) -> None:
-        """
-        GIVEN:
-            - A realistic dated corpus (see the `dated` fixture)
-        WHEN:
-            - A zero-width date form ("now", "noon", "midnight", a quoted
-              "now") or a standalone relative offset ("-1 week") is
-              searched: each resolves to a single instant rather than a
-              span, and quoting does not rescue them the way it rescues
-              other multi-word date expressions, since the problem is the
-              width of the resulting range, not how the value is
-              delimited
-        THEN:
-            - Nothing matches, exactly as the docs warn, rather than
-              presenting these as usable spellings
-        """
-        assert _matched_ids(backend, query) == set()
-
-    def test_bare_timestamp_is_rejected_rather_than_matching_nothing(
-        self,
-        backend: TantivyBackend,
-        dated: dict[str, int],
-    ) -> None:
-        """
-        GIVEN:
-            - A realistic dated corpus, including a document dated at a
-              known full timestamp
-        WHEN:
-            - The bare, unquoted spelling of that full timestamp is
-              searched (the quoted and range-bound spellings pinned above
-              do work and match this fixture's document)
-        THEN:
-            - `InvalidDateQuery` is raised rather than the query silently
-              matching nothing, since this is a user-fixable error the
-              docs tell the user to quote, and the reported value is the
-              whole contiguous fragment the user typed, not just the
-              prefix the date grammar's tokenizer first split on
-        """
-        with pytest.raises(InvalidDateQuery) as exc_info:
-            _matched_ids(backend, "added:2005-03-04T15:30:00Z")
-        assert exc_info.value.field == "added"
-        assert exc_info.value.value == "2005-03-04T15:30:00Z"
-
-    def test_relative_offset_as_a_range_bound_is_a_real_window(
-        self,
-        backend: TantivyBackend,
-        dated: dict[str, int],
-    ) -> None:
-        """
-        GIVEN:
-            - A realistic dated corpus, including a document dated two
-              hours before a "last Monday to now" window opens, and
-              documents dated today and yesterday, inside that window
-        WHEN:
-            - "added:['-1 week' to now]" is searched: the same offset
-              that matches nothing standing alone (see the test above),
-              used here as a range bound instead
-        THEN:
-            - The window matches today and yesterday but excludes the
-              document two hours before it opens, showing the bound is
-              the offset itself and not a whole-day rounding of it, as
-              the docs say next to the warning about the standalone form
-        """
-        assert _matched_ids(backend, "added:['-1 week' to now]") == {
-            dated["today"],
-            dated["yesterday"],
-        }
-
-    def test_double_quoted_range_bound_is_rejected(
-        self,
-        backend: TantivyBackend,
-        dated: dict[str, int],
-    ) -> None:
-        """
-        GIVEN:
-            - A realistic dated corpus
-        WHEN:
-            - A range bound is double-quoted rather than single-quoted
-              ("added:[\"2005-03-04\" to 2005-03-05]")
-        THEN:
-            - `InvalidDateQuery` is raised, pinning which of the two
-              quote characters fails: quoting a range bound is allowed,
-              but only with single quotes, since the double-quoted
-              spelling reaches the date grammar with its quotes still
-              attached and is not a recognizable date
-        """
-        with pytest.raises(InvalidDateQuery) as exc_info:
-            _matched_ids(backend, 'added:["2005-03-04" to 2005-03-05]')
-        assert exc_info.value.value == '"2005-03-04"'

src/documents/tests/search/test_error_routing.py

@@ -1,357 +0,0 @@
-"""Diagnostics route by Cause, and user-facing messages are host-owned.
-
-whoosh-compat documents ``Diagnostic.message`` as developer output with no
-stability guarantee, so it must never reach an HTTP response body.
-"""
-
-from __future__ import annotations
-
-import logging
-from datetime import UTC
-
-import pytest
-import tantivy
-from whoosh_compat.errors import Diagnostic
-from whoosh_compat.errors import DiagnosticKind
-from whoosh_compat.errors import QueryError
-from whoosh_compat.errors import cause_for
-from whoosh_compat.fields import FieldKind
-from whoosh_compat.fields import FieldRef
-
-from documents.search._errors import SearchQueryError
-from documents.search._query import _map_emit_error
-from documents.search._query import _single_diagnostic_to_error
-from documents.search._query import parse_user_query
-from documents.search._schema import build_schema
-from documents.search._tokenizer import register_tokenizers
-
-pytestmark = pytest.mark.search
-
-_LIBRARY_PROSE = "INTERNAL LIBRARY WORDING WITH raw tantivy detail"
-
-
-@pytest.fixture(scope="module")
-def query_index() -> tantivy.Index:
-    """An in-memory, unstemmed index; these tests only parse, never index."""
-    idx = tantivy.Index(build_schema(), path=None)
-    register_tokenizers(idx, "")
-    return idx
-
-
-def _diagnostic(
-    kind: DiagnosticKind,
-    *,
-    field: FieldRef | None = FieldRef("title"),
-    field_kind: FieldKind | None = FieldKind.TEXT,
-) -> Diagnostic:
-    """A Diagnostic shaped like the emitter's, with the library's own
-    kind -> cause mapping rather than a hand-picked cause."""
-    return Diagnostic(
-        kind=kind,
-        cause=cause_for(kind),
-        message=_LIBRARY_PROSE,
-        field=field,
-        field_kind=field_kind,
-    )
-
-
-class TestEmitErrorRouting:
-    """Every Cause gets a distinguishable treatment, not just "a 400"."""
-
-    @pytest.mark.parametrize(
-        "kind",
-        [
-            DiagnosticKind.BACKEND_REJECTED,
-            DiagnosticKind.AST_INVALID_SHAPE,
-            DiagnosticKind.AST_UNKNOWN_FIELD,
-        ],
-    )
-    def test_internal_cause_is_not_converted(self, kind: DiagnosticKind) -> None:
-        """
-        GIVEN:
-            - A QueryError wrapping a Diagnostic whose Cause is INTERNAL
-              (BACKEND_REJECTED/AST_INVALID_SHAPE/AST_UNKNOWN_FIELD)
-        WHEN:
-            - _map_emit_error processes it
-        THEN:
-            - The original QueryError propagates unchanged, so it surfaces
-              as a 500 monitoring can see, never a 400 blaming the user
-        """
-        error = QueryError(_diagnostic(kind))
-        with pytest.raises(QueryError) as excinfo:
-            _map_emit_error(error)
-        assert excinfo.value is error
-
-    def test_misconfigured_cause_is_logged_and_becomes_a_400(
-        self,
-        caplog: pytest.LogCaptureFixture,
-    ) -> None:
-        """
-        GIVEN:
-            - A QueryError for SCHEMA_FIELD_MISSING naming field "asn"
-        WHEN:
-            - _map_emit_error processes it
-        THEN:
-            - It becomes a SearchQueryError, and exactly one ERROR log
-              record is emitted naming the field and the diagnostic kind
-        """
-        kind = DiagnosticKind.SCHEMA_FIELD_MISSING
-        with caplog.at_level(logging.ERROR, logger="paperless.search"):
-            error = _map_emit_error(
-                QueryError(_diagnostic(kind, field=FieldRef("asn"))),
-            )
-        assert isinstance(error, SearchQueryError)
-        errors = [r for r in caplog.records if r.levelno == logging.ERROR]
-        assert len(errors) == 1
-        assert "asn" in errors[0].getMessage()
-        assert kind.name in errors[0].getMessage()
-
-    @pytest.mark.parametrize(
-        "kind",
-        [
-            DiagnosticKind.TEXT_RANGE,
-            DiagnosticKind.PATTERN_TOO_COMPLEX,
-            DiagnosticKind.EXISTS_REQUIRES_FAST,
-        ],
-    )
-    def test_unsupported_cause_is_a_400_with_no_operator_log(
-        self,
-        kind: DiagnosticKind,
-        caplog: pytest.LogCaptureFixture,
-    ) -> None:
-        """
-        GIVEN:
-            - A QueryError for a query tantivy cannot run
-              (TEXT_RANGE/PATTERN_TOO_COMPLEX/EXISTS_REQUIRES_FAST)
-        WHEN:
-            - _map_emit_error processes it
-        THEN:
-            - It becomes a SearchQueryError with no log record at WARNING
-              or above; a query tantivy cannot run is the user's to fix,
-              not an operator alert. EXISTS_REQUIRES_FAST is nominally
-              MISCONFIGURED but belongs here: it is decided from the
-              registry's own FieldSpec, so it never reports a disagreement
-              anyone could resolve
-        """
-        with caplog.at_level(logging.WARNING, logger="paperless.search"):
-            error = _map_emit_error(QueryError(_diagnostic(kind)))
-        assert isinstance(error, SearchQueryError)
-        assert caplog.records == []
-
-    @pytest.mark.parametrize(
-        "kind",
-        [
-            DiagnosticKind.TEXT_RANGE,
-            DiagnosticKind.PATTERN_TOO_COMPLEX,
-            DiagnosticKind.EXISTS_REQUIRES_FAST,
-            DiagnosticKind.SCHEMA_FIELD_MISSING,
-        ],
-    )
-    def test_user_facing_message_never_echoes_library_prose(
-        self,
-        kind: DiagnosticKind,
-    ) -> None:
-        """
-        GIVEN:
-            - A QueryError carrying whoosh-compat's own developer-facing
-              message text
-        WHEN:
-            - _map_emit_error processes it
-        THEN:
-            - The resulting error's string never contains that library
-              prose
-        """
-        error = _map_emit_error(QueryError(_diagnostic(kind)))
-        assert _LIBRARY_PROSE not in str(error)
-
-    @pytest.mark.parametrize(
-        "kind",
-        [
-            DiagnosticKind.TEXT_RANGE,
-            DiagnosticKind.PATTERN_TOO_COMPLEX,
-            DiagnosticKind.EXISTS_REQUIRES_FAST,
-            DiagnosticKind.SCHEMA_FIELD_MISSING,
-        ],
-    )
-    def test_user_facing_message_names_the_field(
-        self,
-        kind: DiagnosticKind,
-    ) -> None:
-        """
-        GIVEN:
-            - A QueryError for a JSON subpath field (custom_fields.value)
-        WHEN:
-            - _map_emit_error processes it
-        THEN:
-            - The resulting error names the field using its canonical
-              dotted form, including the subpath (FieldRef.__str__ yields
-              this dotted name, so every user-reachable emit kind can name
-              it)
-        """
-        diagnostic = _diagnostic(
-            kind,
-            field=FieldRef("custom_fields", "value"),
-            field_kind=FieldKind.JSON,
-        )
-        error = _map_emit_error(QueryError(diagnostic))
-        assert "custom_fields.value" in str(error)
-
-
-class TestParseDiagnosticMessages:
-    """Parse-time diagnostics are host-worded too, off field_kind."""
-
-    def test_too_deep_is_a_400_without_library_prose(self) -> None:
-        """
-        GIVEN:
-            - A parse-time Diagnostic for TOO_DEEP with no field
-        WHEN:
-            - _single_diagnostic_to_error processes it
-        THEN:
-            - It becomes a SearchQueryError with no library prose in its
-              message
-        """
-        error = _single_diagnostic_to_error(
-            _diagnostic(DiagnosticKind.TOO_DEEP, field=None, field_kind=None),
-        )
-        assert isinstance(error, SearchQueryError)
-        assert _LIBRARY_PROSE not in str(error)
-
-    @pytest.mark.parametrize(
-        ("kind", "field_kind"),
-        [
-            (DiagnosticKind.PATTERN_ON_NUMERIC, FieldKind.U64),
-            (DiagnosticKind.PATTERN_ON_BOOLEAN_EXISTS, FieldKind.BOOLEAN_EXISTS),
-            (DiagnosticKind.PATTERN_ON_SUBPATH, FieldKind.JSON),
-        ],
-    )
-    def test_pattern_on_kinds_name_the_field_and_its_kind(
-        self,
-        kind: DiagnosticKind,
-        field_kind: FieldKind,
-    ) -> None:
-        """
-        GIVEN:
-            - A parse-time Diagnostic for a pattern used against a kind
-              that cannot take one
-              (PATTERN_ON_NUMERIC/PATTERN_ON_BOOLEAN_EXISTS/PATTERN_ON_SUBPATH)
-        WHEN:
-            - _single_diagnostic_to_error processes it
-        THEN:
-            - The message names both the field and its kind, with no
-              library prose
-        """
-        error = _single_diagnostic_to_error(
-            _diagnostic(kind, field=FieldRef("asn"), field_kind=field_kind),
-        )
-        message = str(error)
-        assert _LIBRARY_PROSE not in message
-        assert "asn" in message
-        assert field_kind.name.lower() in message
-
-    def test_single_char_bracket_range_names_the_field_and_the_value(self) -> None:
-        """
-        GIVEN:
-            - A SINGLE_CHAR_BRACKET_RANGE diagnostic for "title" with
-              raw_value "200[1-9]"
-        WHEN:
-            - _single_diagnostic_to_error processes it
-        THEN:
-            - The resulting SearchQueryError names both the field and the
-              offending value, with no library prose
-        """
-        diagnostic = Diagnostic(
-            kind=DiagnosticKind.SINGLE_CHAR_BRACKET_RANGE,
-            cause=cause_for(DiagnosticKind.SINGLE_CHAR_BRACKET_RANGE),
-            message=_LIBRARY_PROSE,
-            field=FieldRef("title"),
-            field_kind=FieldKind.TEXT,
-            raw_value="200[1-9]",
-        )
-        error = _single_diagnostic_to_error(diagnostic)
-        message = str(error)
-        assert isinstance(error, SearchQueryError)
-        assert _LIBRARY_PROSE not in message
-        assert "title" in message
-        assert "200[1-9]" in message
-
-
-class TestRealQueriesRouteCorrectly:
-    """The routing table against diagnostics emit() really produces."""
-
-    def test_text_range_is_a_400_naming_the_field(
-        self,
-        query_index: tantivy.Index,
-    ) -> None:
-        """
-        GIVEN:
-            - A real query index
-        WHEN:
-            - parse_user_query is called with a text-range query
-              ("title:[a to b]")
-        THEN:
-            - It raises SearchQueryError naming "title"
-        """
-        with pytest.raises(SearchQueryError) as excinfo:
-            parse_user_query(query_index, "title:[a to b]", UTC)
-        assert "title" in str(excinfo.value)
-
-    def test_wildcard_on_a_numeric_field_is_a_400_naming_the_field(
-        self,
-        query_index: tantivy.Index,
-    ) -> None:
-        """
-        GIVEN:
-            - A real query index
-        WHEN:
-            - parse_user_query is called with a wildcard on a numeric
-              field ("asn:12*")
-        THEN:
-            - It raises SearchQueryError naming "asn"
-        """
-        with pytest.raises(SearchQueryError) as excinfo:
-            parse_user_query(query_index, "asn:12*", UTC)
-        assert "asn" in str(excinfo.value)
-
-    def test_single_char_bracket_range_is_a_400_naming_field_and_value(
-        self,
-        query_index: tantivy.Index,
-    ) -> None:
-        """
-        GIVEN:
-            - A real query index
-        WHEN:
-            - parse_user_query is called with "title:200[1-9]"
-        THEN:
-            - It raises SearchQueryError naming both "title" and
-              "200[1-9]"
-        """
-        with pytest.raises(SearchQueryError) as excinfo:
-            parse_user_query(query_index, "title:200[1-9]", UTC)
-        message = str(excinfo.value)
-        assert "title" in message
-        assert "200[1-9]" in message
-
-    def test_internal_diagnostic_escapes_as_a_query_error(
-        self,
-        query_index: tantivy.Index,
-        monkeypatch: pytest.MonkeyPatch,
-    ) -> None:
-        """
-        GIVEN:
-            - tantivy_emit monkeypatched to raise a QueryError with an
-              INTERNAL-cause diagnostic (BACKEND_REJECTED), the one case
-              with no query text of its own involved
-        WHEN:
-            - parse_user_query runs a normal query ("invoice")
-        THEN:
-            - The QueryError propagates unconverted; emit() reporting a
-              defect in itself must not become a user-facing 400
-        """
-        import documents.search._query as query_mod
-
-        def raise_internal(*args: object, **kwargs: object) -> None:
-            raise QueryError(_diagnostic(DiagnosticKind.BACKEND_REJECTED))
-
-        monkeypatch.setattr(query_mod, "tantivy_emit", raise_internal)
-        with pytest.raises(QueryError):
-            parse_user_query(query_index, "invoice", UTC)

src/documents/tests/search/test_exists_on_json_fields.py

@@ -1,114 +0,0 @@
-"""``field:*`` on a JSON field is user error, not an operator alert.
-
-whoosh-compat classifies EXISTS_REQUIRES_FAST as MISCONFIGURED, and
-_map_emit_error used to route every MISCONFIGURED diagnostic to an ERROR log.
-But the kind is decided from the registry's own FieldSpec (kind plus fast)
-without consulting the index schema, and field_descriptors() builds the JSON
-fields non-fast deliberately, so nothing is misconfigured and no operator
-action can clear the condition. Any authenticated user could otherwise emit
-ERROR lines in a loop by repeating ``notes:*``.
-
-SCHEMA_FIELD_MISSING, the other MISCONFIGURED kind, does compare the registry
-against the live schema, so it stays an ERROR.
-"""
-
-from __future__ import annotations
-
-import logging
-from datetime import UTC
-
-import pytest
-import tantivy
-from whoosh_compat.errors import Diagnostic
-from whoosh_compat.errors import DiagnosticKind
-from whoosh_compat.errors import QueryError
-from whoosh_compat.errors import cause_for
-from whoosh_compat.fields import FieldKind
-from whoosh_compat.fields import FieldRef
-
-from documents.search._errors import SearchQueryError
-from documents.search._query import _map_emit_error
-from documents.search._query import parse_user_query
-from documents.search._schema import build_schema
-from documents.search._tokenizer import register_tokenizers
-
-pytestmark = pytest.mark.search
-
-# Every spelling of "does this JSON field have a value" a user can type.
-EXISTS_QUERIES = [
-    "notes:*",
-    "notes.note:*",
-    "notes.user:*",
-    "custom_fields:*",
-    "custom_fields.name:*",
-    "custom_fields.value:*",
-]
-
-
-@pytest.fixture(scope="module")
-def query_index() -> tantivy.Index:
-    idx = tantivy.Index(build_schema(), path=None)
-    register_tokenizers(idx, "")
-    return idx
-
-
-class TestJsonExistsIsUserError:
-    @pytest.mark.parametrize("query", EXISTS_QUERIES)
-    def test_query_is_a_400_that_emits_no_error_log(
-        self,
-        query_index: tantivy.Index,
-        caplog: pytest.LogCaptureFixture,
-        query: str,
-    ) -> None:
-        """
-        GIVEN:
-            - A real query index, and every spelling of "does this JSON
-              field have a value" (notes:*, notes.note:*, custom_fields:*,
-              etc.)
-        WHEN:
-            - parse_user_query runs the query
-        THEN:
-            - It raises SearchQueryError naming the field, and no
-              ERROR-level log record is emitted; EXISTS_REQUIRES_FAST on a
-              JSON field is by design, not a misconfiguration an operator
-              could act on
-        """
-        with caplog.at_level(logging.WARNING, logger="paperless.search"):
-            with pytest.raises(SearchQueryError) as excinfo:
-                parse_user_query(query_index, query, UTC)
-        assert query.split(":", maxsplit=1)[0] in str(excinfo.value)
-        assert [r for r in caplog.records if r.levelno >= logging.ERROR] == []
-
-
-class TestGenuineMisconfigurationStillLogs:
-    def test_schema_field_missing_is_an_error_log(
-        self,
-        caplog: pytest.LogCaptureFixture,
-    ) -> None:
-        """
-        GIVEN:
-            - A QueryError for SCHEMA_FIELD_MISSING: the registry naming a
-              field the index schema does not have
-        WHEN:
-            - _map_emit_error processes it
-        THEN:
-            - It becomes a SearchQueryError and logs exactly one ERROR
-              record naming the diagnostic kind, since this is a real
-              mismatch an operator can fix and keeps the alert
-        """
-        kind = DiagnosticKind.SCHEMA_FIELD_MISSING
-        error = QueryError(
-            Diagnostic(
-                kind=kind,
-                cause=cause_for(kind),
-                message="field 'asn' is not defined in the index schema",
-                field=FieldRef("asn"),
-                field_kind=FieldKind.U64,
-            ),
-        )
-        with caplog.at_level(logging.ERROR, logger="paperless.search"):
-            mapped = _map_emit_error(error)
-        assert isinstance(mapped, SearchQueryError)
-        records = [r for r in caplog.records if r.levelno == logging.ERROR]
-        assert len(records) == 1
-        assert kind.name in records[0].getMessage()

src/documents/tests/search/test_fuzzy_tokenization.py

@@ -1,225 +0,0 @@
-"""The words the fuzzy blend clause hands back to tantivy's parser.
-
-The clause re-parses a word string through tantivy, which analyzes it
-again, so the words must be the query's raw text rather than the analyzed
-text (analysis is not idempotent), and must still be split into plain
-words so that hyphenated, dotted and quoted terms keep contributing.
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-import pytest
-
-from documents.models import Document
-
-if TYPE_CHECKING:
-    from pytest_django.fixtures import SettingsWrapper
-
-    from documents.search._backend import TantivyBackend
-
-pytestmark = [pytest.mark.search, pytest.mark.django_db]
-
-
-def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
-    return set(backend.search_ids(query, user=None))
-
-
-def _index(backend: TantivyBackend, **kwargs: object) -> Document:
-    doc = Document.objects.create(**kwargs)
-    backend.add_or_update(doc)
-    return doc
-
-
-@pytest.fixture(autouse=True)
-def fuzzy_enabled(settings: SettingsWrapper) -> None:
-    """Enable the fuzzy blend clause. The threshold doubles as a minimum
-    score filter, so it is set to 0.0: every hit passes and the test sees
-    the clause's matching behaviour, not the filter's."""
-    settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.0
-
-
-class TestFuzzyClauseWords:
-    def test_a_stemmed_word_is_not_stemmed_a_second_time(
-        self,
-        backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - Documents whose content contains "universities", a
-              one-transposition typo of it ("universties"), and two
-              unrelated words that share its stem prefix ("univalent",
-              "unicycle")
-        WHEN:
-            - Searching for "universities" with the fuzzy blend enabled
-        THEN:
-            - Only the correctly-spelled document and its typo match; the
-              clause does not widen far enough to reach the unrelated
-              words. 'universities' stems to 'univers'; feeding that back
-              to tantivy would stem it again to 'univ', whose fuzzy prefix
-              reaches unrelated words - the clause must stay wide enough
-              for a typo and no wider
-        """
-        wanted = _index(
-            backend,
-            title="A",
-            content="universities of europe",
-            checksum="fuzz-stem-1",
-        )
-        typo = _index(
-            backend,
-            title="B",
-            content="universties of europe",
-            checksum="fuzz-stem-2",
-        )
-        _index(
-            backend,
-            title="C",
-            content="univalent chemical bonds",
-            checksum="fuzz-stem-3",
-        )
-        _index(
-            backend,
-            title="D",
-            content="unicycle repair manual",
-            checksum="fuzz-stem-4",
-        )
-
-        assert _matched_ids(backend, "universities") == {wanted.pk, typo.pk}
-
-    def test_a_hyphenated_term_still_reaches_the_clause(
-        self,
-        backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - A document whose content contains a near-miss of "COVID-19"
-              ("covidx")
-        WHEN:
-            - Searching for "COVID-19" with the fuzzy blend enabled
-        THEN:
-            - The document matches; 'COVID-19' is one raw token, so unless
-              it is split into words, it carries characters the re-parse
-              would read as grammar, is dropped, and the whole query loses
-              its fuzzy clause
-        """
-        misspelled = _index(
-            backend,
-            title="A",
-            content="covidx testing results",
-            checksum="fuzz-hyphen-1",
-        )
-
-        assert _matched_ids(backend, "COVID-19") == {misspelled.pk}
-
-    def test_a_phrase_still_reaches_the_clause(
-        self,
-        backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - A document whose content near-misses a quoted phrase
-        WHEN:
-            - Searching for the quoted phrase '"tax reports"' with the
-              fuzzy blend enabled
-        THEN:
-            - The document matches; a phrase is one raw token carrying a
-              space, and is the whole query's only free text here, so it
-              must still reach the clause
-        """
-        near_miss = _index(
-            backend,
-            title="A",
-            content="taxation reportage weekly",
-            checksum="fuzz-phrase-1",
-        )
-
-        assert _matched_ids(backend, '"tax reports"') == {near_miss.pk}
-
-
-class TestBooleanKeywordsInRawText:
-    """Tantivy's boolean keywords are word runs, so they survive the cut
-    into words and its own parser reads them as grammar. Raw query text
-    reaches that parser with its case intact, so a quoted phrase can carry
-    them in."""
-
-    @pytest.fixture
-    def corpus(self, backend: TantivyBackend) -> dict[str, int]:
-        both = _index(
-            backend,
-            title="A",
-            content="taxation reportage weekly",
-            checksum="fuzz-kw-1",
-        )
-        tax_only = _index(
-            backend,
-            title="B",
-            content="taxation only here",
-            checksum="fuzz-kw-2",
-        )
-        report_only = _index(
-            backend,
-            title="C",
-            content="reportage only here",
-            checksum="fuzz-kw-3",
-        )
-        return {
-            "both": both.pk,
-            "tax_only": tax_only.pk,
-            "report_only": report_only.pk,
-        }
-
-    @pytest.mark.parametrize(
-        "query",
-        [
-            pytest.param('"tax AND reports"', id="and"),
-            pytest.param('"tax OR reports"', id="or"),
-            pytest.param('"tax NOT reports"', id="not"),
-            pytest.param('"tax IN reports"', id="in"),
-        ],
-    )
-    def test_a_keyword_inside_a_phrase_stays_an_ordinary_word(
-        self,
-        backend: TantivyBackend,
-        corpus: dict[str, int],
-        query: str,
-    ) -> None:
-        """
-        GIVEN:
-            - Three documents: one with both "taxation" and "reportage",
-              one with only "taxation", one with only "reportage"
-        WHEN:
-            - Searching for a quoted phrase carrying a tantivy boolean
-              keyword as one of its words (e.g. '"tax AND reports"')
-        THEN:
-            - The keyword stays an ordinary word inside the phrase, and
-              the fuzzy clause matches all three documents, the same
-              disjunction as the plain '"tax reports"' phrase: AND must
-              not turn it into a conjunction, NOT must not give it its own
-              exclusion, IN must not fail the parse
-        """
-        assert _matched_ids(backend, '"tax reports"') == set(corpus.values())
-        assert _matched_ids(backend, query) == set(corpus.values())
-
-    def test_a_trailing_keyword_does_not_drop_the_clause(
-        self,
-        backend: TantivyBackend,
-        corpus: dict[str, int],
-    ) -> None:
-        """
-        GIVEN:
-            - Three documents: one with both "taxation" and "reportage",
-              one with only "taxation", one with only "reportage"
-        WHEN:
-            - Searching for '"tax AND"', a phrase ending in a tantivy
-              syntax error
-        THEN:
-            - The fuzzy clause still matches on "tax"; 'tax AND' alone is
-              a syntax error to tantivy's parser, which would otherwise
-              cost the whole query its fuzzy clause
-        """
-        assert _matched_ids(backend, '"tax AND"') == {
-            corpus["both"],
-            corpus["tax_only"],
-        }

src/documents/tests/search/test_highlight_query_guard.py

@@ -1,249 +0,0 @@
-"""Regression coverage for the unguarded TEXT-mode highlight query.
-
-parse_simple_text_highlight_query re-parses simple-search tokens through
-Tantivy's query-string parser to build a SnippetGenerator-compatible query.
-Simple-search tokens keep arbitrary punctuation (quotes, colons, brackets,
-slashes), so any token carrying Tantivy query grammar raised an unguarded
-ValueError. The search itself had already succeeded by the time this ran:
-only the highlight step failed, and with the DocumentViewSet.list
-exception handler narrowed elsewhere on this branch, that ValueError now
-reaches the client as a bare 500 rather than a 400.
-
-Covers three angles:
-  - the query builder itself: quoting each token as its own escaped phrase
-    should let it parse instead of raising, for every failure mode a plain-
-    text query can trigger (syntax error, unknown field, unsupported regex).
-  - highlight_hits: even when a token still can't be expressed as a
-    highlight query, the guard must fall back to a query that still
-    produces usable highlight HTML, not silently empty ones.
-  - the real API endpoint: pinning the previously-500 status to 200.
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-import pytest
-import tantivy
-from rest_framework import status
-
-from documents.search._backend import SearchMode
-from documents.search._query import parse_simple_text_highlight_query
-from documents.search._schema import build_schema
-from documents.search._tokenizer import register_tokenizers
-from documents.tests.factories import DocumentFactory
-
-if TYPE_CHECKING:
-    from rest_framework.test import APIClient
-
-    from documents.search._backend import TantivyBackend
-
-pytestmark = [pytest.mark.search, pytest.mark.django_db]
-
-# Each spelling below trips a different Tantivy parser failure mode:
-#   'a"b'     -> Syntax Error (unterminated quote)
-#   foo:bar   -> unknown field
-#   (a        -> Syntax Error (unbalanced group)
-#   [a        -> Syntax Error (unbalanced range)
-#   /a/       -> Unsupported query (regex queries disallowed)
-_MALFORMED_QUERIES = [
-    pytest.param('a"b', id="unterminated_quote"),
-    pytest.param("foo:bar", id="unknown_field"),
-    pytest.param("(a", id="unbalanced_group"),
-    pytest.param("[a", id="unbalanced_range"),
-    pytest.param("/a/", id="unsupported_regex"),
-]
-
-
-@pytest.fixture(scope="module")
-def query_index() -> tantivy.Index:
-    """An in-memory, unstemmed index for parse-only tests."""
-    schema = build_schema()
-    idx = tantivy.Index(schema, path=None)
-    register_tokenizers(idx, "")
-    return idx
-
-
-class TestParseSimpleTextHighlightQueryDoesNotRaise:
-    """The query builder itself must tolerate Tantivy syntax in its tokens."""
-
-    @pytest.mark.parametrize("raw_query", _MALFORMED_QUERIES)
-    def test_malformed_token_does_not_raise(
-        self,
-        query_index: tantivy.Index,
-        raw_query: str,
-    ) -> None:
-        """
-        GIVEN:
-            - A simple-search query token carrying Tantivy query grammar
-              (unterminated quote, unknown field, unbalanced group/range,
-              or unsupported regex)
-        WHEN:
-            - parse_simple_text_highlight_query builds a highlight query
-              from it
-        THEN:
-            - It returns a tantivy.Query instead of raising, since each
-              token is quoted as its own escaped phrase rather than fed
-              to the parser raw
-        """
-        assert isinstance(
-            parse_simple_text_highlight_query(query_index, raw_query),
-            tantivy.Query,
-        )
-
-
-class TestHighlightHitsProducesUsableHighlights:
-    """highlight_hits must keep producing real <b>-wrapped snippet HTML for
-    these queries, not merely avoid raising."""
-
-    @pytest.mark.parametrize(
-        "raw_query",
-        [*_MALFORMED_QUERIES, pytest.param("plain text", id="plain_text_sanity")],
-    )
-    def test_highlight_still_contains_matched_text(
-        self,
-        backend: TantivyBackend,
-        raw_query: str,
-    ) -> None:
-        """
-        GIVEN:
-            - A document whose content contains the raw query text
-              verbatim
-        WHEN:
-            - backend.highlight_hits builds highlights for a TEXT-mode
-              search using that same (possibly Tantivy-grammar-carrying)
-              query text
-        THEN:
-            - The hit still carries a content highlight with real
-              <b>-wrapped matched-term markup, not an empty fallback
-        """
-        doc = DocumentFactory.create(
-            title="probe",
-            content=f"needle content containing {raw_query} literally here",
-        )
-        backend.add_or_update(doc)
-
-        hits = backend.highlight_hits(
-            raw_query,
-            [doc.pk],
-            search_mode=SearchMode.TEXT,
-        )
-
-        assert len(hits) == 1
-        highlights = hits[0]["highlights"]
-        assert "content" in highlights, (
-            f"Expected a content highlight for {raw_query!r}, got: {highlights!r}"
-        )
-        assert "<b>" in highlights["content"], (
-            f"Highlight for {raw_query!r} carries no matched-term markup: "
-            f"{highlights['content']!r}"
-        )
-
-
-class TestHighlightGuardDiscriminatesOnValueError:
-    """The guard added to highlight_hits must catch exactly ValueError, the
-    same shape as the sibling notes_text guard, and let anything else
-    through -- so a real library defect is never mistaken for a harmless
-    syntax error."""
-
-    def test_non_value_error_is_not_swallowed(
-        self,
-        backend: TantivyBackend,
-        monkeypatch: pytest.MonkeyPatch,
-    ) -> None:
-        """
-        GIVEN:
-            - parse_simple_text_highlight_query patched to raise
-              RuntimeError instead of a syntax-related ValueError
-        WHEN:
-            - backend.highlight_hits is called
-        THEN:
-            - The RuntimeError propagates unguarded; the highlight guard
-              must catch exactly ValueError, the same shape as the
-              sibling notes_text guard, never mistaking a real library
-              defect for a harmless syntax error
-        """
-        import documents.search._backend as backend_mod
-
-        def raise_runtime_error(*args: object, **kwargs: object) -> object:
-            raise RuntimeError("synthetic bug, unrelated to query syntax")
-
-        monkeypatch.setattr(
-            backend_mod,
-            "parse_simple_text_highlight_query",
-            raise_runtime_error,
-        )
-
-        doc = DocumentFactory.create(title="probe", content="anything here")
-        backend.add_or_update(doc)
-
-        with pytest.raises(RuntimeError):
-            backend.highlight_hits(
-                "anything",
-                [doc.pk],
-                search_mode=SearchMode.TEXT,
-            )
-
-
-@pytest.mark.usefixtures("_search_index")
-class TestApiNoLongerReturns500:
-    """Pins the actual regression: a matching TEXT-mode search whose query
-    string carries Tantivy syntax must return results, not a server error."""
-
-    @pytest.mark.parametrize("raw_query", _MALFORMED_QUERIES)
-    def test_malformed_text_query_returns_200(
-        self,
-        admin_client: APIClient,
-        raw_query: str,
-    ) -> None:
-        """
-        GIVEN:
-            - A matching document whose content contains the raw query
-              text, indexed via the real search index fixture
-        WHEN:
-            - A TEXT-mode search is issued through the real API with a
-              query string carrying Tantivy syntax
-        THEN:
-            - The response is 200 with the expected result count, not a
-              500 (the regression this file exists to pin)
-        """
-        from documents.search import get_backend
-
-        doc = DocumentFactory.create(
-            title="probe",
-            content=f"needle content containing {raw_query} literally here",
-        )
-        get_backend().add_or_update(doc)
-
-        response = admin_client.get(f"/api/documents/?text={raw_query}")
-
-        assert response.status_code == status.HTTP_200_OK
-        assert response.data["count"] == 1
-
-    def test_plain_text_query_still_returns_200(
-        self,
-        admin_client: APIClient,
-    ) -> None:
-        """
-        GIVEN:
-            - A matching document indexed via the real search index
-              fixture
-        WHEN:
-            - An ordinary TEXT-mode search (no Tantivy syntax) is issued
-        THEN:
-            - The response is 200 with the expected result count; sanity
-              check that the guard does not mask a total failure of the
-              ordinary highlight path
-        """
-        from documents.search import get_backend
-
-        doc = DocumentFactory.create(
-            title="probe",
-            content="needle content containing plain text literally here",
-        )
-        get_backend().add_or_update(doc)
-
-        response = admin_client.get("/api/documents/?text=plain text")
-
-        assert response.status_code == status.HTTP_200_OK
-        assert response.data["count"] == 1

src/documents/tests/search/test_json_field_prefixes.py

@@ -1,206 +0,0 @@
-"""Bare notes:/custom_fields: prefix resolution.
-
-"notes:foo"/"custom_fields:foo" were valid fielded searches before the
-whoosh-compat migration. The registry only exposes them as JSON subpaths, so
-each JSON FieldSpec declares a default subpath (SubpathSpec(default=True)):
-notes: resolves to notes.note:, custom_fields: resolves to
-custom_fields.value:. This replaced an earlier regex-based rewrite
-(_rewrite_bare_json_field_prefixes) that ran on the raw query string before
-parsing and was blind to quoting, so a phrase like
-content:"payment notes: none" was silently corrupted into a notes-field
-search and matched nothing. Resolving the default subpath inside the parser
-instead means quoting is already understood by the time it happens.
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-import pytest
-from django.contrib.auth.models import User
-
-from documents.models import CustomField
-from documents.models import CustomFieldInstance
-from documents.models import Document
-from documents.models import Note
-
-if TYPE_CHECKING:
-    from documents.search._backend import TantivyBackend
-
-pytestmark = [pytest.mark.search, pytest.mark.django_db]
-
-
-def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
-    return set(backend.search_ids(query, user=None))
-
-
-def _index(backend: TantivyBackend, **kwargs: object) -> Document:
-    doc = Document.objects.create(**kwargs)
-    backend.add_or_update(doc)
-    return doc
-
-
-class TestBareJsonFieldPrefixes:
-    def test_bare_notes_prefix_searches_note_text(
-        self,
-        backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - A document with a note whose text contains a word, and a
-              decoy document whose content (not notes) contains the same
-              word
-        WHEN:
-            - A bare "notes:" prefix query is run (notes declares "note"
-              as its default subpath)
-        THEN:
-            - Only the document whose note matches is returned; the
-              decoy's content match does not resurface through a demoted
-              text search
-        """
-        alice = User.objects.create_user(username="alice")
-        with_note = Document.objects.create(
-            title="Has note",
-            content="x",
-            checksum="bare-notes-with",
-        )
-        Note.objects.create(document=with_note, user=alice, note="crocodile")
-        backend.add_or_update(with_note)
-        # This document's CONTENT contains the words a demoted text search
-        # would match; it must NOT match once the prefix addresses notes.
-        _index(
-            backend,
-            title="Notes about things",
-            content="notes crocodile mention",
-            checksum="bare-notes-decoy",
-        )
-        assert _matched_ids(backend, "notes:crocodile") == {with_note.pk}
-
-    def test_bare_custom_fields_prefix_searches_values(
-        self,
-        backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - A document with a custom field instance whose value
-              contains a word, and a decoy document whose content (not a
-              custom field value) contains the same word
-        WHEN:
-            - A bare "custom_fields:" prefix query is run (custom_fields
-              declares "value" as its default subpath)
-        THEN:
-            - Only the document whose custom field value matches is
-              returned
-        """
-        field = CustomField.objects.create(
-            name="Policy Number",
-            data_type=CustomField.FieldDataType.STRING,
-        )
-        with_value = Document.objects.create(
-            title="Has field",
-            content="x",
-            checksum="bare-cf-with",
-        )
-        CustomFieldInstance.objects.create(
-            document=with_value,
-            field=field,
-            value_text="crocodile",
-        )
-        backend.add_or_update(with_value)
-        _index(
-            backend,
-            title="Custom things",
-            content="custom fields crocodile",
-            checksum="bare-cf-decoy",
-        )
-        assert _matched_ids(backend, "custom_fields:crocodile") == {with_value.pk}
-
-    def test_subpath_spellings_are_untouched(
-        self,
-        backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - A document with a note carrying both an author and note text
-        WHEN:
-            - The explicit subpath spellings "notes.user:" and
-              "notes.note:" are queried
-        THEN:
-            - Both resolve to their intended subpath and match the
-              document; the default-subpath resolution for the bare
-              prefix does not interfere with explicit subpath addressing
-        """
-        bob = User.objects.create_user(username="bob")
-        doc = Document.objects.create(
-            title="Bob note",
-            content="x",
-            checksum="bare-subpath",
-        )
-        Note.objects.create(document=doc, user=bob, note="remark")
-        backend.add_or_update(doc)
-        assert _matched_ids(backend, "notes.user:bob") == {doc.pk}
-        assert _matched_ids(backend, "notes.note:remark") == {doc.pk}
-
-
-class TestQuotedPhraseContainingNotesColonIsNotCorrupted:
-    """The regex rewrite this migration removes was blind to quoting: it
-    matched "notes:" anywhere in the raw query string, including inside an
-    already-quoted phrase on an unrelated field, silently turning
-    content:"payment notes: none" into a notes-field search that matched
-    nothing. Resolving the default subpath during parsing (which is
-    quote-aware) fixes this."""
-
-    def test_quoted_phrase_with_notes_colon_matches_by_content(
-        self,
-        backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - A document whose content literally contains the text
-              "payment notes: none" inside a quoted phrase
-        WHEN:
-            - A query quoting that exact phrase against the content
-              field is run
-        THEN:
-            - It matches by content, rather than the "notes:" substring
-              inside the quotes being corrupted into a notes-field search
-              that matches nothing (the bug the deleted regex rewrite
-              caused, since it was blind to quoting)
-        """
-        target = _index(
-            backend,
-            title="Statement",
-            content="payment notes: none",
-            checksum="quoted-phrase-notes-colon",
-        )
-        assert _matched_ids(
-            backend,
-            'content:"payment notes: none"',
-        ) == {target.pk}
-
-    def test_quoted_phrase_matches_the_same_document_unquoted(
-        self,
-        backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - A document whose content contains the same words as the
-              previous test's phrase, but without the colon
-        WHEN:
-            - A query quoting that phrase against the content field is
-              run
-        THEN:
-            - It matches by content, proving the earlier fix is about
-              quote-awareness specifically, not about the words
-              themselves being unsearchable
-        """
-        target = _index(
-            backend,
-            title="Statement",
-            content="payment notes none",
-            checksum="quoted-phrase-no-colon",
-        )
-        assert _matched_ids(
-            backend,
-            'content:"payment notes none"',
-        ) == {target.pk}

src/documents/tests/search/test_json_subpath_completeness.py

@@ -1,92 +0,0 @@
-"""Every declared JSON subpath must actually be written to the index.
-
-PUBLIC_FIELDS declares each JSON field's subpaths (e.g. ``notes`` ->
-{"user", "note"}), but nothing coupled that declaration to what
-``_backend.py``'s document builder actually writes into the JSON blob at
-index time. A subpath declared but never written would be
-queryable-but-always-empty -- syntactically valid, silently matching
-nothing -- with no test failure anywhere.
-
-This indexes one real document carrying values for every JSON field
-(a Note, a CustomFieldInstance) and inspects the document's own stored
-JSON payload, rather than running field-specific queries: that way a
-future JSON field's subpaths are covered automatically, without a new
-per-subpath query having to be added by hand each time.
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-import pytest
-import tantivy
-from django.contrib.auth.models import User
-from whoosh_compat import FieldKind
-
-from documents.models import CustomField
-from documents.models import CustomFieldInstance
-from documents.models import Document
-from documents.models import Note
-from documents.search._fields import PUBLIC_FIELDS
-
-if TYPE_CHECKING:
-    from documents.search._backend import TantivyBackend
-
-pytestmark = [pytest.mark.search, pytest.mark.django_db]
-
-
-class TestJsonSubpathsAreWrittenAtIndexTime:
-    def test_every_declared_json_subpath_appears_in_the_stored_document(
-        self,
-        backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - A document with a Note and a CustomFieldInstance attached
-        WHEN:
-            - The document is indexed via TantivyBackend.add_or_update
-        THEN:
-            - Every subpath PUBLIC_FIELDS declares for notes/custom_fields
-              is present as a key in the document's stored JSON payload
-        """
-        user = User.objects.create_user(username="completeness-user")
-        field = CustomField.objects.create(
-            name="Completeness Field",
-            data_type=CustomField.FieldDataType.STRING,
-        )
-        doc = Document.objects.create(
-            title="Completeness doc",
-            content="x",
-            checksum="json-subpath-completeness",
-        )
-        Note.objects.create(document=doc, user=user, note="a note")
-        CustomFieldInstance.objects.create(
-            document=doc,
-            field=field,
-            value_text="a value",
-        )
-        backend.add_or_update(doc)
-
-        index = backend._index
-        searcher = index.searcher()
-        hits = searcher.search(
-            tantivy.Query.term_query(index.schema, "id", doc.pk),
-            limit=1,
-        ).hits
-        assert hits, "the document was not indexed"
-        stored = searcher.doc(hits[0][1]).to_dict()
-
-        json_fields = [f for f in PUBLIC_FIELDS if f.kind is FieldKind.JSON]
-        assert json_fields, "no JSON fields declared - fixture is stale"
-        for field_spec in json_fields:
-            stored_values = stored.get(field_spec.name)
-            assert stored_values, (
-                f"{field_spec.name} was not written to the index at all"
-            )
-            written_keys = stored_values[0].keys()
-            for subpath in field_spec.subpaths:
-                assert subpath in written_keys, (
-                    f"{field_spec.name}.{subpath} is declared in PUBLIC_FIELDS "
-                    "but _backend.py's document builder never writes it - it "
-                    "would be queryable but always empty"
-                )

src/documents/tests/search/test_keyword_pattern_literal.py

@@ -1,62 +0,0 @@
-"""Wildcard patterns on KEYWORD fields must stay literal.
-
-``checksum`` is the only KEYWORD field: it is indexed with the raw tokenizer,
-so its terms are never lowercased, folded or stemmed. Running its wildcard
-patterns through the stemming normalizer rewrote hex prefixes ("ceded" ->
-"cede") and returned documents whose checksum did not start with what the user
-typed, which for an identity field is a wrong answer.
-
-This covers only the registry-level normalizer, which is all that exists to
-prove at this point in the stack: user queries are not yet routed through
-whoosh-compat (that lands with the query-layer PR), so the same fact proven
-end to end against real indexed documents lives in
-``test_checksum_prefix_queries.py``.
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-import pytest
-
-from documents.search._registry import get_field_registry
-
-if TYPE_CHECKING:
-    from whoosh_compat import FieldRegistry
-    from whoosh_compat import PatternNormalizer
-
-pytestmark = [pytest.mark.search, pytest.mark.django_db]
-
-
-def _normalizer(registry: FieldRegistry, name: str) -> PatternNormalizer:
-    ref = registry.make_ref(name)
-    assert ref is not None
-    resolved = registry.resolve(ref)
-    assert resolved is not None
-    assert resolved.spec.pattern_normalizer is not None
-    return resolved.spec.pattern_normalizer
-
-
-class TestKeywordPatternNormalizer:
-    @pytest.mark.parametrize(
-        "run",
-        [
-            pytest.param("ceded", id="stems_to_cede"),
-            pytest.param("added", id="stems_to_ad"),
-            pytest.param("cafed", id="stems_to_cafe"),
-        ],
-    )
-    def test_keyword_runs_are_folded_not_stemmed(self, run: str) -> None:
-        """
-        GIVEN:
-            - The "checksum" field's registered pattern normalizer
-              (KEYWORD kind, "en" registry)
-        WHEN:
-            - A wildcard pattern run is normalized
-        THEN:
-            - The run is returned unchanged, never widened to a stem (which
-              would return checksums that do not start with what the user
-              typed)
-        """
-        normalize = _normalizer(get_field_registry("en"), "checksum")
-        assert normalize(run) == run

src/documents/tests/search/test_pattern_normalizer.py

@@ -1,156 +0,0 @@
-"""The pattern normalizer's stem-alternates contract, and its consistency
-with the index-side analyzer.
-
-Query patterns are normalized but were not stemmed, while index terms are
-stemmed, so the natural spelling of a prefix search matched nothing:
-``invoice*`` found no document although ``invoic*`` did. v2's index was
-UNSTEMMED (whoosh ``TEXT()`` defaults to ``StandardAnalyzer``), so this
-regressed against both baselines.
-
-These are pure unit tests against ``_make_pattern_normalizer`` and
-``stem_pattern_text`` directly, no query routing involved. The end-to-end
-proof that a real wildcard query actually reaches a stemmed index term
-lives in ``test_pattern_stemming.py``.
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-import pytest
-
-from documents.search._registry import _make_pattern_normalizer
-from documents.search._tokenizer import ascii_fold
-from documents.search._tokenizer import paperless_text_analyzer
-from documents.search._tokenizer import stem_pattern_text
-
-if TYPE_CHECKING:
-    from whoosh_compat import PatternNormalizer
-
-
-class TestStemsMatchTheIndexAnalyzer:
-    """stem_pattern_text rebuilds paperless_text_analyzer's stemming tail rather
-    than sharing it, so a filter added to the index analyzer alone would silently
-    stop patterns from reaching the terms it produces.
-    """
-
-    @pytest.mark.parametrize(
-        "language",
-        ["en", "de", "fr", "es", "sv", None, "klingon"],
-    )
-    @pytest.mark.parametrize(
-        "word",
-        ["Copies", "copyright", "Companies", "Invoices", "laufen", "casas", "Straße"],
-    )
-    def test_stem_equals_the_index_term(self, word: str, language: str | None) -> None:
-        """
-        GIVEN:
-            - A word, across several representative index languages
-              ("en", "de", "fr", "es", "sv"), no language, and an
-              unsupported language ("klingon")
-        WHEN:
-            - `stem_pattern_text` (the pattern-side stemmer) processes the
-              folded word, and `paperless_text_analyzer` (the index-side
-              analyzer) independently processes the same word
-        THEN:
-            - The two produce the identical term. `stem_pattern_text`
-              rebuilds `paperless_text_analyzer`'s stemming tail rather
-              than sharing it, so a filter added to the index analyzer
-              alone would silently stop patterns from reaching the terms
-              it produces; this pins the two staying in sync
-        """
-        indexed = paperless_text_analyzer(language).analyze(word)[0]
-        assert stem_pattern_text(ascii_fold(word.lower()), language) == indexed
-
-
-def _forms(normalize: PatternNormalizer, text: str) -> tuple[str, ...]:
-    """The distinct forms a term may match, in order, the way the emitter reads
-    the normalizer's answer (see whoosh_compat.PatternNormalizer)."""
-    result = normalize(text)
-    if isinstance(result, str):
-        return (result,)
-    return tuple(dict.fromkeys(result))
-
-
-class TestPatternNormalizer:
-    @pytest.mark.parametrize(
-        ("text", "expected"),
-        [
-            ("Invoice", ("invoice", "invoic")),
-            ("companies", ("companies", "compani")),
-            # y -> i is a substitution, so both forms are needed: the index
-            # holds "librari" for "library" and "library" for "librarian".
-            ("library", ("library", "librari")),
-            # A run the stemmer leaves alone collapses back to one form, so it
-            # costs exactly the one regex branch it did before.
-            ("invoic", ("invoic",)),
-            ("Universit", ("universit",)),
-            ("Café", ("cafe",)),
-        ],
-    )
-    def test_offers_the_typed_run_and_its_stem(
-        self,
-        text: str,
-        expected: tuple[str, ...],
-    ) -> None:
-        """
-        GIVEN:
-            - The "en" pattern normalizer
-        WHEN:
-            - It processes a literal run (e.g. "Invoice", "library",
-              "Café")
-        THEN:
-            - It returns the folded run and, where it differs, the
-              stemmed form, as distinct alternatives; a run the stemmer
-              leaves alone (e.g. "invoic") collapses back to the single
-              folded form. "library" needs both forms since y -> i is a
-              substitution: the index holds "librari" for "library" and
-              "library" for "librarian"
-        """
-        assert _forms(_make_pattern_normalizer("en"), text) == expected
-
-    def test_run_that_yields_no_token_falls_back_to_the_typed_run(self) -> None:
-        """
-        GIVEN:
-            - The "en" pattern normalizer
-        WHEN:
-            - It processes a run past the analyzer's remove_long limit
-        THEN:
-            - The run analyzes to zero tokens, so there is no stem to
-              offer, and only the folded run remains
-        """
-        over_long = "invoices" * 20
-        assert _forms(_make_pattern_normalizer("en"), over_long) == (over_long,)
-
-    @pytest.mark.parametrize("language", [None, "klingon"])
-    def test_unstemmed_language_folds_only(self, language: str | None) -> None:
-        """
-        GIVEN:
-            - A pattern normalizer with no language configured, or one
-              this build has no stemmer for ("klingon")
-        WHEN:
-            - It processes "Invoices"
-        THEN:
-            - Only the folded form ("invoices") is offered, since with no
-              stemmer configured the index holds surface forms and the
-              pattern must keep them too
-        """
-        assert _forms(_make_pattern_normalizer(language), "Invoices") == ("invoices",)
-
-    @pytest.mark.parametrize("char", ["a", "Z", "é"])
-    def test_a_single_character_collapses_to_one_folded_form(self, char: str) -> None:
-        """
-        GIVEN:
-            - The "en" pattern normalizer
-        WHEN:
-            - It processes a single character
-        THEN:
-            - Exactly one, one-character form is returned. A bracket
-              class body is normalized one character at a time and the
-              answer is used only when it is a single one-character
-              form, so a stemmer that changed a lone character would
-              silently disable folding inside classes
-        """
-        forms = _forms(_make_pattern_normalizer("en"), char)
-        assert len(forms) == 1
-        assert len(forms[0]) == 1

src/documents/tests/search/test_pattern_stemming.py

@@ -1,220 +0,0 @@
-"""Wildcard patterns must match a stemmed index, end to end.
-
-Query patterns are normalized but were not stemmed, while index terms are
-stemmed, so the natural spelling of a prefix search matched nothing:
-``invoice*`` found no document although ``invoic*`` did. v2's index was
-UNSTEMMED (whoosh ``TEXT()`` defaults to ``StandardAnalyzer``), so this
-regressed against both baselines.
-
-These are end-to-end tests against a real indexed document and a real
-query, proving the pattern normalizer's stem-alternates contract actually
-reaches a stemmed index term. The pure unit tests against the normalizer
-function itself live in ``test_pattern_normalizer.py``.
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-import pytest
-
-from documents.models import Document
-
-if TYPE_CHECKING:
-    from documents.search._backend import TantivyBackend
-
-pytestmark = [pytest.mark.search, pytest.mark.django_db]
-
-CONTENT = (
-    "invoice total due for electricity from both companies, "
-    "payments made to the university library, copies attached"
-)
-
-
-def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
-    return set(backend.search_ids(query, user=None))
-
-
-@pytest.fixture
-def indexed_doc(backend: TantivyBackend) -> Document:
-    doc = Document.objects.create(
-        title="Invoice 2020 productname",
-        content=CONTENT,
-        checksum="pattern-stemming-1",
-        archive_serial_number=900,
-    )
-    backend.add_or_update(doc)
-    return doc
-
-
-class TestPrefixStemming:
-    @pytest.mark.parametrize(
-        "query",
-        [
-            "invoice*",
-            "electricity*",
-            "companies*",
-            "payments*",
-            "library*",
-            "title:Invoice*",
-        ],
-    )
-    def test_full_word_prefix_matches_its_stem(
-        self,
-        backend: TantivyBackend,
-        indexed_doc: Document,
-        query: str,
-    ) -> None:
-        """
-        GIVEN:
-            - A document indexed with content containing "invoice",
-              "electricity", "companies", "payments", "library" and title
-              "Invoice 2020 productname"
-        WHEN:
-            - A prefix wildcard on the full, unstemmed word is queried
-              (e.g. "invoice*", "title:Invoice*")
-        THEN:
-            - The document matches, since the pattern normalizer offers
-              the word's stem as an alternative alongside the typed run,
-              reaching the stemmed index term
-        """
-        assert _matched_ids(backend, query) == {indexed_doc.id}
-
-    @pytest.mark.parametrize("query", ["invoic*", "electr*", "payment*"])
-    def test_already_stemmed_prefix_still_matches(
-        self,
-        backend: TantivyBackend,
-        indexed_doc: Document,
-        query: str,
-    ) -> None:
-        """
-        GIVEN:
-            - The same indexed document
-        WHEN:
-            - A prefix wildcard is typed already in its stemmed spelling
-              (e.g. "invoic*")
-        THEN:
-            - The document still matches, since the typed-run alternative
-              is itself a prefix of the stored stemmed term
-        """
-        assert _matched_ids(backend, query) == {indexed_doc.id}
-
-    @pytest.mark.parametrize("query", ["univers*", "librar*"])
-    def test_partial_prefix_reaches_the_stemmed_term(
-        self,
-        backend: TantivyBackend,
-        indexed_doc: Document,
-        query: str,
-    ) -> None:
-        """
-        GIVEN:
-            - The same indexed document
-        WHEN:
-            - A prefix shorter than a whole word is queried ("univers*",
-              "librar*")
-        THEN:
-            - It still matches, and neither case needs the two-alternative
-              path to do it: measured under "en", the stemmer leaves
-              "librar" alone, so it has one form, and that form is a
-              prefix of the "librari" the index holds for "library";
-              "univers" stems to the *shorter* "univ", and the run as
-              typed and its stem are both prefixes of the "univers" the
-              index holds for "university". The case where the two forms
-              genuinely diverge, and only one of them matches, is
-              test_stem_substitution_reaches_both_the_inflection_and_the_compound
-        """
-        assert _matched_ids(backend, query) == {indexed_doc.id}
-
-    def test_full_word_reaches_the_stem_but_a_fragment_of_it_does_not(
-        self,
-        backend: TantivyBackend,
-        indexed_doc: Document,
-    ) -> None:
-        """
-        GIVEN:
-            - The same indexed document, storing "university" as "univers"
-        WHEN:
-            - "universities*" and "universit*" are each queried
-        THEN:
-            - "universities*" matches, since the stem of "universities" is
-              that same "univers"; "universit*" matches nothing, since
-              "universit" is a prefix of neither its own stem nor the
-              stored term. The alternatives widen recall without turning
-              a wildcard into a prefix search over the original text, and
-              usage.md names this exact pair so a reader told that
-              `universit*` fails is also told which spelling works
-        """
-        assert _matched_ids(backend, "universities*") == {indexed_doc.id}
-        assert _matched_ids(backend, "universit*") == set()
-
-    def test_pattern_past_the_stem_boundary_is_documented_not_fixed(
-        self,
-        backend: TantivyBackend,
-        indexed_doc: Document,
-    ) -> None:
-        """
-        GIVEN:
-            - The same indexed document, with "productname" indexed as
-              "productnam"
-        WHEN:
-            - "produ*name" (a pattern straddling the stem boundary) is
-              queried
-        THEN:
-            - It matches nothing; produ*name cannot match a stemmed
-              index, and usage.md must not advertise it. Pinned so the
-              limitation is deliberate, not accidental
-        """
-        assert _matched_ids(backend, "produ*name") == set()
-
-    def test_stem_substitution_reaches_both_the_inflection_and_the_compound(
-        self,
-        backend: TantivyBackend,
-        indexed_doc: Document,
-    ) -> None:
-        """
-        GIVEN:
-            - The indexed document (containing "copies") plus a second
-              document titled "Copyright notice" with content "copyright
-              notice for the work"
-        WHEN:
-            - "copy*" and "copyright*" are each queried
-        THEN:
-            - "copy*" matches both documents, and "copyright*" matches
-              only the compound one. English stemming substitutes as well
-              as truncates: "copy" and "copies" both index as "copi",
-              while "copyright" keeps its literal "y". Neither form is a
-              prefix of the other, so no single normalized string reaches
-              both; the run is therefore emitted as a disjunction of the
-              folded and stemmed forms, and "copy*" reaches the base
-              word, its inflections and the compound alike
-        """
-        compound = Document.objects.create(
-            title="Copyright notice",
-            content="copyright notice for the work",
-            checksum="pattern-stemming-2",
-            archive_serial_number=901,
-        )
-        backend.add_or_update(compound)
-
-        assert _matched_ids(backend, "copy*") == {indexed_doc.id, compound.id}
-        assert _matched_ids(backend, "copyright*") == {compound.id}
-
-
-class TestBracketClassStillFolds:
-    def test_class_body_matches_case_insensitively(
-        self,
-        backend: TantivyBackend,
-        indexed_doc: Document,
-    ) -> None:
-        """
-        GIVEN:
-            - The indexed document, titled "Invoice 2020 productname"
-        WHEN:
-            - A bracket-class pattern mixing case is queried
-              ("title:[IP]nvoice*")
-        THEN:
-            - It matches: the class body is folded per character, which
-              the alternatives contract preserves only because a lone
-              character stems to itself
-        """
-        assert _matched_ids(backend, "title:[IP]nvoice*") == {indexed_doc.id}

src/documents/tests/search/test_permission_field_isolation.py

@@ -1,198 +0,0 @@
-"""Permission filtering must hold against the real indexed document shape.
-
-Only three of the index's unsigned ``*_id`` columns are load-bearing:
-``owner_id``, ``viewer_id`` and ``viewer_group_id``, all read by
-build_permission_filter. The rest (correspondent/document_type/storage_path/tag
-ids) were written on every document and read by nothing, and were dropped.
-
-These tests index real Documents through the backend's own document builder and
-assert result-level visibility per user, so a mistake about which columns are
-load-bearing shows up as documents leaking across users rather than as a passing
-unit test over a hand-built index.
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-import pytest
-from django.contrib.auth.models import Group
-from django.contrib.auth.models import User
-from guardian.shortcuts import assign_perm
-
-from documents.models import Correspondent
-from documents.models import Document
-from documents.models import DocumentType
-from documents.models import StoragePath
-from documents.models import Tag
-
-if TYPE_CHECKING:
-    from documents.search._backend import TantivyBackend
-
-pytestmark = [pytest.mark.search, pytest.mark.django_db]
-
-
-@pytest.fixture
-def owner() -> User:
-    return User.objects.create_user(username="owner")
-
-
-@pytest.fixture
-def stranger() -> User:
-    return User.objects.create_user(username="stranger")
-
-
-@pytest.fixture
-def viewer() -> User:
-    return User.objects.create_user(username="viewer")
-
-
-@pytest.fixture
-def group_member() -> User:
-    user = User.objects.create_user(username="group_member")
-    user.groups.add(Group.objects.create(name="accounting"))
-    return user
-
-
-class TestPermissionFilteringOnIndexedDocuments:
-    def test_unowned_document_is_visible_to_everyone(
-        self,
-        backend: TantivyBackend,
-        stranger: User,
-    ) -> None:
-        """
-        GIVEN:
-            - A document with no owner, indexed via the backend's real
-              document builder
-        WHEN:
-            - A stranger (no relation to the document) searches
-        THEN:
-            - The document is visible to them
-        """
-        doc = Document.objects.create(
-            title="Public Invoice",
-            content="invoice total due",
-            checksum="perm-unowned",
-        )
-        backend.add_or_update(doc)
-
-        assert backend.search_ids("invoice", user=stranger) == [doc.pk]
-
-    def test_owned_document_is_visible_only_to_its_owner(
-        self,
-        backend: TantivyBackend,
-        owner: User,
-        stranger: User,
-    ) -> None:
-        """
-        GIVEN:
-            - A document owned by one user, indexed via the backend's
-              real document builder
-        WHEN:
-            - The owner and an unrelated stranger each search
-        THEN:
-            - The owner sees the document; the stranger does not
-        """
-        doc = Document.objects.create(
-            title="Private Invoice",
-            content="invoice total due",
-            checksum="perm-owned",
-            owner=owner,
-        )
-        backend.add_or_update(doc)
-
-        assert backend.search_ids("invoice", user=owner) == [doc.pk]
-        assert backend.search_ids("invoice", user=stranger) == []
-
-    def test_explicitly_shared_document_is_visible_to_the_viewer(
-        self,
-        backend: TantivyBackend,
-        owner: User,
-        viewer: User,
-        stranger: User,
-    ) -> None:
-        """
-        GIVEN:
-            - A document owned by one user and explicitly shared with a
-              second user via guardian's view_document permission,
-              indexed via the backend's real document builder
-        WHEN:
-            - The shared viewer and an unrelated stranger each search
-        THEN:
-            - The viewer sees the document; the stranger does not
-        """
-        doc = Document.objects.create(
-            title="Shared Invoice",
-            content="invoice total due",
-            checksum="perm-shared-user",
-            owner=owner,
-        )
-        assign_perm("view_document", viewer, doc)
-        backend.add_or_update(doc)
-
-        assert backend.search_ids("invoice", user=viewer) == [doc.pk]
-        assert backend.search_ids("invoice", user=stranger) == []
-
-    def test_group_shared_document_is_visible_to_group_members(
-        self,
-        backend: TantivyBackend,
-        owner: User,
-        group_member: User,
-        stranger: User,
-    ) -> None:
-        """
-        GIVEN:
-            - A document owned by one user and shared with a group via
-              guardian's view_document permission, indexed via the
-              backend's real document builder
-        WHEN:
-            - A member of that group and an unrelated stranger each
-              search
-        THEN:
-            - The group member sees the document; the stranger does not
-        """
-        doc = Document.objects.create(
-            title="Group Invoice",
-            content="invoice total due",
-            checksum="perm-shared-group",
-            owner=owner,
-        )
-        assign_perm("view_document", group_member.groups.first(), doc)
-        backend.add_or_update(doc)
-
-        assert backend.search_ids("invoice", user=group_member) == [doc.pk]
-        assert backend.search_ids("invoice", user=stranger) == []
-
-    def test_metadata_does_not_widen_visibility(
-        self,
-        backend: TantivyBackend,
-        owner: User,
-        stranger: User,
-    ) -> None:
-        """
-        GIVEN:
-            - A document owned by one user and carrying
-              correspondent/document_type/storage_path/tag metadata,
-              indexed via the backend's real document builder
-        WHEN:
-            - The owner and an unrelated stranger each search
-        THEN:
-            - The owner sees the document; the stranger does not, since
-              the dropped, non-load-bearing metadata *_id columns must
-              not widen visibility beyond the owner_id/viewer_id/
-              viewer_group_id filter
-        """
-        doc = Document.objects.create(
-            title="Tagged Invoice",
-            content="invoice total due",
-            checksum="perm-metadata",
-            owner=owner,
-            correspondent=Correspondent.objects.create(name="ACME"),
-            document_type=DocumentType.objects.create(name="Bill"),
-            storage_path=StoragePath.objects.create(name="Archive", path="archive/"),
-        )
-        doc.tags.add(Tag.objects.create(name="paid"))
-        backend.add_or_update(doc)
-
-        assert backend.search_ids("invoice", user=owner) == [doc.pk]
-        assert backend.search_ids("invoice", user=stranger) == []

src/documents/tests/search/test_query_negation.py

@@ -1,198 +0,0 @@
-"""Negation must survive the blended query.
-
-parse_user_query ORs an exact clause with optional fuzzy and CJK clauses.
-Each of those is built from positive terms only, so unless the query's
-exclusions are applied to the blend as a whole, a document the exact
-clause excluded is re-admitted by whichever other clause is enabled.
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-import pytest
-
-from documents.models import Document
-
-if TYPE_CHECKING:
-    from pytest_django.fixtures import SettingsWrapper
-
-    from documents.search._backend import TantivyBackend
-
-pytestmark = [pytest.mark.search, pytest.mark.django_db]
-
-
-def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
-    return set(backend.search_ids(query, user=None))
-
-
-def _index(backend: TantivyBackend, **kwargs: object) -> Document:
-    doc = Document.objects.create(**kwargs)
-    backend.add_or_update(doc)
-    return doc
-
-
-@pytest.fixture
-def fuzzy_enabled(settings: SettingsWrapper) -> None:
-    """Enable the fuzzy blend clause. The threshold doubles as a minimum
-    score filter, so it is set to 0.0: every hit passes and the test sees
-    the clause's matching behaviour, not the filter's."""
-    settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.0
-
-
-class TestNegationConstrainsEveryClause:
-    @pytest.mark.usefixtures("fuzzy_enabled")
-    def test_fuzzy_clause_does_not_readmit_an_excluded_document(
-        self,
-        backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - Two documents both matching a positive term, one of which
-              also contains a word the query excludes, with the fuzzy
-              blend clause enabled
-        WHEN:
-            - A query combining the positive term with a NOT exclusion is
-              run
-        THEN:
-            - Only the document without the excluded word is returned;
-              the fuzzy clause (built from positive terms only) does not
-              readmit the document the exact clause excluded
-        """
-        secret = _index(
-            backend,
-            title="Invoice A",
-            content="invoice total secret",
-            checksum="neg-fuzzy-1",
-        )
-        public = _index(
-            backend,
-            title="Invoice B",
-            content="invoice total public",
-            checksum="neg-fuzzy-2",
-        )
-
-        assert _matched_ids(backend, "invoice") == {secret.pk, public.pk}
-        assert _matched_ids(backend, "invoice NOT secret") == {public.pk}
-
-    def test_cjk_clause_does_not_readmit_an_excluded_document(
-        self,
-        backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - Two documents both containing a CJK run, one of which also
-              contains a word the query excludes
-        WHEN:
-            - A query combining the CJK term with a NOT exclusion is run
-        THEN:
-            - Only the document without the excluded word is returned;
-              the CJK clause legitimately carries the CJK run, so
-              rebuilding it from the AST cannot help here, only applying
-              the exclusion above the blend keeps the excluded document
-              out
-        """
-        secret = _index(
-            backend,
-            title="Tokyo A",
-            content="東京都の秘密です secret",
-            checksum="neg-cjk-1",
-        )
-        public = _index(
-            backend,
-            title="Tokyo B",
-            content="東京都の報告書です public",
-            checksum="neg-cjk-2",
-        )
-
-        assert _matched_ids(backend, "東京") == {secret.pk, public.pk}
-        assert _matched_ids(backend, "東京 NOT secret") == {public.pk}
-
-    @pytest.mark.usefixtures("fuzzy_enabled")
-    def test_disjunctive_negation_still_admits_the_other_branch(
-        self,
-        backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - A document matching a positive term and also containing a
-              word a disjunctive NOT branch excludes, plus an unrelated
-              document
-        WHEN:
-            - A query of the shape "term OR NOT excluded_word" is run
-        THEN:
-            - Both documents are returned; "invoice OR NOT secret"
-              excludes nothing on its own, so a document matching the
-              left branch stays in even though it contains the excluded
-              word
-        """
-        secret_invoice = _index(
-            backend,
-            title="Invoice A",
-            content="invoice total secret",
-            checksum="neg-or-1",
-        )
-        unrelated = _index(
-            backend,
-            title="Recipe",
-            content="flour and water",
-            checksum="neg-or-2",
-        )
-
-        assert _matched_ids(backend, "invoice OR NOT secret") == {
-            secret_invoice.pk,
-            unrelated.pk,
-        }
-
-    def test_a_negation_under_or_does_not_constrain_the_cjk_clause(
-        self,
-        backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - Two CJK documents, one of which also contains a word an OR
-              branch's own NOT excludes, plus an unrelated latin document
-        WHEN:
-            - The exclusion is under a disjunctive OR branch, versus in
-              conjunctive position
-        THEN:
-            - Under OR, the excluded document still matches through the
-              CJK clause (an exclusion that is one branch's own condition
-              cannot be restated above the blend without dropping
-              documents the other branch matches, so it is left where it
-              is and the CJK clause stays unconstrained by it -- this
-              shows through here in a way it does not for latin text,
-              since the exact clause cannot match a CJK run at all, so
-              the CJK clause is the only thing matching the CJK
-              documents, and the excluded one comes with it)
-            - Under conjunctive "AND NOT", the same exclusion is hoisted
-              and does constrain the CJK clause, pinning the deliberate
-              limit of the hoist
-        """
-        secret = _index(
-            backend,
-            title="Tokyo A",
-            content="東京都の秘密です secret",
-            checksum="neg-or-cjk-1",
-        )
-        public = _index(
-            backend,
-            title="Tokyo B",
-            content="東京都の報告書です public",
-            checksum="neg-or-cjk-2",
-        )
-        bill = _index(
-            backend,
-            title="Bill",
-            content="bill payment received",
-            checksum="neg-or-cjk-3",
-        )
-
-        assert _matched_ids(backend, "(東京 AND NOT secret) OR bill") == {
-            bill.pk,
-            public.pk,
-            secret.pk,
-        }
-        # The same exclusion in conjunctive position is hoisted, and does
-        # constrain the CJK clause.
-        assert _matched_ids(backend, "東京 AND NOT secret") == {public.pk}

src/documents/tests/search/test_registry.py

@@ -1,224 +0,0 @@
-from collections.abc import Sequence
-
-import pytest
-from whoosh_compat import FieldKind
-from whoosh_compat import FieldRegistry
-from whoosh_compat.fields import ResolvedField
-
-from documents.search._fields import PUBLIC_FIELDS
-from documents.search._registry import get_field_registry
-
-
-@pytest.fixture
-def registry() -> FieldRegistry:
-    return get_field_registry(None)
-
-
-def _resolve(registry: FieldRegistry, name: str) -> ResolvedField:
-    ref = registry.make_ref(name)
-    assert ref is not None, f"{name} is not a valid field ref"
-    resolved = registry.resolve(ref)
-    assert resolved is not None, f"{name} did not resolve"
-    return resolved
-
-
-def _distinct_forms(result: str | Sequence[str]) -> tuple[str, ...]:
-    """The forms a term may match, in order, the way whoosh-compat's emitter
-    reads a pattern_normalizer's answer: a bare str is one form, a sequence is
-    several, deduplicated."""
-    if isinstance(result, str):
-        return (result,)
-    return tuple(dict.fromkeys(result))
-
-
-class TestFieldRegistry:
-    def test_no_queryable_field_name_ends_in_id(self) -> None:
-        """
-        GIVEN:
-            - PUBLIC_FIELDS, the canonical query-syntax field table
-        WHEN:
-            - Every declared field name is inspected
-        THEN:
-            - None of them end in "_id" (internal id columns, written for
-              permission filtering and joins, must never reach the query
-              surface; checked against PUBLIC_FIELDS rather than the
-              registry so a leak is caught where it is declared)
-        """
-        leaked = [f.name for f in PUBLIC_FIELDS if f.name.endswith("_id")]
-        assert not leaked, f"internal id fields reached the query surface: {leaked}"
-
-    def test_type_alias_resolves_to_document_type(
-        self,
-        registry: FieldRegistry,
-    ) -> None:
-        """
-        GIVEN:
-            - The field registry
-        WHEN:
-            - The alias "type" is resolved
-        THEN:
-            - It resolves to the canonical "document_type" field
-        """
-        assert _resolve(registry, "type").spec.name == "document_type"
-
-    def test_path_alias_resolves_to_storage_path(self, registry: FieldRegistry) -> None:
-        """
-        GIVEN:
-            - The field registry
-        WHEN:
-            - The alias "path" is resolved
-        THEN:
-            - It resolves to the canonical "storage_path" field
-        """
-        assert _resolve(registry, "path").spec.name == "storage_path"
-
-    def test_notes_json_subpaths_resolve(self, registry: FieldRegistry) -> None:
-        """
-        GIVEN:
-            - The field registry
-        WHEN:
-            - "notes.user" is resolved
-        THEN:
-            - It resolves to the "notes" field with json_path "user"
-        """
-        resolved = _resolve(registry, "notes.user")
-        assert resolved.spec.name == "notes"
-        assert resolved.json_path == "user"
-        assert resolved.is_subpath is True
-
-    def test_custom_fields_json_subpaths_resolve(self, registry: FieldRegistry) -> None:
-        """
-        GIVEN:
-            - The field registry
-        WHEN:
-            - "custom_fields.name" and "custom_fields.value" are resolved
-        THEN:
-            - Both resolve without error
-        """
-        for raw in ("custom_fields.name", "custom_fields.value"):
-            _resolve(registry, raw)
-
-    def test_tag_is_comma_values(self, registry: FieldRegistry) -> None:
-        """
-        GIVEN:
-            - The field registry
-        WHEN:
-            - The "tag" field is resolved
-        THEN:
-            - It is marked comma_values=True
-        """
-        assert _resolve(registry, "tag").spec.comma_values is True
-
-    def test_correspondent_is_not_comma_values(self, registry: FieldRegistry) -> None:
-        """
-        GIVEN:
-            - The field registry
-        WHEN:
-            - The "correspondent" field is resolved
-        THEN:
-            - It is not marked comma_values ("tag" is the only field that
-              opts in; end to end the two readings of
-              "correspondent:foo,bar" agree anyway, since the analyzer
-              splits the literal value on the comma regardless, so this is
-              only observable at the registry level)
-        """
-        assert _resolve(registry, "correspondent").spec.comma_values is False
-
-    def test_created_is_date_kind(self, registry: FieldRegistry) -> None:
-        """
-        GIVEN:
-            - The field registry
-        WHEN:
-            - The "created" field is resolved
-        THEN:
-            - Its kind is DATE and date_only is True
-        """
-        resolved = _resolve(registry, "created")
-        assert resolved.spec.kind is FieldKind.DATE
-        assert resolved.spec.date_only is True
-
-    def test_analyzer_lowercases_and_ascii_folds(self, registry: FieldRegistry) -> None:
-        """
-        GIVEN:
-            - The field registry with no language configured (no stemmer
-              in the analyzer chain)
-        WHEN:
-            - The "title" field's analyzer processes "Café"
-        THEN:
-            - It is lowercased and ASCII-folded to the single token "cafe"
-        """
-        resolved = _resolve(registry, "title")
-        assert resolved.spec.analyzer is not None
-        assert resolved.spec.analyzer("Café") == ["cafe"]
-
-    def test_checksum_analyzer_is_identity_single_token(
-        self,
-        registry: FieldRegistry,
-    ) -> None:
-        """
-        GIVEN:
-            - The field registry
-        WHEN:
-            - The "checksum" field's analyzer (raw tokenizer, no
-              splitting) processes "ABC-123"
-        THEN:
-            - It is returned unchanged as a single token
-        """
-        resolved = _resolve(registry, "checksum")
-        assert resolved.spec.analyzer is not None
-        assert resolved.spec.analyzer("ABC-123") == ["ABC-123"]
-
-    def test_pattern_normalizer_follows_the_registry_language(
-        self,
-        registry: FieldRegistry,
-    ) -> None:
-        """
-        GIVEN:
-            - A registry with no language, and a registry built for "en"
-        WHEN:
-            - The "title" field's pattern normalizer processes "Running"
-        THEN:
-            - With no language, only the folded run is offered
-              ("running"), since the index holds surface forms
-            - With "en", the stem is offered too ("run"), since indexed
-              terms are stemmed and the pattern has to reach them
-        """
-        resolved = _resolve(registry, "title")
-        assert resolved.spec.pattern_normalizer is not None
-        assert _distinct_forms(resolved.spec.pattern_normalizer("Running")) == (
-            "running",
-        )
-
-        resolved_en = _resolve(get_field_registry("en"), "title")
-        assert resolved_en.spec.pattern_normalizer is not None
-        assert _distinct_forms(resolved_en.spec.pattern_normalizer("Running")) == (
-            "running",
-            "run",
-        )
-
-    def test_registry_is_cached_per_language(self) -> None:
-        """
-        GIVEN:
-            - Two calls to get_field_registry("en")
-        WHEN:
-            - Both calls are made
-        THEN:
-            - They return the same registry instance
-        """
-        a = get_field_registry("en")
-        b = get_field_registry("en")
-        assert a is b
-
-    def test_registry_rebuilds_on_language_change(self) -> None:
-        """
-        GIVEN:
-            - A call to get_field_registry("en") and a call to
-              get_field_registry("de")
-        WHEN:
-            - Both calls are made
-        THEN:
-            - They return different registry instances
-        """
-        a = get_field_registry("en")
-        b = get_field_registry("de")
-        assert a is not b

src/documents/tests/search/test_schema.py

@@ -5,17 +5,12 @@ from typing import TYPE_CHECKING
 
 import pytest
 
-from documents.search._fields import PUBLIC_FIELDS
 from documents.search._schema import SCHEMA_VERSION
-from documents.search._schema import build_schema
-from documents.search._schema import field_descriptors
 from documents.search._schema import needs_rebuild
-from documents.search._schema import schema_fingerprint
 
 if TYPE_CHECKING:
     from pathlib import Path
 
-    import tantivy
     from pytest_django.fixtures import Settings
 
 
@@ -35,13 +30,7 @@ class TestNeedsRebuild:
     ) -> None:
         settings.SEARCH_LANGUAGE = "en"
         (index_dir / ".index_settings.json").write_text(
-            json.dumps(
+            json.dumps({"schema_version": SCHEMA_VERSION, "language": "en"}),
-                {
-                    "schema_version": SCHEMA_VERSION,
-                    "language": "en",
-                    "schema_fingerprint": schema_fingerprint(),
-                },
-            ),
         )
         assert needs_rebuild(index_dir) is False
 
@@ -88,61 +77,3 @@ class TestNeedsRebuild:
             json.dumps({"schema_version": SCHEMA_VERSION, "language": "en"}),
         )
         assert needs_rebuild(index_dir) is True
-
-
-def _schema_fields(schema: tantivy.Schema) -> dict[str, dict]:
-    """{name: field-state} for every field declared on a tantivy Schema.
-
-    tantivy-py 0.26 exposes no public introspection API on Schema (no
-    __iter__, get_field, to_json, etc.) -- __reduce__() (used internally for
-    pickling) is the only way to recover the field list, so we lean on it
-    here for test assertions only.
-    """
-    state = schema.__reduce__()[1][0]
-    return {field["name"]: field for field in state["inner"]}
-
-
-class TestSchemaMatchesPublicFields:
-    def test_every_public_field_is_in_the_schema(self) -> None:
-        """
-        GIVEN:
-            - PUBLIC_FIELDS and the tantivy schema built by build_schema()
-        WHEN:
-            - Every field declared in PUBLIC_FIELDS is checked against the
-              schema
-        THEN:
-            - Each one is present as a field in the built schema
-        """
-        schema = build_schema()
-        schema_field_names = set(_schema_fields(schema))
-        for field in PUBLIC_FIELDS:
-            assert field.name in schema_field_names, (
-                f"{field.name} is in PUBLIC_FIELDS but missing from build_schema()"
-            )
-
-
-class TestFastFlagAgreement:
-    def test_every_public_field_fast_flag_matches_the_built_schema(self) -> None:
-        """
-        GIVEN:
-            - PUBLIC_FIELDS and field_descriptors() (the latter is exactly
-              the input build_schema()'s SchemaBuilder consumes for the
-              `fast` kwarg on every field kind, so it pins the agreement
-              without depending on a private tantivy-py pickled
-              representation)
-        WHEN:
-            - Every PUBLIC_FIELDS entry's fast flag is compared against
-              field_descriptors()' fast flag for the same field
-        THEN:
-            - They agree for every field, catching a fast=True
-              PUBLIC_FIELDS entry the builder silently ignores here
-              instead of at a user's field:* existence query, which
-              whoosh-compat's registry trusts PUBLIC_FIELDS' fast flag to
-              resolve
-        """
-        descriptor_fast = {d.name: d.fast for d in field_descriptors()}
-        for public_field in PUBLIC_FIELDS:
-            assert descriptor_fast[public_field.name] == public_field.fast, (
-                f"{public_field.name}: PUBLIC_FIELDS says fast={public_field.fast} but"
-                f" field_descriptors() says fast={descriptor_fast[public_field.name]}"
-            )

src/documents/tests/search/test_schema_fingerprint.py

@@ -1,587 +0,0 @@
-"""The schema fingerprint stamped into .index_settings.json.
-
-tantivy compares schemas by *ordered* field list, and `tantivy.Index(schema,
-path=...)` (what every write path does) raises on any difference. SCHEMA_VERSION
-is the manual guard against that, but build_schema() is edited for *parser*
-reasons - adding an alias, flipping fast=True, adding a subpath - by people not
-thinking about the on-disk index, and forgetting the bump is exactly how this
-branch's bug happened.
-
-The fingerprint is the automatic guard: it hashes the field descriptor list that
-build_schema() itself iterates, so any change to a field's name, kind, options
-or *position* forces a rebuild on its own.
-"""
-
-from __future__ import annotations
-
-import hashlib
-import json
-from typing import TYPE_CHECKING
-
-import pytest
-import tantivy
-
-from documents.search import _schema
-from documents.search._schema import SCHEMA_VERSION
-from documents.search._schema import FieldDescriptor
-from documents.search._schema import _write_sentinels
-from documents.search._schema import build_schema
-from documents.search._schema import field_descriptors
-from documents.search._schema import needs_rebuild
-from documents.search._schema import schema_fingerprint
-
-if TYPE_CHECKING:
-    from pathlib import Path
-
-    from pytest_django.fixtures import SettingsWrapper
-
-pytestmark = pytest.mark.search
-
-# The on-disk field layout of a v2 index, pinned as data. Any edit here is an
-# index-format change: it must come with a rebuild, which the fingerprint now
-# forces automatically. Reproduced from build_schema()'s output as it stood
-# before the descriptor refactor, so it also pins that the refactor changed
-# nothing.
-PINNED_DESCRIPTORS: tuple[FieldDescriptor, ...] = (
-    FieldDescriptor("id", "u64", stored=True, indexed=True, fast=True, tokenizer=None),
-    FieldDescriptor(
-        "title",
-        "text",
-        stored=True,
-        indexed=True,
-        fast=False,
-        tokenizer="paperless_text",
-    ),
-    FieldDescriptor(
-        "content",
-        "text",
-        stored=True,
-        indexed=True,
-        fast=False,
-        tokenizer="paperless_text",
-    ),
-    FieldDescriptor(
-        "correspondent",
-        "text",
-        stored=True,
-        indexed=True,
-        fast=False,
-        tokenizer="paperless_text",
-    ),
-    FieldDescriptor(
-        "document_type",
-        "text",
-        stored=True,
-        indexed=True,
-        fast=False,
-        tokenizer="paperless_text",
-    ),
-    FieldDescriptor(
-        "storage_path",
-        "text",
-        stored=True,
-        indexed=True,
-        fast=False,
-        tokenizer="paperless_text",
-    ),
-    FieldDescriptor(
-        "original_filename",
-        "text",
-        stored=True,
-        indexed=True,
-        fast=False,
-        tokenizer="paperless_text",
-    ),
-    FieldDescriptor(
-        "tag",
-        "text",
-        stored=True,
-        indexed=True,
-        fast=False,
-        tokenizer="paperless_text",
-    ),
-    FieldDescriptor(
-        "checksum",
-        "text",
-        stored=True,
-        indexed=True,
-        fast=False,
-        tokenizer="raw",
-    ),
-    FieldDescriptor("asn", "u64", stored=True, indexed=True, fast=True, tokenizer=None),
-    FieldDescriptor(
-        "page_count",
-        "u64",
-        stored=True,
-        indexed=True,
-        fast=True,
-        tokenizer=None,
-    ),
-    FieldDescriptor(
-        "num_notes",
-        "u64",
-        stored=True,
-        indexed=True,
-        fast=True,
-        tokenizer=None,
-    ),
-    FieldDescriptor(
-        "created",
-        "date",
-        stored=True,
-        indexed=True,
-        fast=True,
-        tokenizer=None,
-    ),
-    FieldDescriptor(
-        "modified",
-        "date",
-        stored=True,
-        indexed=True,
-        fast=True,
-        tokenizer=None,
-    ),
-    FieldDescriptor(
-        "added",
-        "date",
-        stored=True,
-        indexed=True,
-        fast=True,
-        tokenizer=None,
-    ),
-    FieldDescriptor(
-        "notes",
-        "json",
-        stored=True,
-        indexed=True,
-        fast=False,
-        tokenizer="paperless_text",
-    ),
-    FieldDescriptor(
-        "notes_text",
-        "text",
-        stored=True,
-        indexed=True,
-        fast=False,
-        tokenizer="paperless_text",
-    ),
-    FieldDescriptor(
-        "custom_fields",
-        "json",
-        stored=True,
-        indexed=True,
-        fast=False,
-        tokenizer="paperless_text",
-    ),
-    FieldDescriptor(
-        "title_sort",
-        "text",
-        stored=False,
-        indexed=True,
-        fast=True,
-        tokenizer="simple_analyzer",
-    ),
-    FieldDescriptor(
-        "correspondent_sort",
-        "text",
-        stored=False,
-        indexed=True,
-        fast=True,
-        tokenizer="simple_analyzer",
-    ),
-    FieldDescriptor(
-        "type_sort",
-        "text",
-        stored=False,
-        indexed=True,
-        fast=True,
-        tokenizer="simple_analyzer",
-    ),
-    FieldDescriptor(
-        "bigram_content",
-        "text",
-        stored=False,
-        indexed=True,
-        fast=False,
-        tokenizer="bigram_analyzer",
-    ),
-    FieldDescriptor(
-        "bigram_title",
-        "text",
-        stored=False,
-        indexed=True,
-        fast=False,
-        tokenizer="bigram_analyzer",
-    ),
-    FieldDescriptor(
-        "bigram_correspondent",
-        "text",
-        stored=False,
-        indexed=True,
-        fast=False,
-        tokenizer="bigram_analyzer",
-    ),
-    FieldDescriptor(
-        "bigram_document_type",
-        "text",
-        stored=False,
-        indexed=True,
-        fast=False,
-        tokenizer="bigram_analyzer",
-    ),
-    FieldDescriptor(
-        "bigram_tag",
-        "text",
-        stored=False,
-        indexed=True,
-        fast=False,
-        tokenizer="bigram_analyzer",
-    ),
-    FieldDescriptor(
-        "simple_title",
-        "text",
-        stored=False,
-        indexed=True,
-        fast=False,
-        tokenizer="simple_search_analyzer",
-    ),
-    FieldDescriptor(
-        "simple_content",
-        "text",
-        stored=False,
-        indexed=True,
-        fast=False,
-        tokenizer="simple_search_analyzer",
-    ),
-    FieldDescriptor(
-        "autocomplete_word",
-        "text",
-        stored=False,
-        indexed=True,
-        fast=False,
-        tokenizer="raw",
-    ),
-    FieldDescriptor(
-        "owner_id",
-        "u64",
-        stored=False,
-        indexed=True,
-        fast=True,
-        tokenizer=None,
-    ),
-    FieldDescriptor(
-        "viewer_id",
-        "u64",
-        stored=False,
-        indexed=True,
-        fast=True,
-        tokenizer=None,
-    ),
-    FieldDescriptor(
-        "viewer_group_id",
-        "u64",
-        stored=False,
-        indexed=True,
-        fast=True,
-        tokenizer=None,
-    ),
-)
-
-
-def _schema_fields(schema: tantivy.Schema) -> list[dict]:
-    """The tantivy-level field list, in declaration order.
-
-    tantivy-py 0.26 exposes no public introspection API on Schema, so
-    __reduce__() (its pickling hook) is the only way to recover the field list.
-    It is used here, in a test, precisely because it is the representation the
-    persisted fingerprint must NOT depend on.
-    """
-    return schema.__reduce__()[1][0]["inner"]
-
-
-def _sentinels(index_dir: Path, **overrides: object) -> None:
-    data = {
-        "schema_version": SCHEMA_VERSION,
-        "language": None,
-        "schema_fingerprint": schema_fingerprint(),
-    }
-    data.update(overrides)
-    (index_dir / ".index_settings.json").write_text(json.dumps(data))
-
-
-class TestDescriptorsDescribeTheBuiltSchema:
-    def test_descriptors_match_the_pinned_field_layout(self) -> None:
-        """
-        GIVEN:
-            - PINNED_DESCRIPTORS, a frozen snapshot of the v2 on-disk field
-              layout, reproduced from build_schema()'s output as it stood
-              before the descriptor refactor
-        WHEN:
-            - field_descriptors() is called
-        THEN:
-            - It matches the pinned layout exactly, in the same order,
-              pinning that the refactor changed nothing
-        """
-        assert tuple(field_descriptors()) == PINNED_DESCRIPTORS
-
-    def test_built_schema_matches_the_descriptors(self) -> None:
-        """
-        GIVEN:
-            - The schema built by build_schema()
-        WHEN:
-            - Its fields are read back via __reduce__() (schema.__reduce__(),
-              tantivy-py's pickling hook)
-        THEN:
-            - Every field's name, kind, stored/fast flags and tokenizer
-              match what field_descriptors() declared as input; the
-              descriptors are not a parallel description, they are the
-              input, so a descriptor edit cannot claim a shape the
-              SchemaBuilder did not actually build
-        """
-        kinds = {"text": "text", "json": "json_object", "u64": "u64", "date": "date"}
-        built = [
-            (
-                field["name"],
-                field["type"],
-                field["options"]["stored"],
-                bool(field["options"].get("fast")),
-                (field["options"].get("indexing") or {}).get("tokenizer"),
-            )
-            for field in _schema_fields(build_schema())
-        ]
-        expected = [
-            (
-                descriptor.name,
-                kinds[descriptor.kind],
-                descriptor.stored,
-                descriptor.fast,
-                descriptor.tokenizer,
-            )
-            for descriptor in field_descriptors()
-        ]
-        assert built == expected
-
-
-class TestFingerprintSensitivity:
-    def test_a_field_option_change_moves_the_fingerprint(
-        self,
-        monkeypatch: pytest.MonkeyPatch,
-    ) -> None:
-        """
-        GIVEN:
-            - The current schema fingerprint
-        WHEN:
-            - A single field descriptor's "fast" option is changed, with
-              no other change
-        THEN:
-            - The fingerprint changes
-        """
-        before = schema_fingerprint()
-        changed = field_descriptors()
-        changed[1] = changed[1]._replace(fast=True)
-        monkeypatch.setattr(_schema, "field_descriptors", lambda: changed)
-
-        assert schema_fingerprint() != before
-
-    def test_reordering_alone_moves_the_fingerprint(
-        self,
-        monkeypatch: pytest.MonkeyPatch,
-    ) -> None:
-        """
-        GIVEN:
-            - The current schema fingerprint
-        WHEN:
-            - Two field descriptors are swapped, with no other change (the
-              original bug: same fields, different declaration order)
-        THEN:
-            - The fingerprint changes; a set- or dict-based fingerprint
-              would be blind to this, and tantivy would reject every write
-              against the existing index
-        """
-        before = schema_fingerprint()
-        swapped = field_descriptors()
-        swapped[1], swapped[2] = swapped[2], swapped[1]
-        monkeypatch.setattr(_schema, "field_descriptors", lambda: swapped)
-
-        assert schema_fingerprint() != before
-
-
-class TestFingerprintIsIndependentOfTantivy:
-    def test_a_tantivy_option_key_addition_would_not_move_it(self) -> None:
-        """
-        GIVEN:
-            - The built schema's raw field list, and the same list with a
-              new tantivy-internal option key added (simulating a
-              tantivy-py upgrade)
-        WHEN:
-            - Both raw lists are hashed directly, and schema_fingerprint()
-              is compared against a hash of field_descriptors()
-        THEN:
-            - The raw hashes differ (hashing schema.__reduce__() would
-              force a global reindex on every tantivy-py upgrade), but
-              schema_fingerprint() is unaffected, since it hashes
-              field_descriptors(), never tantivy's own representation
-        """
-        fields = _schema_fields(build_schema())
-        upgraded = [
-            {**field, "options": {**field["options"], "coerce": True}}
-            for field in fields
-        ]
-        assert _hash(upgraded) != _hash(fields)
-        assert schema_fingerprint() == _fingerprint_of(field_descriptors())
-
-    def test_fingerprint_never_touches_the_schema_builder(
-        self,
-        monkeypatch: pytest.MonkeyPatch,
-    ) -> None:
-        """
-        GIVEN:
-            - tantivy.SchemaBuilder replaced with a stand-in that raises if
-              constructed
-        WHEN:
-            - build_schema() is called (and raises), then
-              schema_fingerprint() is called again
-        THEN:
-            - schema_fingerprint() still matches its earlier value,
-              proving it never consults SchemaBuilder
-        """
-        before = schema_fingerprint()
-
-        class _RemovedSchemaBuilder:
-            def __init__(self) -> None:
-                raise AssertionError("tantivy.SchemaBuilder was consulted")
-
-        monkeypatch.setattr(tantivy, "SchemaBuilder", _RemovedSchemaBuilder)
-        with pytest.raises(AssertionError):
-            build_schema()
-
-        assert schema_fingerprint() == before
-
-
-def _hash(payload: object) -> str:
-    return hashlib.blake2b(json.dumps(payload).encode()).hexdigest()
-
-
-def _fingerprint_of(descriptors: list[FieldDescriptor]) -> str:
-    return _hash([list(descriptor) for descriptor in descriptors])
-
-
-class TestNeedsRebuildOnFingerprint:
-    def test_matching_fingerprint_does_not_rebuild(
-        self,
-        index_dir: Path,
-        settings: SettingsWrapper,
-    ) -> None:
-        """
-        GIVEN:
-            - An index directory whose sentinel file records the current
-              schema_fingerprint()
-        WHEN:
-            - needs_rebuild() is called
-        THEN:
-            - It returns False
-        """
-        settings.SEARCH_LANGUAGE = None
-        _sentinels(index_dir)
-
-        assert needs_rebuild(index_dir) is False
-
-    def test_stale_fingerprint_rebuilds_despite_a_matching_version(
-        self,
-        index_dir: Path,
-        settings: SettingsWrapper,
-        monkeypatch: pytest.MonkeyPatch,
-    ) -> None:
-        """
-        GIVEN:
-            - An index directory whose sentinel matches SCHEMA_VERSION,
-              but field_descriptors() is patched to add a field the
-              fingerprint never saw (schema edited, version not bumped)
-        WHEN:
-            - needs_rebuild() is called
-        THEN:
-            - It returns True; without the fingerprint check,
-              `reindex --if-needed` would report the index up to date and
-              every subsequent write would raise
-        """
-        settings.SEARCH_LANGUAGE = None
-        _sentinels(index_dir)
-        extended = [
-            *field_descriptors(),
-            FieldDescriptor(
-                "new_field",
-                "u64",
-                stored=False,
-                indexed=True,
-                fast=True,
-                tokenizer=None,
-            ),
-        ]
-        monkeypatch.setattr(_schema, "field_descriptors", lambda: extended)
-
-        assert needs_rebuild(index_dir) is True
-
-    def test_reordered_schema_rebuilds(
-        self,
-        index_dir: Path,
-        settings: SettingsWrapper,
-        monkeypatch: pytest.MonkeyPatch,
-    ) -> None:
-        """
-        GIVEN:
-            - An index directory whose sentinel matches the current
-              fingerprint, but field_descriptors() is patched to swap two
-              fields' order
-        WHEN:
-            - needs_rebuild() is called
-        THEN:
-            - It returns True
-        """
-        settings.SEARCH_LANGUAGE = None
-        _sentinels(index_dir)
-        reordered = field_descriptors()
-        reordered[1], reordered[2] = reordered[2], reordered[1]
-        monkeypatch.setattr(_schema, "field_descriptors", lambda: reordered)
-
-        assert needs_rebuild(index_dir) is True
-
-    def test_missing_fingerprint_rebuilds(
-        self,
-        index_dir: Path,
-        settings: SettingsWrapper,
-    ) -> None:
-        """
-        GIVEN:
-            - An index directory whose sentinel has no "schema_fingerprint"
-              key at all
-        WHEN:
-            - needs_rebuild() is called
-        THEN:
-            - It returns True; an index whose schema shape nobody recorded
-              is rebuilt rather than trusted
-        """
-        settings.SEARCH_LANGUAGE = None
-        (index_dir / ".index_settings.json").write_text(
-            json.dumps({"schema_version": SCHEMA_VERSION, "language": None}),
-        )
-
-        assert needs_rebuild(index_dir) is True
-
-    def test_written_sentinels_satisfy_the_check(
-        self,
-        index_dir: Path,
-        settings: SettingsWrapper,
-    ) -> None:
-        """
-        GIVEN:
-            - An index directory whose sentinels are written by
-              _write_sentinels() itself
-        WHEN:
-            - needs_rebuild() is called
-        THEN:
-            - It returns False
-        """
-        settings.SEARCH_LANGUAGE = "en"
-        _write_sentinels(index_dir)
-
-        assert needs_rebuild(index_dir) is False

src/documents/tests/search/test_schema_version.py

@@ -1,178 +0,0 @@
-"""SCHEMA_VERSION must change whenever build_schema()'s field list or order does.
-
-tantivy compares schemas by *ordered* field list. ``Index.open()`` loads the
-schema from the index's own ``meta.json``, so reads against an index built by an
-older release keep working after a field reorder. Writes do not:
-``WriteBatch.__enter__`` calls ``tantivy.Index(build_schema(), path=...)``, an
-open-or-create that raises ``ValueError`` on any schema difference. Nothing
-catches that ValueError, so consumption, index_document and bulk edit all
-hard-fail while ``/api/status/`` still reports the index healthy.
-
-The only thing that saves such an install is ``needs_rebuild()`` noticing the
-version stamped in ``.index_settings.json`` is stale.
-"""
-
-from __future__ import annotations
-
-import json
-from typing import TYPE_CHECKING
-
-import pytest
-import tantivy
-from django.conf import settings as django_settings
-
-from documents.search._schema import build_schema
-from documents.search._schema import needs_rebuild
-from documents.search._schema import open_or_rebuild_index
-
-if TYPE_CHECKING:
-    from pathlib import Path
-
-pytestmark = [pytest.mark.search]
-
-RELEASED_V1_SCHEMA_VERSION = 1
-
-
-def _build_released_v1_schema() -> tantivy.Schema:
-    """Frozen copy of build_schema() as shipped in v3.0.x (schema version 1).
-
-    Deliberately duplicated rather than imported: it must keep describing the
-    on-disk layout of already-deployed indexes even as build_schema() evolves.
-    """
-    sb = tantivy.SchemaBuilder()
-
-    sb.add_unsigned_field("id", stored=True, indexed=True, fast=True)
-    sb.add_text_field("checksum", stored=True, tokenizer_name="raw")
-
-    for field in (
-        "title",
-        "correspondent",
-        "document_type",
-        "storage_path",
-        "original_filename",
-        "content",
-    ):
-        sb.add_text_field(field, stored=True, tokenizer_name="paperless_text")
-
-    for field in ("title_sort", "correspondent_sort", "type_sort"):
-        sb.add_text_field(
-            field,
-            stored=False,
-            tokenizer_name="simple_analyzer",
-            fast=True,
-        )
-
-    for field in (
-        "bigram_content",
-        "bigram_title",
-        "bigram_correspondent",
-        "bigram_document_type",
-        "bigram_tag",
-    ):
-        sb.add_text_field(field, stored=False, tokenizer_name="bigram_analyzer")
-
-    for field in ("simple_title", "simple_content"):
-        sb.add_text_field(field, stored=False, tokenizer_name="simple_search_analyzer")
-
-    sb.add_text_field("autocomplete_word", stored=False, tokenizer_name="raw")
-    sb.add_text_field("tag", stored=True, tokenizer_name="paperless_text")
-
-    sb.add_json_field("notes", stored=True, tokenizer_name="paperless_text")
-    sb.add_text_field("notes_text", stored=True, tokenizer_name="paperless_text")
-    sb.add_json_field("custom_fields", stored=True, tokenizer_name="paperless_text")
-
-    for field in (
-        "correspondent_id",
-        "document_type_id",
-        "storage_path_id",
-        "tag_id",
-        "owner_id",
-        "viewer_id",
-        "viewer_group_id",
-    ):
-        sb.add_unsigned_field(field, stored=False, indexed=True, fast=True)
-
-    for field in ("created", "modified", "added"):
-        sb.add_date_field(field, stored=True, indexed=True, fast=True)
-
-    for field in ("asn", "page_count", "num_notes"):
-        sb.add_unsigned_field(field, stored=True, indexed=True, fast=True)
-
-    return sb.build()
-
-
-@pytest.fixture
-def released_v1_index(tmp_path: Path) -> Path:
-    """An index directory as a v3.0.x install would leave it on disk."""
-    index_dir = tmp_path / "index"
-    index_dir.mkdir()
-    tantivy.Index(_build_released_v1_schema(), path=str(index_dir))
-    (index_dir / ".index_settings.json").write_text(
-        json.dumps(
-            {
-                "schema_version": RELEASED_V1_SCHEMA_VERSION,
-                "language": django_settings.SEARCH_LANGUAGE,
-            },
-        ),
-    )
-    return index_dir
-
-
-class TestUpgradeFromReleasedV1Index:
-    def test_released_v1_index_is_flagged_for_rebuild(
-        self,
-        released_v1_index: Path,
-    ) -> None:
-        """
-        GIVEN:
-            - An index directory laid out exactly as a v3.0.x (schema
-              version 1) install would leave it
-        WHEN:
-            - needs_rebuild() is called
-        THEN:
-            - It returns True; if this fails,
-              `document_index reindex --if-needed` prints "Search index is
-              up to date" and skips, leaving the mismatched index in place
-        """
-        assert needs_rebuild(released_v1_index) is True
-
-    def test_opening_a_v1_index_leaves_it_writable(
-        self,
-        released_v1_index: Path,
-    ) -> None:
-        """
-        GIVEN:
-            - A v1 index directory
-        WHEN:
-            - open_or_rebuild_index() is called against it
-        THEN:
-            - The directory can be reopened with the current schema
-              without raising; end to end, open_or_rebuild_index must
-              hand back an index the write path can reopen. Before the
-              version bump, needs_rebuild() returned False here, and the
-              stale directory survived untouched, so every subsequent
-              write against it raised tantivy's own schema-mismatch
-              ValueError
-        """
-        open_or_rebuild_index(released_v1_index)
-
-        tantivy.Index(build_schema(), path=str(released_v1_index))
-
-    def test_rebuilt_index_is_not_rebuilt_again(
-        self,
-        released_v1_index: Path,
-    ) -> None:
-        """
-        GIVEN:
-            - A v1 index directory that has just been rebuilt by
-              open_or_rebuild_index()
-        WHEN:
-            - needs_rebuild() is called again
-        THEN:
-            - It returns False; the rebuild must stamp the version it
-              actually wrote, otherwise every startup wipes and reindexes
-              the whole corpus
-        """
-        open_or_rebuild_index(released_v1_index)
-
-        assert needs_rebuild(released_v1_index) is False

src/documents/tests/search/test_stem_pattern_text.py

@@ -1,37 +0,0 @@
-from __future__ import annotations
-
-import pytest
-
-from documents.search._tokenizer import stem_pattern_text
-
-pytestmark = pytest.mark.search
-
-
-class TestStemPatternText:
-    def test_unsupported_language_returns_text_unchanged(self) -> None:
-        """
-        GIVEN:
-            - A language code with no Snowball stemmer mapping
-        WHEN:
-            - A pattern run is stemmed for that language
-        THEN:
-            - The run is returned unchanged, since the stemming gate that
-              disables stemming for an unsupported language also disables
-              the pattern-side stemmer
-        """
-        assert stem_pattern_text("running", "klingon") == "running"
-
-    def test_run_past_remove_long_limit_returns_text_unchanged(self) -> None:
-        """
-        GIVEN:
-            - A supported language and a run longer than the remove_long
-              filter's limit (129 characters, matching Document.title's
-              max_length)
-        WHEN:
-            - The over-long run is stemmed
-        THEN:
-            - The remove_long filter drops the token entirely, leaving no
-              stem to substitute, so the run is returned unchanged
-        """
-        long_run = "a" * 130
-        assert stem_pattern_text(long_run, "en") == long_run

src/documents/tests/search/test_tokenizer.py

@@ -7,8 +7,8 @@ import pytest
 import tantivy
 
 from documents.search._tokenizer import _bigram_analyzer
+from documents.search._tokenizer import _paperless_text
 from documents.search._tokenizer import _simple_search_analyzer
-from documents.search._tokenizer import paperless_text_analyzer
 from documents.search._tokenizer import register_tokenizers
 
 if TYPE_CHECKING:
@@ -25,7 +25,7 @@ class TestTokenizers:
         sb.add_text_field("content", stored=True, tokenizer_name="paperless_text")
         schema = sb.build()
         idx = tantivy.Index(schema, path=None)
-        idx.register_tokenizer("paperless_text", paperless_text_analyzer(""))
+        idx.register_tokenizer("paperless_text", _paperless_text(""))
         return idx
 
     @pytest.fixture

src/documents/tests/search/test_translate.py

@@ -0,0 +1,810 @@
+from __future__ import annotations
+
+from datetime import UTC
+from datetime import datetime
+from typing import TYPE_CHECKING
+from zoneinfo import ZoneInfo
+
+import pytest
+import time_machine
+
+from documents.search._dates import _precision_bounds
+
+if TYPE_CHECKING:
+    import tantivy
+from documents.search._query import _FIELD_BOOSTS
+from documents.search._query import DEFAULT_SEARCH_FIELDS
+from documents.search._translate import OPEN_HI
+from documents.search._translate import OPEN_LO
+from documents.search._translate import Comma
+from documents.search._translate import FieldRange
+from documents.search._translate import FieldValue
+from documents.search._translate import FieldValueList
+from documents.search._translate import InvalidDateQuery
+from documents.search._translate import Passthrough
+from documents.search._translate import resolve_commas
+from documents.search._translate import scan
+from documents.search._translate import translate_query
+from documents.search._translate import translate_range
+from documents.search._translate import translate_scalar
+
+
+@pytest.mark.search
+class TestPrecisionBounds:
+    @pytest.mark.parametrize(
+        ("digits", "expected"),
+        [
+            ("2020", ((2020, 1, 1), (2021, 1, 1))),
+            ("202003", ((2020, 3, 1), (2020, 4, 1))),
+            ("202012", ((2020, 12, 1), (2021, 1, 1))),
+            ("20200115", ((2020, 1, 15), (2020, 1, 16))),
+            ("20201231", ((2020, 12, 31), (2021, 1, 1))),
+        ],
+    )
+    def test_valid(self, digits, expected):
+        lo, hi = _precision_bounds(digits)
+        assert (lo.year, lo.month, lo.day) == expected[0]
+        assert (hi.year, hi.month, hi.day) == expected[1]
+
+    @pytest.mark.parametrize("digits", ["202023", "20200230", "20201301", "20", "abcd"])
+    def test_invalid_returns_none(self, digits):
+        assert _precision_bounds(digits) is None
+
+
+@pytest.mark.search
+class TestScan:
+    def test_plain_words_are_passthrough(self):
+        assert scan("bank statement") == [Passthrough("bank statement")]
+
+    def test_field_value(self):
+        assert scan("created:2020") == [FieldValue("created", "2020")]
+
+    def test_field_value_in_boolean(self):
+        toks = scan("created:2020 OR foo")
+        assert toks == [
+            FieldValue("created", "2020"),
+            Passthrough(" OR foo"),
+        ]
+
+    def test_field_value_in_parens(self):
+        toks = scan("(created:2020 OR foo)")
+        assert toks == [
+            Passthrough("("),
+            FieldValue("created", "2020"),
+            Passthrough(" OR foo)"),
+        ]
+
+    def test_quoted_value(self):
+        assert scan('correspondent:"A B"') == [FieldValue("correspondent", '"A B"')]
+
+    def test_field_range(self):
+        assert scan("created:[2020 TO 2021]") == [
+            FieldRange("created", "[", "2020", "2021", "]"),
+        ]
+
+    @pytest.mark.parametrize(
+        ("query", "expected"),
+        [
+            pytest.param(
+                "created:[2020 to]",
+                FieldRange("created", "[", "2020", "", "]"),
+                id="open_upper",
+            ),
+            pytest.param(
+                "created:[to 2020]",
+                FieldRange("created", "[", "", "2020", "]"),
+                id="open_lower",
+            ),
+        ],
+    )
+    def test_open_range(self, query, expected):
+        assert scan(query) == [expected]
+
+    def test_comma_inside_range_not_split(self):
+        # No depth-0 comma here; the whole thing is one range token.
+        toks = scan("created:[2020 TO 2021]")
+        assert len(toks) == 1
+
+    # --- Edge-case / regression tests (scan must never raise) ---
+
+    def test_url_is_passthrough(self):
+        # "http" is not a known field; the whole URL must pass through verbatim.
+        assert scan("http://example.com") == [Passthrough("http://example.com")]
+
+    def test_unterminated_quote_is_passthrough(self):
+        # title is a known field but the quoted value has no closing quote;
+        # _consume_value returns None so the whole string falls into passthrough.
+        assert scan('title:"abc') == [Passthrough('title:"abc')]
+
+    def test_unterminated_bracket_is_passthrough(self):
+        # created is a known field but the range bracket is never closed;
+        # _consume_range returns None so the whole string falls into passthrough.
+        assert scan("created:[2020") == [Passthrough("created:[2020")]
+
+    def test_empty_value_at_end_is_passthrough(self):
+        # created is a known field but there is no value after the colon
+        # (_consume_value returns None for start >= n), so passthrough.
+        assert scan("created:") == [Passthrough("created:")]
+
+    def test_value_containing_colon(self):
+        # The bare-word value reader stops at whitespace/paren, not at colon,
+        # so "2020:30" is consumed as a single value token.
+        assert scan("created:2020:30") == [FieldValue("created", "2020:30")]
+
+    def test_comma_followed_by_unconsumable_value_stops(self):
+        # A comma followed by whitespace is neither a value-list continuation nor a
+        # clause separator: the value stops and the comma stays as passthrough.
+        assert scan("tag:foo, bar") == [
+            FieldValue("tag", "foo"),
+            Passthrough(", bar"),
+        ]
+
+    def test_bracket_without_to_is_open_upper_bound(self):
+        # A bracketed value with no TO falls back to (value, "") -> open upper bound.
+        assert scan("created:[2020]") == [
+            FieldRange("created", "[", "2020", "", "]"),
+        ]
+
+    def test_known_field_name_midword_is_passthrough(self):
+        # A known field name embedded mid-word is not a field token (the
+        # word-boundary guard); the whole run stays passthrough.
+        assert scan("xtag:foo") == [Passthrough("xtag:foo")]
+
+
+@pytest.mark.search
+class TestCommaResolution:
+    def test_value_list_multi_value_field(self):
+        toks = resolve_commas(scan("tag:foo,bar"))
+        assert toks == [FieldValueList("tag", ("foo", "bar"))]
+
+    def test_value_list_three(self):
+        toks = resolve_commas(scan("tag_id:1,2,3"))
+        assert toks == [FieldValueList("tag_id", ("1", "2", "3"))]
+
+    def test_text_field_comma_is_literal(self):
+        # correspondent is not multi-value: comma stays inside the value.
+        toks = resolve_commas(scan("correspondent:foo,bar"))
+        assert toks == [FieldValue("correspondent", "foo,bar")]
+
+    def test_clause_separator_before_known_field(self):
+        toks = resolve_commas(scan("tag:foo,type:bar"))
+        assert toks == [FieldValue("tag", "foo"), Comma(), FieldValue("type", "bar")]
+
+    def test_clause_separator_after_range(self):
+        toks = resolve_commas(scan("created:[2020 TO 2021],added:[2022 TO 2023]"))
+        assert toks == [
+            FieldRange("created", "[", "2020", "2021", "]"),
+            Comma(),
+            FieldRange("added", "[", "2022", "2023", "]"),
+        ]
+
+    def test_clause_separator_after_quote(self):
+        toks = resolve_commas(scan('correspondent:"A B",created:[2020 TO 2021]'))
+        assert toks == [
+            FieldValue("correspondent", '"A B"'),
+            Comma(),
+            FieldRange("created", "[", "2020", "2021", "]"),
+        ]
+
+    def test_url_comma_is_literal_passthrough(self):
+        toks = resolve_commas(scan("http://example.com/a,b"))
+        assert toks == [Passthrough("http://example.com/a,b")]
+
+    def test_non_multi_value_comma_is_literal(self):
+        # title is not in MULTI_VALUE_FIELDS: comma stays inside the value.
+        toks = resolve_commas(scan("title:10,20"))
+        assert toks == [FieldValue("title", "10,20")]
+
+    def test_clause_separator_before_known_date_field(self):
+        # The comma between a bare value and a known date field acts as a
+        # clause separator; both sides survive as distinct tokens.
+        toks = resolve_commas(scan("correspondent:foo,created:[2020 TO 2021]"))
+        assert toks == [
+            FieldValue("correspondent", "foo"),
+            Comma(),
+            FieldRange("created", "[", "2020", "2021", "]"),
+        ]
+
+
+@pytest.mark.search
+class TestTranslateScalar:
+    @pytest.mark.parametrize(
+        ("field", "value", "expected"),
+        [
+            (
+                "created",
+                "2020",
+                "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",
+            ),
+            (
+                "created",
+                "202003",
+                "created:[2020-03-01T00:00:00Z TO 2020-04-01T00:00:00Z}",
+            ),
+            (
+                "created",
+                "20200115",
+                "created:[2020-01-15T00:00:00Z TO 2020-01-16T00:00:00Z}",
+            ),
+            (
+                "created",
+                "2020-01-15",
+                "created:[2020-01-15T00:00:00Z TO 2020-01-16T00:00:00Z}",
+            ),
+            (
+                "created",
+                "2020-03",
+                "created:[2020-03-01T00:00:00Z TO 2020-04-01T00:00:00Z}",
+            ),
+        ],
+    )
+    def test_partial_and_iso_dates(self, field: str, value: str, expected: str) -> None:
+        assert translate_scalar(field, value, UTC) == expected
+
+    def test_invalid_date_raises(self) -> None:
+        with pytest.raises(InvalidDateQuery) as exc_info:
+            translate_scalar("created", "202023", UTC)
+        assert exc_info.value.field == "created"
+        assert exc_info.value.value == "202023"
+
+    def test_keyword_delegates(self) -> None:
+        # keyword path produces a half-open range; just assert it is a created range
+        out = translate_scalar("created", "today", UTC)
+        assert out.startswith("created:[") and out.endswith("}")
+
+    def test_14digit_compact_datetime(self) -> None:
+        out = translate_scalar("created", "20240115120000", UTC)
+        assert "20240115120000" not in out
+        assert out.startswith("created:")
+        assert out == "created:[2024-01-15T12:00:00Z TO 2024-01-15T12:00:00Z]"
+
+    def test_14digit_invalid_month_raises(self) -> None:
+        with pytest.raises(InvalidDateQuery) as exc_info:
+            translate_scalar("created", "20231300120000", UTC)
+        assert exc_info.value.field == "created"
+        assert exc_info.value.value == "20231300120000"
+
+    def test_unrecognized_value_raises(self) -> None:
+        # A value that is not a keyword, digits, ISO date, or compact timestamp
+        # raises rather than producing invalid Tantivy syntax or silently matching
+        # nothing.
+        with pytest.raises(InvalidDateQuery) as exc_info:
+            translate_scalar("created", "garbage", UTC)
+        assert exc_info.value.field == "created"
+        assert exc_info.value.value == "garbage"
+
+
+@pytest.mark.search
+class TestTranslateRange:
+    @pytest.mark.parametrize(
+        ("lo", "hi", "expected"),
+        [
+            ("2005", "2009", "created:[2005-01-01T00:00:00Z TO 2010-01-01T00:00:00Z}"),
+            (
+                "202001",
+                "202006",
+                "created:[2020-01-01T00:00:00Z TO 2020-07-01T00:00:00Z}",
+            ),
+            (
+                "20200101",
+                "20201231",
+                "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",
+            ),
+            (
+                "2020-01-01",
+                "2020-12-31",
+                "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",
+            ),
+        ],
+    )
+    def test_absolute_ranges(self, lo, hi, expected):
+        assert translate_range("created", lo, hi, UTC) == expected
+
+    def test_reversed_swaps(self):
+        assert translate_range("created", "2009", "2005", UTC) == (
+            "created:[2005-01-01T00:00:00Z TO 2010-01-01T00:00:00Z}"
+        )
+
+    def test_open_upper(self):
+        out = translate_range("created", "2020", "", UTC)
+        assert out == f"created:[2020-01-01T00:00:00Z TO {OPEN_HI}]"
+
+    def test_open_lower(self):
+        out = translate_range("created", "", "2020", UTC)
+        assert out == f"created:[{OPEN_LO} TO 2021-01-01T00:00:00Z}}"
+
+    def test_invalid_bound_raises(self):
+        with pytest.raises(InvalidDateQuery) as exc_info:
+            translate_range("created", "202023", "2025", UTC)
+        assert exc_info.value.field == "created"
+        assert exc_info.value.value == "202023"
+
+    def test_invalid_high_bound_raises(self):
+        # Low bound parses, high bound does not -> raise on the high bound.
+        with pytest.raises(InvalidDateQuery) as exc_info:
+            translate_range("created", "2020", "garbage", UTC)
+        assert exc_info.value.field == "created"
+        assert exc_info.value.value == "garbage"
+
+
+@pytest.mark.search
+class TestTranslateQuery:
+    @pytest.mark.parametrize(
+        ("raw", "expected"),
+        [
+            (
+                "created:2020",
+                "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",
+            ),
+            ("tag:foo,bar", "tag:foo AND tag:bar"),
+            # 'type' is a user-facing alias rewritten to 'document_type' (the real schema field)
+            ("tag:foo,type:bar", "tag:foo AND document_type:bar"),
+            (
+                "created:[2020 TO 2021],added:[2022 TO 2023]",
+                (
+                    "created:[2020-01-01T00:00:00Z TO 2022-01-01T00:00:00Z}"
+                    " AND "
+                    "added:[2022-01-01T00:00:00Z TO 2024-01-01T00:00:00Z}"
+                ),
+            ),
+            # correspondent is not multi-value: comma stays literal inside the value
+            ("correspondent:foo,bar", "correspondent:foo,bar"),
+        ],
+    )
+    def test_golden(self, raw: str, expected: str) -> None:
+        assert translate_query(raw, UTC) == expected
+
+    @pytest.mark.parametrize(
+        "raw",
+        [
+            "created:2020",
+            "created:202003",
+            "created:[20200101 TO 20201231]",
+            "created:[2020-01-01 TO 2020-12-31]",
+            "created:[2020 to]",
+            "created:[to 2020]",
+            "title:x,created:[2020 TO 2021]",
+            "created:2020 OR foo",
+            "(created:2020 OR invoice)",
+            "tag:foo,type:bar",
+            "bank statement",
+        ],
+    )
+    def test_parse_acceptance(self, index: tantivy.Index, raw: str) -> None:
+        translated = translate_query(raw, UTC)
+        # Must not raise:
+        index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
+
+
+@pytest.mark.search
+class TestFieldAliasing:
+    """Whoosh->Tantivy field-name aliasing (type/path -> document_type/storage_path)."""
+
+    def test_type_alias(self) -> None:
+        assert translate_query("type:invoice", UTC) == "document_type:invoice"
+
+    def test_path_alias(self) -> None:
+        assert translate_query("path:/foo/bar", UTC) == "storage_path:/foo/bar"
+
+    def test_type_id_alias(self) -> None:
+        assert translate_query("type_id:5", UTC) == "document_type_id:5"
+
+    def test_path_id_alias(self) -> None:
+        assert translate_query("path_id:7", UTC) == "storage_path_id:7"
+
+    def test_clause_separator_plus_alias(self) -> None:
+        # Comma between known fields acts as AND separator; alias still applied.
+        assert (
+            translate_query("tag:foo,type:bar", UTC) == "tag:foo AND document_type:bar"
+        )
+
+    def test_type_range_alias(self) -> None:
+        # type is not a date field; range passes through verbatim with alias applied.
+        assert (
+            translate_query("type:[2020 TO 2021]", UTC)
+            == "document_type:[2020 TO 2021]"
+        )
+
+    def test_parse_acceptance_type(self, index: tantivy.Index) -> None:
+        # Translated output must be accepted by the real Tantivy parser.
+        translated = translate_query("type:invoice", UTC)
+        index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
+
+    def test_parse_acceptance_path(self, index: tantivy.Index) -> None:
+        translated = translate_query("path:foo", UTC)
+        index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
+
+
+# Freeze time so relative-date tests are deterministic.
+_FROZEN_NOW = datetime(2026, 3, 28, 12, 0, 0, tzinfo=UTC)
+
+
+@pytest.mark.search
+class TestRelativeRanges:
+    """Relative date-range tokens resolved against a frozen clock."""
+
+    @time_machine.travel(_FROZEN_NOW, tick=False)
+    def test_minus_7_days_to_now(self) -> None:
+        assert translate_query("added:[-7 days to now]", UTC) == (
+            "added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]"
+        )
+
+    @time_machine.travel(_FROZEN_NOW, tick=False)
+    def test_minus_1_week_to_now(self) -> None:
+        assert translate_query("added:[-1 week to now]", UTC) == (
+            "added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]"
+        )
+
+    @time_machine.travel(_FROZEN_NOW, tick=False)
+    def test_minus_1_month_to_now(self) -> None:
+        assert translate_query("created:[-1 month to now]", UTC) == (
+            "created:[2026-02-28T12:00:00Z TO 2026-03-28T12:00:00Z]"
+        )
+
+    @time_machine.travel(_FROZEN_NOW, tick=False)
+    def test_minus_1_year_to_now(self) -> None:
+        assert translate_query("modified:[-1 year to now]", UTC) == (
+            "modified:[2025-03-28T12:00:00Z TO 2026-03-28T12:00:00Z]"
+        )
+
+    @time_machine.travel(_FROZEN_NOW, tick=False)
+    def test_minus_3_hours_to_now(self) -> None:
+        assert translate_query("added:[-3 hours to now]", UTC) == (
+            "added:[2026-03-28T09:00:00Z TO 2026-03-28T12:00:00Z]"
+        )
+
+    @time_machine.travel(_FROZEN_NOW, tick=False)
+    def test_uppercase_units(self) -> None:
+        assert translate_query("added:[-1 WEEK TO NOW]", UTC) == (
+            "added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]"
+        )
+
+    @time_machine.travel(_FROZEN_NOW, tick=False)
+    def test_now_minus_7d_compact(self) -> None:
+        assert translate_query("added:[now-7d TO now]", UTC) == (
+            "added:[2026-03-21T12:00:00Z TO 2026-03-28T12:00:00Z]"
+        )
+
+    @time_machine.travel(_FROZEN_NOW, tick=False)
+    def test_reversed_range_swapped(self) -> None:
+        # now+1h TO now-1h is reversed; translate_range swaps -> lo=now-1h, hi=now+1h
+        assert translate_query("added:[now+1h TO now-1h]", UTC) == (
+            "added:[2026-03-28T11:00:00Z TO 2026-03-28T13:00:00Z]"
+        )
+
+    @pytest.mark.parametrize(
+        "raw",
+        [
+            "added:[-7 days to now]",
+            "added:[-1 week to now]",
+            "created:[-1 month to now]",
+            "modified:[-1 year to now]",
+            "added:[-3 hours to now]",
+            "added:[now-7d TO now]",
+            "added:[now+1h TO now-1h]",
+        ],
+    )
+    @time_machine.travel(_FROZEN_NOW, tick=False)
+    def test_parse_acceptance(self, index: tantivy.Index, raw: str) -> None:
+        translated = translate_query(raw, UTC)
+        index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
+
+
+@pytest.mark.search
+class TestWhooshUnitAbbreviations:
+    """
+    Whoosh's PlusMinus date grammar accepted abbreviated unit spellings
+    (e.g. "yrs", "mos", "wks", "hrs", "mins", "secs"); saved views/searches
+    created under the old Whoosh backend can contain those tokens (see
+    https://github.com/paperless-ngx/paperless-ngx/issues/13482), so the
+    Tantivy translator must still accept them.
+    """
+
+    @time_machine.travel(_FROZEN_NOW, tick=False)
+    def test_minus_999_yrs(self) -> None:
+        assert translate_query("created:[-999yrs to now]", UTC) == (
+            "created:[1027-03-28T12:00:00Z TO 2026-03-28T12:00:00Z]"
+        )
+
+    @pytest.mark.parametrize(
+        ("token", "expected_lo"),
+        [
+            ("-1y", "2025-03-28T12:00:00Z"),
+            ("-1yr", "2025-03-28T12:00:00Z"),
+            ("-3mos", "2025-12-28T12:00:00Z"),
+            ("-3mo", "2025-12-28T12:00:00Z"),
+            ("-2wks", "2026-03-14T12:00:00Z"),
+            ("-2wk", "2026-03-14T12:00:00Z"),
+            ("-5dys", "2026-03-23T12:00:00Z"),
+            ("-5dy", "2026-03-23T12:00:00Z"),
+            ("-1hrs", "2026-03-28T11:00:00Z"),
+            ("-1hr", "2026-03-28T11:00:00Z"),
+            ("-10mins", "2026-03-28T11:50:00Z"),
+            ("-10min", "2026-03-28T11:50:00Z"),
+            ("-30secs", "2026-03-28T11:59:30Z"),
+            ("-30sec", "2026-03-28T11:59:30Z"),
+        ],
+    )
+    @time_machine.travel(_FROZEN_NOW, tick=False)
+    def test_abbreviated_units(self, token: str, expected_lo: str) -> None:
+        assert translate_query(f"added:[{token} to now]", UTC) == (
+            f"added:[{expected_lo} TO 2026-03-28T12:00:00Z]"
+        )
+
+    @pytest.mark.parametrize(
+        "raw",
+        [
+            "created:[-999yrs to now]",
+            "added:[-1y to now]",
+            "created:[-3mos to now]",
+            "added:[-2wks to now]",
+            "added:[-5dys to now]",
+            "added:[-1hrs to now]",
+            "added:[-10mins to now]",
+            "added:[-30secs to now]",
+        ],
+    )
+    @time_machine.travel(_FROZEN_NOW, tick=False)
+    def test_parse_acceptance(self, index: tantivy.Index, raw: str) -> None:
+        translated = translate_query(raw, UTC)
+        index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
+
+
+@pytest.mark.search
+class TestOperatorNormalization:
+    """Post-render operator normalization in translate_query."""
+
+    def test_spaced_dash_removed(self) -> None:
+        assert (
+            translate_query("H52.1 - Kurzsichtigkeit", UTC) == "H52.1 Kurzsichtigkeit"
+        )
+
+    def test_spaced_dash_simple(self) -> None:
+        assert translate_query("bar - baz", UTC) == "bar baz"
+
+    def test_trailing_operator_stripped(self) -> None:
+        assert translate_query("foo -", UTC) == "foo"
+
+    def test_date_range_preserved(self) -> None:
+        out = translate_query("created:[2020 TO 2021]", UTC)
+        # Must not corrupt the ISO range
+        assert out == "created:[2020-01-01T00:00:00Z TO 2022-01-01T00:00:00Z}"
+
+    def test_date_scalar_with_or(self) -> None:
+        out = translate_query("created:2020 OR foo", UTC)
+        # The created scalar becomes a range; " OR foo" passes through verbatim.
+        assert out.startswith("created:[")
+        assert "OR foo" in out
+
+    def test_parse_acceptance_spaced_dash(self, index: tantivy.Index) -> None:
+        translated = translate_query("H52.1 - Kurzsichtigkeit", UTC)
+        index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
+
+    def test_parse_acceptance_trailing_op(self, index: tantivy.Index) -> None:
+        translated = translate_query("foo -", UTC)
+        index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
+
+
+@pytest.mark.search
+class TestMultiWordDateKeywords:
+    """scan() must consume multi-word date keywords as a single value."""
+
+    def test_scan_previous_week_as_single_token(self) -> None:
+        # "created:previous week" must produce one FieldValue with value "previous week",
+        # not FieldValue("created","previous") + Passthrough(" week").
+        toks = scan("created:previous week")
+        assert toks == [FieldValue("created", "previous week")]
+
+    def test_scan_this_month_as_single_token(self) -> None:
+        toks = scan("added:this month")
+        assert toks == [FieldValue("added", "this month")]
+
+    def test_scan_previous_month_as_single_token(self) -> None:
+        toks = scan("created:previous month")
+        assert toks == [FieldValue("created", "previous month")]
+
+    def test_scan_this_year_as_single_token(self) -> None:
+        toks = scan("added:this year")
+        assert toks == [FieldValue("added", "this year")]
+
+    def test_scan_previous_year_as_single_token(self) -> None:
+        toks = scan("created:previous year")
+        assert toks == [FieldValue("created", "previous year")]
+
+    def test_scan_previous_quarter_as_single_token(self) -> None:
+        toks = scan("created:previous quarter")
+        assert toks == [FieldValue("created", "previous quarter")]
+
+    def test_quoted_multi_word_keyword_still_works(self) -> None:
+        # The quoted form must continue to work as before.
+        toks = scan('created:"previous week"')
+        assert toks == [FieldValue("created", '"previous week"')]
+
+    def test_non_date_field_not_affected(self) -> None:
+        # "previous" stops at the space for non-date fields; " week" passes through.
+        toks = scan("correspondent:previous week")
+        assert toks == [
+            FieldValue("correspondent", "previous"),
+            Passthrough(" week"),
+        ]
+
+
+@pytest.mark.search
+class TestKeywordDateResolution:
+    """Relative date keywords resolve to exact ISO ranges against a frozen clock.
+
+    Frozen at 2026-03-28 12:00 UTC (a Saturday in Q1) so the week, month,
+    quarter and year rollovers are all exercised by a single anchor.
+    """
+
+    # created is a DateField: bounds are UTC midnight, no timezone offset.
+    @pytest.mark.parametrize(
+        ("keyword", "expected"),
+        [
+            pytest.param(
+                "today",
+                "created:[2026-03-28T00:00:00Z TO 2026-03-29T00:00:00Z}",
+                id="today",
+            ),
+            pytest.param(
+                "yesterday",
+                "created:[2026-03-27T00:00:00Z TO 2026-03-28T00:00:00Z}",
+                id="yesterday",
+            ),
+            pytest.param(
+                "previous week",
+                "created:[2026-03-16T00:00:00Z TO 2026-03-23T00:00:00Z}",
+                id="previous-week",
+            ),
+            pytest.param(
+                "this month",
+                "created:[2026-03-01T00:00:00Z TO 2026-04-01T00:00:00Z}",
+                id="this-month",
+            ),
+            pytest.param(
+                "previous month",
+                "created:[2026-02-01T00:00:00Z TO 2026-03-01T00:00:00Z}",
+                id="previous-month",
+            ),
+            pytest.param(
+                "this year",
+                "created:[2026-01-01T00:00:00Z TO 2027-01-01T00:00:00Z}",
+                id="this-year",
+            ),
+            pytest.param(
+                "previous year",
+                "created:[2025-01-01T00:00:00Z TO 2026-01-01T00:00:00Z}",
+                id="previous-year",
+            ),
+            pytest.param(
+                "previous quarter",
+                "created:[2025-10-01T00:00:00Z TO 2026-01-01T00:00:00Z}",
+                id="previous-quarter",
+            ),
+        ],
+    )
+    @time_machine.travel(_FROZEN_NOW, tick=False)
+    def test_date_only_field_keyword_ranges(
+        self,
+        keyword: str,
+        expected: str,
+    ) -> None:
+        assert translate_query(f"created:{keyword}", UTC) == expected
+
+    # added is a DateTimeField: local-tz midnight converted to UTC. Tokyo
+    # (+09:00, no DST) shifts each midnight boundary back to 15:00Z the day
+    # before, so this also exercises the local-midnight offset path.
+    @pytest.mark.parametrize(
+        ("keyword", "expected"),
+        [
+            pytest.param(
+                "today",
+                "added:[2026-03-27T15:00:00Z TO 2026-03-28T15:00:00Z}",
+                id="today",
+            ),
+            pytest.param(
+                "yesterday",
+                "added:[2026-03-26T15:00:00Z TO 2026-03-27T15:00:00Z}",
+                id="yesterday",
+            ),
+            pytest.param(
+                "previous week",
+                "added:[2026-03-15T15:00:00Z TO 2026-03-22T15:00:00Z}",
+                id="previous-week",
+            ),
+            pytest.param(
+                "this month",
+                "added:[2026-02-28T15:00:00Z TO 2026-03-31T15:00:00Z}",
+                id="this-month",
+            ),
+            pytest.param(
+                "previous month",
+                "added:[2026-01-31T15:00:00Z TO 2026-02-28T15:00:00Z}",
+                id="previous-month",
+            ),
+            pytest.param(
+                "this year",
+                "added:[2025-12-31T15:00:00Z TO 2026-12-31T15:00:00Z}",
+                id="this-year",
+            ),
+            pytest.param(
+                "previous year",
+                "added:[2024-12-31T15:00:00Z TO 2025-12-31T15:00:00Z}",
+                id="previous-year",
+            ),
+            pytest.param(
+                "previous quarter",
+                "added:[2025-09-30T15:00:00Z TO 2025-12-31T15:00:00Z}",
+                id="previous-quarter",
+            ),
+        ],
+    )
+    @time_machine.travel(_FROZEN_NOW, tick=False)
+    def test_datetime_field_keyword_ranges_local_tz(
+        self,
+        keyword: str,
+        expected: str,
+    ) -> None:
+        assert translate_query(f"added:{keyword}", ZoneInfo("Asia/Tokyo")) == expected
+
+
+@pytest.mark.search
+class TestISODatetimeBounds:
+    """Full ISO datetime tokens in range bounds must be parsed directly."""
+
+    def test_translate_range_iso_bounds_passthrough(self) -> None:
+        # Already-ISO datetime bounds must pass through as-is (exact instant).
+        result = translate_range(
+            "created",
+            "2020-01-01T00:00:00Z",
+            "2021-01-01T00:00:00Z",
+            UTC,
+        )
+        assert result == "created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z]"
+
+    def test_translate_query_iso_range_preserved(self) -> None:
+        q = "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
+        assert translate_query(q, UTC) == q
+
+    def test_translate_query_comma_separated_iso_ranges(self) -> None:
+        q = (
+            "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z],"
+            "added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
+        )
+        result = translate_query(q, UTC)
+        assert result == (
+            "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
+            " AND "
+            "added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
+        )
+
+    def test_translate_query_text_before_comma_separated_date_clause(self) -> None:
+        result = translate_query("schäfersee,created:previous year", UTC)
+        assert result == (
+            "schäfersee AND created:[2025-01-01T00:00:00Z TO 2026-01-01T00:00:00Z}"
+        )
+
+    def test_invalid_iso_datetime_raises(self) -> None:
+        # A token with "T" that is not valid ISO datetime -> raise.
+        with pytest.raises(InvalidDateQuery) as exc_info:
+            translate_range(
+                "created",
+                "2020-01-01T99:00:00Z",
+                "2021-01-01T00:00:00Z",
+                UTC,
+            )
+        assert exc_info.value.field == "created"
+        assert exc_info.value.value == "2020-01-01T99:00:00Z"
+
+    def test_parse_acceptance_iso_bounds(self, index: tantivy.Index) -> None:
+        q = "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
+        translated = translate_query(q, UTC)
+        index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)
+
+    def test_parse_acceptance_comma_iso_ranges(self, index: tantivy.Index) -> None:
+        q = (
+            "created:[2026-01-01T00:00:00Z TO 2026-06-01T00:00:00Z],"
+            "added:[2026-05-01T00:00:00Z TO 2026-06-01T00:00:00Z]"
+        )
+        translated = translate_query(q, UTC)
+        index.parse_query(translated, DEFAULT_SEARCH_FIELDS, field_boosts=_FIELD_BOOSTS)

src/documents/tests/test_api_bulk_download.py

@@ -339,29 +339,3 @@ class TestBulkDownload(DirectoriesMixin, SampleDirMixin, APITestCase):
 
         self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
         self.assertEqual(response.content, b"Insufficient permissions")
-
-    def test_bad_search_query_returns_400(self) -> None:
-        """
-        GIVEN:
-            - Bulk download request selects documents via a saved-search
-              query filter
-        WHEN:
-            - The query contains a malformed field value (an invalid date)
-        THEN:
-            - The response is a 400 naming the bad value, exactly like the
-              search list endpoint, never a 500
-        """
-        response = self.client.post(
-            self.ENDPOINT,
-            json.dumps(
-                {
-                    "all": True,
-                    "filters": {"query": "added:notadate"},
-                    "content": "originals",
-                },
-            ),
-            content_type="application/json",
-        )
-
-        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-        self.assertIn(b"notadate", response.content)

src/documents/tests/test_api_bulk_edit.py

@@ -2059,30 +2059,3 @@ class TestBulkEditAPI(DirectoriesMixin, APITestCase):
 
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(LogEntry.objects.filter(object_pk=self.doc1.id).count(), 2)
-
-    def test_api_bulk_edit_with_bad_search_query_returns_400(self) -> None:
-        """
-        GIVEN:
-            - Bulk edit request selects documents via a saved-search query
-              filter
-        WHEN:
-            - The query contains a malformed field value (an invalid date)
-        THEN:
-            - The response is a 400 naming the bad value, exactly like the
-              search list endpoint, never a 500
-        """
-        response = self.client.post(
-            "/api/documents/bulk_edit/",
-            json.dumps(
-                {
-                    "all": True,
-                    "filters": {"query": "added:notadate"},
-                    "method": "set_storage_path",
-                    "parameters": {"storage_path": self.sp1.id},
-                },
-            ),
-            content_type="application/json",
-        )
-
-        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-        self.assertIn(b"notadate", response.content)

src/documents/tests/test_api_search.py

@@ -786,10 +786,6 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
             tick=False,
         ):
             response = self.client.get("/api/documents/?query=added:previous month")
-        assert response.status_code == 200, (
-            f"expected a successful search response, got {response.status_code}: "
-            f"{response.data!r}"
-        )
         results = response.data["results"]
 
         self.assertEqual(len(results), 1)
@@ -822,26 +818,6 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
         self.assertIn("invalid-date", str(response.data["query"]))
 
-    def test_search_multiple_bad_fields_returns_all_messages(self) -> None:
-        """
-        GIVEN:
-            - One document added
-        WHEN:
-            - Query with multiple bad fields (e.g. invalid date and invalid number)
-        THEN:
-            - 400 Bad Request with error messages for every bad field,
-              so the user can fix them all in one round-trip
-        """
-        response = self.client.get(
-            "/api/documents/",
-            {"query": "created:notadate AND asn:notanumber"},
-        )
-        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-        messages = response.data["query"]
-        self.assertEqual(len(messages), 2)
-        self.assertTrue(any("created" in m for m in messages))
-        self.assertTrue(any("asn" in m for m in messages))
-
     @override_settings(
         TIME_ZONE="UTC",
     )
@@ -885,29 +861,6 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         results = response.data["results"]
         self.assertEqual({r["id"] for r in results}, {1, 2})
 
-    @mock.patch("documents.search._backend.parse_user_query")
-    def test_search_parser_bug_surfaces_as_500_not_400(self, m) -> None:
-        """
-        GIVEN:
-            - The query parser itself fails (a whoosh-compat bug, per
-              QueryParserError's own contract: not user-fixable input)
-        WHEN:
-            - Any search request runs
-        THEN:
-            - The error surfaces as a 500 monitoring can see, never a 400
-              blaming the user for a library defect
-        """
-        from whoosh_compat.errors import QueryParserError
-
-        m.side_effect = QueryParserError("synthetic parser bug")
-
-        self.client.raise_request_exception = False
-        response = self.client.get("/api/documents/?query=anything")
-        self.assertEqual(
-            response.status_code,
-            status.HTTP_500_INTERNAL_SERVER_ERROR,
-        )
-
     @mock.patch("documents.search._backend.TantivyBackend.autocomplete")
     def test_search_autocomplete_limits(self, m) -> None:
         """
@@ -2105,77 +2058,3 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
         response = self.client.get("/api/search/?query=no")
         self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
-
-    def _assert_query_finds(self, doc: Document, query: str) -> None:
-        get_backend().add_or_update(doc)
-        response = self.client.get("/api/documents/", {"query": query})
-        self.assertEqual(response.status_code, status.HTTP_200_OK)
-        ids = [r["id"] for r in response.data["results"]]
-        self.assertIn(doc.id, ids)
-
-    def test_search_by_asn(self) -> None:
-        """
-        GIVEN:
-            - A document with an archive serial number, indexed
-        WHEN:
-            - A query filters by "asn:<value>"
-        THEN:
-            - The document is found
-        """
-        doc = Document.objects.create(
-            title="Has ASN",
-            content="content",
-            checksum="asn-checksum",
-            archive_serial_number=555,
-        )
-        self._assert_query_finds(doc, "asn:555")
-
-    def test_search_by_page_count(self) -> None:
-        """
-        GIVEN:
-            - A document with a page count, indexed
-        WHEN:
-            - A query filters by "page_count:<value>"
-        THEN:
-            - The document is found
-        """
-        doc = Document.objects.create(
-            title="Multi-page",
-            content="content",
-            checksum="page-count-checksum",
-            page_count=42,
-        )
-        self._assert_query_finds(doc, "page_count:42")
-
-    def test_search_by_original_filename(self) -> None:
-        """
-        GIVEN:
-            - A document with an original filename, indexed
-        WHEN:
-            - A query filters by "original_filename:<value>"
-        THEN:
-            - The document is found
-        """
-        doc = Document.objects.create(
-            title="Named file",
-            content="content",
-            checksum="filename-checksum",
-            original_filename="quarterly-report.pdf",
-        )
-        self._assert_query_finds(doc, "original_filename:quarterly-report.pdf")
-
-    def test_search_by_checksum(self) -> None:
-        """
-        GIVEN:
-            - A document with a checksum, indexed
-        WHEN:
-            - A query filters by "checksum:<value>"
-        THEN:
-            - The document is found
-        """
-        doc = Document.objects.create(
-            title="Checksum doc",
-            content="content",
-            checksum="deadbeef1234",
-        )
-        self._assert_query_finds(doc, "checksum:deadbeef1234")

src/documents/tests/test_api_search_errors.py

@@ -1,344 +0,0 @@
-"""The search list endpoint's exception handling: what becomes a 400 and
-what a library defect surfaces as instead.
-
-Companion to documents/tests/search/test_error_routing.py, which pins the
-Cause -> SearchQueryError/QueryError routing inside documents/search/_query.py.
-These tests pin the layer above it: DocumentViewSet.list's own except clauses,
-which decide what an already-routed error becomes on the wire.
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-import pytest
-from rest_framework import status
-from whoosh_compat.errors import Cause
-from whoosh_compat.errors import Diagnostic
-from whoosh_compat.errors import DiagnosticKind
-from whoosh_compat.errors import QueryError
-
-from documents.search import SearchQueryError
-from documents.tests.factories import DocumentFactory
-
-if TYPE_CHECKING:
-    from rest_framework.test import APIClient
-
-    from documents.models import Document
-
-pytestmark = [pytest.mark.django_db, pytest.mark.usefixtures("_search_index")]
-
-
-@pytest.fixture
-def indexed_document() -> Document:
-    from documents.search import get_backend
-
-    doc = DocumentFactory.create(title="quarterly invoice", content="acme corp")
-    get_backend().add_or_update(doc)
-    return doc
-
-
-class TestSearchQueryErrorStillBecomesA400:
-    def test_search_query_error_becomes_a_400_naming_the_field(
-        self,
-        admin_client: APIClient,
-        monkeypatch: pytest.MonkeyPatch,
-        indexed_document: Document,
-    ) -> None:
-        """
-        GIVEN:
-            - parse_user_query() raising a SearchQueryError naming a field
-        WHEN:
-            - The document list endpoint is queried
-        THEN:
-            - The response is a 400 whose body names the field
-        """
-        import documents.search._backend as backend_mod
-
-        def raise_search_query_error(*args: object, **kwargs: object) -> object:
-            raise SearchQueryError("bad value for field 'added'")
-
-        monkeypatch.setattr(
-            backend_mod,
-            "parse_user_query",
-            raise_search_query_error,
-        )
-
-        response = admin_client.get("/api/documents/?query=anything")
-
-        assert response.status_code == status.HTTP_400_BAD_REQUEST
-        assert "added" in str(response.data["query"])
-
-
-class TestLibraryDefectsPropagate:
-    """The exact regression this task exists to fix: an unexpected or
-    INTERNAL-cause library error must not be relabeled a 400."""
-
-    def test_unexpected_exception_is_not_converted_to_a_400(
-        self,
-        admin_client: APIClient,
-        monkeypatch: pytest.MonkeyPatch,
-        indexed_document: Document,
-    ) -> None:
-        """
-        GIVEN:
-            - parse_user_query() raising an unrelated exception
-              (ZeroDivisionError), not a SearchQueryError
-        WHEN:
-            - The document list endpoint is queried
-        THEN:
-            - The exception propagates unconverted, rather than being
-              relabeled a 400
-        """
-        import documents.search._backend as backend_mod
-
-        def raise_zero_division(*args: object, **kwargs: object) -> object:
-            raise ZeroDivisionError("synthetic bug, unrelated to search grammar")
-
-        monkeypatch.setattr(
-            backend_mod,
-            "parse_user_query",
-            raise_zero_division,
-        )
-
-        with pytest.raises(ZeroDivisionError):
-            admin_client.get("/api/documents/?query=anything")
-
-    def test_internal_cause_query_error_is_not_converted_to_a_400(
-        self,
-        admin_client: APIClient,
-        monkeypatch: pytest.MonkeyPatch,
-        indexed_document: Document,
-    ) -> None:
-        """
-        GIVEN:
-            - A real query string running through the real parse and
-              routing pipeline (pre-parse rewrites, wc.parse(), and
-              _map_emit_error's own Cause routing all run for real), except
-              the final emit call (tantivy_emit) is forced to report a
-              library-internal defect (Cause.INTERNAL) - the one
-              library-internal failure mode reachable from a real query
-        WHEN:
-            - The document list endpoint is queried
-        THEN:
-            - The QueryError propagates unconverted, rather than being
-              relabeled a 400
-        """
-        import documents.search._query as query_mod
-
-        def raise_internal(*args: object, **kwargs: object) -> object:
-            raise QueryError(
-                Diagnostic(
-                    kind=DiagnosticKind.BACKEND_REJECTED,
-                    cause=Cause.INTERNAL,
-                    message="synthetic whoosh-compat emitter defect",
-                ),
-            )
-
-        monkeypatch.setattr(query_mod, "tantivy_emit", raise_internal)
-
-        with pytest.raises(QueryError):
-            admin_client.get("/api/documents/?query=invoice")
-
-
-class TestSelectionPathsAgreeWithSearch:
-    """DocumentSelectionMixin backs bulk edit, bulk download, and a
-    more_like_id selection filter. It catches only SearchQueryError -- the
-    same contract the search list endpoint enforces above -- so all three
-    must map SearchQueryError to a 400 and let anything else surface."""
-
-    def test_bulk_edit_maps_search_query_error_to_a_400(
-        self,
-        admin_client: APIClient,
-        monkeypatch: pytest.MonkeyPatch,
-        indexed_document: Document,
-    ) -> None:
-        """
-        GIVEN:
-            - parse_user_query() raising a SearchQueryError naming a field
-        WHEN:
-            - The bulk_edit endpoint is called with a query filter
-        THEN:
-            - The response is a 400 whose body names the field
-        """
-        import documents.search._backend as backend_mod
-
-        def raise_search_query_error(*args: object, **kwargs: object) -> object:
-            raise SearchQueryError("bad value for field 'added'")
-
-        monkeypatch.setattr(
-            backend_mod,
-            "parse_user_query",
-            raise_search_query_error,
-        )
-
-        response = admin_client.post(
-            "/api/documents/bulk_edit/",
-            {
-                "documents": [],
-                "all": True,
-                "filters": {"query": "anything"},
-                "method": "set_document_type",
-                "parameters": {"document_type": None},
-            },
-            format="json",
-        )
-
-        assert response.status_code == status.HTTP_400_BAD_REQUEST
-        assert "added" in str(response.data["query"])
-
-    def test_bulk_edit_lets_an_unexpected_exception_surface(
-        self,
-        admin_client: APIClient,
-        monkeypatch: pytest.MonkeyPatch,
-        indexed_document: Document,
-    ) -> None:
-        """
-        GIVEN:
-            - parse_user_query() raising an unrelated exception
-              (ZeroDivisionError), not a SearchQueryError
-        WHEN:
-            - The bulk_edit endpoint is called with a query filter
-        THEN:
-            - The exception propagates unconverted, rather than being
-              relabeled a 400
-        """
-        import documents.search._backend as backend_mod
-
-        def raise_zero_division(*args: object, **kwargs: object) -> object:
-            raise ZeroDivisionError("synthetic bug, unrelated to search grammar")
-
-        monkeypatch.setattr(
-            backend_mod,
-            "parse_user_query",
-            raise_zero_division,
-        )
-
-        with pytest.raises(ZeroDivisionError):
-            admin_client.post(
-                "/api/documents/bulk_edit/",
-                {
-                    "documents": [],
-                    "all": True,
-                    "filters": {"query": "anything"},
-                    "method": "set_document_type",
-                    "parameters": {"document_type": None},
-                },
-                format="json",
-            )
-
-    def test_bulk_download_maps_search_query_error_to_a_400(
-        self,
-        admin_client: APIClient,
-        monkeypatch: pytest.MonkeyPatch,
-        indexed_document: Document,
-    ) -> None:
-        """
-        GIVEN:
-            - parse_user_query() raising a SearchQueryError naming a field
-        WHEN:
-            - The bulk_download endpoint is called with a query filter
-        THEN:
-            - The response is a 400 whose body names the field
-        """
-        import documents.search._backend as backend_mod
-
-        def raise_search_query_error(*args: object, **kwargs: object) -> object:
-            raise SearchQueryError("bad value for field 'added'")
-
-        monkeypatch.setattr(
-            backend_mod,
-            "parse_user_query",
-            raise_search_query_error,
-        )
-
-        response = admin_client.post(
-            "/api/documents/bulk_download/",
-            {
-                "documents": [],
-                "all": True,
-                "filters": {"query": "anything"},
-            },
-            format="json",
-        )
-
-        assert response.status_code == status.HTTP_400_BAD_REQUEST
-        assert "added" in str(response.data["query"])
-
-    def test_more_like_id_selection_filter_maps_search_query_error_to_a_400(
-        self,
-        admin_client: APIClient,
-        monkeypatch: pytest.MonkeyPatch,
-        indexed_document: Document,
-    ) -> None:
-        """
-        GIVEN:
-            - TantivyBackend.more_like_this_ids() raising a
-              SearchQueryError
-        WHEN:
-            - The bulk_download endpoint is called with a more_like_id
-              filter
-        THEN:
-            - The response is a 400
-        """
-        import documents.search._backend as backend_mod
-
-        def raise_search_query_error(*args: object, **kwargs: object) -> object:
-            raise SearchQueryError("similar-document lookup is unavailable")
-
-        monkeypatch.setattr(
-            backend_mod.TantivyBackend,
-            "more_like_this_ids",
-            raise_search_query_error,
-        )
-
-        response = admin_client.post(
-            "/api/documents/bulk_download/",
-            {
-                "documents": [],
-                "all": True,
-                "filters": {"more_like_id": indexed_document.pk},
-            },
-            format="json",
-        )
-
-        assert response.status_code == status.HTTP_400_BAD_REQUEST
-
-    def test_more_like_id_selection_filter_lets_an_unexpected_exception_surface(
-        self,
-        admin_client: APIClient,
-        monkeypatch: pytest.MonkeyPatch,
-        indexed_document: Document,
-    ) -> None:
-        """
-        GIVEN:
-            - TantivyBackend.more_like_this_ids() raising an unrelated
-              exception (ZeroDivisionError), not a SearchQueryError
-        WHEN:
-            - The bulk_download endpoint is called with a more_like_id
-              filter
-        THEN:
-            - The exception propagates unconverted, rather than being
-              relabeled a 400
-        """
-        import documents.search._backend as backend_mod
-
-        def raise_zero_division(*args: object, **kwargs: object) -> object:
-            raise ZeroDivisionError("synthetic bug, unrelated to similarity lookup")
-
-        monkeypatch.setattr(
-            backend_mod.TantivyBackend,
-            "more_like_this_ids",
-            raise_zero_division,
-        )
-
-        with pytest.raises(ZeroDivisionError):
-            admin_client.post(
-                "/api/documents/bulk_download/",
-                {
-                    "documents": [],
-                    "all": True,
-                    "filters": {"more_like_id": indexed_document.pk},
-                },
-                format="json",
-            )

src/documents/tests/test_api_search_query_length.py

@@ -1,266 +0,0 @@
-"""The query-length cap in ``_get_tantivy_query_and_mode``.
-
-whoosh-compat's fieldname tagger is O(n^2) in plain word characters, so an
-unbounded ``query`` (SearchMode.QUERY) string is a CPU-exhaustion vector
-against a single request handler. The GET search endpoint is incidentally
-bounded by the web server's header limit, but the POST selection-filter
-path (bulk edit, bulk download) is not -- that is the real vector, so it
-must be pinned here too, not just the GET path.
-
-The cap is enforced once, in the shared helper both entry points call, so
-these tests exercise the real endpoints rather than the helper directly:
-a construct that looks right in isolation has repeatedly behaved
-differently end to end on this branch.
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-from unittest import mock
-
-import pytest
-from rest_framework import status
-
-from documents.tests.factories import DocumentFactory
-from documents.views import _MAX_QUERY_LENGTH
-
-if TYPE_CHECKING:
-    from rest_framework.test import APIClient
-
-    from documents.models import Document
-
-pytestmark = [pytest.mark.django_db, pytest.mark.usefixtures("_search_index")]
-
-
-@pytest.fixture
-def indexed_document() -> Document:
-    from documents.search import get_backend
-
-    doc = DocumentFactory.create(title="quarterly invoice", content="acme corp")
-    get_backend().add_or_update(doc)
-    return doc
-
-
-class TestGetSearchEndpointEnforcesTheCap:
-    def test_query_one_over_the_cap_is_a_400(
-        self,
-        admin_client: APIClient,
-        indexed_document: Document,
-    ) -> None:
-        """
-        GIVEN:
-            - The GET search endpoint
-        WHEN:
-            - A query one character over `_MAX_QUERY_LENGTH` is submitted
-        THEN:
-            - The response is a 400 naming both the actual length and the
-              cap
-        """
-        query = "a" * (_MAX_QUERY_LENGTH + 1)
-
-        response = admin_client.get("/api/documents/", {"query": query})
-
-        assert response.status_code == status.HTTP_400_BAD_REQUEST
-        message = str(response.data["query"])
-        assert str(_MAX_QUERY_LENGTH) in message
-        assert str(_MAX_QUERY_LENGTH + 1) in message
-
-    def test_query_at_exactly_the_cap_is_accepted(
-        self,
-        admin_client: APIClient,
-        indexed_document: Document,
-    ) -> None:
-        """
-        GIVEN:
-            - The GET search endpoint
-        WHEN:
-            - A query exactly `_MAX_QUERY_LENGTH` characters long is
-              submitted
-        THEN:
-            - The response is a 200 (the cap is inclusive, not exclusive)
-        """
-        query = "a" * _MAX_QUERY_LENGTH
-
-        response = admin_client.get("/api/documents/", {"query": query})
-
-        assert response.status_code == status.HTTP_200_OK
-
-    def test_an_ordinary_query_is_unaffected(
-        self,
-        admin_client: APIClient,
-        indexed_document: Document,
-    ) -> None:
-        """
-        GIVEN:
-            - The GET search endpoint and an indexed document
-        WHEN:
-            - An ordinary, well-under-the-cap query is submitted
-        THEN:
-            - The cap has no effect on a normal search: the matching
-              document is returned
-        """
-        response = admin_client.get("/api/documents/", {"query": "invoice"})
-
-        assert response.status_code == status.HTTP_200_OK
-        assert response.data["count"] == 1
-
-
-class TestPostSelectionPathsEnforceTheCap:
-    """The bulk-edit and bulk-download selection filters share the same
-    helper the GET search path uses. This is the path that actually
-    matters: it is not bounded by a web server's header-length limit the
-    way the GET path incidentally is."""
-
-    def test_bulk_edit_query_one_over_the_cap_is_a_400(
-        self,
-        admin_client: APIClient,
-        indexed_document: Document,
-    ) -> None:
-        """
-        GIVEN:
-            - The bulk-edit selection-filter endpoint
-        WHEN:
-            - Its `filters.query` is one character over `_MAX_QUERY_LENGTH`
-        THEN:
-            - The response is a 400 naming both the actual length and the
-              cap
-        """
-        query = "a" * (_MAX_QUERY_LENGTH + 1)
-
-        response = admin_client.post(
-            "/api/documents/bulk_edit/",
-            {
-                "documents": [],
-                "all": True,
-                "filters": {"query": query},
-                "method": "set_document_type",
-                "parameters": {"document_type": None},
-            },
-            format="json",
-        )
-
-        assert response.status_code == status.HTTP_400_BAD_REQUEST
-        message = str(response.data["query"])
-        assert str(_MAX_QUERY_LENGTH) in message
-        assert str(_MAX_QUERY_LENGTH + 1) in message
-
-    @mock.patch("documents.bulk_edit.bulk_update_documents.apply_async")
-    def test_bulk_edit_query_at_exactly_the_cap_is_accepted(
-        self,
-        bulk_update_task_mock: mock.MagicMock,
-        admin_client: APIClient,
-        indexed_document: Document,
-    ) -> None:
-        """
-        GIVEN:
-            - The bulk-edit selection-filter endpoint
-        WHEN:
-            - Its `filters.query` is exactly `_MAX_QUERY_LENGTH` characters
-              long
-        THEN:
-            - The cap check accepts it and the request reaches the real
-              bulk-edit method (its Celery dispatch is mocked out here,
-              same as every other bulk-edit test, since nothing here is
-              testing that method itself)
-        """
-        query = "a" * _MAX_QUERY_LENGTH
-
-        response = admin_client.post(
-            "/api/documents/bulk_edit/",
-            {
-                "documents": [],
-                "all": True,
-                "filters": {"query": query},
-                "method": "set_document_type",
-                "parameters": {"document_type": None},
-            },
-            format="json",
-        )
-
-        assert response.status_code == status.HTTP_200_OK
-
-    def test_bulk_download_query_one_over_the_cap_is_a_400(
-        self,
-        admin_client: APIClient,
-        indexed_document: Document,
-    ) -> None:
-        """
-        GIVEN:
-            - The bulk-download selection-filter endpoint
-        WHEN:
-            - Its `filters.query` is one character over `_MAX_QUERY_LENGTH`
-        THEN:
-            - The response is a 400 naming both the actual length and the
-              cap
-        """
-        query = "a" * (_MAX_QUERY_LENGTH + 1)
-
-        response = admin_client.post(
-            "/api/documents/bulk_download/",
-            {
-                "documents": [],
-                "all": True,
-                "filters": {"query": query},
-            },
-            format="json",
-        )
-
-        assert response.status_code == status.HTTP_400_BAD_REQUEST
-        message = str(response.data["query"])
-        assert str(_MAX_QUERY_LENGTH) in message
-        assert str(_MAX_QUERY_LENGTH + 1) in message
-
-
-class TestGlobalSearchEnforcesTheCapToo:
-    """GlobalSearchView calls the backend directly, not through the shared helper.
-
-    It hardcodes SearchMode.TEXT, which is linear rather than quadratic, so it
-    was never the CPU-exhaustion vector. It is capped anyway so that "every
-    user query string reaching the backend passes a length check" is an
-    invariant rather than a claim with an exception: the view already bounds
-    the query from below, and a later change letting it select a mode would
-    otherwise reopen the hole silently.
-    """
-
-    def test_query_one_over_the_cap_is_a_400(
-        self,
-        admin_client: APIClient,
-        indexed_document: Document,
-    ) -> None:
-        """
-        GIVEN:
-            - GlobalSearchView, which calls the backend directly in
-              SearchMode.TEXT rather than through the shared cap-checking
-              helper
-        WHEN:
-            - Its query is one character over `_MAX_QUERY_LENGTH`
-        THEN:
-            - The response is still a 400, keeping "every user query
-              string reaching the backend passes a length check" an
-              invariant with no exception, even though TEXT mode is
-              linear and was never itself the CPU-exhaustion vector
-        """
-        response = admin_client.get(
-            "/api/search/",
-            {"query": "a" * (_MAX_QUERY_LENGTH + 1)},
-        )
-        assert response.status_code == status.HTTP_400_BAD_REQUEST
-
-    def test_query_at_exactly_the_cap_is_accepted(
-        self,
-        admin_client: APIClient,
-        indexed_document: Document,
-    ) -> None:
-        """
-        GIVEN:
-            - GlobalSearchView
-        WHEN:
-            - Its query is exactly `_MAX_QUERY_LENGTH` characters long
-        THEN:
-            - The response is a 200 (the cap is inclusive, not exclusive)
-        """
-        response = admin_client.get(
-            "/api/search/",
-            {"query": "a" * _MAX_QUERY_LENGTH},
-        )
-        assert response.status_code == status.HTTP_200_OK

src/documents/tests/test_api_search_unterminated_date_range.py

@@ -1,88 +0,0 @@
-"""An unterminated ``[`` date range bracket at the API level.
-
-``created:[2020`` (with or without a dangling ``to <value>``) now raises
-BAD_DATE and the search endpoint returns HTTP 400, where it used to parse
-past the missing ``]`` and silently pass the malformed range through.
-A 400 is correct: malformed input should fail loudly rather than silently
-matching an unintended query. Pinned at the API level -- the layer a user
-or client actually sees -- rather than only against the parser directly.
-
-The properly closed decoy proves the bracket is what matters, not
-whoosh-compat's date grammar generally: ``created:[2020 to 2021]`` parses
-and searches cleanly.
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-import pytest
-from rest_framework import status
-
-from documents.tests.factories import DocumentFactory
-
-if TYPE_CHECKING:
-    from rest_framework.test import APIClient
-
-    from documents.models import Document
-
-pytestmark = [pytest.mark.django_db, pytest.mark.usefixtures("_search_index")]
-
-
-@pytest.fixture
-def indexed_document() -> Document:
-    from documents.search import get_backend
-
-    doc = DocumentFactory.create(title="quarterly invoice", content="acme corp")
-    get_backend().add_or_update(doc)
-    return doc
-
-
-class TestUnterminatedBracketReturnsA400:
-    @pytest.mark.parametrize(
-        "query",
-        [
-            pytest.param("created:[2020", id="missing_upper_bound_and_bracket"),
-            pytest.param("created:[2020 to 2021", id="missing_closing_bracket"),
-        ],
-    )
-    def test_unterminated_bracket_is_a_400(
-        self,
-        admin_client: APIClient,
-        indexed_document: Document,
-        query: str,
-    ) -> None:
-        """
-        GIVEN:
-            - The search endpoint
-        WHEN:
-            - A date-range query with a missing closing `]` (with or
-              without a dangling upper bound) is submitted
-        THEN:
-            - The response is a 400 naming the field, rather than parsing
-              past the missing bracket and silently passing the malformed
-              range through
-        """
-        response = admin_client.get(f"/api/documents/?query={query}")
-        assert response.status_code == status.HTTP_400_BAD_REQUEST
-        assert "created" in str(response.data["query"])
-
-    def test_properly_closed_bracket_still_searches_cleanly(
-        self,
-        admin_client: APIClient,
-        indexed_document: Document,
-    ) -> None:
-        """
-        GIVEN:
-            - The search endpoint
-        WHEN:
-            - A properly closed date-range query is submitted
-        THEN:
-            - The response is a 200 (the decoy proving the missing
-              bracket, not whoosh-compat's date grammar generally, is
-              what the 400 above is about)
-        """
-        response = admin_client.get(
-            "/api/documents/?query=created:[2020 to 2021]",
-        )
-        assert response.status_code == status.HTTP_200_OK

src/documents/tests/test_api_trash.py

@@ -207,65 +207,3 @@ class TestTrashAPI(DirectoriesMixin, APITestCase):
         )
         self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST)
         self.assertIn("have not yet been deleted", resp.data["documents"][0])
-
-    def _make_versioned_document(self) -> tuple[Document, list[Document]]:
-        root = Document.objects.create(
-            title="root",
-            content="root-content",
-            checksum="root",
-            mime_type="application/pdf",
-        )
-        versions = [
-            Document.objects.create(
-                title=f"v{index}",
-                content=f"v{index}-content",
-                checksum=f"v{index}",
-                mime_type="application/pdf",
-                root_document=root,
-                version_index=index,
-            )
-            for index in range(1, 3)
-        ]
-        return root, versions
-
-    def test_api_trash_restore_document_restores_its_versions(self) -> None:
-        """
-        GIVEN:
-            - Existing document with two versions
-        WHEN:
-            - API request to delete the document
-            - API request to restore it from the trash
-        THEN:
-            - Only the document itself is listed in the trash
-            - A version cannot be restored without its root
-            - The document is restored together with all of its versions
-        """
-        root, versions = self._make_versioned_document()
-
-        self.client.force_login(user=self.user)
-        self.client.delete(f"/api/documents/{root.pk}/")
-        self.assertEqual(Document.deleted_objects.count(), 3)
-
-        resp = self.client.get("/api/trash/")
-        self.assertEqual(resp.status_code, status.HTTP_200_OK)
-        self.assertEqual(resp.data["count"], 1)
-        self.assertEqual(resp.data["results"][0]["id"], root.pk)
-
-        # A version cannot be restored while its root remains in the trash.
-        resp = self.client.post(
-            "/api/trash/",
-            {"action": "restore", "documents": [versions[0].pk]},
-        )
-        self.assertEqual(resp.status_code, status.HTTP_400_BAD_REQUEST)
-        self.assertIn("Restore the root document", resp.data["documents"][0])
-
-        resp = self.client.post(
-            "/api/trash/",
-            {"action": "restore", "documents": [root.pk]},
-        )
-        self.assertEqual(resp.status_code, status.HTTP_200_OK)
-        self.assertEqual(Document.deleted_objects.count(), 0)
-        self.assertCountEqual(
-            Document.objects.filter(root_document=root).values_list("id", flat=True),
-            [version.pk for version in versions],
-        )

src/documents/tests/test_api_update_version_nfc.py

@@ -0,0 +1,69 @@
+import unicodedata
+from typing import TYPE_CHECKING
+from unittest import mock
+
+import celery.result
+import pytest
+from django.core.files.uploadedfile import SimpleUploadedFile
+
+from documents.models import Document
+
+if TYPE_CHECKING:
+    from documents.data_models import ConsumableDocument
+
+
+@pytest.fixture()
+def consume_file_mock():
+    with mock.patch("documents.tasks.consume_file.apply_async") as m:
+        m.return_value = celery.result.AsyncResult(id="test-task-id")
+        yield m
+
+
+@pytest.fixture()
+def directories(tmp_path, settings, _media_settings):
+    scratch = tmp_path / "scratch"
+    scratch.mkdir()
+    settings.SCRATCH_DIR = scratch
+    return scratch
+
+
+@pytest.mark.django_db
+class TestUpdateVersionNFCNormalization:
+    def test_nfd_filename_normalized_to_nfc(
+        self,
+        admin_client,
+        consume_file_mock: mock.MagicMock,
+        directories,
+    ):
+        """Uploaded new-version file with NFD filename must have its temp name stored as NFC."""
+        document = Document.objects.create(
+            title="Test",
+            content="content",
+            checksum="checksum",
+            mime_type="application/pdf",
+        )
+
+        nfd = unicodedata.normalize("NFD", "Rechnung März.pdf")
+        nfc = unicodedata.normalize("NFC", "Rechnung März.pdf")
+
+        assert nfd != nfc
+
+        uploaded = SimpleUploadedFile(
+            nfd,
+            b"%PDF-1.4 test",
+            content_type="application/pdf",
+        )
+        response = admin_client.post(
+            f"/api/documents/{document.pk}/update_version/",
+            {"document": uploaded},
+        )
+
+        assert response.status_code == 200
+
+        task_kwargs = consume_file_mock.call_args.kwargs["kwargs"]
+        input_doc: ConsumableDocument = task_kwargs["input_doc"]
+
+        assert input_doc.original_file.name == nfc, (
+            f"Expected NFC filename {nfc!r}, got {input_doc.original_file.name!r}"
+        )
+        assert unicodedata.is_normalized("NFC", input_doc.original_file.name)

src/documents/tests/test_bulk_edit.py

@@ -392,11 +392,6 @@ class TestBulkEdit(DirectoriesMixin, TestCase):
         self.assertFalse(Document.objects.filter(id=self.doc1.id).exists())
         self.assertFalse(Document.objects.filter(id=version.id).exists())
 
-        Document.deleted_objects.get(id=self.doc1.id).restore(strict=False)
-
-        self.assertTrue(Document.objects.filter(id=self.doc1.id).exists())
-        self.assertTrue(Document.objects.filter(id=version.id).exists())
-
     def test_delete_version_document_keeps_root(self) -> None:
         version = Document.objects.create(
             checksum="A-v1",

src/documents/tests/test_classifier.py

@@ -3,7 +3,6 @@ import warnings
 from pathlib import Path
 from unittest import mock
 
-import numpy as np
 import pytest
 from django.conf import settings
 from django.test import TestCase
@@ -12,7 +11,6 @@ from django.test import override_settings
 from documents.classifier import ClassifierModelCorruptError
 from documents.classifier import DocumentClassifier
 from documents.classifier import IncompatibleClassifierVersionError
-from documents.classifier import _predict_with_threshold
 from documents.classifier import load_classifier
 from documents.models import Correspondent
 from documents.models import Document
@@ -627,103 +625,6 @@ class TestClassifier(DirectoriesMixin, TestCase):
         self.assertEqual(self.classifier.predict_storage_path(doc1.content), sp.pk)
         self.assertIsNone(self.classifier.predict_storage_path(doc2.content))
 
-    def test_predict_rejects_prediction_below_match_threshold(self) -> None:
-        """
-        GIVEN:
-            - Classifiers trained against test data with confident predictions
-        WHEN:
-            - CLASSIFIER_MATCH_THRESHOLD exceeds the model's confidence
-        THEN:
-            - Every predict_* method discards the match in favor of no match
-        """
-        c1 = Correspondent.objects.create(
-            name="c1",
-            matching_algorithm=Correspondent.MATCH_AUTO,
-        )
-        dt1 = DocumentType.objects.create(
-            name="dt1",
-            matching_algorithm=DocumentType.MATCH_AUTO,
-        )
-        sp1 = StoragePath.objects.create(
-            name="sp1",
-            matching_algorithm=StoragePath.MATCH_AUTO,
-        )
-
-        doc1 = Document.objects.create(
-            title="doc1",
-            content="this is a document from c1",
-            correspondent=c1,
-            document_type=dt1,
-            storage_path=sp1,
-            checksum="A",
-        )
-        Document.objects.create(
-            title="doc2",
-            content="this is a document from no one",
-            checksum="B",
-        )
-
-        self.classifier.train()
-
-        predictors = {
-            "correspondent": self.classifier.predict_correspondent,
-            "document_type": self.classifier.predict_document_type,
-            "storage_path": self.classifier.predict_storage_path,
-        }
-        # No real prediction can reach a confidence this high, so this
-        # isolates the threshold check from the model's actual output.
-        with override_settings(CLASSIFIER_MATCH_THRESHOLD=0.999999):
-            for name, predict in predictors.items():
-                with self.subTest(field=name):
-                    self.assertIsNone(predict(doc1.content))
-
-    def test_train_uses_balanced_sample_weight(self) -> None:
-        """
-        GIVEN:
-            - A training set with correspondents, document types and storage paths
-        WHEN:
-            - The classifier is trained
-        THEN:
-            - Each MLP classifier is fit with balanced sample weights, so that
-              over-represented classes don't dominate predictions
-        """
-        c1 = Correspondent.objects.create(
-            name="c1",
-            matching_algorithm=Correspondent.MATCH_AUTO,
-        )
-        dt1 = DocumentType.objects.create(
-            name="dt1",
-            matching_algorithm=DocumentType.MATCH_AUTO,
-        )
-        sp1 = StoragePath.objects.create(
-            name="sp1",
-            matching_algorithm=StoragePath.MATCH_AUTO,
-        )
-
-        Document.objects.create(
-            title="doc1",
-            content="this is a document from c1",
-            correspondent=c1,
-            document_type=dt1,
-            storage_path=sp1,
-            checksum="A",
-        )
-        Document.objects.create(
-            title="doc2",
-            content="this is a document from no one",
-            checksum="B",
-        )
-
-        with mock.patch(
-            "sklearn.utils.class_weight.compute_sample_weight",
-            return_value=None,
-        ) as mocked_compute_sample_weight:
-            self.classifier.train()
-
-        self.assertEqual(mocked_compute_sample_weight.call_count, 3)
-        for call in mocked_compute_sample_weight.call_args_list:
-            self.assertEqual(call.args[0], "balanced")
-
     def test_one_tag_predict(self) -> None:
         t1 = Tag.objects.create(name="t1", matching_algorithm=Tag.MATCH_AUTO, pk=12)
 
@@ -909,52 +810,6 @@ class TestClassifier(DirectoriesMixin, TestCase):
             load_classifier(raise_exception=True)
 
 
-class _StubProbaClassifier:
-    """
-    A fake scikit-learn classifier exposing just enough of the API for
-    `_predict_with_threshold`: `classes_` and `predict_proba`.
-    """
-
-    def __init__(self, classes: list[int], probabilities: list[float]) -> None:
-        self.classes_ = np.array(classes)
-        self._probabilities = np.array([probabilities])
-
-    def predict_proba(self, X) -> np.ndarray:
-        return self._probabilities
-
-
-@pytest.mark.parametrize(
-    ("classes", "probabilities", "threshold", "expected"),
-    [
-        # confident prediction above the threshold is returned
-        ([-1, 3], [0.1, 0.9], 0.6, 3),
-        # prediction below the threshold is discarded
-        ([-1, 3], [0.45, 0.55], 0.6, None),
-        # boundary: exactly at the threshold is accepted, not discarded
-        ([-1, 3], [0.4, 0.6], 0.6, 3),
-        # the winning class is the "no match" pseudo-class, regardless of its
-        # own confidence
-        ([-1, 3], [0.99, 0.01], 0.0, None),
-        # threshold of 0.0 disables the confidence check entirely
-        ([-1, 3], [0.45, 0.55], 0.0, 3),
-    ],
-)
-def test_predict_with_threshold(classes, probabilities, threshold, expected) -> None:
-    classifier = _StubProbaClassifier(classes, probabilities)
-    result = _predict_with_threshold(classifier, X=None, threshold=threshold)
-    assert result == expected
-
-
-def test_classifier_match_threshold_default() -> None:
-    """
-    GIVEN:
-        - No PAPERLESS_CLASSIFIER_MATCH_THRESHOLD environment variable is set
-    THEN:
-        - The classifier match threshold defaults to 0.6
-    """
-    assert settings.CLASSIFIER_MATCH_THRESHOLD == 0.6
-
-
 def test_preprocess_content() -> None:
     """
     GIVEN:

src/documents/tests/test_document_model.py

@@ -110,7 +110,7 @@ class TestDocument(TestCase):
             checksum="checksum",
             mime_type="application/pdf",
         )
-        version = Document.objects.create(
+        Document.objects.create(
             root_document=root,
             correspondent=root.correspondent,
             title="Version",
@@ -124,10 +124,6 @@ class TestDocument(TestCase):
         self.assertEqual(Document.objects.count(), 0)
         self.assertEqual(Document.deleted_objects.count(), 2)
 
-        root.restore(strict=False)
-
-        self.assertTrue(Document.objects.filter(pk=version.pk).exists())
-
     def test_file_name(self) -> None:
         doc = Document(
             mime_type="application/pdf",

src/documents/tests/test_get_public_filename_nfc.py

@@ -0,0 +1,48 @@
+import unicodedata
+from datetime import date
+
+import pytest
+
+from documents.models import Correspondent
+from documents.models import Document
+
+
+@pytest.mark.django_db
+class TestGetPublicFilenameNfc:
+    def test_normalizes_nfd_title_to_nfc(self) -> None:
+        nfd_title = unicodedata.normalize("NFD", "Gehaltserhöhung")
+        assert not unicodedata.is_normalized("NFC", nfd_title)
+
+        doc = Document(
+            mime_type="application/pdf",
+            title=nfd_title,
+            created=date(2025, 10, 17),
+        )
+
+        result = doc.get_public_filename()
+
+        assert unicodedata.is_normalized("NFC", result)
+        assert (
+            result
+            == "2025-10-17 "
+            + unicodedata.normalize(
+                "NFC",
+                nfd_title,
+            )
+            + ".pdf"
+        )
+
+    def test_normalizes_nfd_correspondent_name_to_nfc(self) -> None:
+        nfd_name = unicodedata.normalize("NFD", "Müller GmbH")
+        correspondent = Correspondent.objects.create(name=nfd_name)
+
+        doc = Document.objects.create(
+            mime_type="application/pdf",
+            title="Rechnung",
+            created=date(2025, 10, 17),
+            correspondent=correspondent,
+        )
+
+        result = doc.get_public_filename()
+
+        assert unicodedata.is_normalized("NFC", result)

src/documents/tests/test_management_consumer.py

@@ -136,23 +136,6 @@ def wait_for_mock_call(
     return False
 
 
-def sleep_past_stability(
-    owner: FileStabilityTracker | ConsumerThread,
-    *,
-    windows: float = 1.5,
-) -> None:
-    """
-    Block until a tracked file's stability window has certainly elapsed.
-
-    Args:
-        owner: The tracker, or the consumer thread running one, whose
-               configured stability delay sets the wait.
-        windows: How many stability windows to wait, giving slop for a slow
-                 or loaded test runner.
-    """
-    sleep(owner.stability_delay * windows)
-
-
 class TestTrackedFile:
     """Tests for the TrackedFile dataclass."""
 
@@ -278,56 +261,6 @@ class TestFileStabilityTracker:
         assert len(stable) == 0
         assert stability_tracker.pending_count == 1
 
-    def test_get_stable_files_skips_empty_file(
-        self,
-        stability_tracker: FileStabilityTracker,
-        tmp_path: Path,
-    ) -> None:
-        """
-        GIVEN:
-            - A zero byte file, tracked and past its stability delay
-        WHEN:
-            - Stable files are collected
-        THEN:
-            - The file is not yielded for consumption
-            - The file is dropped from tracking rather than held, so an
-              abandoned placeholder does not keep the watch loop awake
-        """
-        empty = tmp_path / "scan.pdf"
-        empty.write_bytes(b"")
-        stability_tracker.track(empty, Change.added)
-        sleep_past_stability(stability_tracker)
-
-        stable = list(stability_tracker.get_stable_files())
-
-        assert stable == []
-        assert stability_tracker.pending_count == 0
-
-    def test_empty_file_is_yielded_once_content_arrives(
-        self,
-        stability_tracker: FileStabilityTracker,
-        tmp_path: Path,
-    ) -> None:
-        """
-        GIVEN:
-            - A zero byte file which was dropped from tracking while empty
-        WHEN:
-            - The writer fills the file and a new event re-tracks it
-        THEN:
-            - The file is yielded for consumption once it is stable
-        """
-        target = tmp_path / "scan.pdf"
-        target.write_bytes(b"")
-        stability_tracker.track(target, Change.added)
-        sleep_past_stability(stability_tracker)
-        assert list(stability_tracker.get_stable_files()) == []
-
-        target.write_bytes(b"%PDF-1.4 content")
-        stability_tracker.track(target, Change.modified)
-        sleep_past_stability(stability_tracker)
-
-        assert list(stability_tracker.get_stable_files()) == [target]
-
     def test_get_stable_files_deleted_during_check(self, temp_file: Path) -> None:
         """Test deleted file is not returned during stability check."""
         tracker = FileStabilityTracker(stability_delay=0.1)
@@ -946,51 +879,6 @@ class TestCommandWatch:
 
         mock_consume_file_delay.apply_async.assert_called()
 
-    def test_scanner_placeholder_is_not_consumed_while_empty(
-        self,
-        consumption_dir: Path,
-        sample_pdf: Path,
-        mock_consume_file_delay: MagicMock,
-        start_consumer: Callable[..., ConsumerThread],
-    ) -> None:
-        """
-        GIVEN:
-            - A scanner which creates a zero byte placeholder and only writes
-              the page some time later (GH discussion #13969)
-        WHEN:
-            - The placeholder sits untouched well past the stability delay
-            - The scanner then writes the real content
-        THEN:
-            - The empty placeholder is never queued, as it could only fail
-              with "Unsupported mime type inode/x-empty"
-            - The file is queued exactly once, when the content lands
-        """
-        thread = start_consumer(stability_delay=0.2)
-
-        target = consumption_dir / "scan.pdf"
-        target.write_bytes(b"")  # the scanner's placeholder
-
-        # Well past the stability delay: the old behaviour queued it here.
-        sleep_past_stability(thread, windows=5)
-        if thread.exception:
-            raise thread.exception
-        assert mock_consume_file_delay.apply_async.call_count == 0
-
-        shutil.copy(sample_pdf, target)  # the scanner finishes the page
-
-        assert wait_for_mock_call(
-            mock_consume_file_delay.apply_async,
-            timeout_s=5.0,
-        )
-        if thread.exception:
-            raise thread.exception
-
-        assert mock_consume_file_delay.apply_async.call_count == 1
-        queued_doc = mock_consume_file_delay.apply_async.call_args.kwargs["kwargs"][
-            "input_doc"
-        ]
-        assert queued_doc.original_file.name == "scan.pdf"
-
     def test_ignores_macos_files(
         self,
         consumption_dir: Path,

src/documents/tests/test_matching_nfc.py

@@ -0,0 +1,80 @@
+import unicodedata
+
+import pytest
+
+from documents.data_models import ConsumableDocument
+from documents.data_models import DocumentSource
+from documents.matching import consumable_document_matches_workflow
+from documents.matching import existing_document_matches_workflow
+from documents.models import Document
+from documents.models import Workflow
+from documents.models import WorkflowTrigger
+
+
+@pytest.mark.django_db
+class TestMatchingNfcNormalization:
+    def test_consumable_document_filename_nfd_matches_nfc_pattern(
+        self,
+        tmp_path,
+    ) -> None:
+        """
+        GIVEN:
+            - A file on disk whose name is NFD-normalized
+            - A workflow trigger filename filter typed as NFC
+        WHEN:
+            - The consumable document is checked against the trigger
+        THEN:
+            - It matches, because both sides are normalized before comparing
+        """
+        nfd_name = unicodedata.normalize("NFD", "Gehaltserhöhung.pdf")
+        nfc_pattern = unicodedata.normalize("NFC", "*Gehaltserhöhung*")
+        assert nfd_name != unicodedata.normalize("NFC", nfd_name)
+
+        file_path = tmp_path / nfd_name
+        file_path.write_bytes(b"%PDF-1.4 test")
+
+        document = ConsumableDocument(
+            source=DocumentSource.ConsumeFolder,
+            original_file=file_path,
+        )
+        trigger = WorkflowTrigger(
+            type=WorkflowTrigger.WorkflowTriggerType.CONSUMPTION,
+            filter_filename=nfc_pattern,
+            sources=[],
+        )
+
+        matched, reason = consumable_document_matches_workflow(document, trigger)
+
+        assert matched, reason
+
+    def test_existing_document_filename_nfd_matches_nfc_pattern(self) -> None:
+        """
+        GIVEN:
+            - A Document whose original_filename is NFD-normalized (e.g. from
+              before normalization was applied at consumption time)
+            - A workflow trigger filename filter typed as NFC
+        WHEN:
+            - The document is checked against the trigger
+        THEN:
+            - It matches, because both sides are normalized before comparing
+        """
+        nfd_name = unicodedata.normalize("NFD", "Gehaltserhöhung.pdf")
+        nfc_pattern = unicodedata.normalize("NFC", "*Gehaltserhöhung*")
+
+        document = Document.objects.create(
+            title="Test",
+            content="content",
+            checksum="checksum",
+            mime_type="application/pdf",
+            original_filename=nfd_name,
+        )
+        workflow = Workflow.objects.create(name="Test workflow", order=0)
+        trigger = WorkflowTrigger.objects.create(
+            type=WorkflowTrigger.WorkflowTriggerType.DOCUMENT_ADDED,
+            filter_filename=nfc_pattern,
+        )
+        workflow.triggers.add(trigger)
+
+        matched, reason = existing_document_matches_workflow(document, trigger)
+
+        assert matched, reason

src/documents/tests/test_normalize_unicode.py

@@ -0,0 +1,6 @@
+from documents.utils import normalize_unicode
+
+
+class TestNormalizeUnicode:
+    def test_none_passes_through(self) -> None:
+        assert normalize_unicode(None) is None

src/documents/tests/test_tag_hierarchy.py

@@ -2,7 +2,6 @@ from unittest import mock
 
 from django.contrib.auth.models import Permission
 from django.contrib.auth.models import User
-from rest_framework import status
 from rest_framework.test import APITestCase
 
 from documents import bulk_edit
@@ -109,44 +108,6 @@ class TestTagHierarchy(DirectoriesMixin, APITestCase):
         self.document.refresh_from_db()
         assert self.document.tags.count() == 0
 
-    def test_remove_inbox_tags_removes_nested_children(self) -> None:
-        inbox = Tag.objects.create(name="Inbox", is_inbox_tag=True)
-        nested = Tag.objects.create(name="Nested", tn_parent=inbox)
-        self.document.add_nested_tags([nested])
-
-        resp = self.client.patch(
-            f"/api/documents/{self.document.pk}/",
-            {"title": "new title", "remove_inbox_tags": True},
-            format="json",
-        )
-        assert resp.status_code == status.HTTP_200_OK
-        self.document.refresh_from_db()
-        assert self.document.tags.count() == 0
-
-        # A subsequent save must not re-add the inbox tag as an ancestor
-        resp = self.client.patch(
-            f"/api/documents/{self.document.pk}/",
-            {"title": "another title", "tags": [], "remove_inbox_tags": True},
-            format="json",
-        )
-        assert resp.status_code == status.HTTP_200_OK
-        self.document.refresh_from_db()
-        assert self.document.tags.count() == 0
-
-    def test_remove_inbox_tags_keeps_inbox_when_nested_child_added(self) -> None:
-        inbox = Tag.objects.create(name="Inbox", is_inbox_tag=True)
-        nested = Tag.objects.create(name="Nested", tn_parent=inbox)
-        self.document.add_nested_tags([inbox])
-
-        self.client.patch(
-            f"/api/documents/{self.document.pk}/",
-            {"tags": [nested.pk], "remove_inbox_tags": True},
-            format="json",
-        )
-        self.document.refresh_from_db()
-        tags = set(self.document.tags.values_list("pk", flat=True))
-        assert tags == {inbox.pk, nested.pk}
-
     def test_bulk_edit_respects_hierarchy(self) -> None:
         bulk_edit.add_tag([self.document.pk], self.child.pk)
         self.document.refresh_from_db()

src/documents/tests/test_views.py

@@ -32,7 +32,6 @@ from documents.signals.handlers import update_llm_suggestions_cache
 from documents.tests.utils import DirectoriesMixin
 from documents.tests.utils import read_streaming_response
 from paperless.models import ApplicationConfiguration
-from paperless_ai.exceptions import LLMProviderError
 from paperless_ai.exceptions import LLMTimeoutError
 
 
@@ -738,38 +737,6 @@ class TestAISuggestions(DirectoriesMixin, TestCase):
             get_llm_suggestion_cache(self.document.pk, backend="openai-like"),
         )
 
-    @patch("documents.views.get_ai_document_classification")
-    @override_settings(
-        AI_ENABLED=True,
-        LLM_BACKEND="openai-like",
-    )
-    def test_ai_suggestions_with_llm_provider_error(
-        self,
-        mock_get_ai_classification,
-    ) -> None:
-        mock_get_ai_classification.side_effect = LLMProviderError(
-            "confidential provider response",
-        )
-
-        self.client.force_login(user=self.user)
-        response = self.client.get(
-            f"/api/documents/{self.document.pk}/ai_suggestions/",
-        )
-
-        self.assertEqual(response.status_code, status.HTTP_502_BAD_GATEWAY)
-        self.assertEqual(
-            response.json(),
-            {
-                "ai": [
-                    "AI backend rejected the request. Check logs for details.",
-                ],
-            },
-        )
-        self.assertNotIn("confidential provider response", response.content.decode())
-        self.assertIsNone(
-            get_llm_suggestion_cache(self.document.pk, backend="openai-like"),
-        )
-
     @patch("documents.views.get_ai_document_classification")
     @override_settings(
         AI_ENABLED=True,

src/documents/utils.py

@@ -1,6 +1,7 @@
 import hashlib
 import logging
 import shutil
+import unicodedata
 from collections.abc import Callable
 from collections.abc import Iterable
 from collections.abc import Iterator
@@ -31,6 +32,25 @@ def identity(iterable: Iterable[_T]) -> Iterable[_T]:
     return iterable
 
 
+def normalize_unicode(value: str | None) -> str | None:
+    """
+    Normalize a string to Unicode NFC form, or return None unchanged.
+
+    This is the single normalization pass for any user- or filesystem-supplied
+    text that ends up in a filename, path, or is compared/matched against one
+    (titles, correspondent/tag/type names, uploaded filenames, workflow and
+    mail rule filename/path filters). Composed (NFC) and decomposed (NFD)
+    forms of the same visible text are different byte sequences, which breaks
+    exact comparisons and filesystem lookups even though the text looks
+    identical. Always normalize through this function rather than calling
+    unicodedata.normalize() directly, so every call site agrees on the same
+    form.
+    """
+    if value is None:
+        return None
+    return unicodedata.normalize("NFC", value)
+
+
 class QuerySetStream(Generic[_M]):
     """Stream a QuerySet via .iterator(chunk_size=...) instead of
     materializing it (plus any prefetch caches) all at once, while still

src/documents/views.py

@@ -16,7 +16,6 @@ from time import mktime
 from time import sleep
 from typing import TYPE_CHECKING
 from typing import Any
-from typing import Final
 from typing import Literal
 from typing import NamedTuple
 from unicodedata import normalize
@@ -232,6 +231,7 @@ from documents.tasks import sanity_check
 from documents.tasks import train_classifier
 from documents.tasks import update_document_parent_tags
 from documents.utils import get_boolean
+from documents.utils import normalize_unicode
 from documents.versioning import VersionResolutionError
 from documents.versioning import annotate_effective_content
 from documents.versioning import get_latest_version_for_root
@@ -253,7 +253,6 @@ from paperless.views import StandardPagination
 from paperless_ai.ai_classifier import get_ai_document_classification
 from paperless_ai.ai_classifier import get_llm_output_language
 from paperless_ai.chat import stream_chat_with_documents
-from paperless_ai.exceptions import LLMProviderError
 from paperless_ai.exceptions import LLMTimeoutError
 from paperless_ai.matching import extract_unmatched_names
 from paperless_ai.matching import match_correspondents_by_name
@@ -287,40 +286,17 @@ logger = logging.getLogger("paperless.api")
 _TANTIVY_INTERSECT_THRESHOLD = 5_000
 _TANTIVY_SEARCH_PARAM_NAMES = ("text", "title_search", "query", "more_like_id")
 
-# whoosh-compat's fieldname tagger (used only for SearchMode.QUERY, via the
-# whoosh grammar in parse_user_query) is O(n^2) in plain word characters:
-# measured at ~0.96s/10k chars, ~3.67s/20k, ~14.4s/40k against the real field
-# registry. Django's DATA_UPLOAD_MAX_MEMORY_SIZE default (2.5 MB) does not
-# bound this on the POST-body selection-filter path, so an unbounded query
-# is a single-request CPU exhaustion vector. 4096 chars caps the worst case
-# at roughly 0.16s (quadratic extrapolation from the measurements above),
-# far beyond any plausible hand-typed advanced query, while still being fast
-# enough to absorb inside a request handler. Applied to all three modes at
-# this shared choke point: TEXT and TITLE route through simple_search_tokens
-# instead and measure linear even at 20k chars, so the cap is hygiene for
-# them, not a fix, but a single limit here is simpler than one exemption.
-# Not exposed as a PAPERLESS_* setting: this is a hard security boundary,
-# not a tunable, and a raisable ceiling would let a misconfiguration
-# reintroduce the exact hazard this exists to close.
-_MAX_QUERY_LENGTH: Final[int] = 4096
-
 
 def _get_tantivy_query_and_mode(params):
-    from documents.search import QueryTooLongError
     from documents.search import SearchMode
 
     if "text" in params:
-        raw, mode = str(params["text"]), SearchMode.TEXT
+        return str(params["text"]), SearchMode.TEXT
-    elif "title_search" in params:
+    if "title_search" in params:
-        raw, mode = str(params["title_search"]), SearchMode.TITLE
+        return str(params["title_search"]), SearchMode.TITLE
-    elif "query" in params:
+    if "query" in params:
-        raw, mode = str(params["query"]), SearchMode.QUERY
+        return str(params["query"]), SearchMode.QUERY
-    else:
+    return None  # pragma: no cover
-        return None  # pragma: no cover
-
-    if len(raw) > _MAX_QUERY_LENGTH:
-        raise QueryTooLongError(len(raw), _MAX_QUERY_LENGTH)
-    return raw, mode
 
 
 def _get_more_like_id(query_params: dict[str, Any], user: User | None) -> int:
@@ -1628,22 +1604,6 @@ class DocumentViewSet(
                     {"ai": [_("AI backend request timed out.")]},
                     status=status.HTTP_503_SERVICE_UNAVAILABLE,
                 )
-            except LLMProviderError:
-                logger.exception(
-                    "AI backend rejected the request for document %s",
-                    doc.pk,
-                )
-                return Response(
-                    {
-                        "ai": [
-                            _(
-                                "AI backend rejected the request. "
-                                "Check logs for details.",
-                            ),
-                        ],
-                    },
-                    status=status.HTTP_502_BAD_GATEWAY,
-                )
             set_llm_suggestions_cache(
                 doc.pk,
                 llm_suggestions,
@@ -2109,6 +2069,7 @@ class DocumentViewSet(
 
         try:
             doc_name, doc_data = serializer.validated_data.get("document")
+            doc_name = normalize_unicode(doc_name)
             version_label = serializer.validated_data.get("version_label")
 
             t = int(mktime(datetime.now().timetuple()))
@@ -2477,7 +2438,6 @@ class UnifiedSearchViewSet(DocumentViewSet):
         from documents.search import TantivyBackend
         from documents.search import TantivyRelevanceList
         from documents.search import get_backend
-        from documents.search import search_query_error_messages
 
         def parse_search_params() -> SearchParams:
             """Extract query string, search mode, and ordering from request."""
@@ -2668,10 +2628,15 @@ class UnifiedSearchViewSet(DocumentViewSet):
         except ValidationError:
             raise
         except SearchQueryError as e:
-            # User-fixable query error(s) (e.g. unparsable dates/numbers):
+            # User-fixable query error (e.g. an unparsable date): surface the
-            # surface every offending field's message, not just the first,
+            # specific message so the user can correct it, rather than a generic
-            # so the user can fix them all in one round-trip.
+            # 400 or silently empty results.
-            raise ValidationError({"query": search_query_error_messages(e)}) from e
+            raise ValidationError({"query": [str(e)]}) from e
+        except Exception as e:
+            logger.warning(f"An error occurred listing search results: {e!s}")
+            return HttpResponseBadRequest(
+                "Error listing search results, check logs for more detail.",
+            )
 
     @action(detail=False, methods=["GET"], name="Get Next ASN")
     def next_asn(self, request, *args, **kwargs):
@@ -2807,34 +2772,23 @@ class DocumentSelectionMixin:
                 },
             )
 
-        from documents.search import SearchQueryError
         from documents.search import get_backend
-        from documents.search import search_query_error_messages
 
         filter_name = search_filters[0]
         backend = get_backend()
         search_user = None if user.is_superuser else user
 
-        try:
+        if filter_name == "more_like_id":
-            if filter_name == "more_like_id":
+            more_like_doc_id = _get_more_like_id(filters, user)
-                more_like_doc_id = _get_more_like_id(filters, user)
 
-                search_ids = backend.more_like_this_ids(
+            search_ids = backend.more_like_this_ids(more_like_doc_id, user=search_user)
-                    more_like_doc_id,
+        else:
-                    user=search_user,
+            query_str, search_mode = _get_tantivy_query_and_mode(filters)
-                )
+            search_ids = backend.search_ids(
-            else:
+                query_str,
-                query_str, search_mode = _get_tantivy_query_and_mode(filters)
+                user=search_user,
-                search_ids = backend.search_ids(
+                search_mode=search_mode,
-                    query_str,
+            )
-                    user=search_user,
-                    search_mode=search_mode,
-                )
-        except SearchQueryError as e:
-            # Same user-fixable-query mapping as the search list endpoint:
-            # a bad date/number in a bulk selection filter is a 400 naming
-            # the value, never a 500.
-            raise ValidationError({"query": search_query_error_messages(e)}) from e
 
         return search_ids
 
@@ -3382,7 +3336,7 @@ class PostDocumentView(GenericAPIView[Any]):
         serializer.is_valid(raise_exception=True)
 
         doc_name, doc_data = serializer.validated_data.get("document")
-        doc_name = normalize("NFC", doc_name)
+        doc_name = normalize_unicode(doc_name)
         correspondent_id = serializer.validated_data.get("correspondent")
         document_type_id = serializer.validated_data.get("document_type")
         storage_path_id = serializer.validated_data.get("storage_path")
@@ -3675,10 +3629,6 @@ class GlobalSearchView(PassUserMixin):
             return HttpResponseBadRequest("Query required")
         if len(query) < 3:
             return HttpResponseBadRequest("Query must be at least 3 characters")
-        if len(query) > _MAX_QUERY_LENGTH:
-            return HttpResponseBadRequest(
-                f"Query must be at most {_MAX_QUERY_LENGTH} characters",
-            )
 
         db_only = request.query_params.get("db_only", False)
 
@@ -5489,10 +5439,7 @@ class TrashView(ListModelMixin, PassUserMixin):
 
     model = Document
 
-    # A version is listed separately only when its root is not in the trash.
+    queryset = Document.deleted_objects.all()
-    queryset = Document.deleted_objects.exclude(
-        root_document_id__in=Document.deleted_objects.values("id"),
-    )
 
     def get(self, request: Request, format: str | None = None) -> Response:
         self.serializer_class = DocumentSerializer
@@ -5523,15 +5470,7 @@ class TrashView(ListModelMixin, PassUserMixin):
             return HttpResponseForbidden("Insufficient permissions")
         action = serializer.validated_data.get("action")
         if action == "restore":
-            restored = list(self.get_queryset().filter(id__in=doc_ids))
+            restored = list(Document.deleted_objects.filter(id__in=doc_ids))
-            if len(restored) != len(doc_ids):
-                raise ValidationError(
-                    {
-                        "documents": [
-                            "Restore the root document instead of one of its versions.",
-                        ],
-                    },
-                )
             for doc in restored:
                 doc.restore(strict=False)
             if restored:

src/paperless/settings/__init__.py

@@ -96,13 +96,6 @@ MODEL_FILE = get_path_from_env(
     "PAPERLESS_MODEL_FILE",
     DATA_DIR / "classification_model.pickle",
 )
-
-# Minimum confidence (0.0-1.0) for the ML classifier to assign a correspondent,
-# document type, or storage path. 0.0 disables the threshold.
-CLASSIFIER_MATCH_THRESHOLD: Final[float] = get_float_from_env(
-    "PAPERLESS_CLASSIFIER_MATCH_THRESHOLD",
-    0.6,
-)
 LLM_INDEX_DIR = DATA_DIR / "llm_index"
 LLM_INDEX_LOCK = LLM_INDEX_DIR / "index.lock"
 # Cross-process read/write lock guarding the LLM index compaction/migration

src/paperless_ai/ai_classifier.py

@@ -4,24 +4,21 @@ from django.conf import settings
 from django.contrib.auth.models import User
 
 from documents.models import Document
-from documents.permissions import permitted_object_ids
+from documents.permissions import get_objects_for_user_owner_aware
-from documents.permissions import restrict_queryset_to_visible
-from documents.permissions import user_is_unrestricted
 from paperless.config import AIConfig
 from paperless_ai.base_model import ClassificationSuggestions
 from paperless_ai.base_model import TaxonomyChoiceDict
 from paperless_ai.base_model import classification_suggestions_to_model
 from paperless_ai.client import AIClient
 from paperless_ai.db import db_connection_released
+from paperless_ai.indexing import _node_document_ids
 from paperless_ai.indexing import retrieve_similar_nodes
 from paperless_ai.indexing import truncate_content
 from paperless_ai.prompts.context import ClassificationPromptContext
 from paperless_ai.prompts.context import LocalizationPromptContext
 from paperless_ai.prompts.context import RagContextPromptContext
 from paperless_ai.prompts.render import render_prompt
-from paperless_ai.taxonomy import SimilarDocument
 from paperless_ai.taxonomy import TaxonomyCandidates
-from paperless_ai.taxonomy import _node_document_weights
 from paperless_ai.taxonomy import build_taxonomy_candidates
 from paperless_ai.taxonomy import empty_taxonomy_candidates
 from paperless_ai.taxonomy import format_taxonomy_for_prompt
@@ -40,48 +37,6 @@ logger = logging.getLogger("paperless_ai.rag_classifier")
 TAXONOMY_CANDIDATE_TOP_K = 15
 
 
-def _fulltext_similar_documents(
-    document: Document,
-    user: User | None,
-    top_k: int,
-) -> list[SimilarDocument]:
-    """Rank-based fallback when no embedding backend is configured. Uses
-    Tantivy's "More Like This" (term-overlap similarity) instead of vector
-    similarity - cruder, but far better than no candidates at all.
-    more_like_this_ids returns only a ranked ID list, no scores, so weight is
-    synthesized from rank (descending from top_k) rather than claiming a
-    similarity magnitude that doesn't exist. An unrestricted user (none, or an
-    active superuser - see user_is_unrestricted) is normalized to ``None``
-    before calling, since the backend's permission filter has no superuser
-    short-circuit of its own. Results are re-checked with
-    restrict_queryset_to_visible() since Tantivy's indexed permission fields
-    lag the DB via async reindexing.
-    """
-    from documents.search import get_backend
-
-    unrestricted = user_is_unrestricted(user)
-    search_user = None if unrestricted else user
-    backend = get_backend()
-    similar_ids = backend.more_like_this_ids(
-        document.pk,
-        user=search_user,
-        limit=top_k,
-    )
-    if not unrestricted:
-        allowed_ids = set(
-            restrict_queryset_to_visible(
-                Document.objects.filter(pk__in=similar_ids),
-                user,
-                "view_document",
-            ).values_list("pk", flat=True),
-        )
-        similar_ids = [doc_id for doc_id in similar_ids if doc_id in allowed_ids]
-    return [
-        SimilarDocument(document_id=doc_id, weight=float(top_k - rank))
-        for rank, doc_id in enumerate(similar_ids)
-    ]
-
-
 def get_language_name(language_code: str) -> str:
     normalized_language_code = language_code.lower()
     for code, name in settings.LANGUAGES:
@@ -181,52 +136,43 @@ def get_taxonomy_context(
     user: User | None = None,
     max_docs: int = 5,
 ) -> tuple[TaxonomyCandidates, str]:
-    """One retrieval feeds both taxonomy candidates and RAG text context. Uses
+    """One retrieval feeds both taxonomy candidates and RAG text context.
-    vector similarity when an embedding backend is configured, otherwise
+    On any retrieval failure, degrades to empty candidates/context rather than
-    falls back to Tantivy full-text "More Like This" similarity - see
+    propagating the exception - a vector-store outage should not block
-    _fulltext_similar_documents. On any retrieval failure, degrades to empty
+    classification, only its RAG-assisted enrichment.
-    candidates/context rather than propagating the exception - neither a
-    vector-store outage nor a search-index issue should block classification,
-    only its context-assisted enrichment.
     """
-    ai_config = AIConfig()
     try:
-        if ai_config.llm_embedding_backend:
+        # None means "no restriction" to retrieve_similar_nodes. A superuser
-            # None means "no restriction" to retrieve_similar_nodes. An
+        # (like no user at all) can see every document, so skip materializing
-            # unrestricted user (no user at all, or an active superuser -- see
+        # every visible pk into a Python list and passing it through as an IN
-            # user_is_unrestricted) can see every document, so skip
+        # filter: for a large library that is a wasted quadratic scan in the
-            # materializing every visible pk into a Python list and passing it
+        # vector store at best, and past ~32,763 documents a hard
-            # through as an IN filter: for a large library that is a wasted
+        # sqlite3.OperationalError (SQLite's bound-parameter limit) at worst.
-            # quadratic scan in the vector store at best, and past ~32,763
+        # get_objects_for_user_owner_aware() would return every Document for a
-            # documents a hard sqlite3.OperationalError (SQLite's
+        # superuser anyway (guardian's own with_superuser shortcut), so this
-            # bound-parameter limit) at worst.
+        # changes nothing about which documents are considered -- only how we
-            # permitted_object_ids() has its own superuser shortcut that would
+        # get there.
-            # return every Document's id anyway, so this changes nothing about
+        visible_document_ids = (
-            # which documents are considered -- only how we get there.
+            None
-            visible_document_ids = (
+            if user is None or user.is_superuser
-                None
+            else list(
-                if user_is_unrestricted(user)
+                get_objects_for_user_owner_aware(
-                else list(permitted_object_ids(user, Document, "view_document"))
+                    user,
-            )
+                    "view_document",
-            nodes = retrieve_similar_nodes(
+                    Document,
-                document,
+                ).values_list("pk", flat=True),
-                top_k=TAXONOMY_CANDIDATE_TOP_K,
-                document_ids=visible_document_ids,
-            )
-            similar_documents = _node_document_weights(nodes)
-        else:
-            # See _fulltext_similar_documents: it applies its own permission
-            # filter via `user`, so no visible-document-id list is needed here.
-            similar_documents = _fulltext_similar_documents(
-                document,
-                user,
-                top_k=TAXONOMY_CANDIDATE_TOP_K,
             )
+        )
+        nodes = retrieve_similar_nodes(
+            document,
+            top_k=TAXONOMY_CANDIDATE_TOP_K,
+            document_ids=visible_document_ids,
+        )
 
-        candidates = build_taxonomy_candidates(similar_documents, user)
+        candidates = build_taxonomy_candidates(nodes, user)
 
-        # similar_documents is already ordered by descending weight; don't lose it.
+        # ``nodes`` are already ordered by descending vector similarity; don't lose it.
-        similar_document_ids = [s["document_id"] for s in similar_documents]
+        similar_document_ids = list(dict.fromkeys(_node_document_ids(nodes)))
         similar_documents_by_id = Document.objects.in_bulk(similar_document_ids)
         similar_docs = [
             similar_documents_by_id[document_id]
@@ -240,8 +186,8 @@ def get_taxonomy_context(
             context_blocks.append(f"TITLE: {title}\n{text}")
     except Exception:
         logger.exception(
-            "Failed to retrieve similar-document context for document %s; "
+            "Failed to retrieve RAG neighbours for document %s; continuing "
-            "continuing without taxonomy candidates or similar-document context.",
+            "without taxonomy candidates or similar-document context.",
             document.pk,
         )
         return empty_taxonomy_candidates(), ""
@@ -295,13 +241,17 @@ def get_ai_document_classification(
 ) -> ClassificationSuggestions:
     ai_config = AIConfig()
 
-    candidates, context = get_taxonomy_context(document, user)
+    if ai_config.llm_embedding_backend:
-    prompt = build_prompt_with_rag(
+        candidates, context = get_taxonomy_context(document, user)
-        document,
+        prompt = build_prompt_with_rag(
-        ai_config,
+            document,
-        candidates=candidates,
+            ai_config,
-        context=context,
+            candidates=candidates,
-    )
+            context=context,
+        )
+    else:
+        candidates = empty_taxonomy_candidates()
+        prompt = build_prompt_without_rag(document, ai_config, candidates=candidates)
 
     client = AIClient()
     # Hand the pooled DB connection back while the (slow) LLM query runs so it

src/paperless_ai/client.py

@@ -22,7 +22,6 @@ from paperless.network import validate_outbound_http_url
 from paperless_ai.base_model import ClassificationSuggestions
 from paperless_ai.base_model import DocumentClassifierSchema
 from paperless_ai.base_model import model_to_classification_suggestions
-from paperless_ai.exceptions import LLMProviderError
 from paperless_ai.exceptions import LLMTimeoutError
 
 logger = logging.getLogger("paperless_ai.client")
@@ -133,7 +132,7 @@ class AIClient:
         from llama_index.core.llms import ChatMessage
 
         if self.settings.llm_backend == LLMBackend.OLLAMA:
-            with self._normalize_errors():
+            with self._normalize_timeouts():
                 result = self.llm.chat(
                     [ChatMessage(role="user", content=prompt)],
                     format=DocumentClassifierSchema.model_json_schema(),
@@ -154,7 +153,7 @@ class AIClient:
             content=f"{prompt}\n\n"
             f"Answer by calling the {tool.metadata.name} tool. Do not write the answer as text.",
         )
-        with self._normalize_errors():
+        with self._normalize_timeouts():
             result = self.llm.chat_with_tools(
                 tools=[tool],
                 user_msg=user_msg,
@@ -174,7 +173,7 @@ class AIClient:
         )
 
     @contextmanager
-    def _normalize_errors(self) -> Iterator[None]:
+    def _normalize_timeouts(self) -> Iterator[None]:
         try:
             yield
         except httpx.TimeoutException as exc:
@@ -182,23 +181,8 @@ class AIClient:
         except Exception as exc:
             if self._is_openai_timeout(exc):
                 raise LLMTimeoutError from exc
-            if self._is_provider_error(exc):
-                raise LLMProviderError from exc
             raise
 
-    def _is_provider_error(self, exc: Exception) -> bool:
-        if self.settings.llm_backend == LLMBackend.OLLAMA:
-            from ollama import ResponseError
-
-            return isinstance(exc, ResponseError)
-
-        if self.settings.llm_backend == LLMBackend.OPENAI_LIKE:
-            from openai import APIStatusError
-
-            return isinstance(exc, APIStatusError)
-
-        return False
-
     def _is_openai_timeout(self, exc: Exception) -> bool:
         if self.settings.llm_backend != LLMBackend.OPENAI_LIKE:
             return False

src/paperless_ai/exceptions.py

@@ -1,6 +1,2 @@
 class LLMTimeoutError(Exception):
     pass
-
-
-class LLMProviderError(Exception):
-    """The LLM backend rejected the request."""

src/paperless_ai/indexing.py

@@ -721,3 +721,20 @@ def retrieve_similar_nodes(
             continue
         filtered.append(node)
     return filtered
+
+
+def _node_document_ids(nodes: list["NodeWithScore"]) -> list[int]:
+    document_ids: list[int] = []
+    for node in nodes:
+        document_id = node.metadata.get("document_id")
+        if document_id is None:  # pragma: no cover
+            # See the matching guard in retrieve_similar_nodes() above.
+            continue
+        try:
+            document_ids.append(int(document_id))
+        except ValueError:  # pragma: no cover
+            logger.warning(
+                "Skipping LLM index result with invalid document_id %r.",
+                document_id,
+            )
+    return document_ids

src/paperless_ai/taxonomy.py

@@ -31,11 +31,6 @@ class TaxonomyCandidate(TypedDict):
     weight: float
 
 
-class SimilarDocument(TypedDict):
-    document_id: int
-    weight: float
-
-
 class TaxonomyCandidates(TypedDict):
     tags: list[TaxonomyCandidate]
     document_types: list[TaxonomyCandidate]
@@ -54,10 +49,10 @@ def empty_taxonomy_candidates() -> TaxonomyCandidates:
     )
 
 
-def _node_document_weights(nodes: list["NodeWithScore"]) -> list[SimilarDocument]:
+def _node_document_weights(nodes: list["NodeWithScore"]) -> dict[int, float]:
-    """Sum each node's similarity score into its document_id (a document can
+    """document_id -> that node's similarity score, summed if a document_id
-    appear via multiple chunks/nodes) and return one SimilarDocument per
+    appears more than once across the retrieved nodes (e.g. multiple chunks
-    distinct document_id."""
+    of the same source document)."""
     weights: dict[int, float] = defaultdict(float)
     for node in nodes:
         document_id = node.metadata.get("document_id")
@@ -70,14 +65,7 @@ def _node_document_weights(nodes: list["NodeWithScore"]) -> list[SimilarDocument
             weights[int(document_id)] += float(node.score or 0.0)
         except (TypeError, ValueError):  # pragma: no cover
             continue
-    return sorted(
+    return weights
-        (
-            SimilarDocument(document_id=document_id, weight=weight)
-            for document_id, weight in weights.items()
-        ),
-        key=lambda similar: similar["weight"],
-        reverse=True,
-    )
 
 
 def _visible_ranked_candidates(
@@ -113,25 +101,20 @@ def _visible_ranked_candidates(
 
 
 def build_taxonomy_candidates(
-    similar_documents: list[SimilarDocument],
+    nodes: list["NodeWithScore"],
     user: User | None,
 ) -> TaxonomyCandidates:
-    """Resolve each similar document's id to a live Document, read its
+    """Resolve each neighbour node's document_id to a live Document, read its
-    *current* tags/type/correspondent/storage_path via the ORM (never any
+    *current* tags/type/correspondent/storage_path via the ORM (never the
-    possibly-stale names an adapter's source might have cached), weight each
+    possibly-stale names cached in vector-index node metadata), weight each
-    distinct taxonomy object by aggregate similarity weight, permission-filter
+    distinct taxonomy object by aggregate neighbour similarity, permission-filter
     against what ``user`` can see, and return each category ranked by weight
-    and capped. ``similar_documents`` may come from either the vector-RAG
+    and capped.
-    adapter or the full-text fallback adapter - both produce this same shape.
     """
-    if not similar_documents:
-        return empty_taxonomy_candidates()
 
-    # Both adapters guarantee at most one SimilarDocument per document_id, so
+    document_weights = _node_document_weights(nodes)
-    # this never silently drops a duplicate's weight.
+    if not document_weights:
-    document_weights: dict[int, float] = {
+        return empty_taxonomy_candidates()
-        s["document_id"]: s["weight"] for s in similar_documents
-    }
 
     # Only .tags.all() needs prefetching (a reverse M2M, one extra query for
     # the whole batch). document_type/correspondent/storage_path are read

src/paperless_ai/tests/test_ai_classifier.py

@@ -1,5 +1,4 @@
 import datetime
-from collections.abc import Generator
 from types import SimpleNamespace
 from unittest.mock import MagicMock
 from unittest.mock import patch
@@ -7,24 +6,18 @@ from unittest.mock import patch
 import pytest
 import pytest_mock
 from django.test import override_settings
-from guardian.shortcuts import assign_perm
-from guardian.shortcuts import remove_perm
 
 from documents.models import Document
-from documents.search import TantivyBackend
 from documents.tests.factories import DocumentFactory
 from documents.tests.factories import TagFactory
 from documents.tests.factories import UserFactory
 from paperless.config import AIConfig
-from paperless_ai.ai_classifier import TAXONOMY_CANDIDATE_TOP_K
-from paperless_ai.ai_classifier import _fulltext_similar_documents
 from paperless_ai.ai_classifier import build_localization_prompt
 from paperless_ai.ai_classifier import build_prompt_with_rag
 from paperless_ai.ai_classifier import build_prompt_without_rag
 from paperless_ai.ai_classifier import get_ai_document_classification
 from paperless_ai.ai_classifier import get_language_name
 from paperless_ai.ai_classifier import get_taxonomy_context
-from paperless_ai.taxonomy import SimilarDocument
 from paperless_ai.taxonomy import TaxonomyCandidate
 from paperless_ai.taxonomy import TaxonomyCandidates
 
@@ -227,10 +220,12 @@ def test_use_rag_if_configured(
 
 @pytest.mark.django_db
 @patch("paperless_ai.client.AIClient.run_llm_query")
-@patch("paperless_ai.ai_classifier.build_prompt_with_rag")
+@patch("paperless_ai.ai_classifier.build_prompt_without_rag")
+@patch("paperless_ai.ai_classifier.AIConfig")
 @override_settings(LLM_BACKEND="ollama", LLM_MODEL="some_model")
-def test_use_rag_prompt_even_without_embedding_backend(
+def test_use_without_rag_if_not_configured(
-    mock_build_prompt_with_rag,
+    mock_ai_config,
+    mock_build_prompt_without_rag,
     mock_run_llm_query,
     mock_document,
 ):
@@ -240,13 +235,13 @@ def test_use_rag_prompt_even_without_embedding_backend(
     WHEN:
         - get_ai_document_classification() is called
     THEN:
-        - The RAG-context prompt builder is still used (fed by the full-text
+        - The non-RAG prompt builder is used
-          fallback's context/candidates instead of the vector store's)
     """
-    mock_build_prompt_with_rag.return_value = "Prompt with RAG"
+    mock_ai_config.return_value.llm_embedding_backend = None
+    mock_build_prompt_without_rag.return_value = "Prompt without RAG"
     mock_run_llm_query.return_value = NESTED_SUGGESTIONS
     get_ai_document_classification(mock_document)
-    mock_build_prompt_with_rag.assert_called_once()
+    mock_build_prompt_without_rag.assert_called_once()
 
 
 @pytest.mark.django_db
@@ -325,7 +320,6 @@ def test_build_localization_prompt_preserves_unicode_characters():
 
 
 @pytest.mark.django_db
-@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
 def test_get_taxonomy_context_assembles_rag_text_and_candidates():
     """
     GIVEN:
@@ -360,7 +354,6 @@ def test_get_taxonomy_context_assembles_rag_text_and_candidates():
 
 
 @pytest.mark.django_db
-@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
 def test_get_taxonomy_context_preserves_similarity_order_and_distinct_documents():
     """
     GIVEN:
@@ -431,7 +424,6 @@ def test_get_taxonomy_context_preserves_similarity_order_and_distinct_documents(
 
 
 @pytest.mark.django_db
-@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
 def test_get_taxonomy_context_no_similar_docs():
     """
     GIVEN:
@@ -455,67 +447,6 @@ def test_get_taxonomy_context_no_similar_docs():
     }
 
 
-@pytest.mark.django_db
-def test_get_taxonomy_context_uses_fulltext_fallback_when_no_embedding_backend(
-    mocker: pytest_mock.MockerFixture,
-) -> None:
-    """
-    GIVEN:
-        - No LLM embedding backend is configured (the default test settings)
-    WHEN:
-        - get_taxonomy_context() is called
-    THEN:
-        - _fulltext_similar_documents() is called with the document, the user
-          and TAXONOMY_CANDIDATE_TOP_K
-        - retrieve_similar_nodes() (the vector path) is never called
-    """
-    document = DocumentFactory.create(content="Some content")
-    mock_fulltext = mocker.patch(
-        "paperless_ai.ai_classifier._fulltext_similar_documents",
-        return_value=[],
-    )
-    mock_retrieve = mocker.patch("paperless_ai.ai_classifier.retrieve_similar_nodes")
-
-    get_taxonomy_context(document, user=None)
-
-    mock_fulltext.assert_called_once_with(
-        document,
-        None,
-        top_k=TAXONOMY_CANDIDATE_TOP_K,
-    )
-    mock_retrieve.assert_not_called()
-
-
-@pytest.mark.django_db
-@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
-def test_get_taxonomy_context_uses_vector_path_when_embedding_backend_configured(
-    mocker: pytest_mock.MockerFixture,
-) -> None:
-    """
-    GIVEN:
-        - An LLM embedding backend is configured
-    WHEN:
-        - get_taxonomy_context() is called
-    THEN:
-        - retrieve_similar_nodes() (the vector path) is called
-        - _fulltext_similar_documents() (the no-embedding-backend fallback)
-          is never called
-    """
-    document = DocumentFactory.create(content="Some content")
-    mock_retrieve = mocker.patch(
-        "paperless_ai.ai_classifier.retrieve_similar_nodes",
-        return_value=[],
-    )
-    mock_fulltext = mocker.patch(
-        "paperless_ai.ai_classifier._fulltext_similar_documents",
-    )
-
-    get_taxonomy_context(document, user=None)
-
-    mock_retrieve.assert_called_once()
-    mock_fulltext.assert_not_called()
-
-
 class TestGetTaxonomyContextVisibility:
     """get_taxonomy_context must not materialize every visible document id
     for a user who can already see the whole library: a superuser (like no
@@ -528,7 +459,6 @@ class TestGetTaxonomyContextVisibility:
     """
 
     @pytest.mark.django_db
-    @override_settings(LLM_EMBEDDING_BACKEND="huggingface")
     def test_skips_permission_lookup_for_superuser(
         self,
         mocker: pytest_mock.MockerFixture,
@@ -547,18 +477,17 @@ class TestGetTaxonomyContextVisibility:
             "paperless_ai.ai_classifier.retrieve_similar_nodes",
             return_value=[],
         )
-        mock_permitted = mocker.patch(
+        mock_get_objects = mocker.patch(
-            "paperless_ai.ai_classifier.permitted_object_ids",
+            "paperless_ai.ai_classifier.get_objects_for_user_owner_aware",
         )
         user = UserFactory.create(is_superuser=True)
 
         get_taxonomy_context(document, user)
 
-        mock_permitted.assert_not_called()
+        mock_get_objects.assert_not_called()
         assert mock_retrieve.call_args.kwargs["document_ids"] is None
 
     @pytest.mark.django_db
-    @override_settings(LLM_EMBEDDING_BACKEND="huggingface")
     def test_skips_permission_lookup_when_no_user(
         self,
         mocker: pytest_mock.MockerFixture,
@@ -577,17 +506,16 @@ class TestGetTaxonomyContextVisibility:
             "paperless_ai.ai_classifier.retrieve_similar_nodes",
             return_value=[],
         )
-        mock_permitted = mocker.patch(
+        mock_get_objects = mocker.patch(
-            "paperless_ai.ai_classifier.permitted_object_ids",
+            "paperless_ai.ai_classifier.get_objects_for_user_owner_aware",
         )
 
         get_taxonomy_context(document, None)
 
-        mock_permitted.assert_not_called()
+        mock_get_objects.assert_not_called()
         assert mock_retrieve.call_args.kwargs["document_ids"] is None
 
     @pytest.mark.django_db
-    @override_settings(LLM_EMBEDDING_BACKEND="huggingface")
     def test_restricts_to_visible_documents_for_non_superuser(
         self,
         mocker: pytest_mock.MockerFixture,
@@ -598,7 +526,7 @@ class TestGetTaxonomyContextVisibility:
         WHEN:
             - get_taxonomy_context() is called
         THEN:
-            - The user's permitted document ids are looked up and passed to
+            - The user's visible document ids are looked up and passed to
               retrieve_similar_nodes() as a restriction
         """
         document = DocumentFactory.create(content="Some content")
@@ -606,232 +534,21 @@ class TestGetTaxonomyContextVisibility:
             "paperless_ai.ai_classifier.retrieve_similar_nodes",
             return_value=[],
         )
-        mock_permitted = mocker.patch(
+        mock_queryset = mocker.MagicMock()
-            "paperless_ai.ai_classifier.permitted_object_ids",
+        mock_queryset.values_list.return_value = [1, 2, 3]
-            return_value=[1, 2, 3],
+        mock_get_objects = mocker.patch(
+            "paperless_ai.ai_classifier.get_objects_for_user_owner_aware",
+            return_value=mock_queryset,
         )
         user = UserFactory.create(is_superuser=False)
 
         get_taxonomy_context(document, user)
 
-        mock_permitted.assert_called_once_with(user, Document, "view_document")
+        mock_get_objects.assert_called_once_with(user, "view_document", Document)
         assert mock_retrieve.call_args.kwargs["document_ids"] == [1, 2, 3]
 
 
 @pytest.mark.django_db
-class TestFulltextSimilarDocuments:
-    """_fulltext_similar_documents is the no-embedding-backend fallback: it
-    asks the Tantivy full-text index for "More Like This" neighbours instead
-    of the vector store, and synthesizes a rank-based weight since Tantivy's
-    more_like_this_ids returns only an ordered id list, no scores.
-    """
-
-    @pytest.fixture
-    def fulltext_backend(
-        self,
-        mocker: pytest_mock.MockerFixture,
-    ) -> Generator[TantivyBackend, None, None]:
-        """An in-memory Tantivy backend, wired up as the module-level
-        singleton _fulltext_similar_documents resolves via get_backend()."""
-        backend = TantivyBackend(path=None)
-        backend.open()
-        mocker.patch("documents.search.get_backend", return_value=backend)
-        try:
-            yield backend
-        finally:
-            backend.close()
-
-    def test_ranks_by_rank_based_weight_descending(
-        self,
-        fulltext_backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - A source document and two similar documents indexed in Tantivy
-        WHEN:
-            - _fulltext_similar_documents() is called
-        THEN:
-            - Each result's weight reflects its rank (first result weighted
-              higher than the second), not a raw similarity score
-        """
-        source = DocumentFactory.create(content="quarterly financial report details")
-        first = DocumentFactory.create(content="quarterly financial report details")
-        second = DocumentFactory.create(content="financial report")
-        for doc in (source, first, second):
-            fulltext_backend.add_or_update(doc)
-
-        result = _fulltext_similar_documents(source, user=None, top_k=5)
-
-        assert len(result) == 2
-        weight_by_id = {s["document_id"]: s["weight"] for s in result}
-        assert weight_by_id[first.pk] > weight_by_id[second.pk]
-
-    def test_excludes_source_document(
-        self,
-        fulltext_backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - A source document indexed in Tantivy with no other documents
-        WHEN:
-            - _fulltext_similar_documents() is called
-        THEN:
-            - An empty list is returned - the source document is never its
-              own similar document
-        """
-        source = DocumentFactory.create(content="unique unrelated content")
-        fulltext_backend.add_or_update(source)
-
-        result = _fulltext_similar_documents(source, user=None, top_k=5)
-
-        assert result == []
-
-    def test_empty_index_returns_empty_list(
-        self,
-        fulltext_backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - A document that has never been indexed (fresh/empty Tantivy index)
-        WHEN:
-            - _fulltext_similar_documents() is called
-        THEN:
-            - An empty list is returned rather than raising
-        """
-        source = DocumentFactory.create(content="never indexed")
-
-        result = _fulltext_similar_documents(source, user=None, top_k=5)
-
-        assert result == []
-
-    def test_respects_top_k_limit(
-        self,
-        fulltext_backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - A source document and four similar documents indexed
-        WHEN:
-            - _fulltext_similar_documents() is called with top_k=2
-        THEN:
-            - At most 2 results are returned
-        """
-        source = DocumentFactory.create(content="shared overlapping keyword text")
-        fulltext_backend.add_or_update(source)
-        for _ in range(4):
-            fulltext_backend.add_or_update(
-                DocumentFactory.create(content="shared overlapping keyword text"),
-            )
-
-        result = _fulltext_similar_documents(source, user=None, top_k=2)
-
-        assert len(result) == 2
-
-    def test_result_shape_is_similar_document(
-        self,
-        fulltext_backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - A source document and one similar document indexed
-        WHEN:
-            - _fulltext_similar_documents() is called
-        THEN:
-            - Each result is a SimilarDocument (document_id + weight only)
-        """
-        source = DocumentFactory.create(content="shared content phrase")
-        other = DocumentFactory.create(content="shared content phrase")
-        fulltext_backend.add_or_update(source)
-        fulltext_backend.add_or_update(other)
-
-        result = _fulltext_similar_documents(source, user=None, top_k=5)
-
-        # rank 0 (the only/best result) with top_k=5 -> weight = top_k - rank = 5.0,
-        # per the "first result gets top_k, the last gets 1" formula.
-        assert result == [SimilarDocument(document_id=other.pk, weight=5.0)]
-
-    def test_superuser_sees_other_users_documents(
-        self,
-        fulltext_backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - A source document owned by one user and a similar document
-              owned by a different user, with no sharing between them
-        WHEN:
-            - _fulltext_similar_documents() is called with a superuser
-        THEN:
-            - The other user's document is still returned as a similar
-              document - a superuser must not be narrowed by the backend's
-              owner-based permission filter
-        """
-        owner = UserFactory.create()
-        other_owner = UserFactory.create()
-        superuser = UserFactory.create(is_superuser=True)
-        source = DocumentFactory.create(
-            content="shared content phrase",
-            owner=owner,
-        )
-        other = DocumentFactory.create(
-            content="shared content phrase",
-            owner=other_owner,
-        )
-        fulltext_backend.add_or_update(source)
-        fulltext_backend.add_or_update(other)
-
-        result = _fulltext_similar_documents(source, user=superuser, top_k=5)
-
-        assert [s["document_id"] for s in result] == [other.pk]
-
-    def test_excludes_stale_permitted_document_for_regular_user(
-        self,
-        fulltext_backend: TantivyBackend,
-    ) -> None:
-        """
-        GIVEN:
-            - A regular (non-superuser) user
-            - A similar document the user is permitted to view, and another
-              similar document indexed while the user still had view
-              permission but which has since had that permission revoked in
-              the database, i.e. the Tantivy index has stale permission data
-        WHEN:
-            - _fulltext_similar_documents() is called with that user
-        THEN:
-            - Only the still-permitted document is returned - the DB
-              re-check via restrict_queryset_to_visible() must catch the
-              document Tantivy's stale index still thinks is visible
-        """
-        owner = UserFactory.create()
-        viewer = UserFactory.create(is_superuser=False)
-        source = DocumentFactory.create(
-            content="shared content phrase",
-            owner=owner,
-        )
-        permitted = DocumentFactory.create(
-            content="shared content phrase",
-            owner=owner,
-        )
-        now_private = DocumentFactory.create(
-            content="shared content phrase",
-            owner=owner,
-        )
-        assign_perm("view_document", viewer, permitted)
-        assign_perm("view_document", viewer, now_private)
-        fulltext_backend.add_or_update(source)
-        fulltext_backend.add_or_update(permitted)
-        fulltext_backend.add_or_update(now_private)
-
-        # Revoke access after indexing, without reindexing: the index still
-        # carries viewer as a permitted viewer for `now_private`.
-        remove_perm("view_document", viewer, now_private)
-
-        result = _fulltext_similar_documents(source, user=viewer, top_k=5)
-
-        assert [s["document_id"] for s in result] == [permitted.pk]
-
-
-@pytest.mark.django_db
-@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
 @patch("paperless_ai.ai_classifier.retrieve_similar_nodes")
 def test_get_taxonomy_context_retrieval_failure_degrades_to_no_hints(mock_retrieve):
     """
@@ -858,7 +575,6 @@ def test_get_taxonomy_context_retrieval_failure_degrades_to_no_hints(mock_retrie
 
 
 @pytest.mark.django_db
-@override_settings(LLM_EMBEDDING_BACKEND="huggingface")
 @patch("paperless_ai.ai_classifier.build_taxonomy_candidates")
 @patch("paperless_ai.ai_classifier.retrieve_similar_nodes")
 def test_get_taxonomy_context_candidate_building_failure_degrades_to_no_hints(

src/paperless_ai/tests/test_ai_indexing.py

@@ -1188,7 +1188,9 @@ class TestRetrieveSimilarNodesAgainstRealIndex:
 
         nodes = indexing.retrieve_similar_nodes(a, document_ids=[b.id])
 
-        assert all(int(node.metadata["document_id"]) == b.id for node in nodes)
+        assert all(
+            document_id == b.id for document_id in indexing._node_document_ids(nodes)
+        )
 
     def test_excludes_self(
         self,
@@ -1210,7 +1212,7 @@ class TestRetrieveSimilarNodesAgainstRealIndex:
 
         nodes = indexing.retrieve_similar_nodes(a, top_k=5)
 
-        assert {int(node.metadata["document_id"]) for node in nodes} == {b.id}
+        assert set(indexing._node_document_ids(nodes)) == {b.id}
 
     def test_excludes_self_with_multiple_chunks(
         self,
@@ -1233,4 +1235,4 @@ class TestRetrieveSimilarNodesAgainstRealIndex:
 
         nodes = indexing.retrieve_similar_nodes(a, top_k=3)
 
-        assert {int(node.metadata["document_id"]) for node in nodes} == {b.id}
+        assert set(indexing._node_document_ids(nodes)) == {b.id}

src/paperless_ai/tests/test_client.py

@@ -4,7 +4,6 @@ from unittest.mock import MagicMock
 from unittest.mock import patch
 
 import httpx
-import ollama
 import openai
 import pytest
 from llama_index.core.llms.llm import ToolSelection
@@ -12,7 +11,6 @@ from llama_index.core.llms.llm import ToolSelection
 from paperless_ai.client import LLM_SYSTEM_PROMPT
 from paperless_ai.client import PLACEHOLDER_API_KEY
 from paperless_ai.client import AIClient
-from paperless_ai.exceptions import LLMProviderError
 from paperless_ai.exceptions import LLMTimeoutError
 
 
@@ -216,52 +214,6 @@ def test_run_llm_query_openai_timeout_raises_local_error(
         client.run_llm_query("test_prompt")
 
 
-def test_run_llm_query_openai_status_error_raises_provider_error(
-    mock_ai_config,
-    mock_openai_llm,
-):
-    mock_ai_config.llm_backend = "openai-like"
-    mock_ai_config.llm_model = "test_model"
-    mock_ai_config.llm_endpoint = "http://test-url"
-
-    request = httpx.Request("POST", "http://test-url/v1/chat/completions")
-    body = {"error": {"message": "Thinking mode does not support this tool_choice"}}
-    mock_openai_llm.return_value.chat_with_tools.side_effect = openai.BadRequestError(
-        "Error code: 400",
-        response=httpx.Response(400, request=request, json=body),
-        body=body,
-    )
-
-    client = AIClient()
-
-    with pytest.raises(LLMProviderError) as exc_info:
-        client.run_llm_query("test_prompt")
-    assert str(exc_info.value) == ""
-    assert isinstance(exc_info.value.__cause__, openai.BadRequestError)
-
-
-def test_run_llm_query_ollama_response_error_raises_provider_error(
-    mock_ai_config,
-    mock_ollama_llm,
-):
-    mock_ai_config.llm_backend = "ollama"
-    mock_ai_config.llm_model = "test_model"
-    mock_ai_config.llm_endpoint = "http://test-url"
-
-    response_error = ollama.ResponseError(
-        "confidential provider response",
-        status_code=400,
-    )
-    mock_ollama_llm.return_value.chat.side_effect = response_error
-
-    client = AIClient()
-
-    with pytest.raises(LLMProviderError) as exc_info:
-        client.run_llm_query("test_prompt")
-    assert str(exc_info.value) == ""
-    assert exc_info.value.__cause__ is response_error
-
-
 def test_run_llm_query_httpx_timeout_raises_local_error(
     mock_ai_config,
     mock_ollama_llm,

src/paperless_ai/tests/test_taxonomy.py

@@ -1,4 +1,5 @@
 import json
+from types import SimpleNamespace
 
 import pytest
 import pytest_mock
@@ -9,14 +10,14 @@ from documents.tests.factories import DocumentTypeFactory
 from documents.tests.factories import StoragePathFactory
 from documents.tests.factories import TagFactory
 from documents.tests.factories import UserFactory
-from paperless_ai.taxonomy import SimilarDocument
 from paperless_ai.taxonomy import TaxonomyCandidates
 from paperless_ai.taxonomy import build_taxonomy_candidates
 from paperless_ai.taxonomy import format_taxonomy_for_prompt
 
 
-def make_similar(document_id: int, weight: float) -> SimilarDocument:
+def make_node(document_id: int, score: float) -> SimpleNamespace:
-    return SimilarDocument(document_id=document_id, weight=weight)
+    """A stand-in for NodeWithScore: only ``.metadata``/``.score`` are read."""
+    return SimpleNamespace(metadata={"document_id": str(document_id)}, score=score)
 
 
 @pytest.mark.django_db
@@ -52,9 +53,9 @@ class TestBuildTaxonomyCandidates:
         doc_a.tags.add(tag)
         doc_b = DocumentFactory.create()
         doc_b.tags.add(tag)
-        similar_documents = [make_similar(doc_a.pk, 0.9), make_similar(doc_b.pk, 0.4)]
+        nodes = [make_node(doc_a.pk, 0.9), make_node(doc_b.pk, 0.4)]
 
-        result = build_taxonomy_candidates(similar_documents, user=None)
+        result = build_taxonomy_candidates(nodes, user=None)
 
         assert len(result["tags"]) == 1
         assert result["tags"][0]["id"] == tag.pk
@@ -79,9 +80,9 @@ class TestBuildTaxonomyCandidates:
         document.tags.add(tag)
         tag.name = "New Name"
         tag.save()
-        similar_documents = [make_similar(document.pk, 0.5)]
+        nodes = [make_node(document.pk, 0.5)]
 
-        result = build_taxonomy_candidates(similar_documents, user=None)
+        result = build_taxonomy_candidates(nodes, user=None)
 
         assert result["tags"][0]["name"] == "New Name"
 
@@ -101,9 +102,9 @@ class TestBuildTaxonomyCandidates:
         document = DocumentFactory.create()
         document.tags.add(tag)
         tag.delete()
-        similar_documents = [make_similar(document.pk, 0.5)]
+        nodes = [make_node(document.pk, 0.5)]
 
-        result = build_taxonomy_candidates(similar_documents, user=None)
+        result = build_taxonomy_candidates(nodes, user=None)
 
         assert result["tags"] == []
 
@@ -122,12 +123,9 @@ class TestBuildTaxonomyCandidates:
         strong_doc.tags.add(strong_tag)
         weak_doc = DocumentFactory.create()
         weak_doc.tags.add(weak_tag)
-        similar_documents = [
+        nodes = [make_node(strong_doc.pk, 0.9), make_node(weak_doc.pk, 0.1)]
-            make_similar(strong_doc.pk, 0.9),
-            make_similar(weak_doc.pk, 0.1),
-        ]
 
-        result = build_taxonomy_candidates(similar_documents, user=None)
+        result = build_taxonomy_candidates(nodes, user=None)
 
         assert [c["name"] for c in result["tags"]] == ["Strong", "Weak"]
 
@@ -143,9 +141,9 @@ class TestBuildTaxonomyCandidates:
         document = DocumentFactory.create()
         for i in range(15):
             document.tags.add(TagFactory.create(name=f"Tag{i}"))
-        similar_documents = [make_similar(document.pk, 0.5)]
+        nodes = [make_node(document.pk, 0.5)]
 
-        result = build_taxonomy_candidates(similar_documents, user=None)
+        result = build_taxonomy_candidates(nodes, user=None)
 
         assert len(result["tags"]) == 10
 
@@ -159,12 +157,12 @@ class TestBuildTaxonomyCandidates:
             - Only 5 correspondents are returned
         """
         correspondents = CorrespondentFactory.create_batch(7)
-        similar_documents = [
+        nodes = [
-            make_similar(DocumentFactory.create(correspondent=c).pk, 0.5)
+            make_node(DocumentFactory.create(correspondent=c).pk, 0.5)
             for c in correspondents
         ]
 
-        result = build_taxonomy_candidates(similar_documents, user=None)
+        result = build_taxonomy_candidates(nodes, user=None)
 
         assert len(result["correspondents"]) == 5
 
@@ -179,9 +177,9 @@ class TestBuildTaxonomyCandidates:
         """
         document_type = DocumentTypeFactory.create(name="Invoice")
         document = DocumentFactory.create(document_type=document_type)
-        similar_documents = [make_similar(document.pk, 0.5)]
+        nodes = [make_node(document.pk, 0.5)]
 
-        result = build_taxonomy_candidates(similar_documents, user=None)
+        result = build_taxonomy_candidates(nodes, user=None)
 
         assert len(result["document_types"]) == 1
         assert result["document_types"][0]["id"] == document_type.pk
@@ -197,12 +195,12 @@ class TestBuildTaxonomyCandidates:
             - Only 5 document_types are returned
         """
         document_types = DocumentTypeFactory.create_batch(7)
-        similar_documents = [
+        nodes = [
-            make_similar(DocumentFactory.create(document_type=dt).pk, 0.5)
+            make_node(DocumentFactory.create(document_type=dt).pk, 0.5)
             for dt in document_types
         ]
 
-        result = build_taxonomy_candidates(similar_documents, user=None)
+        result = build_taxonomy_candidates(nodes, user=None)
 
         assert len(result["document_types"]) == 5
 
@@ -217,9 +215,9 @@ class TestBuildTaxonomyCandidates:
         """
         storage_path = StoragePathFactory.create(name="Invoices")
         document = DocumentFactory.create(storage_path=storage_path)
-        similar_documents = [make_similar(document.pk, 0.5)]
+        nodes = [make_node(document.pk, 0.5)]
 
-        result = build_taxonomy_candidates(similar_documents, user=None)
+        result = build_taxonomy_candidates(nodes, user=None)
 
         assert len(result["storage_paths"]) == 1
         assert result["storage_paths"][0]["id"] == storage_path.pk
@@ -235,12 +233,12 @@ class TestBuildTaxonomyCandidates:
             - Only 5 storage_paths are returned
         """
         storage_paths = StoragePathFactory.create_batch(7)
-        similar_documents = [
+        nodes = [
-            make_similar(DocumentFactory.create(storage_path=sp).pk, 0.5)
+            make_node(DocumentFactory.create(storage_path=sp).pk, 0.5)
             for sp in storage_paths
         ]
 
-        result = build_taxonomy_candidates(similar_documents, user=None)
+        result = build_taxonomy_candidates(nodes, user=None)
 
         assert len(result["storage_paths"]) == 5
 
@@ -260,14 +258,14 @@ class TestBuildTaxonomyCandidates:
         tag = TagFactory.create(name="Restricted")
         document = DocumentFactory.create()
         document.tags.add(tag)
-        similar_documents = [make_similar(document.pk, 0.5)]
+        nodes = [make_node(document.pk, 0.5)]
         user = UserFactory.create()
         mocker.patch(
             "documents.permissions.permitted_object_ids",
             return_value=[],  # user cannot see this tag
         )
 
-        result = build_taxonomy_candidates(similar_documents, user=user)
+        result = build_taxonomy_candidates(nodes, user=user)
 
         assert result["tags"] == []
 
@@ -297,10 +295,10 @@ class TestBuildTaxonomyCandidates:
         tag.save()
         document = DocumentFactory.create()
         document.tags.add(tag)
-        similar_documents = [make_similar(document.pk, 0.5)]
+        nodes = [make_node(document.pk, 0.5)]
         spy = mocker.patch("documents.permissions.permitted_object_ids")
 
-        result = build_taxonomy_candidates(similar_documents, user=None)
+        result = build_taxonomy_candidates(nodes, user=None)
 
         assert result["tags"][0]["name"] == "Owned"
         spy.assert_not_called()

src/paperless_mail/mail.py

@@ -6,7 +6,6 @@ import socket
 import ssl
 import tempfile
 import traceback
-import unicodedata
 from datetime import date
 from datetime import timedelta
 from fnmatch import fnmatch
@@ -45,6 +44,7 @@ from documents.models import Correspondent
 from documents.models import PaperlessTask
 from documents.parsers import is_mime_type_supported
 from documents.tasks import consume_file
+from documents.utils import normalize_unicode
 from paperless.network import is_public_ip
 from paperless.network import resolve_hostname_ips
 from paperless_mail.models import MailAccount
@@ -617,10 +617,10 @@ class MailAccountHandler(LoggingMixin):
         rule: MailRule,
     ) -> str | None:
         if rule.assign_title_from == MailRule.TitleSource.FROM_SUBJECT:
-            return unicodedata.normalize("NFC", message.subject)
+            return normalize_unicode(message.subject)
 
         elif rule.assign_title_from == MailRule.TitleSource.FROM_FILENAME:
-            return unicodedata.normalize("NFC", Path(att.filename).stem)
+            return normalize_unicode(Path(att.filename).stem)
 
         elif rule.assign_title_from == MailRule.TitleSource.NONE:
             return None
@@ -1004,6 +1004,8 @@ class MailAccountHandler(LoggingMixin):
         consume_tasks = []
 
         for att in message.attachments:
+            attachment_filename = normalize_unicode(att.filename)
+
             if (
                 att.content_disposition != "attachment"
                 and rule.attachment_type
@@ -1018,7 +1020,7 @@ class MailAccountHandler(LoggingMixin):
 
             if not self.filename_inclusion_matches(
                 rule.filter_attachment_filename_include,
-                att.filename,
+                attachment_filename,
             ):
                 # Force the filename and pattern to the lowercase
                 # as this is system dependent otherwise
@@ -1030,7 +1032,7 @@ class MailAccountHandler(LoggingMixin):
                 continue
             elif self.filename_exclusion_matches(
                 rule.filter_attachment_filename_exclude,
-                att.filename,
+                attachment_filename,
             ):
                 self.log.debug(
                     f"Rule {rule}: "
@@ -1064,7 +1066,7 @@ class MailAccountHandler(LoggingMixin):
                 )
 
                 attachment_name = pathvalidate.sanitize_filename(
-                    unicodedata.normalize("NFC", att.filename),
+                    attachment_filename,
                 )
                 if attachment_name:
                     temp_filename = temp_dir / attachment_name
@@ -1175,7 +1177,7 @@ class MailAccountHandler(LoggingMixin):
         doc_overrides = DocumentMetadataOverrides(
             title=message.subject,
             filename=pathvalidate.sanitize_filename(
-                unicodedata.normalize("NFC", f"{message.subject}.eml"),
+                normalize_unicode(f"{message.subject}.eml"),
             ),
             correspondent_id=correspondent.id if correspondent else None,
             document_type_id=doc_type.id if doc_type else None,

src/paperless_mail/serialisers.py

@@ -8,6 +8,7 @@ from documents.serialisers import CorrespondentField
 from documents.serialisers import DocumentTypeField
 from documents.serialisers import OwnedObjectSerializer
 from documents.serialisers import TagsField
+from documents.utils import normalize_unicode
 from paperless_mail.models import MailAccount
 from paperless_mail.models import MailRule
 from paperless_mail.models import ProcessedMail
@@ -161,6 +162,12 @@ class MailRuleSerializer(OwnedObjectSerializer):
             raise serializers.ValidationError("Maximum mail age is unreasonably large.")
         return value
 
+    def validate_filter_attachment_filename_include(self, value):
+        return normalize_unicode(value)
+
+    def validate_filter_attachment_filename_exclude(self, value):
+        return normalize_unicode(value)
+
 
 class ProcessedMailSerializer(OwnedObjectSerializer):
     class Meta: