Feature: match CJK terms through their bigram fields in place (#14156)

QUERY-mode searches blended a separate bigram clause in at the top of the
query, built from the parsed AST's free-text tokens. Because it sat beside
the exact clause rather than inside the query, nothing around a CJK term
constrained its bigram match: an exclusion that was one OR branch's own
condition could never reach it, so "(東京 AND NOT secret) OR bill" still
returned the secret document.

Widen each CJK leaf where it sits instead, through emit()'s rewrite_leaf
hook, so every AND, NOT, REQUIRE, boost and field restriction around the
leaf applies to its bigram match too. Negated leaves are widened on
purpose, so "NOT X" excludes exactly what "X" matches.

pyproject.toml

@@ -76,7 +76,7 @@ dependencies = [
   "torch~=2.13.0",
   "watchfiles>=1.2",
   "whitenoise~=6.11",
-  "whoosh-compat[tantivy]==0.2",
+  "whoosh-compat[tantivy]==0.3",
   "zxing-cpp~=3.1.0",
 ]
 [project.optional-dependencies]

src/documents/search/_backend.py

@@ -23,6 +23,7 @@ from django.conf import settings
 from django.utils.timezone import get_current_timezone
 
 from documents.search._query import extract_cjk_text
+from documents.search._query import normalize_search_text
 from documents.search._query import parse_simple_text_highlight_query
 from documents.search._query import parse_simple_text_query
 from documents.search._query import parse_simple_title_query
@@ -462,6 +463,7 @@ class TantivyBackend:
     ) -> tantivy.Query:
         """Parse a user query string into a Tantivy Query object."""
         tz = get_current_timezone()
+        query = normalize_search_text(query)
         if search_mode is SearchMode.TEXT:
             return parse_simple_text_query(self._index, query)
         elif search_mode is SearchMode.TITLE:
@@ -510,54 +512,67 @@ class TantivyBackend:
         from guardian.shortcuts import get_groups_with_perms
         from guardian.shortcuts import get_users_with_perms
 
-        content = document.get_effective_content() or ""
+        # Every searchable string is normalized on the way in, and every
+        # query string on the way out (_parse_query), so the two agree on
+        # how a composed character is spelled. See normalize_search_text.
+        content = normalize_search_text(document.get_effective_content() or "")
+        title = normalize_search_text(document.title)
 
         doc = tantivy.Document()
 
         # Basic fields
         doc.add_unsigned("id", document.pk)
         doc.add_text("checksum", document.checksum)
-        doc.add_text("title", document.title)
-        doc.add_text("title_sort", document.title)
-        doc.add_text("simple_title", document.title)
+        doc.add_text("title", title)
+        doc.add_text("title_sort", title)
+        doc.add_text("simple_title", title)
         doc.add_text("content", content)
         doc.add_text("simple_content", content)
         # Bigram (character-ngram) fields exist for CJK substring search,
         # no need to bloat the bigram index with latin characters.
-        if cjk_title := extract_cjk_text(document.title):
+        if cjk_title := extract_cjk_text(title):
             doc.add_text("bigram_title", cjk_title)
         if content and (cjk_content := extract_cjk_text(content)):
             doc.add_text("bigram_content", cjk_content)
 
         # Original filename - only add if not None/empty
         if document.original_filename:
-            doc.add_text("original_filename", document.original_filename)
+            doc.add_text(
+                "original_filename",
+                normalize_search_text(document.original_filename),
+            )
 
         # Correspondent
         if document.correspondent:
-            doc.add_text("correspondent", document.correspondent.name)
-            doc.add_text("correspondent_sort", document.correspondent.name)
-            if cjk_corr := extract_cjk_text(document.correspondent.name):
+            correspondent = normalize_search_text(document.correspondent.name)
+            doc.add_text("correspondent", correspondent)
+            doc.add_text("correspondent_sort", correspondent)
+            if cjk_corr := extract_cjk_text(correspondent):
                 doc.add_text("bigram_correspondent", cjk_corr)
 
         # Document type
         if document.document_type:
-            doc.add_text("document_type", document.document_type.name)
-            doc.add_text("type_sort", document.document_type.name)
-            if cjk_type := extract_cjk_text(document.document_type.name):
+            document_type = normalize_search_text(document.document_type.name)
+            doc.add_text("document_type", document_type)
+            doc.add_text("type_sort", document_type)
+            if cjk_type := extract_cjk_text(document_type):
                 doc.add_text("bigram_document_type", cjk_type)
 
         # Storage path
         if document.storage_path:
-            doc.add_text("storage_path", document.storage_path.name)
+            doc.add_text(
+                "storage_path",
+                normalize_search_text(document.storage_path.name),
+            )
 
         # Tags — collect names for autocomplete in the same pass
         tag_names: list[str] = []
         for tag in document.tags.all():
-            doc.add_text("tag", tag.name)
-            if cjk_tag := extract_cjk_text(tag.name):
+            tag_name = normalize_search_text(tag.name)
+            doc.add_text("tag", tag_name)
+            if cjk_tag := extract_cjk_text(tag_name):
                 doc.add_text("bigram_tag", cjk_tag)
-            tag_names.append(tag.name)
+            tag_names.append(tag_name)
 
         # Notes — JSON for structured queries (notes.user:alice, notes.note:text).
         # notes_text is a plain-text companion for snippet/highlight generation;
@@ -568,14 +583,17 @@ class TantivyBackend:
         note_texts: list[str] = []
         for note in document.notes.all():
             num_notes += 1
+            note_text = normalize_search_text(note.note)
             doc.add_json(
                 "notes",
                 {
-                    "note": note.note,
-                    "user": note.user.username if note.user else None,
+                    "note": note_text,
+                    "user": (
+                        normalize_search_text(note.user.username) if note.user else None
+                    ),
                 },
             )
-            note_texts.append(note.note)
+            note_texts.append(note_text)
         if note_texts:
             doc.add_text("notes_text", " ".join(note_texts))
 
@@ -590,8 +608,8 @@ class TantivyBackend:
             doc.add_json(
                 "custom_fields",
                 {
-                    "name": cfi.field.name,
-                    "value": search_value,
+                    "name": normalize_search_text(cfi.field.name),
+                    "value": normalize_search_text(search_value),
                 },
             )
 
@@ -645,11 +663,11 @@ class TantivyBackend:
             doc.add_unsigned("viewer_group_id", viewer_group_id)
 
         # Autocomplete words
-        text_sources = [document.title, content]
+        text_sources = [title, content]
         if document.correspondent:
-            text_sources.append(document.correspondent.name)
+            text_sources.append(correspondent)
         if document.document_type:
-            text_sources.append(document.document_type.name)
+            text_sources.append(document_type)
         text_sources.extend(tag_names)
 
         for word in sorted(_extract_autocomplete_words(text_sources)):
@@ -746,6 +764,9 @@ class TantivyBackend:
 
         self._ensure_open()
         user_query = self._parse_query(query, search_mode)
+        # _parse_query normalizes its own copy; the snippet queries below are
+        # built from the string directly, so normalize it here too.
+        query = normalize_search_text(query)
         highlight_query = user_query
         if search_mode is SearchMode.TEXT:
             try:

src/documents/search/_query.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
 import logging
+import unicodedata
+from functools import cache
 from typing import TYPE_CHECKING
 from typing import Final
 
@@ -19,6 +21,7 @@ from documents.search._errors import InvalidNumberQuery
 from documents.search._errors import MultipleSearchQueryErrors
 from documents.search._errors import SearchQueryError
 from documents.search._registry import get_field_registry
+from documents.search._tokenizer import _bigram_analyzer
 from documents.search._tokenizer import simple_search_tokens
 
 if TYPE_CHECKING:
@@ -34,7 +37,37 @@ _REGEX_TIMEOUT: Final[float] = 1.0
 
 # Matches CJK/Hangul characters so queries can be routed to bigram fields.
 # Uses Unicode properties to cover all blocks including Extension B+ planes.
-_CJK_RE: Final = regex.compile(r"[\p{Han}\p{Hiragana}\p{Katakana}\p{Hangul}]+")
+# The marks that sit inside a Japanese word are listed explicitly, because
+# their Unicode script is Common and the script classes therefore miss
+# them: the katakana prolonged sound mark ー (U+30FC) and its halfwidth form
+# ｰ (U+FF70), the closing mark 〆 (U+3006), and the halfwidth voiced and
+# semi-voiced sound marks ﾞ (U+FF9E) and ﾟ (U+FF9F). Without them a word
+# splits into one-character runs, which have no bigrams: コーヒー becomes
+# コ + ヒ, and halfwidth ﾊﾟﾝ becomes ﾊ + ﾝ, leaving nothing to index or
+# search at all.
+#
+# The combining marks U+3099/U+309A are deliberately absent: everything
+# entering the index and every query string is put through
+# normalize_search_text first, so decomposed kana is composed away before
+# this pattern ever sees it. The halfwidth marks are not, and cannot be:
+# unlike パ (U+30D1), halfwidth katakana has no precomposed voiced form, so
+# NFC leaves ﾊ + ﾟ as two codepoints where it folds か + U+3099 into が.
+_CJK_RE: Final = regex.compile(
+    r"[\p{Han}\p{Hiragana}\p{Katakana}\p{Hangul}ーｰ〆ﾞﾟ]+",
+)
+
+
+def normalize_search_text(text: str) -> str:
+    """Put text into the one Unicode normal form the index is built in.
+
+    Both the indexed text and the query string go through this, because a
+    bigram is a pair of codepoints: NFD がっこう is four where NFC is three,
+    so an unnormalized document never matches a normalized query.
+
+    NFC, not NFKC: folding ﾊﾟﾝ to パン would be a search-behavior decision
+    rather than an encoding one.
+    """
+    return unicodedata.normalize("NFC", text)
 
 
 def _user_facing_emit_message(d: Diagnostic) -> str:
@@ -113,8 +146,8 @@ def extract_cjk_text(text: str) -> str:
     """Join the CJK runs in ``text`` for indexing into bigram (char-ngram) fields.
 
     Mirrors the query side, which extracts the CJK runs of whatever it is
-    about to search for (the raw string in simple modes, the parsed query's
-    free-text tokens in query mode): only CJK runs are ever searched against
+    about to search for (the raw string in simple modes, each CJK term's
+    own text in query mode): only CJK runs are ever searched against
     the bigram fields, so only CJK runs are worth indexing there. Latin text
     fed to a character-bigram field is never matched and only bloats the
     index and slows indexing/merge. Returns "" when there is no CJK text.
@@ -163,52 +196,6 @@ def _build_cjk_query(
     return _parse_cjk_text(index, cjk_text, fields)
 
 
-def _build_ast_cjk_query(
-    index: tantivy.Index,
-    ast: wc.ast.Node,
-    registry: wc.FieldRegistry,
-) -> tantivy.Query | None:
-    """Build the bigram clause of a QUERY-mode search from the parsed AST.
-
-    Same discipline as the fuzzy clause (see _try_parse_fuzzy_query): the CJK
-    runs come from whoosh_compat's ``free_text_tokens`` over the parsed tree,
-    never from the raw query string, so a term the user negated or restricted
-    to a field outside the default search fields contributes nothing, instead
-    of resurfacing as a top-level clause matching every bigram field.
-
-    ``free_text_tokens`` reports no field of its own, so the tokens are
-    collected one default field at a time: a bare term, which the parser has
-    already copied onto every default field, is therefore searched across
-    every bigram field, while ``title:東京`` reaches ``bigram_title`` alone.
-    Fields whose CJK text is identical (the bare-term case) share a single
-    parse over all of their bigram fields at once.
-
-    Raw (``analyzed=False``) tokens are used because the bigram fields have
-    their own character-ngram analyzer: the default fields' word analyzers
-    have no useful say over a CJK run, and running them first would only
-    risk dropping it (remove_long) before the run is ever extracted.
-    Returns None when the query has no CJK free text.
-    """
-    fields_by_text: dict[str, list[str]] = {}
-    for field, bigram_field in _CJK_BIGRAM_FIELDS.items():
-        tokens = wc.free_text_tokens(
-            ast,
-            registry=registry,
-            fields=[field],
-            analyzed=False,
-        )
-        cjk_text = extract_cjk_text(" ".join(tokens))
-        if cjk_text:
-            fields_by_text.setdefault(cjk_text, []).append(bigram_field)
-
-    clauses: list[tuple[tantivy.Occur, tantivy.Query]] = [
-        (tantivy.Occur.Should, query)
-        for cjk_text, bigram_fields in fields_by_text.items()
-        if (query := _parse_cjk_text(index, cjk_text, bigram_fields)) is not None
-    ]
-    return _any_of(clauses) if clauses else None
-
-
 # A joined fuzzy word string must stay plain words: it goes back through
 # tantivy's own query parser, and the raw query text the clause collects
 # routinely carries characters that parser reads as grammar (a colon, a
@@ -278,7 +265,7 @@ def _try_parse_fuzzy_query(
 
     The ValueError guard stays as insurance (the word string is plain
     tokens, so tantivy accepting it is expected, not assumed): on a parse
-    failure the fuzzy clause is skipped and the exact/CJK clauses stand,
+    failure the fuzzy clause is skipped and the exact clause stands,
     rather than the whole query failing.
     """
     tokens = wc.free_text_tokens(
@@ -332,6 +319,131 @@ _FIELD_BOOSTS = {"title": 2.0}
 _SIMPLE_FIELD_BOOSTS = {"simple_title": 2.0}
 
 
+@cache
+def _get_emit_field_registry(language: str | None) -> wc.FieldRegistry:
+    """The parse registry plus the CJK bigram fields, for analyzing and
+    emitting a query whose CJK leaves _widen_cjk_leaf has widened. Cached
+    per language, on the same trigger get_field_registry() rebuilds on.
+
+    Never used to parse: the bigram fields are internal (absent from
+    PUBLIC_FIELDS), and queries are still parsed against
+    get_field_registry(), so ``bigram_content:...`` never becomes query
+    syntax.
+
+    The bigram specs set ``multitoken=Multitoken.AND`` explicitly. A
+    widened leaf's bigram side always sits inside the widening Or, so under
+    Multitoken.DEFAULT a run's bigrams would inherit that Or, and 東京都
+    would match a document containing only 京都.
+    """
+    bigram_analyze = _bigram_analyzer().analyze
+    return wc.FieldRegistry(
+        [
+            *get_field_registry(language),
+            *(
+                wc.FieldSpec(
+                    bigram_field,
+                    wc.FieldKind.TEXT,
+                    analyzer=bigram_analyze,
+                    multitoken=wc.Multitoken.AND,
+                )
+                for bigram_field in _CJK_BIGRAM_FIELDS.values()
+            ),
+        ],
+    )
+
+
+# Splits text exactly where the content analyzer's simple tokenizer does,
+# with none of its filters, so each piece is the raw text of one token the
+# index could hold. No remove_long either: a long CJK run must still reach
+# the bigram side.
+_TOKEN_SPLITTER: Final = tantivy.TextAnalyzerBuilder(tantivy.Tokenizer.simple()).build()
+
+
+def _collapse(
+    node_cls: type[wc.ast.Node],
+    children: list[wc.ast.Node],
+    span: dict[str, int | None],
+) -> wc.ast.Node:
+    """Return the single child as it is, or wrap several in node_cls."""
+    if len(children) == 1:
+        return children[0]
+    return node_cls(children=tuple(children), **span)
+
+
+def _widen_cjk_leaf(leaf: wc.ast.Term | wc.ast.Phrase) -> wc.ast.Node:
+    """``rewrite_leaf`` hook for emit(): widen a CJK term or phrase on a
+    default search field to ``Or(leaf, alternative)`` where it sits, and
+    leave every other leaf as it is.
+
+    The content analyzer keeps an unspaced CJK run as one token, so only
+    the bigram fields can find a CJK term inside running text. Widening in
+    place, rather than OR-ing a separate bigram clause in at the top, keeps
+    every AND, NOT, REQUIRE, boost and field restriction around the leaf
+    applying to its bigram match too. analyze() calls this for negated
+    leaves as well, and they are widened on purpose, so ``NOT X`` excludes
+    exactly what ``X`` matches.
+
+    The alternative is built from the leaf's own text, split where the
+    content analyzer would split it. What each kind of piece contributes,
+    and how the pieces combine, is commented at the step that decides it.
+
+    analyze() only offers Term and Phrase leaves, so Prefix and Wildcard
+    patterns are never widened.
+    """
+    if leaf.field is None or leaf.field.name not in _CJK_BIGRAM_FIELDS:
+        return leaf
+    text = str(leaf.text)
+    if not _has_cjk(text):
+        return leaf
+    span = {"startchar": leaf.startchar, "endchar": leaf.endchar}
+    bigram_field = wc.FieldRef(_CJK_BIGRAM_FIELDS[leaf.field.name])
+    cjk_terms: list[wc.ast.Node] = []
+    latin_terms: list[wc.ast.Node] = []
+    for token in _TOKEN_SPLITTER.analyze(text):
+        runs = _CJK_RE.findall(token)
+        if runs:
+            # One bigram Term per run, never a joined string, which would
+            # produce bigrams spanning the join. A run's own bigrams stay
+            # jointly required through multitoken=AND on the bigram
+            # FieldSpec (see _get_emit_field_registry), which no enclosing
+            # group can loosen. A one-character run has no bigram at all
+            # and analyzes away to nothing.
+            cjk_terms.extend(
+                wc.ast.Term(field=bigram_field, text=run, **span) for run in runs
+            )
+        else:
+            # Latin the analyzer split off on its own. Latin glued to CJK
+            # inside one token (東京report) never reaches here, and must
+            # not: the index holds it only inside that whole unspaced
+            # token, so requiring it would lose documents the run finds.
+            latin_terms.append(wc.ast.Term(field=leaf.field, text=token, **span))
+    # A Term's runs are alternatives to each other, the way the separate
+    # bigram clause treated them. A Phrase's are required together: quoting
+    # asks for more than the bare words, and the parser's default group is
+    # And, so an Or here would make "東京都 大阪府" match strictly more
+    # than 東京都 大阪府 does. And is also the tightest thing available,
+    # since the bigram analyzer puts every token at position 0 and no
+    # alternative built from it can enforce adjacency.
+    cjk_group = wc.ast.And if isinstance(leaf, wc.ast.Phrase) else wc.ast.Or
+    pieces: list[wc.ast.Node] = []
+    if cjk_terms:
+        pieces.append(_collapse(cjk_group, cjk_terms, span))
+    pieces.extend(latin_terms)
+    if not pieces:
+        # A few hundred codepoints match _CJK_RE but yield no token at all
+        # from the simple tokenizer (CJK radicals, circled and squared
+        # forms), leaving nothing to widen with. The leaf then analyzes to
+        # the same nothing it does today.
+        return leaf
+    # Separated latin is required alongside the CJK side, which is what
+    # stops "invoice NOT 東京-report" from excluding every 東京 document.
+    alternative = _collapse(wc.ast.And, pieces, span)
+    # The leaf itself, not a copy: analyze() then keeps it combined the way
+    # its enclosing group says rather than the way this Or would, and a long
+    # run its analyzer drops to nothing leaves just the alternative.
+    return wc.ast.Or(children=(leaf, alternative), **span)
+
+
 class _ConjunctiveNegations(wc.ast.Visitor[tuple["wc.ast.Node", ...]]):
     """Collect the subtrees an AST excludes from every document it matches.
 
@@ -383,13 +495,17 @@ def _negation_clauses(
     tantivy accepts a bare one.
 
     The except branch has no reachable trigger under the current control
-    flow: this only runs after ``exact = tantivy_emit(result.ast, ...)``
-    (parse_user_query) has already emitted the *whole* AST successfully,
-    and every subtree ``_ConjunctiveNegations`` collects here is a piece
-    of that same tree. Kept as insurance, not dead weight: re-emitting a
-    subtree in isolation is not proven identical to emitting it in
-    context, just believed to be, and this is the seam that finds out if
-    that belief is ever wrong.
+    flow: this only runs after parse_user_query has already emitted the
+    exact clause from the whole parsed AST (widened, when the query has CJK
+    text), and every subtree ``_ConjunctiveNegations`` collects here is a
+    piece of that same parsed tree. The re-emit here uses the public
+    registry, without the widening hook, while the exact clause was emitted
+    against ``_get_emit_field_registry()``; the two registries agree on
+    every public field, and the hook only adds nodes, so this does not
+    reopen the branch. Kept as insurance, not dead weight: re-emitting a
+    subtree in isolation is not proven identical to emitting it in context,
+    just believed to be, and this is the seam that finds out if that
+    belief is ever wrong.
     """
     try:
         return [
@@ -451,7 +567,8 @@ def parse_user_query(
     tz: tzinfo,
 ) -> tantivy.Query:
     """
-    Parse user query through whoosh-compat, then blend in fuzzy/CJK clauses.
+    Parse user query through whoosh-compat, widen CJK terms, then blend in
+    the optional fuzzy clause.
 
     1. wc.parse() against the shared FieldRegistry (whoosh grammar -> AST).
        Bare notes:/custom_fields: prefixes resolve to their default subpath
@@ -460,11 +577,18 @@ def parse_user_query(
     2. Any diagnostics (bad dates/numbers) map to SearchQueryError subclasses
        and raise, the view returns HTTP 400 with every offending field
        listed, not just the first.
-    3. emit() turns the AST into a tantivy.Query directly (no string
+    3. When the query has CJK text, emit()'s rewrite_leaf hook
+       (_widen_cjk_leaf) rewrites each CJK term in the AST to also match
+       its bigram field, in place, so the rest of the query constrains the
+       bigram match too. The tree is analyzed and emitted against
+       _get_emit_field_registry(), which adds the bigram fields; the query
+       itself was parsed without them.
+       emit() turns the AST into a tantivy.Query directly (no string
        round-trip). A QueryError is routed by its Diagnostic's Cause
        (_map_emit_error): a construct that parses but can't execute against
        tantivy (e.g. a text-field range) is a 400, a registry/schema
-       mismatch is logged and a 400, and an INTERNAL defect is re-raised.
+       mismatch is logged and re-raised, and an INTERNAL defect is
+       re-raised.
     4. Optional fuzzy blend (ADVANCED_FUZZY_SEARCH_THRESHOLD) builds a
        plain word string from the parsed AST's free-text tokens
        (whoosh_compat.free_text_tokens) and feeds THAT to
@@ -472,13 +596,12 @@ def parse_user_query(
        keywords, bracket-class wildcards, etc.) tantivy's parser rejects,
        which used to silently knock the fuzzy clause out of any mixed
        query (see _try_parse_fuzzy_query).
-    5. Optional CJK bigram clause, built from the same parsed AST for the
-       same reason (see _build_ast_cjk_query): a CJK term the query negated
-       or fielded must not resurface through it.
-    6. When any optional clause was added, the query's conjunctive
-       exclusions are restated as MustNot above the blend
-       (_negation_clauses): a clause built from positive terms cannot
-       express them, and as a bare Should it would undo them.
+    5. When the fuzzy clause was added, the query's conjunctive exclusions
+       are restated as MustNot above the blend (_negation_clauses): a
+       clause built from positive terms cannot express them, and as a bare
+       Should it would undo them. The restated exclusions come from the
+       unwidened AST, so they are content-only for CJK terms: a known gap
+       that goes away once fuzzy is also widened in the tree.
     """
     registry = get_field_registry(settings.SEARCH_LANGUAGE)
     result = wc.parse(
@@ -491,17 +614,20 @@ def parse_user_query(
     if result.diagnostics:
         raise _diagnostics_to_error(result.diagnostics)
 
+    emit_registry, rewrite_leaf = registry, None
+    if _has_cjk(raw_query):
+        emit_registry = _get_emit_field_registry(settings.SEARCH_LANGUAGE)
+        rewrite_leaf = _widen_cjk_leaf
     try:
-        exact = tantivy_emit(result.ast, index=index, registry=registry)
+        exact = tantivy_emit(
+            result.ast,
+            index=index,
+            registry=emit_registry,
+            rewrite_leaf=rewrite_leaf,
+        )
     except QueryError as e:
         raise _map_emit_error(e) from e
 
-    cjk_query = (
-        _build_ast_cjk_query(index, result.ast, registry)
-        if _has_cjk(raw_query)
-        else None
-    )
-
     clauses: list[tuple[tantivy.Occur, tantivy.Query]] = [
         (tantivy.Occur.Should, exact),
     ]
@@ -514,16 +640,13 @@ def parse_user_query(
                 (tantivy.Occur.Should, tantivy.Query.boost_query(fuzzy, 0.1)),
             )
 
-    if cjk_query is not None:
-        clauses.append((tantivy.Occur.Should, cjk_query))
-
     if len(clauses) == 1:
         return exact
-    # The fuzzy and CJK clauses are built from positive terms only, so as
-    # plain Shoulds beside the exact clause they re-admit exactly the
-    # documents the query excluded. Restate the exclusions once, above the
-    # whole blend. Redundant against the exact clause, which already
-    # carries them, but idempotently so.
+    # The fuzzy clause is built from positive terms only, so as a plain
+    # Should beside the exact clause it re-admits exactly the documents the
+    # query excluded. Restate the exclusions once, above the whole blend.
+    # Redundant against the exact clause, which already carries them, but
+    # idempotently so.
     negations = _negation_clauses(index, result.ast, registry)
     if not negations:
         return _any_of(clauses)

src/documents/tests/search/conftest.py

@@ -6,13 +6,17 @@ import pytest
 
 from documents.search._backend import TantivyBackend
 from documents.search._backend import reset_backend
+from documents.tests.factories import DocumentFactory
 
 if TYPE_CHECKING:
+    from collections.abc import Callable
     from collections.abc import Generator
     from pathlib import Path
 
     from pytest_django.fixtures import Settings
 
+    from documents.models import Document
+
 
 @pytest.fixture
 def index_dir(tmp_path: Path, settings: Settings) -> Path:
@@ -31,3 +35,40 @@ def backend() -> Generator[TantivyBackend, None, None]:
     finally:
         b.close()
         reset_backend()
+
+
+@pytest.fixture
+def long_cjk_run() -> str:
+    """A CJK run too long for the content analyzer to keep.
+
+    48 characters, 144 UTF-8 bytes: one token longer than remove_long's
+    129-byte limit, so the content side analyzes to nothing and only the
+    bigram field can match it.
+    """
+    return "東京都の公共文書について" * 4
+
+
+@pytest.fixture
+def index_document(backend: TantivyBackend) -> Callable[..., Document]:
+    """Build a Document with DocumentFactory and add it to the index.
+
+    The factory supplies a unique checksum, so a test only passes the
+    fields its assertion actually depends on.
+    """
+
+    def _index(**kwargs: object) -> Document:
+        doc = DocumentFactory(**kwargs)
+        backend.add_or_update(doc)
+        return doc
+
+    return _index
+
+
+@pytest.fixture
+def matched_ids(backend: TantivyBackend) -> Callable[[str], set[int]]:
+    """Run a query as no particular user and return the matching pks."""
+
+    def _matched_ids(query: str) -> set[int]:
+        return set(backend.search_ids(query, user=None))
+
+    return _matched_ids

src/documents/tests/search/test_cjk_clause.py

@@ -1,9 +1,10 @@
-"""The CJK bigram clause blended into QUERY-mode searches.
+"""CJK bigram matching in QUERY-mode searches.
 
-The clause exists so CJK runs are matchable at all (the default analyzers
-keep a whitespace-free CJK run as one indivisible token), but it must not
-widen the query beyond what the user asked for: a CJK term the query
-excludes, or restricts to one field, must not come back through it.
+The bigram fields exist so CJK runs are matchable at all (the default
+analyzers keep a whitespace-free CJK run as one indivisible token), but
+matching them must not widen the query beyond what the user asked for: a
+CJK term the query excludes, or restricts to one field, must not come back
+through them.
 """
 
 from __future__ import annotations
@@ -12,26 +13,17 @@ from typing import TYPE_CHECKING
 
 import pytest
 
-from documents.models import Document
-
 if TYPE_CHECKING:
+    from collections.abc import Callable
+
     from pytest_django.fixtures import SettingsWrapper
 
-    from documents.search._backend import TantivyBackend
+    from documents.models import Document
+
 
 pytestmark = [pytest.mark.search, pytest.mark.django_db]
 
 
-def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
-    return set(backend.search_ids(query, user=None))
-
-
-def _index(backend: TantivyBackend, **kwargs: object) -> Document:
-    doc = Document.objects.create(**kwargs)
-    backend.add_or_update(doc)
-    return doc
-
-
 class TestCjkParseFailureDegradesGracefully:
     def test_a_cjk_run_tantivy_cannot_parse_drops_the_clause_only(self) -> None:
         """
@@ -76,7 +68,11 @@ class TestCjkParseFailureDegradesGracefully:
 
 
 class TestCjkClauseFollowsTheParsedQuery:
-    def test_negated_cjk_term_is_excluded(self, backend: TantivyBackend) -> None:
+    def test_negated_cjk_term_is_excluded(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
         """
         GIVEN:
             - Two documents both matching "invoice", one whose content
@@ -87,21 +83,17 @@ class TestCjkClauseFollowsTheParsedQuery:
             - Only the document without 漢字 matches; 'invoice NOT 漢字'
               must not return the document containing 漢字
         """
-        with_cjk = _index(
-            backend,
+        with_cjk = index_document(
             title="Invoice A",
             content="invoice total 漢字",
-            checksum="cjk-neg-1",
         )
-        without_cjk = _index(
-            backend,
+        without_cjk = index_document(
             title="Invoice B",
             content="invoice total only",
-            checksum="cjk-neg-2",
         )
 
-        assert _matched_ids(backend, "invoice") == {with_cjk.pk, without_cjk.pk}
-        assert _matched_ids(backend, "invoice NOT 漢字") == {without_cjk.pk}
+        assert matched_ids("invoice") == {with_cjk.pk, without_cjk.pk}
+        assert matched_ids("invoice NOT 漢字") == {without_cjk.pk}
 
     @pytest.mark.parametrize(
         ("threshold", "expected"),
@@ -112,7 +104,8 @@ class TestCjkClauseFollowsTheParsedQuery:
     )
     def test_fielded_cjk_term_searches_only_that_field(
         self,
-        backend: TantivyBackend,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
         settings: SettingsWrapper,
         threshold: float | None,
         expected: set[str],
@@ -135,26 +128,23 @@ class TestCjkClauseFollowsTheParsedQuery:
               it stays deliberate
         """
         settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = threshold
-        content_only = _index(
-            backend,
+        content_only = index_document(
             title="Tokyo report",
             content="東京都の人口は約1400万人です",
-            checksum="cjk-field-1",
         )
-        titled = _index(
-            backend,
+        titled = index_document(
             title="東京都の報告書",
             content="an english summary",
-            checksum="cjk-field-2",
         )
         pks = {"titled": titled.pk, "content_only": content_only.pk}
 
-        assert _matched_ids(backend, "東京") == set(pks.values())
-        assert _matched_ids(backend, "title:東京") == {pks[label] for label in expected}
+        assert matched_ids("東京") == set(pks.values())
+        assert matched_ids("title:東京") == {pks[label] for label in expected}
 
-    def test_cjk_on_a_non_default_field_builds_no_clause(
+    def test_cjk_on_a_non_default_field_is_not_widened(
         self,
-        backend: TantivyBackend,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
     ) -> None:
         """
         GIVEN:
@@ -164,22 +154,21 @@ class TestCjkClauseFollowsTheParsedQuery:
               search fields)
         THEN:
             - Nothing matches; a CJK term restricted to a field outside
-              the default search fields has nothing to contribute to the
-              bigram clause, so it must not fall back to matching 東京 in
-              the content
+              the default search fields has no bigram companion to widen
+              to, so it must not fall back to matching 東京 in the
+              content
         """
-        _index(
-            backend,
+        index_document(
             title="Tokyo report",
             content="東京都の人口は約1400万人です",
-            checksum="cjk-notes-1",
         )
 
-        assert _matched_ids(backend, "notes:東京") == set()
+        assert matched_ids("notes:東京") == set()
 
     def test_bare_cjk_term_still_matches_every_default_field(
         self,
-        backend: TantivyBackend,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
     ) -> None:
         """
         GIVEN:
@@ -188,25 +177,21 @@ class TestCjkClauseFollowsTheParsedQuery:
         WHEN:
             - "重要" and "重要 OR report" are each searched unfielded
         THEN:
-            - Both documents match either way; the clause's reason for
-              existing is that an unfielded CJK run matches wherever it
-              is indexed, and does so alongside a latin term
+            - Both documents match either way; bigram matching exists
+              so that an unfielded CJK run matches wherever it is
+              indexed, and does so alongside a latin term
         """
-        in_content = _index(
-            backend,
+        in_content = index_document(
             title="report",
             content="本文に重要な情報",
-            checksum="cjk-bare-1",
         )
-        in_title = _index(
-            backend,
+        in_title = index_document(
             title="重要な報告書",
             content="english only",
-            checksum="cjk-bare-2",
         )
 
-        assert _matched_ids(backend, "重要") == {in_content.pk, in_title.pk}
-        assert _matched_ids(backend, "重要 OR report") == {
+        assert matched_ids("重要") == {in_content.pk, in_title.pk}
+        assert matched_ids("重要 OR report") == {
             in_content.pk,
             in_title.pk,
         }

src/documents/tests/search/test_cjk_emit_registry.py

@@ -0,0 +1,122 @@
+"""The emit-only FieldRegistry a CJK-widened query tree is emitted against.
+
+It is the parse registry plus the internal bigram fields. These tests pin
+what the bigram half has to get right on its own: combining a run's bigrams
+with AND wherever the leaf sits, producing exactly the terms the index
+holds, and naming only fields the index schema actually has.
+"""
+
+from __future__ import annotations
+
+import pytest
+import tantivy
+import whoosh_compat as wc
+import whoosh_compat.ast as wc_ast
+
+from documents.search._query import _CJK_BIGRAM_FIELDS
+from documents.search._query import _get_emit_field_registry
+from documents.search._schema import field_descriptors
+from documents.search._tokenizer import register_tokenizers
+
+pytestmark = pytest.mark.search
+
+_BIGRAM_CONTENT = wc.FieldRef("bigram_content")
+_CONTENT = wc.FieldRef("content")
+
+
+class TestEmitFieldRegistry:
+    def test_a_multi_bigram_run_requires_every_bigram_even_under_an_or(
+        self,
+    ) -> None:
+        """
+        GIVEN:
+            - A three-character CJK run on bigram_content, sitting inside
+              an Or (where the CJK widener always places it)
+        WHEN:
+            - The tree is analyzed against the emit registry
+        THEN:
+            - The run's two bigrams are combined with And, not Or. Left
+              to Multitoken.DEFAULT they would inherit the enclosing Or,
+              so "東京都" would match a document containing only 京都
+        """
+        tree = wc_ast.Or(
+            children=(
+                wc_ast.Term(field=_BIGRAM_CONTENT, text="東京都"),
+                wc_ast.Term(field=_CONTENT, text="report"),
+            ),
+        )
+
+        analyzed = wc_ast.analyze(tree, _get_emit_field_registry(None))
+
+        assert analyzed == wc_ast.Or(
+            children=(
+                wc_ast.And(
+                    children=(
+                        wc_ast.Term(field=_BIGRAM_CONTENT, text="東京"),
+                        wc_ast.Term(field=_BIGRAM_CONTENT, text="京都"),
+                    ),
+                ),
+                wc_ast.Term(field=_CONTENT, text="report"),
+            ),
+        )
+
+    @pytest.mark.parametrize(
+        "text",
+        [
+            pytest.param("東京都の公共文書", id="japanese"),
+            pytest.param("北京市人民政府", id="chinese"),
+            pytest.param("서울특별시", id="korean"),
+        ],
+    )
+    def test_every_query_bigram_is_a_term_the_index_holds(self, text: str) -> None:
+        """
+        GIVEN:
+            - A document whose bigram_content was indexed by the tokenizer
+              register_tokenizers() installs
+        WHEN:
+            - The emit registry's bigram analyzer tokenizes the same text
+        THEN:
+            - Every token it produces is found as an indexed term. The
+              query side and the index side each build their own bigram
+              analyzer, so this pins the two staying equivalent
+        """
+        sb = tantivy.SchemaBuilder()
+        sb.add_text_field(
+            "bigram_content",
+            stored=False,
+            tokenizer_name="bigram_analyzer",
+        )
+        index = tantivy.Index(sb.build(), path=None)
+        register_tokenizers(index, None)
+        writer = index.writer()
+        doc = tantivy.Document()
+        doc.add_text("bigram_content", text)
+        writer.add_document(doc)
+        writer.commit()
+        index.reload()
+
+        resolved = _get_emit_field_registry(None).resolve(_BIGRAM_CONTENT)
+        assert resolved is not None
+        tokens = resolved.spec.analyzer(text)
+
+        assert tokens
+        searcher = index.searcher()
+        for token in tokens:
+            query = tantivy.Query.term_query(index.schema, "bigram_content", token)
+            assert searcher.search(query, limit=1).count == 1, token
+
+    def test_every_bigram_field_is_in_the_index_schema(self) -> None:
+        """
+        GIVEN:
+            - The bigram fields the emit registry declares, and the index
+              schema's field descriptors
+        WHEN:
+            - Each bigram field name is looked up in the schema
+        THEN:
+            - Every one is present. Every index is built from these
+              descriptors, or rebuilt when its stored fingerprint of them
+              differs, so this is what guarantees each widened bigram leaf
+              names a field the index has
+        """
+        schema_names = {descriptor.name for descriptor in field_descriptors()}
+        assert set(_CJK_BIGRAM_FIELDS.values()) <= schema_names

src/documents/tests/search/test_cjk_leaf_rewrite.py

@@ -0,0 +1,500 @@
+"""_widen_cjk_leaf, the rewrite_leaf hook that makes CJK runs matchable in
+QUERY mode.
+
+Unit tests against hand-built trees, no index needed: the hook on its own,
+then the tree wc.analyze() builds around it. Result-level proof against a
+real corpus lives in test_cjk_widening.py.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+import whoosh_compat as wc
+import whoosh_compat.ast as wc_ast
+
+from documents.search._query import _DEFAULT_SEARCH_FIELDS
+from documents.search._query import _FIELD_BOOSTS
+from documents.search._query import _get_emit_field_registry
+from documents.search._query import _widen_cjk_leaf
+from documents.search._registry import get_field_registry
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+pytestmark = pytest.mark.search
+
+_CONTENT = wc.FieldRef("content")
+_TITLE = wc.FieldRef("title")
+_NOTES = wc.FieldRef("notes", "note")
+_BIGRAM_CONTENT = wc.FieldRef("bigram_content")
+_BIGRAM_TITLE = wc.FieldRef("bigram_title")
+
+
+def _content(text: str) -> wc_ast.Term:
+    return wc_ast.Term(field=_CONTENT, text=text)
+
+
+def _bigram(text: str) -> wc_ast.Term:
+    return wc_ast.Term(field=_BIGRAM_CONTENT, text=text)
+
+
+def _widened(original: wc_ast.Node, bigram_side: wc_ast.Node) -> wc_ast.Or:
+    return wc_ast.Or(children=(original, bigram_side))
+
+
+def _analyze(tree: wc_ast.Node) -> wc_ast.Node:
+    return wc.analyze(
+        tree,
+        _get_emit_field_registry(None),
+        rewrite_leaf=_widen_cjk_leaf,
+    )
+
+
+class TestTheHook:
+    @pytest.mark.parametrize(
+        "leaf",
+        [
+            pytest.param(_content("invoice"), id="latin_term"),
+            pytest.param(
+                wc_ast.Term(field=_NOTES, text="東京"),
+                id="non_default_field",
+            ),
+            pytest.param(wc_ast.Term(field=None, text="東京"), id="unfielded"),
+        ],
+    )
+    def test_leaf_is_returned_unchanged(self, leaf: wc_ast.Term) -> None:
+        """
+        GIVEN:
+            - A leaf with no CJK text, a CJK term on a field outside the
+              default search fields, or an unfielded CJK term
+        WHEN:
+            - The hook is called with it
+        THEN:
+            - It returns the very same object, which tells analyze() to
+              keep the leaf's ordinary analysis
+        """
+        assert _widen_cjk_leaf(leaf) is leaf
+
+    def test_a_leaf_the_tokenizer_drops_entirely_is_returned_unchanged(self) -> None:
+        """
+        GIVEN:
+            - A term whose text matches _CJK_RE (U+2E80, a CJK Radicals
+              Supplement codepoint, so _has_cjk is True) but which the
+              content analyzer's simple tokenizer yields no token for at
+              all: a few hundred codepoints across the CJK blocks share
+              this gap between the regex and the tokenizer
+        WHEN:
+            - The hook is called with it
+        THEN:
+            - Neither cjk_terms nor latin_terms gains a piece, and the
+              leaf is returned unchanged rather than becoming an empty Or
+        """
+        leaf = _content("⺀")
+
+        assert _widen_cjk_leaf(leaf) is leaf
+
+    def test_the_original_leaf_object_is_the_first_alternative(self) -> None:
+        """
+        GIVEN:
+            - A CJK term on content
+        WHEN:
+            - The hook widens it
+        THEN:
+            - The Or's first child is the leaf object itself, not an equal
+              copy: analyze() only keeps the leaf's own enclosing-group
+              analysis for that exact object
+        """
+        leaf = _content("東京")
+
+        widened = _widen_cjk_leaf(leaf)
+
+        assert isinstance(widened, wc_ast.Or)
+        assert widened.children[0] is leaf
+
+    def test_a_cjk_term_gains_a_bigram_alternative_on_its_own_field(self) -> None:
+        """
+        GIVEN:
+            - A CJK term on title
+        WHEN:
+            - The hook widens it
+        THEN:
+            - The bigram side targets bigram_title, the leaf's own field's
+              companion, so a fielded term stays fielded
+        """
+        leaf = wc_ast.Term(field=_TITLE, text="東京")
+
+        assert _widen_cjk_leaf(leaf) == _widened(
+            leaf,
+            wc_ast.Term(field=_BIGRAM_TITLE, text="東京"),
+        )
+
+    def test_each_cjk_run_gets_its_own_bigram_leaf(self) -> None:
+        """
+        GIVEN:
+            - One term whose text holds two CJK runs split by an
+              interpunct (東京・大阪)
+        WHEN:
+            - The hook widens it
+        THEN:
+            - The bigram side is Or(bigram 東京, bigram 大阪), one leaf
+              per run. Joining the runs with a space into one leaf would
+              produce bigrams spanning the space, which only exist when
+              the two runs sit side by side in a document. The runs are
+              alternatives, not requirements: a multi-run term matches on
+              any one of its runs, which is what the separate bigram
+              clause did before this change
+        """
+        leaf = _content("東京・大阪")
+
+        assert _widen_cjk_leaf(leaf) == _widened(
+            leaf,
+            wc_ast.Or(children=(_bigram("東京"), _bigram("大阪"))),
+        )
+
+    def test_each_run_keeps_its_own_bigrams_required(self) -> None:
+        """
+        GIVEN:
+            - A term whose two runs each produce more than one bigram
+              (東京都・大阪府)
+        WHEN:
+            - The hook widens it and the tree is analyzed
+        THEN:
+            - Each run's bigrams stay And-combined inside the cross-run
+              Or. The Or is between runs only; multitoken=AND on the
+              bigram FieldSpec is what requires a run's own bigrams, and
+              it does not inherit from the enclosing group. Without this,
+              東京都 would match a document holding only 京都
+        """
+        assert _analyze(_content("東京都・大阪府")) == wc_ast.Or(
+            children=(
+                wc_ast.And(children=(_content("東京都"), _content("大阪府"))),
+                wc_ast.And(children=(_bigram("東京"), _bigram("京都"))),
+                wc_ast.And(children=(_bigram("大阪"), _bigram("阪府"))),
+            ),
+        )
+
+    def test_a_phrase_widens_to_a_bigram_term_not_a_bigram_phrase(self) -> None:
+        """
+        GIVEN:
+            - A quoted CJK phrase on content
+        WHEN:
+            - The hook widens it
+        THEN:
+            - The bigram side is a plain Term: the bigram analyzer puts
+              every token at position 0, so a positional Phrase against it
+              could never match
+        """
+        leaf = wc_ast.Phrase(field=_CONTENT, text="東京都")
+
+        assert _widen_cjk_leaf(leaf) == _widened(leaf, _bigram("東京都"))
+
+    def test_a_multi_word_phrase_requires_every_run(self) -> None:
+        """
+        GIVEN:
+            - A quoted CJK phrase holding two words
+        WHEN:
+            - The hook widens it
+        THEN:
+            - The runs are And-combined, where a Term's runs are
+              Or-combined (test_each_cjk_run_gets_its_own_bigram_leaf).
+              The parser's default group is And, so an Or here would let
+              the quoted phrase match strictly more than the same two
+              words unquoted, and quoting must not widen a search
+        """
+        leaf = wc_ast.Phrase(field=_CONTENT, text="東京都 大阪府")
+
+        assert _widen_cjk_leaf(leaf) == _widened(
+            leaf,
+            wc_ast.And(children=(_bigram("東京都"), _bigram("大阪府"))),
+        )
+
+    @pytest.mark.parametrize(
+        ("leaf", "alternative"),
+        [
+            pytest.param(
+                _content("東京-report"),
+                wc_ast.And(children=(_bigram("東京"), _content("report"))),
+                id="separated_term",
+            ),
+            pytest.param(
+                wc_ast.Phrase(field=_CONTENT, text="東京 report"),
+                wc_ast.And(children=(_bigram("東京"), _content("report"))),
+                id="phrase",
+            ),
+            pytest.param(_content("東京report"), _bigram("東京"), id="glued"),
+        ],
+    )
+    def test_latin_is_required_only_where_the_analyzer_splits_it_off(
+        self,
+        leaf: wc_ast.Term | wc_ast.Phrase,
+        alternative: wc_ast.Node,
+    ) -> None:
+        """
+        GIVEN:
+            - A leaf mixing CJK and latin text, either split into separate
+              tokens by the content analyzer (東京-report, "東京 report")
+              or glued into one token (東京report)
+        WHEN:
+            - The hook widens it
+        THEN:
+            - A separate latin token is required on the leaf's own field
+              beside the CJK run's bigrams. Glued latin is not required:
+              the index only holds it inside the whole unspaced token, so
+              requiring it would lose documents the CJK run finds
+        """
+        assert _widen_cjk_leaf(leaf) == _widened(leaf, alternative)
+
+    def test_only_latin_pieces_after_a_tokenizer_drop_still_widens(self) -> None:
+        """
+        GIVEN:
+            - A term combining a tokenizer-dropped CJK codepoint (U+2E80)
+              with a latin word (⺀report): _has_cjk matches on the
+              leading codepoint, but the tokenizer discards it and keeps
+              only the "report" token, so no run ever reaches cjk_terms
+        WHEN:
+            - The hook widens it
+        THEN:
+            - The alternative is the lone latin term itself, not wrapped
+              in And (a single piece collapses to itself), even though
+              the CJK side contributed nothing
+        """
+        leaf = _content("⺀report")
+
+        assert _widen_cjk_leaf(leaf) == _widened(leaf, _content("report"))
+
+    def test_new_nodes_carry_the_leaf_span(self) -> None:
+        """
+        GIVEN:
+            - A CJK term with a source span, split into two pieces
+        WHEN:
+            - The hook widens it
+        THEN:
+            - The Or, the alternative and each piece carry the leaf's
+              span, so an emit-time diagnostic still points into the
+              query text
+        """
+        leaf = wc_ast.Term(field=_CONTENT, text="東京・大阪", startchar=3, endchar=8)
+
+        widened = _widen_cjk_leaf(leaf)
+
+        assert isinstance(widened, wc_ast.Or)
+        alternative = widened.children[1]
+        assert isinstance(alternative, wc_ast.Or)
+        spans = {
+            (node.startchar, node.endchar)
+            for node in (widened, alternative, *alternative.children)
+        }
+        assert spans == {(3, 8)}
+
+
+class TestAnalyzedTree:
+    @pytest.mark.parametrize(
+        "node",
+        [
+            pytest.param(wc_ast.Prefix(field=_CONTENT, text="東京"), id="cjk_prefix"),
+            pytest.param(
+                wc_ast.Wildcard(field=_CONTENT, pattern="東*"),
+                id="cjk_wildcard",
+            ),
+        ],
+    )
+    def test_patterns_are_never_widened(self, node: wc_ast.Node) -> None:
+        """
+        GIVEN:
+            - A CJK prefix or wildcard pattern on content
+        WHEN:
+            - The tree is analyzed with the hook
+        THEN:
+            - It comes back unchanged: analyze() only offers Term and
+              Phrase leaves to the hook
+        """
+        assert _analyze(node) == node
+
+    @pytest.mark.parametrize(
+        "build",
+        [
+            pytest.param(lambda node: node, id="bare"),
+            pytest.param(lambda node: wc_ast.Not(child=node), id="negated"),
+        ],
+    )
+    def test_a_run_past_the_length_limit_keeps_its_bigram_side(
+        self,
+        build: Callable[[wc_ast.Node], wc_ast.Node],
+        long_cjk_run: str,
+    ) -> None:
+        """
+        GIVEN:
+            - A CJK run longer than the content analyzer's remove_long
+              limit, bare or under NOT
+        WHEN:
+            - The tree is analyzed with the hook
+        THEN:
+            - The content side drops out and the bigram side stands alone,
+              still negated under NOT. A NOT left holding nothing would
+              turn into "match everything" instead
+        """
+        resolved = _get_emit_field_registry(None).resolve(_BIGRAM_CONTENT)
+        assert resolved is not None
+        bigrams = resolved.spec.analyzer(long_cjk_run)
+
+        assert _analyze(build(_content(long_cjk_run))) == build(
+            wc_ast.And(
+                children=tuple(_bigram(token) for token in dict.fromkeys(bigrams)),
+            ),
+        )
+
+    def test_a_one_character_run_drops_out_of_the_alternative(self) -> None:
+        """
+        GIVEN:
+            - A term whose runs are 東 (one character) and 大阪
+        WHEN:
+            - The tree is analyzed with the hook
+        THEN:
+            - Only 大阪 is required on the bigram side: a one-character run
+              has no bigram. An accepted gap, the same on the positive and
+              the negated side
+        """
+        analyzed = _analyze(_content("東・大阪"))
+
+        assert analyzed == _widened(
+            wc_ast.And(children=(_content("東"), _content("大阪"))),
+            _bigram("大阪"),
+        )
+
+    def test_the_title_boost_wraps_the_widened_title_leaf(self) -> None:
+        """
+        GIVEN:
+            - An unfielded CJK query parsed the way parse_user_query parses
+              it: copied onto every default field, title boosted
+        WHEN:
+            - The tree is analyzed with the hook
+        THEN:
+            - Each field copy is widened on its own bigram field, and the
+              title boost wraps the title copy's whole widened Or, so a
+              title bigram match scores like a title match
+        """
+        parsed = wc.parse(
+            "東京",
+            registry=get_field_registry(None),
+            default_fields=_DEFAULT_SEARCH_FIELDS,
+            field_boosts=_FIELD_BOOSTS,
+        ).ast
+
+        analyzed = _analyze(parsed)
+
+        assert isinstance(analyzed, wc_ast.Or)
+        assert (
+            wc_ast.Boosted(
+                child=_widened(
+                    wc_ast.Term(field=_TITLE, text="東京"),
+                    wc_ast.Term(field=_BIGRAM_TITLE, text="東京"),
+                ),
+                boost=_FIELD_BOOSTS["title"],
+            )
+            in analyzed.children
+        )
+        assert {_BIGRAM_CONTENT, wc.FieldRef("bigram_tag")} <= {
+            child.field for child in analyzed.children if isinstance(child, wc_ast.Term)
+        }
+
+    def test_under_and_the_original_side_keeps_and(self) -> None:
+        """
+        GIVEN:
+            - A term the content analyzer splits into two tokens
+              (東京・大阪), inside an And group
+        WHEN:
+            - The tree is analyzed with the hook
+        THEN:
+            - The original side still ANDs its two tokens. Wrapping the
+              leaf in the hook's Or must not turn that into an Or of them.
+              The cross-run Or flattens into the widening Or, so the
+              alternative is not a single child here and _widened() does
+              not fit; the tree is written out literally
+        """
+        tree = wc_ast.And(children=(_content("東京・大阪"), _content("report")))
+
+        assert _analyze(tree) == wc_ast.And(
+            children=(
+                wc_ast.Or(
+                    children=(
+                        wc_ast.And(children=(_content("東京"), _content("大阪"))),
+                        _bigram("東京"),
+                        _bigram("大阪"),
+                    ),
+                ),
+                _content("report"),
+            ),
+        )
+
+    def test_under_or_the_original_side_keeps_or(self) -> None:
+        """
+        GIVEN:
+            - The same two-token term inside an Or group
+        WHEN:
+            - The tree is analyzed with the hook
+        THEN:
+            - The original side ORs its tokens, as the group says, and the
+              hook's Or flattens into the group
+        """
+        tree = wc_ast.Or(children=(_content("東京・大阪"), _content("report")))
+
+        assert _analyze(tree) == wc_ast.Or(
+            children=(
+                _content("東京"),
+                _content("大阪"),
+                _bigram("東京"),
+                _bigram("大阪"),
+                _content("report"),
+            ),
+        )
+
+    @pytest.mark.parametrize(
+        "build",
+        [
+            pytest.param(lambda leaf: wc_ast.Not(child=leaf), id="not"),
+            pytest.param(
+                lambda leaf: wc_ast.AndNot(positive=_content("invoice"), negative=leaf),
+                id="andnot",
+            ),
+            pytest.param(
+                lambda leaf: wc_ast.AndMaybe(
+                    required=_content("invoice"),
+                    optional=leaf,
+                ),
+                id="andmaybe",
+            ),
+            pytest.param(
+                lambda leaf: wc_ast.Require(
+                    scored=leaf,
+                    filter_only=_content("invoice"),
+                ),
+                id="require",
+            ),
+            pytest.param(
+                lambda leaf: wc_ast.Boosted(child=leaf, boost=2.0),
+                id="boosted",
+            ),
+        ],
+    )
+    def test_the_widened_leaf_stays_where_it_was(
+        self,
+        build: Callable[[wc_ast.Node], wc_ast.Node],
+        long_cjk_run: str,
+    ) -> None:
+        """
+        GIVEN:
+            - A CJK term under Not, AndNot, AndMaybe, Require or Boosted,
+              beside a latin term
+        WHEN:
+            - The tree is analyzed with the hook
+        THEN:
+            - The node keeps its type, its latin operand is untouched, and
+              only the CJK leaf is replaced by its widened Or, in the same
+              position. A negated CJK leaf is widened too, so a negation
+              excludes exactly what the positive search would match
+        """
+        assert _analyze(build(_content("東京"))) == build(
+            _widened(_content("東京"), _bigram("東京")),
+        )

src/documents/tests/search/test_cjk_run_extraction.py

@@ -0,0 +1,200 @@
+"""Which characters count as part of a CJK run.
+
+_CJK_RE decides both what is indexed into the bigram fields
+(extract_cjk_text) and how a query's CJK text is cut into runs
+(_widen_cjk_leaf). A character it misses splits a word in two on both
+sides. For the katakana prolonged sound mark that leaves one-character
+runs, which have no bigrams, so the word could not be found through the
+bigram fields at all.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from documents.search._query import extract_cjk_text
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from documents.models import Document
+
+
+pytestmark = pytest.mark.search
+
+
+class TestRunExtraction:
+    @pytest.mark.parametrize(
+        ("text", "expected"),
+        [
+            pytest.param("コーヒーを飲む", "コーヒーを飲む", id="prolonged_sound_mark"),
+            pytest.param("ｺｰﾋｰ", "ｺｰﾋｰ", id="halfwidth_prolonged_sound_mark"),
+            pytest.param("ｺﾞﾙﾌ", "ｺﾞﾙﾌ", id="halfwidth_voiced_mark"),
+            pytest.param("ﾊﾟﾝ", "ﾊﾟﾝ", id="halfwidth_semi_voiced_mark"),
+            pytest.param("締め切り〆日", "締め切り〆日", id="closing_mark"),
+            pytest.param("人々の生活", "人々の生活", id="iteration_mark"),
+            pytest.param("東京・大阪", "東京 大阪", id="interpunct_still_splits"),
+            pytest.param("東京、大阪", "東京 大阪", id="comma_still_splits"),
+        ],
+    )
+    def test_japanese_word_marks_stay_inside_the_run(
+        self,
+        text: str,
+        expected: str,
+    ) -> None:
+        """
+        GIVEN:
+            - Japanese text containing ー (U+30FC), its halfwidth form
+              (U+FF70), the halfwidth voiced and semi-voiced marks
+              (U+FF9E, U+FF9F), 〆 or 々, or a separator between two runs
+        WHEN:
+            - Its CJK runs are extracted for the bigram fields
+        THEN:
+            - The marks stay inside their word's run. Every one of them
+              has Unicode script Common, so the script classes alone miss
+              them. Punctuation such as ・ and 、 still separates runs
+        """
+        assert extract_cjk_text(text) == expected
+
+
+@pytest.mark.django_db
+class TestProlongedSoundMarkSearch:
+    @pytest.mark.parametrize(
+        ("query", "content"),
+        [
+            pytest.param("コーヒー", "コーヒーを飲む", id="fullwidth"),
+            pytest.param("サーバー", "サーバーの設定を変更", id="fullwidth_two_marks"),
+            pytest.param("ｺｰﾋｰ", "ｺｰﾋｰを飲む", id="halfwidth"),
+        ],
+    )
+    def test_a_word_with_the_mark_is_found_inside_running_text(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+        query: str,
+        content: str,
+    ) -> None:
+        """
+        GIVEN:
+            - A document containing a katakana word with ー inside a
+              longer unspaced run, and an unrelated latin document
+        WHEN:
+            - The word is searched
+        THEN:
+            - The document matches through the bigram fields. Without the
+              mark in _CJK_RE, the word splits into one-character runs
+              with no bigrams, and only a standalone content token could
+              match
+        """
+        match = index_document(title="A", content=content)
+        index_document(title="B", content="invoice only")
+
+        assert matched_ids(query) == {match.pk}
+
+    def test_not_excludes_a_word_with_the_mark(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - An "invoice" document containing コーヒー inside a longer
+              run, and a latin-only "invoice" document
+        WHEN:
+            - "invoice NOT コーヒー" is searched
+        THEN:
+            - Only the latin document matches: the negation excludes what
+              the positive search finds
+        """
+        index_document(
+            title="A",
+            content="コーヒーを飲む invoice",
+        )
+        latin = index_document(title="B", content="invoice only")
+
+        assert matched_ids("invoice NOT コーヒー") == {latin.pk}
+
+    def test_a_different_word_sharing_only_the_edges_does_not_match(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document with コード and ヒント, two separate words
+        WHEN:
+            - "コーヒー" is searched
+        THEN:
+            - Nothing matches. With ー outside _CJK_RE, both sides reduce
+              to runs like コ and ヒ joined by spaces, and the space
+              bigrams that produces could match unrelated words
+        """
+        index_document(title="A", content="コード ヒント")
+
+        assert matched_ids("コーヒー") == set()
+
+
+@pytest.mark.django_db
+class TestHalfwidthVoicedMarkSearch:
+    """Halfwidth katakana, as legacy systems and bank statements write it.
+
+    Its voiced marks are separate codepoints with no precomposed form, so
+    NFC leaves them where they are and the character class has to cover
+    them. A word like ﾊﾟﾝ is three codepoints, and splitting it at the mark
+    leaves two one-character runs with no bigrams, so it is not merely
+    imprecise but unfindable.
+    """
+
+    @pytest.mark.parametrize(
+        ("query", "content"),
+        [
+            pytest.param("ｺﾞﾙﾌ", "ｺﾞﾙﾌ場の利用料金", id="voiced_mark"),
+            pytest.param("ﾊﾟﾝ", "ﾊﾟﾝと牛乳の購入", id="semi_voiced_mark"),
+        ],
+    )
+    def test_a_word_with_the_mark_is_found_inside_running_text(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+        query: str,
+        content: str,
+    ) -> None:
+        """
+        GIVEN:
+            - A document holding a halfwidth katakana word inside a longer
+              unspaced run, and an unrelated latin document
+        WHEN:
+            - The word is searched
+        THEN:
+            - The document matches. Without the mark in _CJK_RE the word
+              splits at it, and ﾊﾟﾝ in particular loses both halves to the
+              one-character rule and can never be found
+        """
+        match = index_document(title="A", content=content)
+        index_document(title="B", content="invoice only")
+
+        assert matched_ids(query) == {match.pk}
+
+    def test_the_whole_word_is_searched_not_just_the_tail(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document holding ｺﾞﾙﾌ, and one holding ﾙﾌﾅﾞｰ , which
+              shares the ﾙﾌ pair that splitting ｺﾞﾙﾌ at its mark leaves
+              behind
+        WHEN:
+            - "ｺﾞﾙﾌ" is searched
+        THEN:
+            - Only the first matches. Splitting at the mark would search
+              the ﾙﾌ fragment alone, which silently degrades the query
+              into a two-character substring search
+        """
+        golf = index_document(title="A", content="ｺﾞﾙﾌ場の利用料金")
+        index_document(title="B", content="ﾙﾌﾅﾞｰの修理")
+
+        assert matched_ids("ｺﾞﾙﾌ") == {golf.pk}

src/documents/tests/search/test_cjk_widening.py

@@ -0,0 +1,625 @@
+"""CJK terms in QUERY-mode searches, matched through their bigram fields
+in place inside the parsed query.
+
+The content analyzer keeps an unspaced CJK run as one token, so a CJK
+term is only findable through the bigram fields. Each CJK leaf is widened
+where it sits, so everything around it (AND, NOT, REQUIRE, boosts,
+fielding) constrains the bigram match exactly as it constrains the
+original term.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from documents.models import Correspondent
+from documents.models import Document
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from pytest_django.fixtures import SettingsWrapper
+
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+
+class TestNegationSymmetry:
+    def test_not_excludes_what_the_positive_term_matches(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document whose 東京 sits inside a longer unspaced run
+              (indexed as one content token), and a latin-only document,
+              both matching "invoice"
+        WHEN:
+            - "東京" and "invoice NOT 東京" are searched
+        THEN:
+            - 東京 finds the CJK document, and NOT 東京 excludes that same
+              document. A content-only negation never fires here, since
+              the content field holds the whole run, not 東京
+        """
+        cjk = index_document(
+            title="A",
+            content="東京都の公共文書について invoice",
+        )
+        latin = index_document(title="B", content="invoice only")
+
+        assert matched_ids("東京") == {cjk.pk}
+        assert matched_ids("invoice NOT 東京") == {latin.pk}
+
+    def test_bigram_approximation_is_the_same_on_both_sides(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document mentioning 東京 and 京都 separately, never 東京都
+        WHEN:
+            - "東京都" and "invoice NOT 東京都" are searched
+        THEN:
+            - 東京都 matches it (both of its bigrams are present, and the
+              bigram match ignores position), and NOT 東京都 excludes it.
+              An accepted approximation: a query and its negation agree
+        """
+        separate = index_document(
+            title="A",
+            content="東京 と 京都 invoice",
+        )
+        latin = index_document(title="B", content="invoice only")
+
+        assert matched_ids("東京都") == {separate.pk}
+        assert matched_ids("invoice NOT 東京都") == {latin.pk}
+
+    @pytest.mark.parametrize(
+        "query",
+        [
+            pytest.param("invoice NOT 東京都", id="term"),
+            pytest.param('invoice NOT "東京都"', id="phrase"),
+        ],
+    )
+    def test_a_negated_multi_bigram_run_needs_every_bigram_to_exclude(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+        query: str,
+    ) -> None:
+        """
+        GIVEN:
+            - A document containing 東京都, and one containing only 京都
+              (one of 東京都's two bigrams)
+        WHEN:
+            - 東京都 is excluded, as a bare term or a quoted phrase
+        THEN:
+            - Only the 東京都 document is excluded. Sharing one bigram
+              with the excluded run is not enough to be excluded
+        """
+        full = index_document(
+            title="A",
+            content="東京都の公共文書について invoice",
+        )
+        partial = index_document(
+            title="B",
+            content="京都の invoice",
+        )
+
+        assert matched_ids("invoice") == {full.pk, partial.pk}
+        assert matched_ids(query) == {partial.pk}
+
+    def test_a_run_past_the_length_limit_is_matched_and_excluded(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+        long_cjk_run: str,
+    ) -> None:
+        """
+        GIVEN:
+            - A document holding a CJK run longer than the content
+              analyzer's remove_long limit, and a latin-only document
+        WHEN:
+            - The long run is searched bare, and excluded with NOT
+        THEN:
+            - Bare, it matches the document; negated, it excludes that
+              document and nothing else. The content side of the run
+              analyzes to nothing, which must not erase the NOT
+        """
+        long = index_document(
+            title="A",
+            content=f"{long_cjk_run} invoice",
+        )
+        latin = index_document(title="B", content="invoice only")
+
+        assert matched_ids(long_cjk_run) == {long.pk}
+        assert matched_ids(f"invoice NOT {long_cjk_run}") == {latin.pk}
+
+
+class TestTokenizerDroppedCodepoints:
+    @pytest.fixture
+    def invoice(self, index_document: Callable[..., Document]) -> Document:
+        """One latin "invoice" document, with no U+2E80 anywhere in it."""
+        return index_document(title="A", content="invoice only")
+
+    def test_a_regex_only_match_finds_nothing(
+        self,
+        invoice: Document,
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document whose content has nothing resembling U+2E80 (a
+              CJK Radicals Supplement codepoint that matches _CJK_RE but
+              that the content analyzer's simple tokenizer yields no
+              token for at all)
+        WHEN:
+            - "⺀" is searched
+        THEN:
+            - Nothing matches: _widen_cjk_leaf finds no pieces to widen
+              with and returns the leaf as it is, which analyzes to the
+              same nothing it always did
+        """
+        assert matched_ids("⺀") == set()
+
+    def test_not_a_regex_only_match_excludes_nothing(
+        self,
+        invoice: Document,
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - An "invoice" document with no U+2E80 in it
+        WHEN:
+            - "invoice NOT ⺀" is searched
+        THEN:
+            - The document still matches: the negated leaf widens to
+              nothing extra, exactly as an unwidened NOT of a term with
+              no matches would behave
+        """
+        assert matched_ids("invoice NOT ⺀") == {invoice.pk}
+
+
+class TestStructureConstrainsTheCjkMatch:
+    @pytest.fixture
+    def pks(self, index_document: Callable[..., Document]) -> dict[str, int]:
+        """One document with both 東京 and "invoice", one with only 東京."""
+        return {
+            "both": index_document(title="A", content="東京都の請求書 invoice").pk,
+            "cjk_only": index_document(title="B", content="東京 report").pk,
+        }
+
+    @pytest.mark.parametrize(
+        ("query", "expected"),
+        [
+            pytest.param("東京 AND invoice", {"both"}, id="and"),
+            pytest.param("東京 REQUIRE invoice", {"both"}, id="require"),
+            pytest.param("東京^2 AND invoice", {"both"}, id="boosted"),
+            pytest.param("東京 ANDMAYBE invoice", {"both", "cjk_only"}, id="andmaybe"),
+        ],
+    )
+    def test_a_latin_operand_still_constrains_the_cjk_term(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        pks: dict[str, int],
+        query: str,
+        expected: set[str],
+    ) -> None:
+        """
+        GIVEN:
+            - One document with both 東京 and "invoice", one with only 東京
+        WHEN:
+            - 東京 is combined with "invoice" by AND, REQUIRE, a boosted
+              AND, or ANDMAYBE
+        THEN:
+            - AND and REQUIRE (boosted or not) need "invoice" too, so only
+              the document with both matches; ANDMAYBE leaves "invoice"
+              optional, so both match
+        """
+        assert matched_ids(query) == {pks[label] for label in expected}
+
+
+class TestFieldedTerms:
+    def test_a_fielded_multi_run_term_matches_through_either_run(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document with 東京 and 大阪, and one with only 東京, both
+              containing "report"
+        WHEN:
+            - "content:東京・大阪 AND report" is searched (the content
+              analyzer splits 東京・大阪 into two tokens)
+        THEN:
+            - Both match. The original side still ANDs its two content
+              tokens, pinned in test_cjk_leaf_rewrite.py, but the bigram
+              side accepts either run, so the 東京-only document comes
+              back through it. The AND still binds: "report" is required
+              of both. This is what the separate bigram clause returned
+              before this change
+        """
+        both = index_document(title="A", content="東京 大阪 report")
+        one = index_document(title="B", content="東京 report")
+
+        assert matched_ids("content:東京・大阪 AND report") == {
+            both.pk,
+            one.pk,
+        }
+
+    def test_a_run_is_matched_wherever_it_appears_in_the_text(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document holding 大阪 and 東京 apart, inside one unspaced
+              run (大阪府と東京都), a document with only 東京, and one
+              with neither
+        WHEN:
+            - "content:東京・大阪" is searched
+        THEN:
+            - The first two match and the third does not. Each run is
+              searched on its own, so no bigram spanning the gap between
+              the runs is required, and either run on its own is enough
+        """
+        apart = index_document(title="A", content="大阪府と東京都")
+        one = index_document(title="B", content="東京 report")
+        index_document(title="C", content="report only")
+
+        assert matched_ids("content:東京・大阪") == {apart.pk, one.pk}
+
+    def test_an_unfielded_multi_run_term_matches_on_either_run(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document with both CJK runs, one with only the first run,
+              and one with neither, all containing "invoice"
+        WHEN:
+            - The two runs are searched as one unfielded term (東京・大阪)
+        THEN:
+            - Both CJK documents match. An unfielded term is copied across
+              the default fields inside an Or, so the leaf's own semantics
+              are "either"; the bigram side matches that, and returns what
+              the separate bigram clause returned before this change
+        """
+        both = index_document(
+            title="A",
+            content="大阪府と東京都の報告 invoice",
+        )
+        one = index_document(
+            title="B",
+            content="東京都の報告書 invoice",
+        )
+        index_document(title="C", content="invoice only")
+
+        assert matched_ids("東京・大阪") == {both.pk, one.pk}
+
+    def test_a_negated_multi_run_term_excludes_on_either_run(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - Documents containing both runs of a glued mixed term, only
+              its first run, only its second run, and neither, all
+              containing "invoice"
+        WHEN:
+            - "invoice NOT 東京report資料" is searched
+        THEN:
+            - Only the document with neither run survives. The negation
+              excludes exactly what the positive term matches, and the
+              positive term matches on either run, so the exclusion is
+              wider than an "only documents holding both" reading. Today
+              this query excludes nothing at all
+        """
+        index_document(
+            title="A",
+            content="東京都のreport資料です invoice",
+        )
+        index_document(title="B", content="東京都の報告 invoice")
+        index_document(title="C", content="参考資料の一覧 invoice")
+        neither = index_document(title="D", content="invoice only")
+
+        assert matched_ids("invoice NOT 東京report資料") == {neither.pk}
+
+    def test_a_bigram_field_name_is_not_query_syntax(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document holding 東京 in its content
+        WHEN:
+            - A query naming an internal bigram field as a field prefix is
+              searched end to end (bigram_content:東京)
+        THEN:
+            - Nothing matches. The prefix is not query syntax, so the
+              whole thing is plain text: the literal token
+              "bigram_content" is required on the leaf's own field, and no
+              document holds it. Asserted through the real search path, so
+              that parsing with the emit registry instead of the public
+              one would fail this test. A unit-level wc.parse() assertion
+              would not: it pins whoosh-compat's handling of an unknown
+              prefix rather than our choice of parse registry
+        """
+        index_document(title="A", content="東京都の報告書")
+
+        assert matched_ids("bigram_content:東京") == set()
+
+
+class TestMixedScript:
+    def test_separated_latin_is_required(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document with 東京 and "report" as separate words, and one
+              with 東京 but no "report"
+        WHEN:
+            - "東京-report" is searched (the content analyzer splits it into
+              東京 and report)
+        THEN:
+            - Only the document with both matches: the latin piece is
+              required beside the CJK run's bigrams
+        """
+        both = index_document(title="A", content="東京都の report")
+        index_document(title="B", content="東京都の invoice")
+
+        assert matched_ids("東京-report") == {both.pk}
+
+    def test_glued_latin_is_not_required(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document where the latin text is glued to other CJK text
+              (東京都のreport資料, one content token), and one with 東京
+              but no "report" at all
+        WHEN:
+            - "東京report" is searched (one token: latin glued to CJK)
+        THEN:
+            - Both match through 東京's bigrams. Glued latin is only ever
+              indexed inside a whole unspaced token, so requiring it would
+              have lost the first document
+        """
+        glued = index_document(
+            title="A",
+            content="東京都のreport資料",
+        )
+        other = index_document(
+            title="B",
+            content="東京都の invoice",
+        )
+
+        assert matched_ids("東京report") == {glued.pk, other.pk}
+
+    def test_not_a_separated_mixed_term_excludes_only_documents_with_both(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - Three "invoice" documents: with 東京 and "report", with 東京
+              only, and latin only
+        WHEN:
+            - "invoice NOT 東京-report" is searched
+        THEN:
+            - Only the document with both is excluded, not every document
+              mentioning 東京
+        """
+        index_document(
+            title="A",
+            content="東京都の report invoice",
+        )
+        cjk_only = index_document(
+            title="B",
+            content="東京都の invoice",
+        )
+        latin = index_document(title="C", content="invoice only")
+
+        assert matched_ids("invoice NOT 東京-report") == {
+            cjk_only.pk,
+            latin.pk,
+        }
+
+    def test_a_one_character_run_is_not_required(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+    ) -> None:
+        """
+        GIVEN:
+            - A document containing 大阪 but not 東
+        WHEN:
+            - "content:東・大阪" is searched
+        THEN:
+            - It matches: a one-character run has no bigram, so only 大阪
+              is required. An accepted gap, the same under NOT
+        """
+        osaka = index_document(title="A", content="大阪府の報告書")
+
+        assert matched_ids("content:東・大阪") == {osaka.pk}
+
+
+class TestQuotedPhrase:
+    """A quoted CJK phrase must not match more than the same words unquoted.
+
+    The bigram fields put every token at position 0, so no alternative
+    built from them can enforce adjacency. Requiring both runs is the
+    tightest thing available, and the floor: the parser's default group is
+    And, so anything looser would make quoting widen the search.
+    """
+
+    @pytest.fixture
+    def pks(self, index_document: Callable[..., Document]) -> dict[str, int]:
+        return {
+            "tokyo": index_document(
+                title="A",
+                content="東京都の報告書 invoice",
+            ).pk,
+            "osaka": index_document(
+                title="B",
+                content="大阪府の報告書 invoice",
+            ).pk,
+            "both": index_document(
+                title="C",
+                content="東京都と大阪府の報告書 invoice",
+            ).pk,
+        }
+
+    @pytest.mark.parametrize(
+        "query",
+        [
+            pytest.param("東京都 大阪府", id="unquoted"),
+            pytest.param('"東京都 大阪府"', id="quoted"),
+        ],
+    )
+    def test_both_words_are_required_either_way(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        pks: dict[str, int],
+        query: str,
+    ) -> None:
+        """
+        GIVEN:
+            - Three documents, holding only the first word, only the
+              second, and both
+        WHEN:
+            - The two words are searched unquoted and as a quoted phrase
+        THEN:
+            - Only the document holding both matches, either way. Quoting
+              asks for more than the bare words, never less
+        """
+        assert matched_ids(query) == {pks["both"]}
+
+    @pytest.mark.parametrize(
+        "query",
+        [
+            pytest.param("invoice NOT (東京都 大阪府)", id="unquoted"),
+            pytest.param('invoice NOT "東京都 大阪府"', id="quoted"),
+        ],
+    )
+    def test_the_negation_excludes_only_what_the_phrase_matches(
+        self,
+        matched_ids: Callable[[str], set[int]],
+        pks: dict[str, int],
+        query: str,
+    ) -> None:
+        """
+        GIVEN:
+            - The same three documents, all matching "invoice"
+        WHEN:
+            - The two words are excluded, grouped and as a quoted phrase
+        THEN:
+            - Only the document holding both words is excluded. The
+              negation is the mirror of the positive side, so a phrase
+              that required either word would take the two single-word
+              documents out with it
+        """
+        assert matched_ids(query) == {pks["tokyo"], pks["osaka"]}
+
+
+class TestOtherDefaultFields:
+    @pytest.mark.parametrize("where", ["title", "correspondent"])
+    def test_not_excludes_a_cjk_match_outside_content(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+        where: str,
+    ) -> None:
+        """
+        GIVEN:
+            - An "invoice" document whose only 東京 is inside a longer run
+              in its title or its correspondent, and a latin-only one
+        WHEN:
+            - "invoice NOT 東京" is searched
+        THEN:
+            - The CJK document is excluded: every default field's bigram
+              companion is widened, not only content's
+        """
+        kwargs: dict[str, object] = {
+            "title": "A",
+            "content": "invoice",
+        }
+        if where == "title":
+            kwargs["title"] = "東京都の報告書"
+        else:
+            kwargs["correspondent"] = Correspondent.objects.create(name="東京電力")
+        index_document(**kwargs)
+        latin = index_document(
+            title="B",
+            content="invoice only",
+        )
+
+        assert matched_ids("invoice NOT 東京") == {latin.pk}
+
+
+class TestFuzzyOnGap:
+    @pytest.mark.parametrize(
+        ("query", "threshold", "expected"),
+        [
+            pytest.param("invoice NOT 東京", None, {"latin"}, id="not_fuzzy_off"),
+            pytest.param("invoice NOT 東京", 0.0, {"latin", "cjk"}, id="not_fuzzy_on"),
+            pytest.param("東京 AND invoice", None, {"cjk"}, id="and_fuzzy_off"),
+            pytest.param(
+                "東京 AND invoice",
+                0.0,
+                {"cjk", "cjk_only", "latin"},
+                id="and_fuzzy_on",
+            ),
+        ],
+    )
+    def test_fuzzy_on_readmits_what_the_structure_excludes(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+        settings: SettingsWrapper,
+        query: str,
+        threshold: float | None,
+        expected: set[str],
+    ) -> None:
+        """
+        GIVEN:
+            - An "invoice" document whose 東京 is inside a longer unspaced
+              run, a 東京-only document and a latin-only "invoice"
+              document, with fuzzy search off or on
+        WHEN:
+            - "invoice NOT 東京" or "東京 AND invoice" is searched
+        THEN:
+            - With fuzzy off, the query's structure decides. With fuzzy on,
+              the separate fuzzy clause matches any one of the query's
+              words on its own, and the exclusion restated above it is
+              content-only, so it re-admits what NOT and AND excluded. A
+              known interim gap, not a regression (both settings returned
+              these before), removed when fuzzy moves into the query tree
+              too; pinned so that change flips it deliberately
+        """
+        settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = threshold
+        cjk = index_document(
+            title="A",
+            content="東京都の公共文書について invoice",
+        )
+        cjk_only = index_document(
+            title="B",
+            content="東京都の報告書",
+        )
+        latin = index_document(title="C", content="invoice only")
+        pks = {"cjk": cjk.pk, "cjk_only": cjk_only.pk, "latin": latin.pk}
+
+        assert matched_ids(query) == {pks[label] for label in expected}

src/documents/tests/search/test_conjunctive_negations.py

@@ -3,7 +3,7 @@ query excludes from every document it matches, and _any_of, the clause-list
 collapsing helper it feeds into.
 
 Result-level proof that a negation reached through NOT/AND survives the
-fuzzy/CJK blend lives in test_query_negation.py. These are direct unit
+fuzzy blend lives in test_query_negation.py. These are direct unit
 tests of the visitor's dispatch for the rarer grammar shapes
 (AndNot/Boosted/AndMaybe/Require) that file's real-corpus queries don't
 happen to exercise, plus the empty-clause-list case of _any_of.

src/documents/tests/search/test_default_search_fields_guard.py

@@ -3,9 +3,9 @@ field names.
 
 Nothing enforced this before: a rename in PUBLIC_FIELDS not mirrored in
 ``_DEFAULT_SEARCH_FIELDS`` (documents/search/_query.py) would 400 every
-unfielded search at request time, since ``index.parse_query`` and the
-fuzzy/CJK clause builders are handed a field name the schema no longer
-has.
+unfielded search at request time, since ``index.parse_query``, the
+fuzzy clause builder and the CJK bigram widening are handed a field name
+the schema no longer has.
 """
 
 from __future__ import annotations

src/documents/tests/search/test_fuzzy_tokenization.py

@@ -51,7 +51,7 @@ class TestFuzzyClauseParseFailureDegradesGracefully:
         THEN:
             - It returns None instead of propagating, so a fuzzy word
               string tantivy's own parser rejects only drops the fuzzy
-              clause: the exact/CJK clauses still stand rather than the
+              clause: the exact clause still stands rather than the
               whole query failing. The ValueError guard is insurance (the
               word string is plain tokens, so tantivy accepting it is
               expected, not assumed)

src/documents/tests/search/test_query.py

@@ -522,8 +522,8 @@ class TestEmitErrorContract:
         THEN:
             - The emit failure is caught and mapped to a SearchQueryError
               (400) rather than propagating as a raw QueryError. This
-              fails at the whole-tree "exact" clause emission
-              (`exact = tantivy_emit(result.ast, ...)`), the same path
+              fails at the whole-tree "exact" clause emission (the
+              `tantivy_emit` call in parse_user_query), the same path
               TestRealQueriesRouteCorrectly::test_text_range_is_a_400_naming_the_field
               already covers without the NOT wrapper: `_negation_clauses`
               never runs here, since the whole-tree emit already raises

src/documents/tests/search/test_query_negation.py

@@ -1,9 +1,10 @@
 """Negation must survive the blended query.
 
-parse_user_query ORs an exact clause with optional fuzzy and CJK clauses.
-Each of those is built from positive terms only, so unless the query's
-exclusions are applied to the blend as a whole, a document the exact
-clause excluded is re-admitted by whichever other clause is enabled.
+parse_user_query ORs an exact clause with an optional fuzzy clause. That
+clause is built from positive terms only, so unless the query's exclusions
+are applied to the blend as a whole, a document the exact clause excluded
+is re-admitted by it. CJK terms are widened inside the exact clause itself,
+so the query's own structure constrains them.
 """
 
 from __future__ import annotations
@@ -12,26 +13,17 @@ from typing import TYPE_CHECKING
 
 import pytest
 
-from documents.models import Document
-
 if TYPE_CHECKING:
+    from collections.abc import Callable
+
     from pytest_django.fixtures import SettingsWrapper
 
-    from documents.search._backend import TantivyBackend
+    from documents.models import Document
+
 
 pytestmark = [pytest.mark.search, pytest.mark.django_db]
 
 
-def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:
-    return set(backend.search_ids(query, user=None))
-
-
-def _index(backend: TantivyBackend, **kwargs: object) -> Document:
-    doc = Document.objects.create(**kwargs)
-    backend.add_or_update(doc)
-    return doc
-
-
 @pytest.fixture
 def fuzzy_enabled(settings: SettingsWrapper) -> None:
     """Enable the fuzzy blend clause. The threshold doubles as a minimum
@@ -44,7 +36,8 @@ class TestNegationConstrainsEveryClause:
     @pytest.mark.usefixtures("fuzzy_enabled")
     def test_fuzzy_clause_does_not_readmit_an_excluded_document(
         self,
-        backend: TantivyBackend,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
     ) -> None:
         """
         GIVEN:
@@ -59,25 +52,22 @@ class TestNegationConstrainsEveryClause:
               the fuzzy clause (built from positive terms only) does not
               readmit the document the exact clause excluded
         """
-        secret = _index(
-            backend,
+        secret = index_document(
             title="Invoice A",
             content="invoice total secret",
-            checksum="neg-fuzzy-1",
         )
-        public = _index(
-            backend,
+        public = index_document(
             title="Invoice B",
             content="invoice total public",
-            checksum="neg-fuzzy-2",
         )
 
-        assert _matched_ids(backend, "invoice") == {secret.pk, public.pk}
-        assert _matched_ids(backend, "invoice NOT secret") == {public.pk}
+        assert matched_ids("invoice") == {secret.pk, public.pk}
+        assert matched_ids("invoice NOT secret") == {public.pk}
 
-    def test_cjk_clause_does_not_readmit_an_excluded_document(
+    def test_an_excluded_document_stays_out_of_a_cjk_match(
         self,
-        backend: TantivyBackend,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
     ) -> None:
         """
         GIVEN:
@@ -87,31 +77,26 @@ class TestNegationConstrainsEveryClause:
             - A query combining the CJK term with a NOT exclusion is run
         THEN:
             - Only the document without the excluded word is returned;
-              the CJK clause legitimately carries the CJK run, so
-              rebuilding it from the AST cannot help here, only applying
-              the exclusion above the blend keeps the excluded document
-              out
+              the CJK term's bigram match sits beside the NOT inside the
+              same query, so the exclusion applies to it
         """
-        secret = _index(
-            backend,
+        secret = index_document(
             title="Tokyo A",
             content="東京都の秘密です secret",
-            checksum="neg-cjk-1",
         )
-        public = _index(
-            backend,
+        public = index_document(
             title="Tokyo B",
             content="東京都の報告書です public",
-            checksum="neg-cjk-2",
         )
 
-        assert _matched_ids(backend, "東京") == {secret.pk, public.pk}
-        assert _matched_ids(backend, "東京 NOT secret") == {public.pk}
+        assert matched_ids("東京") == {secret.pk, public.pk}
+        assert matched_ids("東京 NOT secret") == {public.pk}
 
     @pytest.mark.usefixtures("fuzzy_enabled")
     def test_disjunctive_negation_still_admits_the_other_branch(
         self,
-        backend: TantivyBackend,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
     ) -> None:
         """
         GIVEN:
@@ -126,73 +111,54 @@ class TestNegationConstrainsEveryClause:
               left branch stays in even though it contains the excluded
               word
         """
-        secret_invoice = _index(
-            backend,
+        secret_invoice = index_document(
             title="Invoice A",
             content="invoice total secret",
-            checksum="neg-or-1",
         )
-        unrelated = _index(
-            backend,
+        unrelated = index_document(
             title="Recipe",
             content="flour and water",
-            checksum="neg-or-2",
         )
 
-        assert _matched_ids(backend, "invoice OR NOT secret") == {
+        assert matched_ids("invoice OR NOT secret") == {
             secret_invoice.pk,
             unrelated.pk,
         }
 
-    def test_a_negation_under_or_does_not_constrain_the_cjk_clause(
+    def test_a_negation_under_or_constrains_its_own_cjk_term(
         self,
-        backend: TantivyBackend,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
     ) -> None:
         """
         GIVEN:
             - Two CJK documents, one of which also contains a word an OR
               branch's own NOT excludes, plus an unrelated latin document
         WHEN:
-            - The exclusion is under a disjunctive OR branch, versus in
+            - The exclusion is under a disjunctive OR branch, and in
               conjunctive position
         THEN:
-            - Under OR, the excluded document still matches through the
-              CJK clause (an exclusion that is one branch's own condition
-              cannot be restated above the blend without dropping
-              documents the other branch matches, so it is left where it
-              is and the CJK clause stays unconstrained by it -- this
-              shows through here in a way it does not for latin text,
-              since the exact clause cannot match a CJK run at all, so
-              the CJK clause is the only thing matching the CJK
-              documents, and the excluded one comes with it)
-            - Under conjunctive "AND NOT", the same exclusion is hoisted
-              and does constrain the CJK clause, pinning the deliberate
-              limit of the hoist
+            - Either way the excluded document is left out. The CJK
+              term's bigram match is widened in place inside its own OR
+              branch, so that branch's NOT applies to it; the other
+              branch still admits the latin document
         """
-        secret = _index(
-            backend,
+        secret = index_document(
             title="Tokyo A",
             content="東京都の秘密です secret",
-            checksum="neg-or-cjk-1",
         )
-        public = _index(
-            backend,
+        public = index_document(
             title="Tokyo B",
             content="東京都の報告書です public",
-            checksum="neg-or-cjk-2",
         )
-        bill = _index(
-            backend,
+        bill = index_document(
             title="Bill",
             content="bill payment received",
-            checksum="neg-or-cjk-3",
         )
 
-        assert _matched_ids(backend, "(東京 AND NOT secret) OR bill") == {
+        assert matched_ids("(東京 AND NOT secret) OR bill") == {
             bill.pk,
             public.pk,
-            secret.pk,
         }
-        # The same exclusion in conjunctive position is hoisted, and does
-        # constrain the CJK clause.
-        assert _matched_ids(backend, "東京 AND NOT secret") == {public.pk}
+        assert matched_ids("東京 AND NOT secret") == {public.pk}
+        assert secret.pk in matched_ids("東京")

src/documents/tests/search/test_unicode_normalization.py

@@ -0,0 +1,184 @@
+"""Search text is held in one Unicode normal form on both sides.
+
+A bigram is a pair of codepoints, so decomposed and composed spellings of
+the same Japanese word produce different bigrams. Unless the indexed text
+and the query string are normalized the same way, a document written one
+way is invisible to a query written the other.
+"""
+
+from __future__ import annotations
+
+import unicodedata
+from typing import TYPE_CHECKING
+
+import pytest
+
+from documents.models import Correspondent
+from documents.models import CustomField
+from documents.models import CustomFieldInstance
+from documents.models import DocumentType
+from documents.models import StoragePath
+from documents.models import Tag
+from documents.search._query import normalize_search_text
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from documents.models import Document
+    from documents.search._backend import TantivyBackend
+
+pytestmark = [pytest.mark.search, pytest.mark.django_db]
+
+# がっこうの書類 ("school documents"). The が is precomposed in NFC and
+# か + U+3099 in NFD, so the two spellings differ by one codepoint.
+_NFC = "がっこうの書類"
+_NFD = unicodedata.normalize("NFD", _NFC)
+# The bare word, for fielded queries against a name or filename.
+_NFC_WORD = "がっこう"
+_NFD_WORD = unicodedata.normalize("NFD", _NFC_WORD)
+
+
+class TestTheNormalizer:
+    def test_it_composes_decomposed_kana(self) -> None:
+        """
+        GIVEN:
+            - The same word spelled decomposed and composed
+        WHEN:
+            - Each is normalized
+        THEN:
+            - Both become the composed spelling. The inputs really do
+              differ, so the fixture is not vacuous
+        """
+        assert _NFD != _NFC
+        assert normalize_search_text(_NFD) == _NFC
+        assert normalize_search_text(_NFC) == _NFC
+
+    def test_it_leaves_halfwidth_katakana_alone(self) -> None:
+        """
+        GIVEN:
+            - A halfwidth katakana word carrying a voiced sound mark
+        WHEN:
+            - It is normalized
+        THEN:
+            - It is unchanged. Halfwidth katakana has no precomposed
+              voiced form, which is why _CJK_RE still has to list the
+              marks rather than rely on this
+        """
+        assert normalize_search_text("ﾊﾟﾝ") == "ﾊﾟﾝ"
+
+
+class TestEitherSpellingFindsEither:
+    @pytest.mark.parametrize(
+        "content",
+        [
+            pytest.param(_NFC, id="composed_document"),
+            pytest.param(_NFD, id="decomposed_document"),
+        ],
+    )
+    @pytest.mark.parametrize(
+        "query",
+        [
+            pytest.param("がっこう", id="composed_query"),
+            pytest.param(
+                unicodedata.normalize("NFD", "がっこう"),
+                id="decomposed_query",
+            ),
+        ],
+    )
+    def test_a_document_is_found_whichever_way_each_side_is_spelled(
+        self,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+        content: str,
+        query: str,
+    ) -> None:
+        """
+        GIVEN:
+            - A document holding the word inside a longer unspaced run,
+              spelled composed or decomposed, and an unrelated document
+        WHEN:
+            - The word is searched, spelled composed or decomposed
+        THEN:
+            - It matches in all four combinations. Without normalizing
+              both sides, the decomposed run splits at the combining mark
+              and the two spellings produce different bigrams
+        """
+        match = index_document(title="A", content=content)
+        index_document(title="B", content="invoice only")
+
+        assert matched_ids(query) == {match.pk}
+
+
+class TestEverySearchableFieldIsNormalized:
+    """The query side is normalized in _parse_query, so every searchable
+    field has to be normalized on the way in as well.
+
+    A field left out is worse than normalizing nothing: both sides used to
+    be decomposed and matched each other, so normalizing only the query
+    turns a working search into no results. original_filename is the one
+    most likely to hold NFD in practice, since macOS stores filenames
+    decomposed.
+    """
+
+    @pytest.mark.parametrize(
+        "field",
+        [
+            pytest.param("title", id="title"),
+            pytest.param("content", id="content"),
+            pytest.param("original_filename", id="original_filename"),
+            pytest.param("correspondent", id="correspondent"),
+            pytest.param("document_type", id="document_type"),
+            pytest.param("storage_path", id="storage_path"),
+            pytest.param("tag", id="tag"),
+            pytest.param("custom_fields.value", id="custom_field"),
+        ],
+    )
+    def test_a_composed_query_finds_a_decomposed_value(
+        self,
+        backend: TantivyBackend,
+        index_document: Callable[..., Document],
+        matched_ids: Callable[[str], set[int]],
+        field: str,
+    ) -> None:
+        """
+        GIVEN:
+            - A document carrying a decomposed Japanese word in one
+              searchable field, and an unrelated document
+        WHEN:
+            - The composed spelling is searched, fielded to that field
+        THEN:
+            - The document matches. The query is normalized either way, so
+              a field left unnormalized on the way in can never be found
+        """
+        kwargs: dict[str, object] = {"title": "A", "content": "invoice"}
+        if field == "correspondent":
+            kwargs["correspondent"] = Correspondent.objects.create(name=_NFD_WORD)
+        elif field == "document_type":
+            kwargs["document_type"] = DocumentType.objects.create(name=_NFD_WORD)
+        elif field == "storage_path":
+            kwargs["storage_path"] = StoragePath.objects.create(
+                name=_NFD_WORD,
+                path="archive/",
+            )
+        elif field in {"title", "content", "original_filename"}:
+            kwargs[field] = _NFD_WORD
+
+        doc = index_document(**kwargs)
+
+        if field == "tag":
+            doc.tags.add(Tag.objects.create(name=_NFD_WORD))
+        elif field == "custom_fields.value":
+            CustomFieldInstance.objects.create(
+                document=doc,
+                field=CustomField.objects.create(
+                    name="Note",
+                    data_type=CustomField.FieldDataType.STRING,
+                ),
+                value_text=_NFD_WORD,
+            )
+        # The relations above are attached after the factory built the
+        # document, so the index needs the newer state.
+        backend.add_or_update(doc)
+        index_document(title="B", content="invoice only")
+
+        assert matched_ids(f"{field}:{_NFC_WORD}") == {doc.pk}

uv.lock

@@ -3089,7 +3089,7 @@ requires-dist = [
     { name = "torch", specifier = "~=2.13.0", index = "https://download.pytorch.org/whl/cpu" },
     { name = "watchfiles", specifier = ">=1.2" },
     { name = "whitenoise", specifier = "~=6.11" },
-    { name = "whoosh-compat", extras = ["tantivy"], specifier = "==0.2" },
+    { name = "whoosh-compat", extras = ["tantivy"], specifier = "==0.3" },
     { name = "zxing-cpp", specifier = "~=3.1.0" },
 ]
 provides-extras = ["mariadb", "postgres", "webserver"]
@@ -5640,14 +5640,14 @@ wheels = [
 
 [[package]]
 name = "whoosh-compat"
-version = "0.2.0"
+version = "0.3.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "python-dateutil" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/d7/b2/ef410aa5297d61e9e98448f88ea9385c92840d811e0878de2f9ed2710620/whoosh_compat-0.2.0.tar.gz", hash = "sha256:f5d1b8bf2956a304c9b9c147ec840f2487d976cb7fa872bea767dcdeed7c3e45", size = 605669, upload-time = "2026-08-27T20:30:08.154Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/23/6b/945311159351411dd5a1de82f2948fe1ef6bf36308ee65ed76f568b94a7a/whoosh_compat-0.3.0.tar.gz", hash = "sha256:a7590ea0f178c60ca83774752d0d511c5d72b20357bbf361223aaab12e3d1813", size = 665922, upload-time = "2026-09-16T15:46:29.852Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/e2/3f/78e37cd794ae26ee9b94d81d608906a8aefa31a08897ef3aaeafcbb3a55a/whoosh_compat-0.2.0-py3-none-any.whl", hash = "sha256:891e98508042673862516d3811e010a62205998a31ae6e09b80a221eb8d544d3", size = 158514, upload-time = "2026-08-27T20:30:06.764Z" },
+    { url = "https://files.pythonhosted.org/packages/53/b0/538c7bdcd294ae32f1f6a13ab6486887e208567355820e4a24a56c4558c2/whoosh_compat-0.3.0-py3-none-any.whl", hash = "sha256:91e22de08fc22be50fc90f836c72d45de9c7a552f61465ffd1ebee98a6b28cfb", size = 170602, upload-time = "2026-09-16T15:46:28.227Z" },
 ]
 
 [package.optional-dependencies]