mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-08-19 17:23:21 +00:00
refactor(search): make PUBLIC_FIELDS a tuple[FieldSpec, ...], drop PublicField
PublicField duplicated seven fields whoosh-compat's own FieldSpec already has (name/kind/aliases/comma_values/date_only/fast/subpaths), and _registry.py hand-copied all of them across on every registry build. FieldSpec is a frozen dataclass with analyzer/pattern_normalizer already optional (default None), so PUBLIC_FIELDS can just BE the FieldSpec tuple - _schema.py only ever read name/kind/fast off it and needs no changes. _registry.py now attaches the per-language analyzer/pattern_normalizer via dataclasses.replace() instead of reconstructing every field from scratch. FieldSpec.__post_init__ normalizes subpaths into a MappingProxyType, so test_fields.py's exact-tuple-equality subpath assertions become set comparisons; a genuinely empty subpaths is now `not field.subpaths` rather than `== ()`. Verified test_api_trash.py::test_api_trash's "Schema error: An index exists but the schema does not match" failure is a pre-existing, unrelated local environment issue (a stale, untracked data/index/ directory in this checkout) - reproduces identically with this commit's changes stashed out.
This commit is contained in:
@@ -1,45 +1,33 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from whoosh_compat import FieldKind
|
||||
from whoosh_compat import FieldSpec
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PublicField:
|
||||
"""One query-syntax-addressable field, shared by the Tantivy schema
|
||||
builder (_schema.py) and the whoosh-compat FieldRegistry (_registry.py).
|
||||
|
||||
Internal-only schema fields with no query-syntax meaning of their own
|
||||
(sort shadow fields, bigram CJK fields, simple_title/simple_content,
|
||||
autocomplete_word, notes_text) are NOT represented here — they stay
|
||||
hardcoded in _schema.py's build_schema().
|
||||
"""
|
||||
|
||||
name: str
|
||||
kind: FieldKind
|
||||
aliases: tuple[str, ...] = ()
|
||||
comma_values: bool = False
|
||||
date_only: bool = False
|
||||
fast: bool = False
|
||||
subpaths: tuple[str, ...] = () # JSON kind only
|
||||
|
||||
|
||||
PUBLIC_FIELDS: tuple[PublicField, ...] = (
|
||||
PublicField("title", FieldKind.TEXT),
|
||||
PublicField("content", FieldKind.TEXT),
|
||||
PublicField("correspondent", FieldKind.TEXT),
|
||||
PublicField("document_type", FieldKind.TEXT, aliases=("type",)),
|
||||
PublicField("storage_path", FieldKind.TEXT, aliases=("path",)),
|
||||
PublicField("original_filename", FieldKind.TEXT),
|
||||
PublicField("tag", FieldKind.TEXT, comma_values=True),
|
||||
PublicField("checksum", FieldKind.KEYWORD),
|
||||
PublicField("asn", FieldKind.U64, fast=True),
|
||||
PublicField("page_count", FieldKind.U64, fast=True),
|
||||
PublicField("num_notes", FieldKind.U64, fast=True),
|
||||
PublicField("created", FieldKind.DATE, date_only=True, fast=True),
|
||||
PublicField("modified", FieldKind.DATETIME, fast=True),
|
||||
PublicField("added", FieldKind.DATETIME, fast=True),
|
||||
PublicField("notes", FieldKind.JSON, subpaths=("user", "note")),
|
||||
PublicField("custom_fields", FieldKind.JSON, subpaths=("name", "value")),
|
||||
# Internal-only schema fields with no query-syntax meaning of their own
|
||||
# (sort shadow fields, bigram CJK fields, simple_title/simple_content,
|
||||
# autocomplete_word, notes_text) are NOT represented here — they stay
|
||||
# hardcoded in _schema.py's build_schema().
|
||||
#
|
||||
# analyzer/pattern_normalizer are deliberately left at FieldSpec's default
|
||||
# (None): they're language-specific and only meaningful to whoosh-compat's
|
||||
# parser, so _registry.py attaches them per-language via dataclasses.replace()
|
||||
# rather than PUBLIC_FIELDS declaring them itself. _schema.py only reads
|
||||
# name/kind/fast and never sees the analyzer at all.
|
||||
PUBLIC_FIELDS: tuple[FieldSpec, ...] = (
|
||||
FieldSpec("title", FieldKind.TEXT),
|
||||
FieldSpec("content", FieldKind.TEXT),
|
||||
FieldSpec("correspondent", FieldKind.TEXT),
|
||||
FieldSpec("document_type", FieldKind.TEXT, aliases=("type",)),
|
||||
FieldSpec("storage_path", FieldKind.TEXT, aliases=("path",)),
|
||||
FieldSpec("original_filename", FieldKind.TEXT),
|
||||
FieldSpec("tag", FieldKind.TEXT, comma_values=True),
|
||||
FieldSpec("checksum", FieldKind.KEYWORD),
|
||||
FieldSpec("asn", FieldKind.U64, fast=True),
|
||||
FieldSpec("page_count", FieldKind.U64, fast=True),
|
||||
FieldSpec("num_notes", FieldKind.U64, fast=True),
|
||||
FieldSpec("created", FieldKind.DATE, date_only=True, fast=True),
|
||||
FieldSpec("modified", FieldKind.DATETIME, fast=True),
|
||||
FieldSpec("added", FieldKind.DATETIME, fast=True),
|
||||
FieldSpec("notes", FieldKind.JSON, subpaths=("user", "note")),
|
||||
FieldSpec("custom_fields", FieldKind.JSON, subpaths=("name", "value")),
|
||||
)
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
|
||||
from whoosh_compat import FieldKind
|
||||
from whoosh_compat import FieldRegistry
|
||||
from whoosh_compat import FieldSpec
|
||||
|
||||
from documents.search._fields import PUBLIC_FIELDS
|
||||
from documents.search._tokenizer import ascii_fold
|
||||
@@ -39,25 +40,16 @@ def get_field_registry(language: str | None) -> FieldRegistry:
|
||||
|
||||
text_analyzer = paperless_text_analyzer(language).analyze
|
||||
|
||||
specs = []
|
||||
for field in PUBLIC_FIELDS:
|
||||
if field.kind is FieldKind.KEYWORD:
|
||||
analyzer = _identity_analyzer
|
||||
else:
|
||||
analyzer = text_analyzer
|
||||
specs.append(
|
||||
FieldSpec(
|
||||
name=field.name,
|
||||
kind=field.kind,
|
||||
aliases=field.aliases,
|
||||
comma_values=field.comma_values,
|
||||
analyzer=analyzer,
|
||||
pattern_normalizer=_pattern_normalizer,
|
||||
date_only=field.date_only,
|
||||
fast=field.fast,
|
||||
subpaths=field.subpaths,
|
||||
),
|
||||
specs = [
|
||||
dataclasses.replace(
|
||||
field,
|
||||
analyzer=_identity_analyzer
|
||||
if field.kind is FieldKind.KEYWORD
|
||||
else text_analyzer,
|
||||
pattern_normalizer=_pattern_normalizer,
|
||||
)
|
||||
for field in PUBLIC_FIELDS
|
||||
]
|
||||
|
||||
registry = FieldRegistry(specs)
|
||||
_registry_cache[language] = registry
|
||||
|
||||
@@ -20,7 +20,7 @@ class TestPublicFields:
|
||||
def test_non_json_fields_have_no_subpaths(self) -> None:
|
||||
for field in PUBLIC_FIELDS:
|
||||
if field.kind is not FieldKind.JSON:
|
||||
assert field.subpaths == ()
|
||||
assert not field.subpaths
|
||||
|
||||
def test_document_type_alias_is_type(self) -> None:
|
||||
field = next(f for f in PUBLIC_FIELDS if f.name == "document_type")
|
||||
@@ -36,11 +36,11 @@ class TestPublicFields:
|
||||
|
||||
def test_notes_subpaths(self) -> None:
|
||||
field = next(f for f in PUBLIC_FIELDS if f.name == "notes")
|
||||
assert field.subpaths == ("user", "note")
|
||||
assert set(field.subpaths) == {"user", "note"}
|
||||
|
||||
def test_custom_fields_subpaths(self) -> None:
|
||||
field = next(f for f in PUBLIC_FIELDS if f.name == "custom_fields")
|
||||
assert field.subpaths == ("name", "value")
|
||||
assert set(field.subpaths) == {"name", "value"}
|
||||
|
||||
def test_no_internal_id_fields_present(self) -> None:
|
||||
# tag_id/owner_id/viewer_id/etc. are permission-filter-only fields,
|
||||
|
||||
@@ -130,7 +130,7 @@ class TestSchemaMatchesPublicFields:
|
||||
|
||||
class TestFastFlagAgreement:
|
||||
def test_every_public_field_fast_flag_matches_the_built_schema(self) -> None:
|
||||
# whoosh-compat's registry trusts PublicField.fast when resolving
|
||||
# whoosh-compat's registry trusts PUBLIC_FIELDS' fast flag when resolving
|
||||
# field:* existence checks (its FAST_FIELD strategy); a fast=True
|
||||
# entry whose actual tantivy column is not fast would make those
|
||||
# searches silently match nothing at search time. build_schema()
|
||||
@@ -144,6 +144,6 @@ class TestFastFlagAgreement:
|
||||
}
|
||||
for public_field in PUBLIC_FIELDS:
|
||||
assert schema_fast[public_field.name] == public_field.fast, (
|
||||
f"{public_field.name}: PublicField.fast={public_field.fast} but the"
|
||||
f"{public_field.name}: PUBLIC_FIELDS says fast={public_field.fast} but the"
|
||||
f" built schema says fast={schema_fast[public_field.name]}"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user