mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-09-19 16:08:32 +00:00
feat(search): natural date keyword rewriting with Whoosh compat shims
Implement date/timezone boundary math for natural language date queries: - `created` (DateField): local calendar date to UTC midnight boundaries - `added`/`modified` (DateTimeField): local day boundaries with full offset arithmetic - Whoosh compat shims: compact dates (YYYYMMDDHHmmss) → ISO 8601 - Relative ranges: `[now-7d TO now]` → concrete ISO timestamps - Natural keywords: today, yesterday, this_week, last_week, etc. - Timezone-aware: handles UTC offset arithmetic for datetime fields - Passthrough: bare keywords without field prefixes unchanged Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Trenton H
co-authored by
Claude Sonnet 4.6
Claude Sonnet 4.6
parent
2cf85d9b58
commit
cbeb7469a1
@@ -0,0 +1,207 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import UTC
|
||||
from datetime import date
|
||||
from datetime import datetime
|
||||
from datetime import timedelta
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from datetime import tzinfo
|
||||
|
||||
_DATE_ONLY_FIELDS = frozenset({"created"})
|
||||
|
||||
_DATE_KEYWORDS = frozenset(
|
||||
{
|
||||
"today",
|
||||
"yesterday",
|
||||
"this_week",
|
||||
"last_week",
|
||||
"this_month",
|
||||
"last_month",
|
||||
"this_year",
|
||||
"last_year",
|
||||
},
|
||||
)
|
||||
|
||||
_FIELD_DATE_RE = re.compile(
|
||||
r"(\w+):(" + "|".join(_DATE_KEYWORDS) + r")\b",
|
||||
)
|
||||
_COMPACT_DATE_RE = re.compile(r"\b(\d{14})\b")
|
||||
_RELATIVE_RANGE_RE = re.compile(
|
||||
r"\[now([+-]\d+[dhm])?\s+TO\s+now([+-]\d+[dhm])?\]",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _fmt(dt: datetime) -> str:
|
||||
return dt.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def _iso_range(lo: datetime, hi: datetime) -> str:
|
||||
return f"[{_fmt(lo)} TO {_fmt(hi)}]"
|
||||
|
||||
|
||||
def _date_only_range(keyword: str, tz: tzinfo) -> str:
|
||||
"""
|
||||
For `created` (DateField): use the local calendar date, converted to
|
||||
midnight UTC boundaries. No offset arithmetic — date only.
|
||||
"""
|
||||
|
||||
today = datetime.now(tz).date()
|
||||
|
||||
if keyword == "today":
|
||||
lo = datetime(today.year, today.month, today.day, tzinfo=UTC)
|
||||
return _iso_range(lo, lo + timedelta(days=1))
|
||||
if keyword == "yesterday":
|
||||
y = today - timedelta(days=1)
|
||||
lo = datetime(y.year, y.month, y.day, tzinfo=UTC)
|
||||
hi = datetime(today.year, today.month, today.day, tzinfo=UTC)
|
||||
return _iso_range(lo, hi)
|
||||
if keyword == "this_week":
|
||||
mon = today - timedelta(days=today.weekday())
|
||||
lo = datetime(mon.year, mon.month, mon.day, tzinfo=UTC)
|
||||
return _iso_range(lo, lo + timedelta(weeks=1))
|
||||
if keyword == "last_week":
|
||||
this_mon = today - timedelta(days=today.weekday())
|
||||
last_mon = this_mon - timedelta(weeks=1)
|
||||
lo = datetime(last_mon.year, last_mon.month, last_mon.day, tzinfo=UTC)
|
||||
hi = datetime(this_mon.year, this_mon.month, this_mon.day, tzinfo=UTC)
|
||||
return _iso_range(lo, hi)
|
||||
if keyword == "this_month":
|
||||
lo = datetime(today.year, today.month, 1, tzinfo=UTC)
|
||||
if today.month == 12:
|
||||
hi = datetime(today.year + 1, 1, 1, tzinfo=UTC)
|
||||
else:
|
||||
hi = datetime(today.year, today.month + 1, 1, tzinfo=UTC)
|
||||
return _iso_range(lo, hi)
|
||||
if keyword == "last_month":
|
||||
if today.month == 1:
|
||||
lo = datetime(today.year - 1, 12, 1, tzinfo=UTC)
|
||||
else:
|
||||
lo = datetime(today.year, today.month - 1, 1, tzinfo=UTC)
|
||||
hi = datetime(today.year, today.month, 1, tzinfo=UTC)
|
||||
return _iso_range(lo, hi)
|
||||
if keyword == "this_year":
|
||||
lo = datetime(today.year, 1, 1, tzinfo=UTC)
|
||||
return _iso_range(lo, datetime(today.year + 1, 1, 1, tzinfo=UTC))
|
||||
if keyword == "last_year":
|
||||
lo = datetime(today.year - 1, 1, 1, tzinfo=UTC)
|
||||
return _iso_range(lo, datetime(today.year, 1, 1, tzinfo=UTC))
|
||||
raise ValueError(f"Unknown keyword: {keyword}")
|
||||
|
||||
|
||||
def _datetime_range(keyword: str, tz: tzinfo) -> str:
|
||||
"""
|
||||
For `added` / `modified` (DateTimeField, stored as UTC): convert local day
|
||||
boundaries to UTC — full offset arithmetic required.
|
||||
"""
|
||||
|
||||
now_local = datetime.now(tz)
|
||||
today = now_local.date()
|
||||
|
||||
def _midnight(d: date) -> datetime:
|
||||
return datetime(d.year, d.month, d.day, tzinfo=tz).astimezone(UTC)
|
||||
|
||||
if keyword == "today":
|
||||
return _iso_range(_midnight(today), _midnight(today + timedelta(days=1)))
|
||||
if keyword == "yesterday":
|
||||
y = today - timedelta(days=1)
|
||||
return _iso_range(_midnight(y), _midnight(today))
|
||||
if keyword == "this_week":
|
||||
mon = today - timedelta(days=today.weekday())
|
||||
return _iso_range(_midnight(mon), _midnight(mon + timedelta(weeks=1)))
|
||||
if keyword == "last_week":
|
||||
this_mon = today - timedelta(days=today.weekday())
|
||||
last_mon = this_mon - timedelta(weeks=1)
|
||||
return _iso_range(_midnight(last_mon), _midnight(this_mon))
|
||||
if keyword == "this_month":
|
||||
first = today.replace(day=1)
|
||||
if today.month == 12:
|
||||
next_first = date(today.year + 1, 1, 1)
|
||||
else:
|
||||
next_first = date(today.year, today.month + 1, 1)
|
||||
return _iso_range(_midnight(first), _midnight(next_first))
|
||||
if keyword == "last_month":
|
||||
this_first = today.replace(day=1)
|
||||
if today.month == 1:
|
||||
last_first = date(today.year - 1, 12, 1)
|
||||
else:
|
||||
last_first = date(today.year, today.month - 1, 1)
|
||||
return _iso_range(_midnight(last_first), _midnight(this_first))
|
||||
if keyword == "this_year":
|
||||
return _iso_range(
|
||||
_midnight(date(today.year, 1, 1)),
|
||||
_midnight(date(today.year + 1, 1, 1)),
|
||||
)
|
||||
if keyword == "last_year":
|
||||
return _iso_range(
|
||||
_midnight(date(today.year - 1, 1, 1)),
|
||||
_midnight(date(today.year, 1, 1)),
|
||||
)
|
||||
raise ValueError(f"Unknown keyword: {keyword}")
|
||||
|
||||
|
||||
def _rewrite_compact_date(query: str) -> str:
|
||||
def _sub(m: re.Match) -> str:
|
||||
raw = m.group(1)
|
||||
try:
|
||||
dt = datetime(
|
||||
int(raw[0:4]),
|
||||
int(raw[4:6]),
|
||||
int(raw[6:8]),
|
||||
int(raw[8:10]),
|
||||
int(raw[10:12]),
|
||||
int(raw[12:14]),
|
||||
tzinfo=UTC,
|
||||
)
|
||||
return dt.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
except ValueError:
|
||||
return m.group(0)
|
||||
|
||||
return _COMPACT_DATE_RE.sub(_sub, query)
|
||||
|
||||
|
||||
def _rewrite_relative_range(query: str) -> str:
|
||||
def _sub(m: re.Match) -> str:
|
||||
now = datetime.now(UTC)
|
||||
|
||||
def _offset(s: str | None) -> timedelta:
|
||||
if not s:
|
||||
return timedelta(0)
|
||||
sign = 1 if s[0] == "+" else -1
|
||||
n, unit = int(s[1:-1]), s[-1]
|
||||
return (
|
||||
sign
|
||||
* {
|
||||
"d": timedelta(days=n),
|
||||
"h": timedelta(hours=n),
|
||||
"m": timedelta(minutes=n),
|
||||
}[unit]
|
||||
)
|
||||
|
||||
lo, hi = now + _offset(m.group(1)), now + _offset(m.group(2))
|
||||
if lo > hi:
|
||||
lo, hi = hi, lo
|
||||
return f"[{_fmt(lo)} TO {_fmt(hi)}]"
|
||||
|
||||
return _RELATIVE_RANGE_RE.sub(_sub, query)
|
||||
|
||||
|
||||
def rewrite_natural_date_keywords(query: str, tz: tzinfo) -> str:
|
||||
"""
|
||||
Preprocessing stage 1: rewrite Whoosh compact dates, relative ranges,
|
||||
and natural date keywords (field:today etc.) to ISO 8601.
|
||||
Bare keywords without a field: prefix pass through unchanged.
|
||||
"""
|
||||
query = _rewrite_compact_date(query)
|
||||
query = _rewrite_relative_range(query)
|
||||
|
||||
def _replace(m: re.Match) -> str:
|
||||
field, keyword = m.group(1), m.group(2)
|
||||
if field in _DATE_ONLY_FIELDS:
|
||||
return f"{field}:{_date_only_range(keyword, tz)}"
|
||||
return f"{field}:{_datetime_range(keyword, tz)}"
|
||||
|
||||
return _FIELD_DATE_RE.sub(_replace, query)
|
||||
@@ -31,7 +31,7 @@ def build_schema() -> tantivy.Schema:
|
||||
):
|
||||
sb.add_text_field(field, stored=True, tokenizer_name="paperless_text")
|
||||
|
||||
# Shadow sort fields — fast, not stored/indexed
|
||||
# Shadow sort fields - fast, not stored/indexed
|
||||
for field in ("title_sort", "correspondent_sort", "type_sort"):
|
||||
sb.add_text_field(
|
||||
field,
|
||||
@@ -40,10 +40,10 @@ def build_schema() -> tantivy.Schema:
|
||||
fast=True,
|
||||
)
|
||||
|
||||
# CJK support — not stored, indexed only
|
||||
# CJK support - not stored, indexed only
|
||||
sb.add_text_field("bigram_content", stored=False, tokenizer_name="bigram_analyzer")
|
||||
|
||||
# Autocomplete prefix scan — stored, not indexed
|
||||
# Autocomplete prefix scan - stored, not indexed
|
||||
sb.add_text_field("autocomplete_word", stored=True, tokenizer_name="raw")
|
||||
|
||||
sb.add_text_field("tag", stored=True, tokenizer_name="paperless_text")
|
||||
@@ -75,17 +75,17 @@ def _needs_rebuild(index_dir: Path) -> bool:
|
||||
return True
|
||||
try:
|
||||
if int(version_file.read_text().strip()) != SCHEMA_VERSION:
|
||||
logger.info("Search index schema version mismatch — rebuilding.")
|
||||
logger.info("Search index schema version mismatch - rebuilding.")
|
||||
return True
|
||||
except ValueError:
|
||||
return True
|
||||
|
||||
language_file = index_dir / ".schema_language"
|
||||
if not language_file.exists():
|
||||
logger.info("Search index language sentinel missing — rebuilding.")
|
||||
logger.info("Search index language sentinel missing - rebuilding.")
|
||||
return True
|
||||
if language_file.read_text().strip() != settings.SEARCH_LANGUAGE:
|
||||
logger.info("Search index language changed — rebuilding.")
|
||||
logger.info("Search index language changed - rebuilding.")
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
@@ -6,7 +6,7 @@ import tantivy
|
||||
|
||||
logger = logging.getLogger("paperless.search")
|
||||
|
||||
# Mapping of ISO 639-1 codes (and common aliases) → Tantivy Snowball name
|
||||
# Mapping of ISO 639-1 codes (and common aliases) -> Tantivy Snowball name
|
||||
_LANGUAGE_MAP: dict[str, str] = {
|
||||
"ar": "Arabic",
|
||||
"arabic": "Arabic",
|
||||
@@ -52,7 +52,7 @@ SUPPORTED_LANGUAGES: frozenset[str] = frozenset(_LANGUAGE_MAP)
|
||||
def register_tokenizers(index: tantivy.Index, language: str) -> None:
|
||||
"""
|
||||
Register all custom tokenizers on *index*. Must be called on every Index
|
||||
instance — tantivy requires re-registration at each open.
|
||||
instance - tantivy requires re-registration at each open.
|
||||
"""
|
||||
index.register_tokenizer("paperless_text", _paperless_text(language))
|
||||
index.register_tokenizer("simple_analyzer", _simple_analyzer())
|
||||
@@ -60,7 +60,7 @@ def register_tokenizers(index: tantivy.Index, language: str) -> None:
|
||||
|
||||
|
||||
def _paperless_text(language: str) -> tantivy.TextAnalyzer:
|
||||
"""simple → remove_long(65) → lowercase → ascii_fold [→ stemmer]"""
|
||||
"""simple -> remove_long(65) -> lowercase -> ascii_fold [-> stemmer]"""
|
||||
builder = (
|
||||
tantivy.TextAnalyzerBuilder(tantivy.Tokenizer.simple())
|
||||
.filter(tantivy.Filter.remove_long(65))
|
||||
@@ -73,7 +73,7 @@ def _paperless_text(language: str) -> tantivy.TextAnalyzer:
|
||||
builder = builder.filter(tantivy.Filter.stemmer(tantivy_lang))
|
||||
else:
|
||||
logger.warning(
|
||||
"Unsupported search language '%s' — stemming disabled. Supported: %s",
|
||||
"Unsupported search language '%s' - stemming disabled. Supported: %s",
|
||||
language,
|
||||
", ".join(sorted(SUPPORTED_LANGUAGES)),
|
||||
)
|
||||
@@ -81,7 +81,7 @@ def _paperless_text(language: str) -> tantivy.TextAnalyzer:
|
||||
|
||||
|
||||
def _simple_analyzer() -> tantivy.TextAnalyzer:
|
||||
"""simple → lowercase → ascii_fold. Used for shadow sort fields."""
|
||||
"""simple -> lowercase -> ascii_fold. Used for shadow sort fields."""
|
||||
return (
|
||||
tantivy.TextAnalyzerBuilder(tantivy.Tokenizer.simple())
|
||||
.filter(tantivy.Filter.lowercase())
|
||||
@@ -91,7 +91,7 @@ def _simple_analyzer() -> tantivy.TextAnalyzer:
|
||||
|
||||
|
||||
def _bigram_analyzer() -> tantivy.TextAnalyzer:
|
||||
"""ngram(2,2) → lowercase. CJK / no-whitespace language support."""
|
||||
"""ngram(2,2) -> lowercase. CJK / no-whitespace language support."""
|
||||
return (
|
||||
tantivy.TextAnalyzerBuilder(
|
||||
tantivy.Tokenizer.ngram(min_gram=2, max_gram=2, prefix_only=False),
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import UTC
|
||||
from datetime import datetime
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import pytest
|
||||
import time_machine
|
||||
|
||||
from documents.search._query import rewrite_natural_date_keywords
|
||||
|
||||
pytestmark = pytest.mark.search
|
||||
|
||||
UTC = UTC
|
||||
EASTERN = ZoneInfo("America/New_York") # UTC-5 / UTC-4 (DST)
|
||||
AUCKLAND = ZoneInfo("Pacific/Auckland") # UTC+13 in southern-hemisphere summer
|
||||
|
||||
|
||||
def _range(result: str, field: str) -> tuple[str, str]:
|
||||
m = re.search(rf"{field}:\[(.+?) TO (.+?)\]", result)
|
||||
assert m, f"No range for {field!r} in: {result!r}"
|
||||
return m.group(1), m.group(2)
|
||||
|
||||
|
||||
class TestCreatedDateField:
|
||||
"""
|
||||
created is a Django DateField: indexed as midnight UTC of the local calendar
|
||||
date. No offset arithmetic needed - the local calendar date is what matters.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tz", "expected_lo", "expected_hi"),
|
||||
[
|
||||
pytest.param(UTC, "2026-03-28T00:00:00Z", "2026-03-29T00:00:00Z", id="utc"),
|
||||
pytest.param(
|
||||
EASTERN,
|
||||
"2026-03-28T00:00:00Z",
|
||||
"2026-03-29T00:00:00Z",
|
||||
id="eastern_same_calendar_date",
|
||||
),
|
||||
],
|
||||
)
|
||||
@time_machine.travel(datetime(2026, 3, 28, 15, 30, tzinfo=UTC), tick=False)
|
||||
def test_today(self, tz, expected_lo, expected_hi):
|
||||
lo, hi = _range(rewrite_natural_date_keywords("created:today", tz), "created")
|
||||
assert lo == expected_lo
|
||||
assert hi == expected_hi
|
||||
|
||||
@time_machine.travel(datetime(2026, 3, 28, 3, 0, tzinfo=UTC), tick=False)
|
||||
def test_today_auckland_ahead_of_utc(self):
|
||||
# UTC 03:00 -> Auckland (UTC+13) = 16:00 same date; local date = 2026-03-28
|
||||
lo, _ = _range(
|
||||
rewrite_natural_date_keywords("created:today", AUCKLAND),
|
||||
"created",
|
||||
)
|
||||
assert lo == "2026-03-28T00:00:00Z"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "keyword", "expected_lo", "expected_hi"),
|
||||
[
|
||||
pytest.param(
|
||||
"created",
|
||||
"yesterday",
|
||||
"2026-03-27T00:00:00Z",
|
||||
"2026-03-28T00:00:00Z",
|
||||
id="yesterday",
|
||||
),
|
||||
pytest.param(
|
||||
"created",
|
||||
"this_week",
|
||||
"2026-03-23T00:00:00Z",
|
||||
"2026-03-30T00:00:00Z",
|
||||
id="this_week_mon_sun",
|
||||
),
|
||||
pytest.param(
|
||||
"created",
|
||||
"last_week",
|
||||
"2026-03-16T00:00:00Z",
|
||||
"2026-03-23T00:00:00Z",
|
||||
id="last_week",
|
||||
),
|
||||
pytest.param(
|
||||
"created",
|
||||
"this_month",
|
||||
"2026-03-01T00:00:00Z",
|
||||
"2026-04-01T00:00:00Z",
|
||||
id="this_month",
|
||||
),
|
||||
pytest.param(
|
||||
"created",
|
||||
"last_month",
|
||||
"2026-02-01T00:00:00Z",
|
||||
"2026-03-01T00:00:00Z",
|
||||
id="last_month",
|
||||
),
|
||||
pytest.param(
|
||||
"created",
|
||||
"this_year",
|
||||
"2026-01-01T00:00:00Z",
|
||||
"2027-01-01T00:00:00Z",
|
||||
id="this_year",
|
||||
),
|
||||
pytest.param(
|
||||
"created",
|
||||
"last_year",
|
||||
"2025-01-01T00:00:00Z",
|
||||
"2026-01-01T00:00:00Z",
|
||||
id="last_year",
|
||||
),
|
||||
],
|
||||
)
|
||||
@time_machine.travel(datetime(2026, 3, 28, 15, 0, tzinfo=UTC), tick=False)
|
||||
def test_date_keywords(self, field, keyword, expected_lo, expected_hi):
|
||||
# 2026-03-28 is Saturday; Mon-Sun week calculation built into expectations
|
||||
query = f"{field}:{keyword}"
|
||||
lo, hi = _range(rewrite_natural_date_keywords(query, UTC), field)
|
||||
assert lo == expected_lo
|
||||
assert hi == expected_hi
|
||||
|
||||
|
||||
class TestDateTimeFields:
|
||||
"""
|
||||
added/modified store full UTC datetimes. Natural keywords must convert
|
||||
the local day boundaries to UTC - timezone offset arithmetic IS required.
|
||||
"""
|
||||
|
||||
@time_machine.travel(datetime(2026, 3, 28, 15, 30, tzinfo=UTC), tick=False)
|
||||
def test_added_today_eastern(self):
|
||||
# EDT = UTC-4; local midnight 2026-03-28 00:00 EDT = 2026-03-28 04:00 UTC
|
||||
lo, hi = _range(rewrite_natural_date_keywords("added:today", EASTERN), "added")
|
||||
assert lo == "2026-03-28T04:00:00Z"
|
||||
assert hi == "2026-03-29T04:00:00Z"
|
||||
|
||||
@time_machine.travel(datetime(2026, 3, 29, 2, 0, tzinfo=UTC), tick=False)
|
||||
def test_added_today_auckland_midnight_crossing(self):
|
||||
# UTC 02:00 on 2026-03-29 -> Auckland (UTC+13) = 2026-03-29 15:00 local
|
||||
# Auckland midnight = UTC 2026-03-28 11:00
|
||||
lo, hi = _range(rewrite_natural_date_keywords("added:today", AUCKLAND), "added")
|
||||
assert lo == "2026-03-28T11:00:00Z"
|
||||
assert hi == "2026-03-29T11:00:00Z"
|
||||
|
||||
@time_machine.travel(datetime(2026, 3, 28, 15, 0, tzinfo=UTC), tick=False)
|
||||
def test_modified_today_utc(self):
|
||||
lo, hi = _range(
|
||||
rewrite_natural_date_keywords("modified:today", UTC),
|
||||
"modified",
|
||||
)
|
||||
assert lo == "2026-03-28T00:00:00Z"
|
||||
assert hi == "2026-03-29T00:00:00Z"
|
||||
|
||||
|
||||
class TestWhooshCompatShims:
|
||||
"""Whoosh compact dates and relative ranges must be converted to ISO format."""
|
||||
|
||||
@time_machine.travel(datetime(2026, 3, 28, 15, 0, tzinfo=UTC), tick=False)
|
||||
def test_compact_date_shim_rewrites_to_iso(self):
|
||||
# Whoosh compact: YYYYMMDDHHmmss
|
||||
result = rewrite_natural_date_keywords("created:20240115120000", UTC)
|
||||
assert "2024-01-15" in result
|
||||
assert "20240115120000" not in result
|
||||
|
||||
@time_machine.travel(datetime(2026, 3, 28, 15, 0, tzinfo=UTC), tick=False)
|
||||
def test_relative_range_shim_removes_now(self):
|
||||
result = rewrite_natural_date_keywords("added:[now-7d TO now]", UTC)
|
||||
assert "now" not in result
|
||||
assert "2026-03-" in result
|
||||
|
||||
|
||||
class TestPassthrough:
|
||||
"""Queries without field prefixes or unrelated content pass through unchanged."""
|
||||
|
||||
def test_bare_keyword_no_field_prefix_unchanged(self):
|
||||
# Bare 'today' with no field: prefix passes through unchanged
|
||||
result = rewrite_natural_date_keywords("bank statement today", UTC)
|
||||
assert "today" in result
|
||||
|
||||
def test_unrelated_query_unchanged(self):
|
||||
assert rewrite_natural_date_keywords("title:invoice", UTC) == "title:invoice"
|
||||
Reference in New Issue
Block a user