amorfo77/paperless-ngx
1
0
Fork 0
mirror of https://github.com/paperless-ngx/paperless-ngx.git synced 2026-07-28 23:04:56 +00:00
Code Issues Projects Releases Wiki Activity

Fix: exclude next-period start from relative date-range filters (#13381)

Browse Source 
* Fix: exclude next-period start from relative date-range filters

Tantivy's [lo TO hi] range is inclusive on both ends, but computed upper
bounds (keyword ranges, YYYY/YYYYMM/YYYYMMDD tokens) represent the start of
the next period. Use half-open [lo TO hi} for those so e.g. "previous month"
no longer matches the 1st of the current month.

* Adds a regression test down to the second check for the hi range
This commit is contained in:
Trenton H
2026-07-28 16:58:38 +00:00
committed by GitHub
parent 12d318deff
commit 1b32b9d678
5 changed files with 102 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/documents/search/_dates.py
+10 -2
View File
@@ -43,8 +43,16 @@ def _fmt(dt: datetime) -> str:
def _iso_range(lo: datetime, hi: datetime) -> str:
"""Format a [lo TO hi] range string in ISO 8601 for Tantivy query syntax."""
return f"[{_fmt(lo)} TO {_fmt(hi)}]"
"""
Format a half-open ``[lo TO hi)`` range in ISO 8601 for Tantivy query syntax.
``hi`` is always the exclusive ceiling of a computed period (the start of
the *next* day/week/month/quarter/year), so the closing bracket must be
the Tantivy exclusive-range brace ``}`` rather than ``]`` — otherwise the
first instant of the following period (e.g. the 1st of next month) is
incorrectly included in the match.
"""
return f"[{_fmt(lo)} TO {_fmt(hi)}}}"
def _quarter_start(d: date) -> date:
src/documents/search/_translate.py
+13 -2
View File
@@ -566,6 +566,17 @@ def translate_range(field: str, lo: str, hi: str, tz: tzinfo) -> str:
lo_pair, hi_pair = hi_pair, lo_pair
lo_iso = _fmt(lo_pair[0]) if lo_pair is not None else OPEN_LO
hi_iso = _fmt(hi_pair[1]) if hi_pair is not None else OPEN_HI
return f"{field}:[{lo_iso} TO {hi_iso}]"
# A bound resolves to (floor, ceil) where floor == ceil for an exact instant
# (a full ISO datetime, "now", or a "+/-N unit" offset) and floor != ceil for
# a coarser period token (year/month/day precision). Only the latter needs a
# half-open close: its ceil is the start of the *next* period and must be
# excluded, or that instant (e.g. the 1st of next month) wrongly matches.
if hi_pair is not None:
hi_iso = _fmt(hi_pair[1])
hi_close = "]" if hi_pair[0] == hi_pair[1] else "}"
else:
hi_iso = OPEN_HI
hi_close = "]"
return f"{field}:[{lo_iso} TO {hi_iso}{hi_close}"
src/documents/tests/search/test_query.py
+3 -1
View File
@@ -32,7 +32,9 @@ AUCKLAND = ZoneInfo("Pacific/Auckland") # UTC+13 in southern-hemisphere summer
def _range(result: str, field: str) -> tuple[str, str]:
m = re.search(rf"{field}:\[(.+?) TO (.+?)\]", result)
# Half-open period ranges close with "}" (exclusive); exact-instant ranges
# (full ISO datetimes, "now", relative offsets) close with "]" (inclusive).
m = re.search(rf"{field}:\[(.+?) TO (.+?)[\]}}]", result)
assert m, f"No range for {field!r} in: {result!r}"
return m.group(1), m.group(2)
src/documents/tests/search/test_translate.py
+34 -34
View File
@@ -214,27 +214,27 @@ class TestTranslateScalar:
(
"created",
"2020",
"created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z]",
"created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",
),
(
"created",
"202003",
"created:[2020-03-01T00:00:00Z TO 2020-04-01T00:00:00Z]",
"created:[2020-03-01T00:00:00Z TO 2020-04-01T00:00:00Z}",
),
(
"created",
"20200115",
"created:[2020-01-15T00:00:00Z TO 2020-01-16T00:00:00Z]",
"created:[2020-01-15T00:00:00Z TO 2020-01-16T00:00:00Z}",
),
(
"created",
"2020-01-15",
"created:[2020-01-15T00:00:00Z TO 2020-01-16T00:00:00Z]",
"created:[2020-01-15T00:00:00Z TO 2020-01-16T00:00:00Z}",
),
(
"created",
"2020-03",
"created:[2020-03-01T00:00:00Z TO 2020-04-01T00:00:00Z]",
"created:[2020-03-01T00:00:00Z TO 2020-04-01T00:00:00Z}",
),
],
)
@@ -248,9 +248,9 @@ class TestTranslateScalar:
assert exc_info.value.value == "202023"
def test_keyword_delegates(self) -> None:
# keyword path produces a range; just assert it is a created range
# keyword path produces a half-open range; just assert it is a created range
out = translate_scalar("created", "today", UTC)
assert out.startswith("created:[") and out.endswith("]")
assert out.startswith("created:[") and out.endswith("}")
def test_14digit_compact_datetime(self) -> None:
out = translate_scalar("created", "20240115120000", UTC)
@@ -279,21 +279,21 @@ class TestTranslateRange:
@pytest.mark.parametrize(
("lo", "hi", "expected"),
[
("2005", "2009", "created:[2005-01-01T00:00:00Z TO 2010-01-01T00:00:00Z]"),
("2005", "2009", "created:[2005-01-01T00:00:00Z TO 2010-01-01T00:00:00Z}"),
(
"202001",
"202006",
"created:[2020-01-01T00:00:00Z TO 2020-07-01T00:00:00Z]",
"created:[2020-01-01T00:00:00Z TO 2020-07-01T00:00:00Z}",
),
(
"20200101",
"20201231",
"created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z]",
"created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",
),
(
"2020-01-01",
"2020-12-31",
"created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z]",
"created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",
),
],
)
@@ -302,7 +302,7 @@ class TestTranslateRange:
def test_reversed_swaps(self):
assert translate_range("created", "2009", "2005", UTC) == (
"created:[2005-01-01T00:00:00Z TO 2010-01-01T00:00:00Z]"
"created:[2005-01-01T00:00:00Z TO 2010-01-01T00:00:00Z}"
)
def test_open_upper(self):
@@ -311,7 +311,7 @@ class TestTranslateRange:
def test_open_lower(self):
out = translate_range("created", "", "2020", UTC)
assert out == f"created:[{OPEN_LO} TO 2021-01-01T00:00:00Z]"
assert out == f"created:[{OPEN_LO} TO 2021-01-01T00:00:00Z}}"
def test_invalid_bound_raises(self):
with pytest.raises(InvalidDateQuery) as exc_info:
@@ -334,16 +334,16 @@ class TestTranslateQuery:
[
(
"created:2020",
"created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z]",
"created:[2020-01-01T00:00:00Z TO 2021-01-01T00:00:00Z}",
),
("tag:foo,bar", "tag:foo AND tag:bar"),
# 'type' is a user-facing alias rewritten to 'document_type' (the real schema field)
("tag:foo,type:bar", "tag:foo AND document_type:bar"),
(
"created:[2020 TO 2021],added:[2022 TO 2023]",
"created:[2020-01-01T00:00:00Z TO 2022-01-01T00:00:00Z]"
"created:[2020-01-01T00:00:00Z TO 2022-01-01T00:00:00Z}"
" AND "
"added:[2022-01-01T00:00:00Z TO 2024-01-01T00:00:00Z]",
"added:[2022-01-01T00:00:00Z TO 2024-01-01T00:00:00Z}",
),
# correspondent is not multi-value: comma stays literal inside the value
("correspondent:foo,bar", "correspondent:foo,bar"),
@@ -506,7 +506,7 @@ class TestOperatorNormalization:
def test_date_range_preserved(self) -> None:
out = translate_query("created:[2020 TO 2021]", UTC)
# Must not corrupt the ISO range
assert out == "created:[2020-01-01T00:00:00Z TO 2022-01-01T00:00:00Z]"
assert out == "created:[2020-01-01T00:00:00Z TO 2022-01-01T00:00:00Z}"
def test_date_scalar_with_or(self) -> None:
out = translate_query("created:2020 OR foo", UTC)
@@ -581,42 +581,42 @@ class TestKeywordDateResolution:
[
pytest.param(
"today",
"created:[2026-03-28T00:00:00Z TO 2026-03-29T00:00:00Z]",
"created:[2026-03-28T00:00:00Z TO 2026-03-29T00:00:00Z}",
id="today",
),
pytest.param(
"yesterday",
"created:[2026-03-27T00:00:00Z TO 2026-03-28T00:00:00Z]",
"created:[2026-03-27T00:00:00Z TO 2026-03-28T00:00:00Z}",
id="yesterday",
),
pytest.param(
"previous week",
"created:[2026-03-16T00:00:00Z TO 2026-03-23T00:00:00Z]",
"created:[2026-03-16T00:00:00Z TO 2026-03-23T00:00:00Z}",
id="previous-week",
),
pytest.param(
"this month",
"created:[2026-03-01T00:00:00Z TO 2026-04-01T00:00:00Z]",
"created:[2026-03-01T00:00:00Z TO 2026-04-01T00:00:00Z}",
id="this-month",
),
pytest.param(
"previous month",
"created:[2026-02-01T00:00:00Z TO 2026-03-01T00:00:00Z]",
"created:[2026-02-01T00:00:00Z TO 2026-03-01T00:00:00Z}",
id="previous-month",
),
pytest.param(
"this year",
"created:[2026-01-01T00:00:00Z TO 2027-01-01T00:00:00Z]",
"created:[2026-01-01T00:00:00Z TO 2027-01-01T00:00:00Z}",
id="this-year",
),
pytest.param(
"previous year",
"created:[2025-01-01T00:00:00Z TO 2026-01-01T00:00:00Z]",
"created:[2025-01-01T00:00:00Z TO 2026-01-01T00:00:00Z}",
id="previous-year",
),
pytest.param(
"previous quarter",
"created:[2025-10-01T00:00:00Z TO 2026-01-01T00:00:00Z]",
"created:[2025-10-01T00:00:00Z TO 2026-01-01T00:00:00Z}",
id="previous-quarter",
),
],
@@ -637,42 +637,42 @@ class TestKeywordDateResolution:
[
pytest.param(
"today",
"added:[2026-03-27T15:00:00Z TO 2026-03-28T15:00:00Z]",
"added:[2026-03-27T15:00:00Z TO 2026-03-28T15:00:00Z}",
id="today",
),
pytest.param(
"yesterday",
"added:[2026-03-26T15:00:00Z TO 2026-03-27T15:00:00Z]",
"added:[2026-03-26T15:00:00Z TO 2026-03-27T15:00:00Z}",
id="yesterday",
),
pytest.param(
"previous week",
"added:[2026-03-15T15:00:00Z TO 2026-03-22T15:00:00Z]",
"added:[2026-03-15T15:00:00Z TO 2026-03-22T15:00:00Z}",
id="previous-week",
),
pytest.param(
"this month",
"added:[2026-02-28T15:00:00Z TO 2026-03-31T15:00:00Z]",
"added:[2026-02-28T15:00:00Z TO 2026-03-31T15:00:00Z}",
id="this-month",
),
pytest.param(
"previous month",
"added:[2026-01-31T15:00:00Z TO 2026-02-28T15:00:00Z]",
"added:[2026-01-31T15:00:00Z TO 2026-02-28T15:00:00Z}",
id="previous-month",
),
pytest.param(
"this year",
"added:[2025-12-31T15:00:00Z TO 2026-12-31T15:00:00Z]",
"added:[2025-12-31T15:00:00Z TO 2026-12-31T15:00:00Z}",
id="this-year",
),
pytest.param(
"previous year",
"added:[2024-12-31T15:00:00Z TO 2025-12-31T15:00:00Z]",
"added:[2024-12-31T15:00:00Z TO 2025-12-31T15:00:00Z}",
id="previous-year",
),
pytest.param(
"previous quarter",
"added:[2025-09-30T15:00:00Z TO 2025-12-31T15:00:00Z]",
"added:[2025-09-30T15:00:00Z TO 2025-12-31T15:00:00Z}",
id="previous-quarter",
),
],
@@ -719,7 +719,7 @@ class TestISODatetimeBounds:
def test_translate_query_text_before_comma_separated_date_clause(self) -> None:
result = translate_query("schäfersee,created:previous year", UTC)
assert result == (
"schäfersee AND created:[2025-01-01T00:00:00Z TO 2026-01-01T00:00:00Z]"
"schäfersee AND created:[2025-01-01T00:00:00Z TO 2026-01-01T00:00:00Z}"
)
def test_invalid_iso_datetime_raises(self) -> None:
src/documents/tests/test_api_search.py
+42
View File
@@ -720,6 +720,48 @@ class TestDocumentSearchApi(DirectoriesMixin, APITestCase):
self.assertEqual(results[0]["id"], 3)
self.assertEqual(results[0]["title"], "bank statement 3")
def test_search_added_previous_month_excludes_next_period_start(self) -> None:
"""
GIVEN:
- One document added at the last instant of last month
- One document added exactly at the first instant of this month
WHEN:
- Query for documents added in the previous month
THEN:
- Only the document from last month is returned; the document dated
exactly at the start of this month (the exclusive upper bound of
the range) is not
"""
d1 = DocumentFactory.create(
title="end of last month",
content="last instant of last month",
checksum="A",
pk=1,
added=timezone.make_aware(datetime.datetime(2024, 1, 31, 23, 59, 59)),
)
d2 = DocumentFactory.create(
title="start of this month",
content="first instant of this month",
checksum="B",
pk=2,
added=timezone.make_aware(datetime.datetime(2024, 2, 1, 0, 0, 0)),
)
backend = get_backend()
backend.add_or_update(d1)
backend.add_or_update(d2)
with time_machine.travel(
timezone.make_aware(datetime.datetime(2024, 2, 15, 12, 0, 0)),
tick=False,
):
response = self.client.get("/api/documents/?query=added:previous month")
results = response.data["results"]
self.assertEqual(len(results), 1)
self.assertEqual(results[0]["id"], 1)
self.assertEqual(results[0]["title"], "end of last month")
def test_search_added_invalid_date(self) -> None:
"""
GIVEN: