mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-11-19 20:25:21 +00:00
Feature: find dates after a label in OCR zones
A crop may include a label, such as "Datum: 13.04.2026", and those dates never parsed unless they were ISO.
This commit is contained in:
1 parent
acf7ba346c
commit
0e6d06c255
2 files changed
+197
-18
No files matched your search
@@ -1614,6 +1614,161 @@ class TestParseDate:
|
||||
"""
|
||||
assert _parse_date(text, _date_rules(ocr_language)) is None
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "ocr_language", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
"Datum: 13.04.2026",
|
||||
"deu",
|
||||
date(2026, 4, 13),
|
||||
id="german-label-colon",
|
||||
),
|
||||
pytest.param(
|
||||
"Rechnungsdatum 12.05.2024",
|
||||
"deu",
|
||||
date(2024, 5, 12),
|
||||
id="german-compound-label",
|
||||
),
|
||||
pytest.param(
|
||||
"Invoice date 05/12/2024",
|
||||
"eng",
|
||||
date(2024, 5, 12),
|
||||
id="english-label-numeric",
|
||||
),
|
||||
pytest.param(
|
||||
"Date: 13 April 2026",
|
||||
"eng",
|
||||
date(2026, 4, 13),
|
||||
id="english-label-month-name",
|
||||
),
|
||||
pytest.param(
|
||||
"Rechnung 2024-001 vom 12.05.2024",
|
||||
"deu",
|
||||
date(2024, 5, 12),
|
||||
id="german-label-before-invoice-number",
|
||||
),
|
||||
pytest.param(
|
||||
"No. 12 Datum 13.04.2026",
|
||||
"deu",
|
||||
date(2026, 4, 13),
|
||||
id="german-label-after-number",
|
||||
),
|
||||
pytest.param(
|
||||
"Kd-Nr 12345 05.06.2024",
|
||||
"deu",
|
||||
date(2024, 6, 5),
|
||||
id="german-label-after-customer-number",
|
||||
),
|
||||
pytest.param(
|
||||
"Lieferdatum: 12. Mai 2024",
|
||||
"deu",
|
||||
date(2024, 5, 12),
|
||||
id="german-label-month-name",
|
||||
),
|
||||
pytest.param(
|
||||
"Total 5",
|
||||
"eng",
|
||||
None,
|
||||
id="single-number-after-label",
|
||||
),
|
||||
pytest.param(
|
||||
"Total 5 items 2024",
|
||||
"eng",
|
||||
None,
|
||||
id="number-run-without-month",
|
||||
),
|
||||
pytest.param(
|
||||
"Page 3 of 2024",
|
||||
"eng",
|
||||
None,
|
||||
id="page-number-without-month",
|
||||
),
|
||||
pytest.param(
|
||||
"Invoice 12.5",
|
||||
"eng",
|
||||
None,
|
||||
id="partial-numeric-date",
|
||||
),
|
||||
pytest.param(
|
||||
"Page 3 of 12 2024",
|
||||
"eng",
|
||||
None,
|
||||
id="page-of-number-run-with-year",
|
||||
),
|
||||
pytest.param(
|
||||
"Total 3 10 2024",
|
||||
"eng",
|
||||
None,
|
||||
id="space-separated-numbers-with-year",
|
||||
),
|
||||
pytest.param(
|
||||
"Order 12 3 2024",
|
||||
"eng",
|
||||
None,
|
||||
id="order-numbers-with-year",
|
||||
),
|
||||
pytest.param(
|
||||
"Ref 1 2 2024",
|
||||
"eng",
|
||||
None,
|
||||
id="ref-numbers-with-year",
|
||||
),
|
||||
pytest.param(
|
||||
"Invoice #2024 from 5 May",
|
||||
"eng",
|
||||
None,
|
||||
id="year-token-before-day-month",
|
||||
),
|
||||
pytest.param(
|
||||
"Page 1 of 3 03 2024",
|
||||
"eng",
|
||||
None,
|
||||
id="page-numbers-ending-in-year",
|
||||
),
|
||||
pytest.param(
|
||||
"Version 1.2.34",
|
||||
"deu",
|
||||
None,
|
||||
id="version-number-with-two-digit-tail",
|
||||
),
|
||||
pytest.param(
|
||||
"v1.2.24",
|
||||
"eng",
|
||||
None,
|
||||
id="short-version-number",
|
||||
),
|
||||
pytest.param(
|
||||
"Art. 12.3.45",
|
||||
"deu",
|
||||
None,
|
||||
id="article-number",
|
||||
),
|
||||
pytest.param(
|
||||
"Ticket 1-2-34",
|
||||
"eng",
|
||||
None,
|
||||
id="dashed-ticket-number",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_labelled_text_yields_the_date_after_the_label(
|
||||
self,
|
||||
text: str,
|
||||
ocr_language: str,
|
||||
expected: date | None,
|
||||
) -> None:
|
||||
"""
|
||||
GIVEN:
|
||||
- OCR text with a label before a date, or a label-like number
|
||||
with no complete date
|
||||
WHEN:
|
||||
- The date is parsed
|
||||
THEN:
|
||||
- The date after the label is returned in the zone's day/month order
|
||||
- Text without a complete date after a label returns no date
|
||||
"""
|
||||
assert _parse_date(text, _date_rules(ocr_language)) == expected
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "date_format", "expected"),
|
||||
[
|
||||
|
||||
+42
-18
@@ -60,6 +60,14 @@ YEAR_FIRST_DATE_RE = re.compile(
|
||||
r"(?<!\d)(\d{4})([-/.])(\d{2})\2(\d{2})(?!\d)",
|
||||
)
|
||||
|
||||
# Date-shaped tokens inside labelled text: a numeric day/month/year with a
|
||||
# separator, or a day, a month name and a four-digit year. Only these tokens are
|
||||
# handed to dateparser, so bare numbers around a label are never read as dates.
|
||||
DATE_TOKEN_RE = re.compile(
|
||||
r"(?<!\d)\d{1,2}[./-]\d{1,2}[./-]\d{4}(?!\d)"
|
||||
r"|(?<!\d)\d{1,2}\.?\s+[^\W\d_]{3,}\.?\s+\d{4}(?!\d)",
|
||||
)
|
||||
|
||||
# Earlier dates are far more likely OCR noise than a real document date.
|
||||
MIN_ZONE_DATE_YEAR = 1900
|
||||
|
||||
@@ -541,19 +549,8 @@ def _date_rules(ocr_language: str, date_format: str = "") -> DateRules:
|
||||
return DateRules(_zone_language(ocr_language), date_format)
|
||||
|
||||
|
||||
def _candidate_dates(text: str, rules: DateRules) -> Iterator[date]:
|
||||
"""Yield the dates the text may hold, most specific reading first."""
|
||||
if rules.date_format:
|
||||
try:
|
||||
yield datetime.strptime(text, rules.date_format).date()
|
||||
except ValueError:
|
||||
pass
|
||||
for match in YEAR_FIRST_DATE_RE.finditer(text):
|
||||
year, _, month, day = match.groups()
|
||||
try:
|
||||
yield date(int(year), int(month), int(day))
|
||||
except ValueError:
|
||||
continue
|
||||
def _dateparser_date(text: str, rules: DateRules) -> date | None:
|
||||
"""Read a whole-text absolute date with dateparser in the zone's language."""
|
||||
# Only absolute dates: dateparser would also read "now" or "yesterday"
|
||||
date_settings: dict[str, Any] = {
|
||||
"PARSERS": ["custom-formats", "absolute-time"],
|
||||
@@ -570,17 +567,44 @@ def _candidate_dates(text: str, rules: DateRules) -> Iterator[date]:
|
||||
)
|
||||
except (ValueError, OverflowError):
|
||||
logger.debug("Zone OCR: date parsing failed")
|
||||
return
|
||||
if parsed:
|
||||
yield parsed.date()
|
||||
return None
|
||||
return parsed.date() if parsed else None
|
||||
|
||||
|
||||
def _candidate_dates(text: str, rules: DateRules) -> Iterator[date]:
|
||||
"""Yield the dates the text may hold, most specific reading first."""
|
||||
if rules.date_format:
|
||||
try:
|
||||
yield datetime.strptime(text, rules.date_format).date()
|
||||
except ValueError:
|
||||
pass
|
||||
for match in YEAR_FIRST_DATE_RE.finditer(text):
|
||||
year, _, month, day = match.groups()
|
||||
try:
|
||||
yield date(int(year), int(month), int(day))
|
||||
except ValueError:
|
||||
continue
|
||||
whole_text = _dateparser_date(text, rules)
|
||||
if whole_text is not None:
|
||||
yield whole_text
|
||||
# Labelled text such as "Datum: 13.04.2026": try each date-shaped token in order
|
||||
for token in DATE_TOKEN_RE.finditer(text):
|
||||
labelled = _dateparser_date(token.group(), rules)
|
||||
if labelled is not None:
|
||||
yield labelled
|
||||
|
||||
|
||||
def _parse_date(text: str, rules: DateRules) -> date | None:
|
||||
"""Read a calendar date from OCR text.
|
||||
|
||||
Tries the zone's date format, year-first dates, then dateparser in the zone's
|
||||
language (or the configured date order when the language is unknown).
|
||||
Dates before MIN_ZONE_DATE_YEAR are treated as OCR noise.
|
||||
language (or the configured date order when the language is unknown) on the
|
||||
whole text. As a last resort, labelled text such as "Datum: 13.04.2026" is
|
||||
searched for date-shaped tokens with a four-digit year (numeric, or day,
|
||||
month name and year), and the first token dateparser accepts is used. Bare
|
||||
numbers and two-digit years in surrounding text are never read as dates, so
|
||||
version or article numbers like "1.2.34" are ignored. Dates before
|
||||
MIN_ZONE_DATE_YEAR are treated as OCR noise.
|
||||
"""
|
||||
return next(
|
||||
(
|
||||
|
||||
Reference in new issue
Block a user