mirror of
https://github.com/paperless-ngx/paperless-ngx.git
synced 2026-04-03 06:38:51 +00:00
dda05a7c00
- Make sure we're always using regex with timeouts for user controlled data - Adds rate limiting to the token endpoint (configurable) - Signs the classifier pickle file with the SECRET_KEY and refuse to load one which doesn't verify. - Require the user to set a secret key, instead of falling back to our old hard coded one
68 lines
2.4 KiB
Python
68 lines
2.4 KiB
Python
import datetime
|
|
from collections.abc import Iterator
|
|
|
|
import regex
|
|
from regex import Match
|
|
|
|
from documents.plugins.date_parsing.base import DateParserPluginBase
|
|
from documents.regex import safe_regex_finditer
|
|
|
|
|
|
class RegexDateParserPlugin(DateParserPluginBase):
|
|
"""
|
|
The default date parser, using a series of regular expressions.
|
|
|
|
It is configured entirely by the DateParserConfig object
|
|
passed to its constructor.
|
|
"""
|
|
|
|
DATE_REGEX = regex.compile(
|
|
r"(\b|(?!=([_-])))(\d{1,2})[\.\/-](\d{1,2})[\.\/-](\d{4}|\d{2})(\b|(?=([_-])))|"
|
|
r"(\b|(?!=([_-])))(\d{4}|\d{2})[\.\/-](\d{1,2})[\.\/-](\d{1,2})(\b|(?=([_-])))|"
|
|
r"(\b|(?!=([_-])))(\d{1,2}[\. ]+[a-zéûäëčžúřěáíóńźçŞğü]{3,9} \d{4}|[a-zéûäëčžúřěáíóńźçŞğü]{3,9} \d{1,2}, \d{4})(\b|(?=([_-])))|"
|
|
r"(\b|(?!=([_-])))([^\W\d_]{3,9} \d{1,2}, (\d{4}))(\b|(?=([_-])))|"
|
|
r"(\b|(?!=([_-])))([^\W\d_]{3,9} \d{4})(\b|(?=([_-])))|"
|
|
r"(\b|(?!=([_-])))(\d{1,2}[^ 0-9]{2}[\. ]+[^ ]{3,9}[ \.\/-]\d{4})(\b|(?=([_-])))|"
|
|
r"(\b|(?!=([_-])))(\b\d{1,2}[ \.\/-][a-zéûäëčžúřěáíóńźçŞğü]{3}[ \.\/-]\d{4})(\b|(?=([_-])))",
|
|
regex.IGNORECASE,
|
|
)
|
|
|
|
def _process_match(
|
|
self,
|
|
match: Match[str],
|
|
date_order: str,
|
|
) -> datetime.datetime | None:
|
|
"""
|
|
Processes a single regex match using the base class helpers.
|
|
"""
|
|
date_string = match.group(0)
|
|
date = self._parse_string(date_string, date_order)
|
|
return self._filter_date(date)
|
|
|
|
def _process_content(
|
|
self,
|
|
content: str,
|
|
date_order: str,
|
|
) -> Iterator[datetime.datetime]:
|
|
"""
|
|
Finds all regex matches in content and yields valid dates.
|
|
"""
|
|
for m in safe_regex_finditer(self.DATE_REGEX, content):
|
|
date = self._process_match(m, date_order)
|
|
if date is not None:
|
|
yield date
|
|
|
|
def parse(self, filename: str, content: str) -> Iterator[datetime.datetime]:
|
|
"""
|
|
Implementation of the abstract parse method.
|
|
|
|
Reads its configuration from `self.config`.
|
|
"""
|
|
if self.config.filename_date_order:
|
|
yield from self._process_content(
|
|
filename,
|
|
self.config.filename_date_order,
|
|
)
|
|
|
|
yield from self._process_content(content, self.config.content_date_order)
|