diff --git a/CHANGELOG.md b/CHANGELOG.md index 69a1405..e293257 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +6.7.1 +----- + +- Parse forensic email samples with non-standard date headers +- Graceful handling of a failure to download the GeoIP database (issue #123) + 6.7.0 ----- diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 9d14a8e..4caf818 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -33,7 +33,7 @@ from parsedmarc.utils import is_outlook_msg, convert_outlook_msg from parsedmarc.utils import timestamp_to_human, human_timestamp_to_datetime from parsedmarc.utils import parse_email -__version__ = "6.7.0" +__version__ = "6.7.1" logging.basicConfig( format='%(levelname)8s:%(filename)s:%(lineno)d:' diff --git a/parsedmarc/utils.py b/parsedmarc/utils.py index 7536fb5..dca076a 100644 --- a/parsedmarc/utils.py +++ b/parsedmarc/utils.py @@ -17,6 +17,7 @@ import base64 import platform import atexit import mailbox +import re import dateparser import dns.reversename @@ -32,6 +33,8 @@ USER_AGENT = "Mozilla/5.0 ((0 {1})) parsedmarc".format( platform.release(), ) +parenthesis_regex = re.compile(r'\s*\(.*\)\s*') + null_file = open(os.devnull, "w") logger = logging.getLogger("parsedmarc") mailparser_logger = logging.getLogger("mailparser") @@ -236,6 +239,8 @@ def human_timestamp_to_datetime(human_timestamp, to_utc=False): DateTime: The converted timestamp """ + human_timestamp = human_timestamp.replace("-0000", "") + human_timestamp = parenthesis_regex.sub("", human_timestamp) settings = {} if to_utc: @@ -320,14 +325,22 @@ def get_ip_address_country(ip_address, parallel=False, offline=False): if db_path is None: db_path = os.path.join(tempdir, "GeoLite2-Country.mmdb") if not os.path.exists(db_path): - download_country_database(db_path) + try: + download_country_database() + except Exception as e: + logger.error(e.__str__()) + return None if not os.path.exists(db_path): return None else: db_age = datetime.now() - datetime.fromtimestamp( os.stat(db_path).st_mtime) if db_age > timedelta(days=7): - download_country_database() + try: + download_country_database() + except Exception as e: + logger.error(e.__str__()) + return None db_path = db_path db_reader = geoip2.database.Reader(db_path) diff --git a/requirements.txt b/requirements.txt index ad59ab0..e8d3c66 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ xmltodict>=0.12.0 geoip2>=2.9.0 imapclient>=2.1.0 mail-parser>=3.9.2 -dateparser>=0.7.1 +dateparser>=0.7.2 elasticsearch>=6.3.1,<7.0.0 elasticsearch-dsl>=6.3.1,<7.0.0 kafka-python>=1.4.4 diff --git a/setup.py b/setup.py index 8913a87..47e9b4f 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ from setuptools import setup from codecs import open from os import path -__version__ = "6.7.0" +__version__ = "6.7.1" description = "A Python package and CLI for parsing aggregate and " \ "forensic DMARC reports" @@ -99,7 +99,7 @@ setup( 'geoip2>=2.9.0', 'urllib3<1.25,>=1.21.1', 'requests>=2.2.16.0', 'imapclient>=2.1.0', 'mail-parser>=3.9.2', - 'dateparser>=0.7.1', + 'dateparser>=0.7.2', 'mailsuite>=1.3.0', 'elasticsearch>=6.3.1,<7.0.0', 'elasticsearch-dsl>=6.3.1,<7.0.0',