From 4813f7bc87929683fbb3567f18dcce1296114d68 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Sun, 24 Feb 2019 14:32:41 -0500 Subject: [PATCH] 6.2.0 --- CHANGELOG.md | 6 ++++++ parsedmarc/__init__.py | 3 +-- parsedmarc/elastic.py | 8 +++++--- parsedmarc/utils.py | 30 ++++++++++++++++++------------ setup.py | 2 +- 5 files changed, 31 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0ad0b9..3dbde0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +6.2.0 +----- + +- Add support for multi-process parallelized processing via CLI (Thanks zscholl - PR #62) +- Save sha256 hashes of attachments in forensic samples to Elasticsearch + 6.1.8 ----- diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index aab42e2..8a00678 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -38,7 +38,7 @@ from parsedmarc.utils import is_outlook_msg, convert_outlook_msg from parsedmarc.utils import timestamp_to_human, human_timestamp_to_datetime from parsedmarc.utils import parse_email -__version__ = "6.1.8" +__version__ = "6.2.0" logging.basicConfig( format='%(levelname)8s:%(filename)s:%(lineno)d:' @@ -769,7 +769,6 @@ def parse_report_email(input_, nameservers=None, dns_timeout=2.0, 'forensic DMARC report: {1}'.format(subject, e) raise InvalidForensicReport(error) except Exception as e: - print("DEBUGGGING: {}".format(e)) raise InvalidForensicReport(e.__str__()) result = OrderedDict([("report_type", "forensic"), diff --git a/parsedmarc/elastic.py b/parsedmarc/elastic.py index 72db2ec..593313e 100644 --- a/parsedmarc/elastic.py +++ b/parsedmarc/elastic.py @@ -102,6 +102,7 @@ class _EmailAddressDoc(InnerDoc): class _EmailAttachmentDoc(Document): filename = Text() content_type = Text() + sha256 = Text() class _ForensicSampleDoc(InnerDoc): @@ -135,9 +136,9 @@ class _ForensicSampleDoc(InnerDoc): self.bcc.append(_EmailAddressDoc(display_name=display_name, address=address)) - def add_attachment(self, filename, content_type): + def add_attachment(self, filename, content_type, sha256): self.attachments.append(_EmailAttachmentDoc(filename=filename, - content_type=content_type)) + content_type=content_type, sha256=sha256)) class _ForensicReportDoc(Document): @@ -467,7 +468,8 @@ def save_forensic_report_to_elasticsearch(forensic_report, address=address["address"]) for attachment in forensic_report["parsed_sample"]["attachments"]: sample.add_attachment(filename=attachment["filename"], - content_type=attachment["mail_content_type"]) + content_type=attachment["mail_content_type"], + sha256=attachment["sha256"]) forensic_doc = _ForensicReportDoc( feedback_type=forensic_report["feedback_type"], diff --git a/parsedmarc/utils.py b/parsedmarc/utils.py index e86dfca..26b016d 100644 --- a/parsedmarc/utils.py +++ b/parsedmarc/utils.py @@ -250,13 +250,14 @@ def human_timestamp_to_timestamp(human_timestamp): return human_timestamp_to_datetime(human_timestamp).timestamp() -def get_ip_address_country(ip_address): +def get_ip_address_country(ip_address, parallel=False): """ Uses the MaxMind Geolite2 Country database to return the ISO code for the country associated with the given IPv4 or IPv6 address Args: - ip_address (str): The IP address to query for + ip_address (str): The IP address to query for, + parallel (bool): parallel processing Returns: str: And ISO country code associated with the given IP address @@ -267,6 +268,9 @@ def get_ip_address_country(ip_address): Args: location (str): Local location for the database file """ + if parallel: + logging.warning("Cannot download GeoIP database in parallel mode") + return url = "https://geolite.maxmind.com/download/geoip/database/" \ "GeoLite2-Country.tar.gz" # Use a browser-like user agent string to bypass some proxy blocks @@ -286,11 +290,15 @@ def get_ip_address_country(ip_address): logger.warning("Error downloading {0}: {1}".format(url, e.__str__())) - system_paths = ["/usr/local/share/GeoIP/GeoLite2-Country.mmdb", - "/usr/share/GeoIP/GeoLite2-Country.mmdb", - "/var/lib/GeoIP/GeoLite2-Country.mmdb", - "/var/local/lib/GeoIP/GeoLite2-Country.mmdb", - "C:\\GeoIP\\GeoLite2-Country.mmdb"] + system_paths = [ + "GeoLite2-Country.mmdb", + "/usr/local/share/GeoIP/GeoLite2-Country.mmdb", + "/usr/share/GeoIP/GeoLite2-Country.mmdb", + "/var/lib/GeoIP/GeoLite2-Country.mmdb", + "/var/local/lib/GeoIP/GeoLite2-Country.mmdb", + "C:\\GeoIP\\GeoLite2-Country.mmdb" + ] + db_path = None for system_path in system_paths: @@ -334,6 +342,7 @@ def get_ip_address_info(ip_address, cache=None, nameservers=None, nameservers (list): A list of one or more nameservers to use (Cloudflare's public DNS resolvers by default) timeout (float): Sets the DNS timeout in seconds + parallel (bool): parallel processing Returns: OrderedDict: ``ip_address``, ``reverse_dns`` @@ -349,11 +358,8 @@ def get_ip_address_info(ip_address, cache=None, nameservers=None, reverse_dns = get_reverse_dns(ip_address, nameservers=nameservers, timeout=timeout) - if not parallel: - country = get_ip_address_country(ip_address) - info["country"] = country - else: - info["country"] = None + country = get_ip_address_country(ip_address, parallel=parallel) + info["country"] = country info["reverse_dns"] = reverse_dns info["base_domain"] = None if reverse_dns is not None: diff --git a/setup.py b/setup.py index 2959e13..97847d3 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ from setuptools import setup from codecs import open from os import path -__version__ = "6.1.8" +__version__ = "6.2.0" description = "A Python package and CLI for parsing aggregate and " \ "forensic DMARC reports"