6.2.0

CHANGELOG.md

@@ -1,3 +1,9 @@
+6.2.0
+-----
+
+- Add support for multi-process parallelized processing via CLI (Thanks zscholl - PR #62)
+- Save sha256 hashes of attachments in forensic samples to Elasticsearch
+
 6.1.8
 -----
 

parsedmarc/__init__.py

@@ -38,7 +38,7 @@ from parsedmarc.utils import is_outlook_msg, convert_outlook_msg
 from parsedmarc.utils import timestamp_to_human, human_timestamp_to_datetime
 from parsedmarc.utils import parse_email
 
-__version__ = "6.1.8"
+__version__ = "6.2.0"
 
 logging.basicConfig(
     format='%(levelname)8s:%(filename)s:%(lineno)d:'
@@ -769,7 +769,6 @@ def parse_report_email(input_, nameservers=None, dns_timeout=2.0,
                     'forensic DMARC report: {1}'.format(subject, e)
             raise InvalidForensicReport(error)
         except Exception as e:
-            print("DEBUGGGING: {}".format(e))
             raise InvalidForensicReport(e.__str__())
 
         result = OrderedDict([("report_type", "forensic"),

parsedmarc/elastic.py

@@ -102,6 +102,7 @@ class _EmailAddressDoc(InnerDoc):
 class _EmailAttachmentDoc(Document):
     filename = Text()
     content_type = Text()
+    sha256 = Text()
 
 
 class _ForensicSampleDoc(InnerDoc):
@@ -135,9 +136,9 @@ class _ForensicSampleDoc(InnerDoc):
         self.bcc.append(_EmailAddressDoc(display_name=display_name,
                                          address=address))
 
-    def add_attachment(self, filename, content_type):
+    def add_attachment(self, filename, content_type, sha256):
         self.attachments.append(_EmailAttachmentDoc(filename=filename,
-                                content_type=content_type))
+                                content_type=content_type, sha256=sha256))
 
 
 class _ForensicReportDoc(Document):
@@ -467,7 +468,8 @@ def save_forensic_report_to_elasticsearch(forensic_report,
                        address=address["address"])
     for attachment in forensic_report["parsed_sample"]["attachments"]:
         sample.add_attachment(filename=attachment["filename"],
-                              content_type=attachment["mail_content_type"])
+                              content_type=attachment["mail_content_type"],
+                              sha256=attachment["sha256"])
 
     forensic_doc = _ForensicReportDoc(
         feedback_type=forensic_report["feedback_type"],

parsedmarc/utils.py

@@ -250,13 +250,14 @@ def human_timestamp_to_timestamp(human_timestamp):
     return human_timestamp_to_datetime(human_timestamp).timestamp()
 
 
-def get_ip_address_country(ip_address):
+def get_ip_address_country(ip_address, parallel=False):
     """
     Uses the MaxMind Geolite2 Country database to return the ISO code for the
     country associated with the given IPv4 or IPv6 address
 
     Args:
-        ip_address (str): The IP address to query for
+        ip_address (str): The IP address to query for,
+        parallel (bool): parallel processing
 
     Returns:
         str: And ISO country code associated with the given IP address
@@ -267,6 +268,9 @@ def get_ip_address_country(ip_address):
         Args:
             location (str): Local location for the database file
         """
+        if parallel:
+            logging.warning("Cannot download GeoIP database in parallel mode")
+            return
         url = "https://geolite.maxmind.com/download/geoip/database/" \
               "GeoLite2-Country.tar.gz"
         # Use a browser-like user agent string to bypass some proxy blocks
@@ -286,11 +290,15 @@ def get_ip_address_country(ip_address):
             logger.warning("Error downloading {0}: {1}".format(url,
                                                                e.__str__()))
 
-    system_paths = ["/usr/local/share/GeoIP/GeoLite2-Country.mmdb",
-                    "/usr/share/GeoIP/GeoLite2-Country.mmdb",
-                    "/var/lib/GeoIP/GeoLite2-Country.mmdb",
-                    "/var/local/lib/GeoIP/GeoLite2-Country.mmdb",
-                    "C:\\GeoIP\\GeoLite2-Country.mmdb"]
+    system_paths = [
+        "GeoLite2-Country.mmdb",
+        "/usr/local/share/GeoIP/GeoLite2-Country.mmdb",
+        "/usr/share/GeoIP/GeoLite2-Country.mmdb",
+        "/var/lib/GeoIP/GeoLite2-Country.mmdb",
+        "/var/local/lib/GeoIP/GeoLite2-Country.mmdb",
+        "C:\\GeoIP\\GeoLite2-Country.mmdb"
+    ]
+
     db_path = None
 
     for system_path in system_paths:
@@ -334,6 +342,7 @@ def get_ip_address_info(ip_address, cache=None, nameservers=None,
         nameservers (list): A list of one or more nameservers to use
         (Cloudflare's public DNS resolvers by default)
         timeout (float): Sets the DNS timeout in seconds
+        parallel (bool): parallel processing
 
     Returns:
         OrderedDict: ``ip_address``, ``reverse_dns``
@@ -349,11 +358,8 @@ def get_ip_address_info(ip_address, cache=None, nameservers=None,
     reverse_dns = get_reverse_dns(ip_address,
                                   nameservers=nameservers,
                                   timeout=timeout)
-    if not parallel:
-        country = get_ip_address_country(ip_address)
-        info["country"] = country
-    else:
-        info["country"] = None
+    country = get_ip_address_country(ip_address, parallel=parallel)
+    info["country"] = country
     info["reverse_dns"] = reverse_dns
     info["base_domain"] = None
     if reverse_dns is not None:

setup.py

@@ -14,7 +14,7 @@ from setuptools import setup
 from codecs import open
 from os import path
 
-__version__ = "6.1.8"
+__version__ = "6.2.0"
 
 description = "A Python package and CLI for parsing aggregate and " \
               "forensic DMARC reports"