Merge pull request #32 from vitalvas/master

Cache reverse DNS lookups

parsedmarc/__init__.py

@@ -50,6 +50,8 @@ MAGIC_ZIP = b"\x50\x4B\x03\x04"
 MAGIC_GZIP = b"\x1F\x8B"
 MAGIC_XML = b"\x3c\x3f\x78\x6d\x6c\x20"
 
+DNS_CACHE = dict()
+
 
 class ParserError(RuntimeError):
     """Raised whenever the parser fails for some reason"""
@@ -93,9 +95,13 @@ def _parse_report_record(record, nameservers=None, timeout=2.0):
         nameservers = ["8.8.8.8", "4.4.4.4"]
     record = record.copy()
     new_record = OrderedDict()
-    new_record["source"] = get_ip_address_info(record["row"]["source_ip"],
-                                               nameservers=nameservers,
-                                               timeout=timeout)
+    new_record_source = DNS_CACHE.get(record["row"]["source_ip"], None)
+    if not new_record_source:
+        new_record_source = get_ip_address_info(record["row"]["source_ip"],
+                                                nameservers=nameservers,
+                                                timeout=timeout)
+        DNS_CACHE[record["row"]["source_ip"]] = new_record_source
+    new_record["source"] = new_record_source
     new_record["count"] = int(record["row"]["count"])
     policy_evaluated = record["row"]["policy_evaluated"].copy()
     new_policy_evaluated = OrderedDict([("disposition", "none"),
@@ -536,9 +542,13 @@ def parse_forensic_report(feedback_report, sample, msg_date,
         parsed_report["arrival_date_utc"] = arrival_utc
 
         ip_address = parsed_report["source_ip"]
-        parsed_report["source"] = get_ip_address_info(ip_address,
-                                                      nameservers=nameservers,
-                                                      timeout=timeout)
+        parsed_report_source = DNS_CACHE.get(ip_address, None)
+        if not parsed_report_source:
+            parsed_report_source = get_ip_address_info(ip_address,
+                                                       nameservers=nameservers,
+                                                       timeout=timeout)
+            DNS_CACHE[ip_address] = parsed_report_source
+        parsed_report["source"] = parsed_report_source
         del parsed_report["source_ip"]
 
         if "identity_alignment" not in parsed_report:

parsedmarc/utils.py

@@ -426,11 +426,15 @@ def parse_email(data, strip_attachment_payloads=False):
     headers = json.loads(parsed_email.headers_json).copy()
     parsed_email = json.loads(parsed_email.mail_json).copy()
     parsed_email["headers"] = headers
+
     if "received" in parsed_email:
         for received in parsed_email["received"]:
             if "date_utc" in received:
-                received["date_utc"] = received["date_utc"].replace("T",
-                                                                    " ")
+                if received["date_utc"] is None:
+                    del received["date_utc"]
+                else:
+                    received["date_utc"] = received["date_utc"].replace("T",
+                                                                        " ")
 
     if "from" not in parsed_email:
         if "From" in parsed_email["headers"]: