[SMTP TLS] some minor bug fixes (#477)

* fix minor bugs during smtp-tls parsing, add docker-compose for local elasticsearch, add smtp-tls tests

* fix wrong log message parameter

* fix wrong log message

* add contact-info to smtp tls report, fix wrong fieldnames

* fix wrong fieldnames

* fix wrong index name for search

* at least for some reporting organizations the field sending-mta-ip is optional...

* add missing fields to elasticsearch for smtp tls

* failure_details is a list, add more test cases

* fix wrong name in ci.ini

ci.ini

@@ -1,6 +1,7 @@
 [general]
 save_aggregate = True
 save_forensic = True
+save_smtp_tls = True
 debug = True
 
 [elasticsearch]

docker-compose.yml

@@ -0,0 +1,30 @@
+version: '3.7'
+
+services:
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.3.1
+    environment:
+      - network.host=127.0.0.1
+      - http.host=0.0.0.0
+      - node.name=elasticsearch
+      - discovery.type=single-node
+      - cluster.name=parsedmarc-cluster
+      - discovery.seed_hosts=elasticsearch
+      - bootstrap.memory_lock=true
+      - xpack.security.enabled=false
+      - xpack.license.self_generated.type=basic
+    ports:
+      - 127.0.0.1:9200:9200
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl -s -XGET http://localhost:9200/_cluster/health?pretty | grep status | grep -q '\\(green\\|yellow\\)'"
+        ]
+      interval: 10s
+      timeout: 10s
+      retries: 24

parsedmarc/__init__.py

@@ -214,10 +214,12 @@ def _parse_smtp_tls_failure_details(failure_details):
         new_failure_details = OrderedDict(
             result_type=failure_details["result-type"],
             failed_session_count=failure_details["failed-session-count"],
-            sending_mta_ip=failure_details["sending-mta-ip"],
-            receiving_ip=failure_details["receiving-ip"]
         )
 
+        if "sending-mta-ip" in failure_details:
+            new_failure_details["sending_mta_ip"] = failure_details["sending-mta-ip"]
+        if "receiving-ip" in failure_details:
+            new_failure_details["receiving_ip"] = failure_details["receiving-ip"]
         if "receiving-mx-hostname" in failure_details:
             new_failure_details["receiving_mx_hostname"] = failure_details[
                 "receiving-mx-hostname"]
@@ -303,6 +305,7 @@ def parse_smtp_tls_report_json(report):
             organization_name=report["organization-name"],
             begin_date=report["date-range"]["start-datetime"],
             end_date=report["date-range"]["end-datetime"],
+            contact_info=report["contact-info"],
             report_id=report["report-id"],
             policies=policies
         )
@@ -363,7 +366,7 @@ def parsed_smtp_tls_reports_to_csv(reports):
     """
 
     fields = ["organization_name", "begin_date", "end_date", "report_id",
-              "successful_session_count", "failed_session_count",
+              "result_type", "successful_session_count", "failed_session_count",
               "policy_domain", "policy_type", "policy_strings",
               "mx_host_patterns", "sending_mta_ip", "receiving_ip",
               "receiving_mx_hostname", "receiving_mx_helo",
@@ -1453,7 +1456,7 @@ def get_dmarc_reports_from_mailbox(connection: MailboxConnection,
                     message = "Moving message"
                     logger.debug("{0} {1} of {2}: UID {3}".format(
                         message,
-                        i + 1, smtp_tls_msg_uids, msg_uid))
+                        i + 1, number_of_smtp_tls_uids, msg_uid))
                     try:
                         connection.move_message(msg_uid,
                                                 smtp_tls_reports_folder)

parsedmarc/elastic.py

@@ -187,12 +187,14 @@ class _SMTPTLSPolicyDoc(InnerDoc):
                             receiving_ip,
                             receiving_mx_helo,
                             failed_session_count,
+                            sending_mta_ip=None,
                             receiving_mx_hostname=None,
                             additional_information_uri=None,
                             failure_reason_code=None):
-        self.failure_details.append(
+        _details = _SMTPTLSFailureDetailsDoc(
             result_type=result_type,
             ip_address=ip_address,
+            sending_mta_ip=sending_mta_ip,
             receiving_mx_hostname=receiving_mx_hostname,
             receiving_mx_helo=receiving_mx_helo,
             receiving_ip=receiving_ip,
@@ -200,9 +202,10 @@ class _SMTPTLSPolicyDoc(InnerDoc):
             additional_information=additional_information_uri,
             failure_reason_code=failure_reason_code
         )
+        self.failure_details.append(_details)
 
 
-class _SMTPTLSFailureReportDoc(Document):
+class _SMTPTLSReportDoc(Document):
 
     class Index:
         name = "smtp_tls"
@@ -639,8 +642,8 @@ def save_smtp_tls_report_to_elasticsearch(report,
     Raises:
             AlreadySaved
     """
-    logger.info("Saving aggregate report to Elasticsearch")
+    logger.info("Saving smtp tls report to Elasticsearch")
-    org_name = report["org_name"]
+    org_name = report["organization_name"]
     report_id = report["report_id"]
     begin_date = human_timestamp_to_datetime(report["begin_date"],
                                              to_utc=True)
@@ -663,7 +666,7 @@ def save_smtp_tls_report_to_elasticsearch(report,
     if index_suffix is not None:
         search = Search(index="smtp_tls_{0}*".format(index_suffix))
     else:
-        search = Search(index="smtp_tls")
+        search = Search(index="smtp_tls*")
     query = org_name_query & report_id_query
     query = query & begin_date_query & end_date_query
     search.query = query
@@ -688,11 +691,11 @@ def save_smtp_tls_report_to_elasticsearch(report,
     index_settings = dict(number_of_shards=number_of_shards,
                           number_of_replicas=number_of_replicas)
 
-    smtp_tls_doc = _SMTPTLSFailureReportDoc(
+    smtp_tls_doc = _SMTPTLSReportDoc(
-        organization_name=report["organization_name"],
+        org_name=report["organization_name"],
-        date_range=[report["date_begin"], report["date_end"]],
+        date_range=[report["begin_date"], report["end_date"]],
-        date_begin=report["date_begin"],
+        date_begin=report["begin_date"],
-        date_end=report["date_end"],
+        date_end=report["end_date"],
         contact_info=report["contact_info"],
         report_id=report["report_id"]
     )
@@ -707,32 +710,48 @@ def save_smtp_tls_report_to_elasticsearch(report,
         policy_doc = _SMTPTLSPolicyDoc(
             policy_domain=policy["policy_domain"],
             policy_type=policy["policy_type"],
+            succesful_session_count=policy["successful_session_count"],
+            failed_session_count=policy["failed_session_count"],
             policy_string=policy_strings,
             mx_host_patterns=mx_host_patterns
         )
         if "failure_details" in policy:
-            failure_details = policy["failure_details"]
+            for failure_detail in policy["failure_details"]:
-            receiving_mx_hostname = None
+                receiving_mx_hostname = None
-            additional_information_uri = None
+                additional_information_uri = None
-            failure_reason_code = None
+                failure_reason_code = None
-            if "receiving_mx_hostname" in failure_details:
+                ip_address = None
-                receiving_mx_hostname = failure_details[
+                receiving_ip = None
-                    "receiving_mx_hostname"]
+                receiving_mx_helo = None
-            if "additional_information_uri" in failure_details:
+                sending_mta_ip = None
-                additional_information_uri = failure_details[
+
-                    "additional_information_uri"]
+                if "receiving_mx_hostname" in failure_detail:
-            if "failure_reason_code" in failure_details:
+                    receiving_mx_hostname = failure_detail[
-                failure_reason_code = failure_details["failure_reason_code"]
+                        "receiving_mx_hostname"]
-            policy_doc.add_failure_details(
+                if "additional_information_uri" in failure_detail:
-                result_type=failure_details["result_type"],
+                    additional_information_uri = failure_detail[
-                ip_address=failure_details["ip_address"],
+                        "additional_information_uri"]
-                receiving_ip=failure_details["receiving_ip"],
+                if "failure_reason_code" in failure_detail:
-                receiving_mx_helo=failure_details["receiving_mx_helo"],
+                    failure_reason_code = failure_detail["failure_reason_code"]
-                failed_session_count=failure_details["failed_session_count"],
+                if "ip_address" in failure_detail:
-                receiving_mx_hostname=receiving_mx_hostname,
+                    ip_address = failure_detail["ip_address"]
-                additional_information_uri=additional_information_uri,
+                if "receiving_ip" in failure_detail:
-                failure_reason_code=failure_reason_code
+                    receiving_ip = failure_detail["receiving_ip"]
-            )
+                if "receiving_mx_helo" in failure_detail:
+                    receiving_mx_helo = failure_detail["receiving_mx_helo"]
+                if "sending_mta_ip" in failure_detail:
+                    sending_mta_ip = failure_detail["sending_mta_ip"]
+                policy_doc.add_failure_details(
+                    result_type=failure_detail["result_type"],
+                    ip_address=ip_address,
+                    receiving_ip=receiving_ip,
+                    receiving_mx_helo=receiving_mx_helo,
+                    failed_session_count=failure_detail["failed_session_count"],
+                    sending_mta_ip=sending_mta_ip,
+                    receiving_mx_hostname=receiving_mx_hostname,
+                    additional_information_uri=additional_information_uri,
+                    failure_reason_code=failure_reason_code
+                )
         smtp_tls_doc.policies.append(policy_doc)
 
     create_indexes([index], index_settings)

samples/smtp_tls/mail.ru.json

@@ -0,0 +1,33 @@
+{
+   "contact-info": "tls_support@corp.mail.ru",
+   "date-range": {
+      "end-datetime": "2024-02-23T00:00:00Z",
+      "start-datetime": "2024-02-22T00:00:00Z"
+   },
+   "organization-name": "Mail.ru",
+   "policies": [
+      {
+         "failure-details": [
+            {
+               "failed-session-count": 1,
+               "failure-reason-code": "bad https response code: 404",
+               "result-type": "sts-policy-fetch-error"
+            },
+            {
+               "failed-session-count": 1,
+               "failure-reason-code": "bad https response code: 500",
+               "result-type": "sts-policy-fetch-error"
+            }
+         ],
+         "policy": {
+            "policy-domain": "example.com",
+            "policy-type": "sts"
+         },
+         "summary": {
+            "total-failure-session-count": 1,
+            "total-successful-session-count": 0
+         }
+      }
+   ],
+   "report-id": "b28254de-7b2e-be36-bb5c-4c3b92da8b25@mail.ru"
+}

samples/smtp_tls/rfc8460.json

@@ -0,0 +1,42 @@
+{
+     "organization-name": "Company-X",
+     "date-range": {
+       "start-datetime": "2016-04-01T00:00:00Z",
+       "end-datetime": "2016-04-01T23:59:59Z"
+     },
+     "contact-info": "sts-reporting@company-x.example",
+     "report-id": "5065427c-23d3-47ca-b6e0-946ea0e8c4be",
+     "policies": [{
+       "policy": {
+         "policy-type": "sts",
+         "policy-string": ["version: STSv1","mode: testing",
+               "mx: *.mail.company-y.example","max_age: 86400"],
+         "policy-domain": "company-y.example",
+         "mx-host": "*.mail.company-y.example"
+       },
+       "summary": {
+         "total-successful-session-count": 5326,
+         "total-failure-session-count": 303
+       },
+       "failure-details": [{
+         "result-type": "certificate-expired",
+         "sending-mta-ip": "2001:db8:abcd:0012::1",
+         "receiving-mx-hostname": "mx1.mail.company-y.example",
+         "failed-session-count": 100
+       }, {
+         "result-type": "starttls-not-supported",
+         "sending-mta-ip": "2001:db8:abcd:0013::1",
+         "receiving-mx-hostname": "mx2.mail.company-y.example",
+         "receiving-ip": "203.0.113.56",
+         "failed-session-count": 200,
+         "additional-information": "https://reports.company-x.example/report_info?id=5065427c-23d3#StarttlsNotSupported"
+       }, {
+         "result-type": "validation-failure",
+         "sending-mta-ip": "198.51.100.62",
+         "receiving-ip": "203.0.113.58",
+         "receiving-mx-hostname": "mx-backup.mail.company-y.example",
+         "failed-session-count": 3,
+         "failure-reason-code": "X509_V_ERR_PROXY_PATH_LENGTH_EXCEEDED"
+       }]
+     }]
+   }

tests.py

@@ -59,6 +59,19 @@ class Test(unittest.TestCase):
             parsedmarc.parsed_forensic_reports_to_csv(parsed_report)
             print("Passed!")
 
+    def testSmtpTlsSamples(self):
+        """Test sample SMTP TLS reports"""
+        print()
+        sample_paths = glob("samples/smtp_tls/*")
+        for sample_path in sample_paths:
+            if os.path.isdir(sample_path):
+                continue
+            print("Testing {0}: " .format(sample_path), end="")
+            parsed_report = parsedmarc.parse_report_file(
+                sample_path)["report"]
+            parsedmarc.parsed_smtp_tls_reports_to_csv(parsed_report)
+            print("Passed!")
+
 
 if __name__ == "__main__":
     unittest.main(verbosity=2)