diff --git a/_modules/index.html b/_modules/index.html index 370330a..b2fd7a6 100644 --- a/_modules/index.html +++ b/_modules/index.html @@ -5,14 +5,14 @@ - Overview: module code — parsedmarc 8.16.1 documentation + Overview: module code — parsedmarc 8.17.0 documentation - + diff --git a/_modules/parsedmarc.html b/_modules/parsedmarc.html index 60157b9..3ed0d03 100644 --- a/_modules/parsedmarc.html +++ b/_modules/parsedmarc.html @@ -5,14 +5,14 @@ - parsedmarc — parsedmarc 8.16.1 documentation + parsedmarc — parsedmarc 8.17.0 documentation - + @@ -120,7 +120,7 @@ from parsedmarc.utils import parse_email from parsedmarc.utils import timestamp_to_human, human_timestamp_to_datetime -__version__ = "8.16.1" +__version__ = "8.17.0" logger.debug("parsedmarc v{0}".format(__version__)) @@ -135,6 +135,7 @@ MAGIC_JSON = b"\7b" IP_ADDRESS_CACHE = ExpiringDict(max_len=10000, max_age_seconds=14400) +SEEN_AGGREGATE_REPORT_IDS = ExpiringDict(max_len=100000000, max_age_seconds=3600) REVERSE_DNS_MAP = dict() @@ -1610,7 +1611,17 @@ strip_attachment_payloads=sa, ) if parsed_email["report_type"] == "aggregate": - aggregate_reports.append(parsed_email["report"]) + report_org = parsed_email["report"]["report_metadata"]["org_name"] + report_id = parsed_email["report"]["report_metadata"]["report_id"] + report_key = f"{report_org}_{report_id}" + if report_key not in SEEN_AGGREGATE_REPORT_IDS: + SEEN_AGGREGATE_REPORT_IDS[report_key] = True + aggregate_reports.append(parsed_email["report"]) + else: + logger.debug( + "Skipping duplicate aggregate report " + f"from {report_org} with ID: {report_id}" + ) elif parsed_email["report_type"] == "forensic": forensic_reports.append(parsed_email["report"]) elif parsed_email["report_type"] == "smtp_tls": @@ -1790,7 +1801,16 @@ keep_alive=connection.keepalive, ) if parsed_email["report_type"] == "aggregate": - aggregate_reports.append(parsed_email["report"]) + report_org = parsed_email["report"]["report_metadata"]["org_name"] + report_id = parsed_email["report"]["report_metadata"]["report_id"] + report_key = f"{report_org}_{report_id}" + if report_key not in SEEN_AGGREGATE_REPORT_IDS: + SEEN_AGGREGATE_REPORT_IDS[report_key] = True + aggregate_reports.append(parsed_email["report"]) + else: + logger.debug( + "Skipping duplicate aggregate report " f"with ID: {report_id}" + ) aggregate_report_msg_uids.append(msg_uid) elif parsed_email["report_type"] == "forensic": forensic_reports.append(parsed_email["report"]) diff --git a/_modules/parsedmarc/elastic.html b/_modules/parsedmarc/elastic.html index 338cd72..b1906ed 100644 --- a/_modules/parsedmarc/elastic.html +++ b/_modules/parsedmarc/elastic.html @@ -5,14 +5,14 @@ - parsedmarc.elastic — parsedmarc 8.16.1 documentation + parsedmarc.elastic — parsedmarc 8.17.0 documentation - + diff --git a/_modules/parsedmarc/opensearch.html b/_modules/parsedmarc/opensearch.html index 530af72..7f1dc94 100644 --- a/_modules/parsedmarc/opensearch.html +++ b/_modules/parsedmarc/opensearch.html @@ -5,14 +5,14 @@ - parsedmarc.opensearch — parsedmarc 8.16.1 documentation + parsedmarc.opensearch — parsedmarc 8.17.0 documentation - + @@ -287,13 +287,15 @@ receiving_ip, receiving_mx_helo, failed_session_count, + sending_mta_ip=None, receiving_mx_hostname=None, additional_information_uri=None, failure_reason_code=None, ): - self.failure_details.append( + _details = _SMTPTLSFailureDetailsDoc( result_type=result_type, ip_address=ip_address, + sending_mta_ip=sending_mta_ip, receiving_mx_hostname=receiving_mx_hostname, receiving_mx_helo=receiving_mx_helo, receiving_ip=receiving_ip, @@ -301,9 +303,10 @@ additional_information=additional_information_uri, failure_reason_code=failure_reason_code, ) + self.failure_details.append(_details) -class _SMTPTLSFailureReportDoc(Document): +class _SMTPTLSReportDoc(Document): class Index: name = "smtp_tls" @@ -598,6 +601,7 @@ index = "{0}_{1}".format(index, index_suffix) if index_prefix: index = "{0}{1}".format(index_prefix, index) + index = "{0}-{1}".format(index, index_date) index_settings = dict( number_of_shards=number_of_shards, number_of_replicas=number_of_replicas @@ -790,7 +794,7 @@ AlreadySaved """ logger.info("Saving aggregate report to OpenSearch") - org_name = report["org_name"] + org_name = report["organization_name"] report_id = report["report_id"] begin_date = human_timestamp_to_datetime(report["begin_date"], to_utc=True) end_date = human_timestamp_to_datetime(report["end_date"], to_utc=True) @@ -846,11 +850,11 @@ number_of_shards=number_of_shards, number_of_replicas=number_of_replicas ) - smtp_tls_doc = _SMTPTLSFailureReportDoc( - organization_name=report["organization_name"], - date_range=[report["date_begin"], report["date_end"]], - date_begin=report["date_begin"], - date_end=report["date_end"], + smtp_tls_doc = _SMTPTLSReportDoc( + org_name=report["organization_name"], + date_range=[report["begin_date"], report["end_date"]], + date_begin=report["begin_date"], + date_end=report["end_date"], contact_info=report["contact_info"], report_id=report["report_id"], ) @@ -865,32 +869,48 @@ policy_doc = _SMTPTLSPolicyDoc( policy_domain=policy["policy_domain"], policy_type=policy["policy_type"], + succesful_session_count=policy["successful_session_count"], + failed_session_count=policy["failed_session_count"], policy_string=policy_strings, mx_host_patterns=mx_host_patterns, ) if "failure_details" in policy: - failure_details = policy["failure_details"] - receiving_mx_hostname = None - additional_information_uri = None - failure_reason_code = None - if "receiving_mx_hostname" in failure_details: - receiving_mx_hostname = failure_details["receiving_mx_hostname"] - if "additional_information_uri" in failure_details: - additional_information_uri = failure_details[ - "additional_information_uri" - ] - if "failure_reason_code" in failure_details: - failure_reason_code = failure_details["failure_reason_code"] - policy_doc.add_failure_details( - result_type=failure_details["result_type"], - ip_address=failure_details["ip_address"], - receiving_ip=failure_details["receiving_ip"], - receiving_mx_helo=failure_details["receiving_mx_helo"], - failed_session_count=failure_details["failed_session_count"], - receiving_mx_hostname=receiving_mx_hostname, - additional_information_uri=additional_information_uri, - failure_reason_code=failure_reason_code, - ) + for failure_detail in policy["failure_details"]: + receiving_mx_hostname = None + additional_information_uri = None + failure_reason_code = None + ip_address = None + receiving_ip = None + receiving_mx_helo = None + sending_mta_ip = None + + if "receiving_mx_hostname" in failure_detail: + receiving_mx_hostname = failure_detail["receiving_mx_hostname"] + if "additional_information_uri" in failure_detail: + additional_information_uri = failure_detail[ + "additional_information_uri" + ] + if "failure_reason_code" in failure_detail: + failure_reason_code = failure_detail["failure_reason_code"] + if "ip_address" in failure_detail: + ip_address = failure_detail["ip_address"] + if "receiving_ip" in failure_detail: + receiving_ip = failure_detail["receiving_ip"] + if "receiving_mx_helo" in failure_detail: + receiving_mx_helo = failure_detail["receiving_mx_helo"] + if "sending_mta_ip" in failure_detail: + sending_mta_ip = failure_detail["sending_mta_ip"] + policy_doc.add_failure_details( + result_type=failure_detail["result_type"], + ip_address=ip_address, + receiving_ip=receiving_ip, + receiving_mx_helo=receiving_mx_helo, + failed_session_count=failure_detail["failed_session_count"], + sending_mta_ip=sending_mta_ip, + receiving_mx_hostname=receiving_mx_hostname, + additional_information_uri=additional_information_uri, + failure_reason_code=failure_reason_code, + ) smtp_tls_doc.policies.append(policy_doc) create_indexes([index], index_settings) diff --git a/_modules/parsedmarc/splunk.html b/_modules/parsedmarc/splunk.html index 65e1c4f..d61cfcf 100644 --- a/_modules/parsedmarc/splunk.html +++ b/_modules/parsedmarc/splunk.html @@ -5,14 +5,14 @@ - parsedmarc.splunk — parsedmarc 8.16.1 documentation + parsedmarc.splunk — parsedmarc 8.17.0 documentation - + diff --git a/_modules/parsedmarc/utils.html b/_modules/parsedmarc/utils.html index d3e19a3..2bd1f31 100644 --- a/_modules/parsedmarc/utils.html +++ b/_modules/parsedmarc/utils.html @@ -5,14 +5,14 @@ - parsedmarc.utils — parsedmarc 8.16.1 documentation + parsedmarc.utils — parsedmarc 8.17.0 documentation - + diff --git a/_static/documentation_options.js b/_static/documentation_options.js index a00b9b0..8153123 100644 --- a/_static/documentation_options.js +++ b/_static/documentation_options.js @@ -1,5 +1,5 @@ const DOCUMENTATION_OPTIONS = { - VERSION: '8.16.1', + VERSION: '8.17.0', LANGUAGE: 'en', COLLAPSE_INDEX: false, BUILDER: 'html', diff --git a/api.html b/api.html index 382224f..a0ad0f7 100644 --- a/api.html +++ b/api.html @@ -6,14 +6,14 @@ - API reference — parsedmarc 8.16.1 documentation + API reference — parsedmarc 8.17.0 documentation - + diff --git a/contributing.html b/contributing.html index abbb746..f0481ab 100644 --- a/contributing.html +++ b/contributing.html @@ -6,14 +6,14 @@ - Contributing to parsedmarc — parsedmarc 8.16.1 documentation + Contributing to parsedmarc — parsedmarc 8.17.0 documentation - + diff --git a/davmail.html b/davmail.html index 4528104..4a7474e 100644 --- a/davmail.html +++ b/davmail.html @@ -6,14 +6,14 @@ - Accessing an inbox using OWA/EWS — parsedmarc 8.16.1 documentation + Accessing an inbox using OWA/EWS — parsedmarc 8.17.0 documentation - + diff --git a/dmarc.html b/dmarc.html index 62d0feb..851f868 100644 --- a/dmarc.html +++ b/dmarc.html @@ -6,14 +6,14 @@ - Understanding DMARC — parsedmarc 8.16.1 documentation + Understanding DMARC — parsedmarc 8.17.0 documentation - + diff --git a/elasticsearch.html b/elasticsearch.html index b01721b..5f3421b 100644 --- a/elasticsearch.html +++ b/elasticsearch.html @@ -6,14 +6,14 @@ - Elasticsearch and Kibana — parsedmarc 8.16.1 documentation + Elasticsearch and Kibana — parsedmarc 8.17.0 documentation - + diff --git a/genindex.html b/genindex.html index 9f35d18..a1be1b0 100644 --- a/genindex.html +++ b/genindex.html @@ -5,14 +5,14 @@ - Index — parsedmarc 8.16.1 documentation + Index — parsedmarc 8.17.0 documentation - + diff --git a/index.html b/index.html index 13c50bf..3325566 100644 --- a/index.html +++ b/index.html @@ -6,14 +6,14 @@ - parsedmarc documentation - Open source DMARC report analyzer and visualizer — parsedmarc 8.16.1 documentation + parsedmarc documentation - Open source DMARC report analyzer and visualizer — parsedmarc 8.17.0 documentation - + diff --git a/installation.html b/installation.html index 0705ced..00f508d 100644 --- a/installation.html +++ b/installation.html @@ -6,14 +6,14 @@ - Installation — parsedmarc 8.16.1 documentation + Installation — parsedmarc 8.17.0 documentation - + diff --git a/kibana.html b/kibana.html index 21e4d7a..0805e47 100644 --- a/kibana.html +++ b/kibana.html @@ -6,14 +6,14 @@ - Using the Kibana dashboards — parsedmarc 8.16.1 documentation + Using the Kibana dashboards — parsedmarc 8.17.0 documentation - + diff --git a/mailing-lists.html b/mailing-lists.html index f0a9350..55e5ad1 100644 --- a/mailing-lists.html +++ b/mailing-lists.html @@ -6,14 +6,14 @@ - What about mailing lists? — parsedmarc 8.16.1 documentation + What about mailing lists? — parsedmarc 8.17.0 documentation - + diff --git a/objects.inv b/objects.inv index 2c9c937..96384d9 100644 Binary files a/objects.inv and b/objects.inv differ diff --git a/opensearch.html b/opensearch.html index de159cf..fb5e3ed 100644 --- a/opensearch.html +++ b/opensearch.html @@ -6,14 +6,14 @@ - OpenSearch and Grafana — parsedmarc 8.16.1 documentation + OpenSearch and Grafana — parsedmarc 8.17.0 documentation - + diff --git a/output.html b/output.html index b59604e..3cb62d8 100644 --- a/output.html +++ b/output.html @@ -6,14 +6,14 @@ - Sample outputs — parsedmarc 8.16.1 documentation + Sample outputs — parsedmarc 8.17.0 documentation - + diff --git a/py-modindex.html b/py-modindex.html index 336415b..e31fa86 100644 --- a/py-modindex.html +++ b/py-modindex.html @@ -5,14 +5,14 @@ - Python Module Index — parsedmarc 8.16.1 documentation + Python Module Index — parsedmarc 8.17.0 documentation - + diff --git a/search.html b/search.html index 1e944cc..8ce7d43 100644 --- a/search.html +++ b/search.html @@ -5,7 +5,7 @@ - Search — parsedmarc 8.16.1 documentation + Search — parsedmarc 8.17.0 documentation @@ -13,7 +13,7 @@ - + diff --git a/splunk.html b/splunk.html index 0188a98..447d342 100644 --- a/splunk.html +++ b/splunk.html @@ -6,14 +6,14 @@ - Splunk — parsedmarc 8.16.1 documentation + Splunk — parsedmarc 8.17.0 documentation - + diff --git a/usage.html b/usage.html index 84a7935..be28fc5 100644 --- a/usage.html +++ b/usage.html @@ -6,14 +6,14 @@ - Using parsedmarc — parsedmarc 8.16.1 documentation + Using parsedmarc — parsedmarc 8.17.0 documentation - +