diff --git a/_modules/index.html b/_modules/index.html index 8c7c103..d625010 100644 --- a/_modules/index.html +++ b/_modules/index.html @@ -5,14 +5,14 @@ - Overview: module code — parsedmarc 9.0.0 documentation + Overview: module code — parsedmarc 9.0.1 documentation - + diff --git a/_modules/parsedmarc.html b/_modules/parsedmarc.html index 4744dd6..04de4da 100644 --- a/_modules/parsedmarc.html +++ b/_modules/parsedmarc.html @@ -5,14 +5,14 @@ - parsedmarc — parsedmarc 9.0.0 documentation + parsedmarc — parsedmarc 9.0.1 documentation - + diff --git a/_modules/parsedmarc/elastic.html b/_modules/parsedmarc/elastic.html index 6843946..c9c6474 100644 --- a/_modules/parsedmarc/elastic.html +++ b/_modules/parsedmarc/elastic.html @@ -5,14 +5,14 @@ - parsedmarc.elastic — parsedmarc 9.0.0 documentation + parsedmarc.elastic — parsedmarc 9.0.1 documentation - + @@ -494,7 +494,51 @@ org_name = metadata["org_name"] report_id = metadata["report_id"] domain = aggregate_report["policy_published"]["domain"] + begin_date = human_timestamp_to_datetime(metadata["begin_date"], to_utc=True) + end_date = human_timestamp_to_datetime(metadata["end_date"], to_utc=True) + if monthly_indexes: + index_date = begin_date.strftime("%Y-%m") + else: + index_date = begin_date.strftime("%Y-%m-%d") + + org_name_query = Q(dict(match_phrase=dict(org_name=org_name))) + report_id_query = Q(dict(match_phrase=dict(report_id=report_id))) + domain_query = Q(dict(match_phrase={"published_policy.domain": domain})) + begin_date_query = Q(dict(match=dict(date_begin=begin_date))) + end_date_query = Q(dict(match=dict(date_end=end_date))) + + if index_suffix is not None: + search_index = "dmarc_aggregate_{0}*".format(index_suffix) + else: + search_index = "dmarc_aggregate*" + if index_prefix is not None: + search_index = "{0}{1}".format(index_prefix, search_index) + search = Search(index=search_index) + query = org_name_query & report_id_query & domain_query + query = query & begin_date_query & end_date_query + search.query = query + + try: + existing = search.execute() + except Exception as error_: + begin_date_human = begin_date.strftime("%Y-%m-%d %H:%M:%SZ") + end_date_human = end_date.strftime("%Y-%m-%d %H:%M:%SZ") + + raise ElasticsearchError( + "Elasticsearch's search for existing report \ + error: {}".format(error_.__str__()) + ) + + if len(existing) > 0: + raise AlreadySaved( + "An aggregate report ID {0} from {1} about {2} " + "with a date range of {3} UTC to {4} UTC already " + "exists in " + "Elasticsearch".format( + report_id, org_name, domain, begin_date_human, end_date_human + ) + ) published_policy = _PublishedPolicy( domain=aggregate_report["policy_published"]["domain"], adkim=aggregate_report["policy_published"]["adkim"], @@ -508,8 +552,8 @@ for record in aggregate_report["records"]: begin_date = human_timestamp_to_datetime(record["interval_begin"], to_utc=True) end_date = human_timestamp_to_datetime(record["interval_end"], to_utc=True) - begin_date_human = begin_date.strftime("%Y-%m-%d %H:%M:%SZ") - end_date_human = end_date.strftime("%Y-%m-%d %H:%M:%SZ") + normalized_timespan = record["normalized_timespan"] + if monthly_indexes: index_date = begin_date.strftime("%Y-%m") else: @@ -517,41 +561,6 @@ aggregate_report["begin_date"] = begin_date aggregate_report["end_date"] = end_date date_range = [aggregate_report["begin_date"], aggregate_report["end_date"]] - - org_name_query = Q(dict(match_phrase=dict(org_name=org_name))) - report_id_query = Q(dict(match_phrase=dict(report_id=report_id))) - domain_query = Q(dict(match_phrase={"published_policy.domain": domain})) - begin_date_query = Q(dict(match=dict(date_begin=begin_date))) - end_date_query = Q(dict(match=dict(date_end=end_date))) - - if index_suffix is not None: - search_index = "dmarc_aggregate_{0}*".format(index_suffix) - else: - search_index = "dmarc_aggregate*" - if index_prefix is not None: - search_index = "{0}{1}".format(index_prefix, search_index) - search = Search(index=search_index) - query = org_name_query & report_id_query & domain_query - query = query & begin_date_query & end_date_query - search.query = query - - try: - existing = search.execute() - except Exception as error_: - raise ElasticsearchError( - "Elasticsearch's search for existing report \ - error: {}".format(error_.__str__()) - ) - - if len(existing) > 0: - raise AlreadySaved( - "An aggregate report ID {0} from {1} about {2} " - "with a date range of {3} UTC to {4} UTC already " - "exists in " - "Elasticsearch".format( - report_id, org_name, domain, begin_date_human, end_date_human - ) - ) agg_doc = _AggregateReportDoc( xml_schema=aggregate_report["xml_schema"], org_name=metadata["org_name"], @@ -559,9 +568,9 @@ org_extra_contact_info=metadata["org_extra_contact_info"], report_id=metadata["report_id"], date_range=date_range, - date_begin=aggregate_report["begin_date"], - date_end=aggregate_report["end_date"], - normalized_timespan=record["normalized_timespan"], + date_begin=begin_date, + date_end=end_date, + normalized_timespan=normalized_timespan, errors=metadata["errors"], published_policy=published_policy, source_ip_address=record["source"]["ip_address"], @@ -890,7 +899,7 @@ policy_doc = _SMTPTLSPolicyDoc( policy_domain=policy["policy_domain"], policy_type=policy["policy_type"], - succesful_session_count=policy["successful_session_count"], + successful_session_count=policy["successful_session_count"], failed_session_count=policy["failed_session_count"], policy_string=policy_strings, mx_host_patterns=mx_host_patterns, diff --git a/_modules/parsedmarc/opensearch.html b/_modules/parsedmarc/opensearch.html index 796c90b..eb68293 100644 --- a/_modules/parsedmarc/opensearch.html +++ b/_modules/parsedmarc/opensearch.html @@ -5,14 +5,14 @@ - parsedmarc.opensearch — parsedmarc 9.0.0 documentation + parsedmarc.opensearch — parsedmarc 9.0.1 documentation - + @@ -494,7 +494,51 @@ org_name = metadata["org_name"] report_id = metadata["report_id"] domain = aggregate_report["policy_published"]["domain"] + begin_date = human_timestamp_to_datetime(metadata["begin_date"], to_utc=True) + end_date = human_timestamp_to_datetime(metadata["end_date"], to_utc=True) + if monthly_indexes: + index_date = begin_date.strftime("%Y-%m") + else: + index_date = begin_date.strftime("%Y-%m-%d") + + org_name_query = Q(dict(match_phrase=dict(org_name=org_name))) + report_id_query = Q(dict(match_phrase=dict(report_id=report_id))) + domain_query = Q(dict(match_phrase={"published_policy.domain": domain})) + begin_date_query = Q(dict(match=dict(date_begin=begin_date))) + end_date_query = Q(dict(match=dict(date_end=end_date))) + + if index_suffix is not None: + search_index = "dmarc_aggregate_{0}*".format(index_suffix) + else: + search_index = "dmarc_aggregate*" + if index_prefix is not None: + search_index = "{0}{1}".format(index_prefix, search_index) + search = Search(index=search_index) + query = org_name_query & report_id_query & domain_query + query = query & begin_date_query & end_date_query + search.query = query + + try: + existing = search.execute() + except Exception as error_: + begin_date_human = begin_date.strftime("%Y-%m-%d %H:%M:%SZ") + end_date_human = end_date.strftime("%Y-%m-%d %H:%M:%SZ") + + raise OpenSearchError( + "Opensearch's search for existing report \ + error: {}".format(error_.__str__()) + ) + + if len(existing) > 0: + raise AlreadySaved( + "An aggregate report ID {0} from {1} about {2} " + "with a date range of {3} UTC to {4} UTC already " + "exists in " + "Opensearch".format( + report_id, org_name, domain, begin_date_human, end_date_human + ) + ) published_policy = _PublishedPolicy( domain=aggregate_report["policy_published"]["domain"], adkim=aggregate_report["policy_published"]["adkim"], @@ -508,8 +552,8 @@ for record in aggregate_report["records"]: begin_date = human_timestamp_to_datetime(record["interval_begin"], to_utc=True) end_date = human_timestamp_to_datetime(record["interval_end"], to_utc=True) - begin_date_human = begin_date.strftime("%Y-%m-%d %H:%M:%SZ") - end_date_human = end_date.strftime("%Y-%m-%d %H:%M:%SZ") + normalized_timespan = record["normalized_timespan"] + if monthly_indexes: index_date = begin_date.strftime("%Y-%m") else: @@ -517,41 +561,6 @@ aggregate_report["begin_date"] = begin_date aggregate_report["end_date"] = end_date date_range = [aggregate_report["begin_date"], aggregate_report["end_date"]] - - org_name_query = Q(dict(match_phrase=dict(org_name=org_name))) - report_id_query = Q(dict(match_phrase=dict(report_id=report_id))) - domain_query = Q(dict(match_phrase={"published_policy.domain": domain})) - begin_date_query = Q(dict(match=dict(date_begin=begin_date))) - end_date_query = Q(dict(match=dict(date_end=end_date))) - - if index_suffix is not None: - search_index = "dmarc_aggregate_{0}*".format(index_suffix) - else: - search_index = "dmarc_aggregate*" - if index_prefix is not None: - search_index = "{0}{1}".format(index_prefix, search_index) - search = Search(index=search_index) - query = org_name_query & report_id_query & domain_query - query = query & begin_date_query & end_date_query - search.query = query - - try: - existing = search.execute() - except Exception as error_: - raise OpenSearchError( - "OpenSearch's search for existing report \ - error: {}".format(error_.__str__()) - ) - - if len(existing) > 0: - raise AlreadySaved( - "An aggregate report ID {0} from {1} about {2} " - "with a date range of {3} UTC to {4} UTC already " - "exists in " - "OpenSearch".format( - report_id, org_name, domain, begin_date_human, end_date_human - ) - ) agg_doc = _AggregateReportDoc( xml_schema=aggregate_report["xml_schema"], org_name=metadata["org_name"], @@ -559,8 +568,9 @@ org_extra_contact_info=metadata["org_extra_contact_info"], report_id=metadata["report_id"], date_range=date_range, - date_begin=aggregate_report["begin_date"], - date_end=aggregate_report["end_date"], + date_begin=begin_date, + date_end=end_date, + normalized_timespan=normalized_timespan, errors=metadata["errors"], published_policy=published_policy, source_ip_address=record["source"]["ip_address"], diff --git a/_modules/parsedmarc/splunk.html b/_modules/parsedmarc/splunk.html index 5f15088..0de7f9d 100644 --- a/_modules/parsedmarc/splunk.html +++ b/_modules/parsedmarc/splunk.html @@ -5,14 +5,14 @@ - parsedmarc.splunk — parsedmarc 9.0.0 documentation + parsedmarc.splunk — parsedmarc 9.0.1 documentation - + diff --git a/_modules/parsedmarc/utils.html b/_modules/parsedmarc/utils.html index a66ef67..7e7e543 100644 --- a/_modules/parsedmarc/utils.html +++ b/_modules/parsedmarc/utils.html @@ -5,14 +5,14 @@ - parsedmarc.utils — parsedmarc 9.0.0 documentation + parsedmarc.utils — parsedmarc 9.0.1 documentation - + diff --git a/_static/documentation_options.js b/_static/documentation_options.js index 0f0cd60..1a83982 100644 --- a/_static/documentation_options.js +++ b/_static/documentation_options.js @@ -1,5 +1,5 @@ const DOCUMENTATION_OPTIONS = { - VERSION: '9.0.0', + VERSION: '9.0.1', LANGUAGE: 'en', COLLAPSE_INDEX: false, BUILDER: 'html', diff --git a/api.html b/api.html index 781033e..660f546 100644 --- a/api.html +++ b/api.html @@ -6,14 +6,14 @@ - API reference — parsedmarc 9.0.0 documentation + API reference — parsedmarc 9.0.1 documentation - + diff --git a/contributing.html b/contributing.html index b31de9c..0d88a4c 100644 --- a/contributing.html +++ b/contributing.html @@ -6,14 +6,14 @@ - Contributing to parsedmarc — parsedmarc 9.0.0 documentation + Contributing to parsedmarc — parsedmarc 9.0.1 documentation - + diff --git a/davmail.html b/davmail.html index 2e65bf3..5ecbc91 100644 --- a/davmail.html +++ b/davmail.html @@ -6,14 +6,14 @@ - Accessing an inbox using OWA/EWS — parsedmarc 9.0.0 documentation + Accessing an inbox using OWA/EWS — parsedmarc 9.0.1 documentation - + diff --git a/dmarc.html b/dmarc.html index b759608..9f645c3 100644 --- a/dmarc.html +++ b/dmarc.html @@ -6,14 +6,14 @@ - Understanding DMARC — parsedmarc 9.0.0 documentation + Understanding DMARC — parsedmarc 9.0.1 documentation - + diff --git a/elasticsearch.html b/elasticsearch.html index a74e574..be4038e 100644 --- a/elasticsearch.html +++ b/elasticsearch.html @@ -6,14 +6,14 @@ - Elasticsearch and Kibana — parsedmarc 9.0.0 documentation + Elasticsearch and Kibana — parsedmarc 9.0.1 documentation - + diff --git a/genindex.html b/genindex.html index d58fe08..ba0b0b3 100644 --- a/genindex.html +++ b/genindex.html @@ -5,14 +5,14 @@ - Index — parsedmarc 9.0.0 documentation + Index — parsedmarc 9.0.1 documentation - + diff --git a/index.html b/index.html index 569764f..dff7e41 100644 --- a/index.html +++ b/index.html @@ -6,14 +6,14 @@ - parsedmarc documentation - Open source DMARC report analyzer and visualizer — parsedmarc 9.0.0 documentation + parsedmarc documentation - Open source DMARC report analyzer and visualizer — parsedmarc 9.0.1 documentation - + diff --git a/installation.html b/installation.html index e4cf5c9..df03b08 100644 --- a/installation.html +++ b/installation.html @@ -6,14 +6,14 @@ - Installation — parsedmarc 9.0.0 documentation + Installation — parsedmarc 9.0.1 documentation - + diff --git a/kibana.html b/kibana.html index 4d15b44..6f26dd0 100644 --- a/kibana.html +++ b/kibana.html @@ -6,14 +6,14 @@ - Using the Kibana dashboards — parsedmarc 9.0.0 documentation + Using the Kibana dashboards — parsedmarc 9.0.1 documentation - + diff --git a/mailing-lists.html b/mailing-lists.html index d418eae..8cbf21e 100644 --- a/mailing-lists.html +++ b/mailing-lists.html @@ -6,14 +6,14 @@ - What about mailing lists? — parsedmarc 9.0.0 documentation + What about mailing lists? — parsedmarc 9.0.1 documentation - + diff --git a/objects.inv b/objects.inv index 93d3160..4ec7155 100644 Binary files a/objects.inv and b/objects.inv differ diff --git a/opensearch.html b/opensearch.html index e3dfe3b..7a0dc0a 100644 --- a/opensearch.html +++ b/opensearch.html @@ -6,14 +6,14 @@ - OpenSearch and Grafana — parsedmarc 9.0.0 documentation + OpenSearch and Grafana — parsedmarc 9.0.1 documentation - + diff --git a/output.html b/output.html index d9f2753..cc3bd0c 100644 --- a/output.html +++ b/output.html @@ -6,14 +6,14 @@ - Sample outputs — parsedmarc 9.0.0 documentation + Sample outputs — parsedmarc 9.0.1 documentation - + diff --git a/py-modindex.html b/py-modindex.html index e81c5e1..0108ef4 100644 --- a/py-modindex.html +++ b/py-modindex.html @@ -5,14 +5,14 @@ - Python Module Index — parsedmarc 9.0.0 documentation + Python Module Index — parsedmarc 9.0.1 documentation - + diff --git a/search.html b/search.html index b0fe352..f404f30 100644 --- a/search.html +++ b/search.html @@ -5,7 +5,7 @@ - Search — parsedmarc 9.0.0 documentation + Search — parsedmarc 9.0.1 documentation @@ -13,7 +13,7 @@ - + diff --git a/splunk.html b/splunk.html index 20c6231..af7434a 100644 --- a/splunk.html +++ b/splunk.html @@ -6,14 +6,14 @@ - Splunk — parsedmarc 9.0.0 documentation + Splunk — parsedmarc 9.0.1 documentation - + diff --git a/usage.html b/usage.html index 1248b45..2cd1349 100644 --- a/usage.html +++ b/usage.html @@ -6,14 +6,14 @@ - Using parsedmarc — parsedmarc 9.0.0 documentation + Using parsedmarc — parsedmarc 9.0.1 documentation - +