From 0aa7d84d0df7d6e20ddd99e7a8b3403d897d2acb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matth=C3=A4us=20Wander?= Date: Sun, 6 Jun 2021 18:41:23 +0200 Subject: [PATCH 1/2] Use UTC datetimes for Elastic. Elastic by default expects UTC. --- parsedmarc/elastic.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/parsedmarc/elastic.py b/parsedmarc/elastic.py index d6c3ae0..4eb64f0 100644 --- a/parsedmarc/elastic.py +++ b/parsedmarc/elastic.py @@ -301,10 +301,10 @@ def save_aggregate_report_to_elasticsearch(aggregate_report, org_name = metadata["org_name"] report_id = metadata["report_id"] domain = aggregate_report["policy_published"]["domain"] - begin_date = human_timestamp_to_datetime(metadata["begin_date"]) - end_date = human_timestamp_to_datetime(metadata["end_date"]) - begin_date_human = begin_date.strftime("%Y-%m-%d %H:%M:%S") - end_date_human = end_date.strftime("%Y-%m-%d %H:%M:%S") + begin_date = human_timestamp_to_datetime(metadata["begin_date"], to_utc=True) + end_date = human_timestamp_to_datetime(metadata["end_date"], to_utc=True) + begin_date_human = begin_date.strftime("%Y-%m-%d %H:%M:%SZ") + end_date_human = end_date.strftime("%Y-%m-%d %H:%M:%SZ") if monthly_indexes: index_date = begin_date.strftime("%Y-%m") else: From 4bc7b0b62c4423a79a451aeb4d00f790b2ba647b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matth=C3=A4us=20Wander?= Date: Mon, 7 Jun 2021 00:03:39 +0200 Subject: [PATCH 2/2] deduplicate over date_begin and date_end instead of date_range --- parsedmarc/elastic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parsedmarc/elastic.py b/parsedmarc/elastic.py index 4eb64f0..32cdea4 100644 --- a/parsedmarc/elastic.py +++ b/parsedmarc/elastic.py @@ -317,8 +317,8 @@ def save_aggregate_report_to_elasticsearch(aggregate_report, org_name_query = Q(dict(match_phrase=dict(org_name=org_name))) report_id_query = Q(dict(match_phrase=dict(report_id=report_id))) domain_query = Q(dict(match_phrase={"published_policy.domain": domain})) - begin_date_query = Q(dict(match=dict(date_range=begin_date))) - end_date_query = Q(dict(match=dict(date_range=end_date))) + begin_date_query = Q(dict(match=dict(date_begin=begin_date))) + end_date_query = Q(dict(match=dict(date_end=end_date))) if index_suffix is not None: search = Search(index="dmarc_aggregate_{0}*".format(index_suffix))