diff --git a/CHANGELOG.md b/CHANGELOG.md index f5eaa31..57ce317 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ 5.1.0 ----- +- Bugfix: Submit aggregate dates to Elasticsearch as lists, not tuples +- Support `elasticsearch-dsl<=6.3.0` - Add support for TLS/SSL and uerename/password auth to Kafka 5.0.2 diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index 927c0ea..e5997fa 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -157,13 +157,9 @@ def _main(): arg_parser.add_argument("-K", "--kafka-hosts", nargs="*", help="A list of one or more Kafka hostnames") arg_parser.add_argument("--kafka-username", - help='An optional Kafka username\nNote: For ' - 'Azure Event hub, this is literally ' - '"$ConnectionString"') + help='An optional Kafka username') arg_parser.add_argument("--kafka-password", - help="An optional Kafka password\n" - "Note: For Azure Event hub, this is the " - "Azure-provided connection string") + help="An optional Kafka password") arg_parser.add_argument("--kafka-use-ssl", action="store_true", help="Use SSL/TLS to connect to Kafka " diff --git a/parsedmarc/elastic.py b/parsedmarc/elastic.py index 87d8f79..d4ab59e 100644 --- a/parsedmarc/elastic.py +++ b/parsedmarc/elastic.py @@ -5,7 +5,7 @@ from collections import OrderedDict from elasticsearch_dsl.search import Q from elasticsearch_dsl import connections, Object, Document, Index, Nested, \ - InnerDoc, Integer, Text, Boolean, DateRange, Ip, Date, Search + InnerDoc, Integer, Text, Boolean, Ip, Date, Search from elasticsearch.helpers import reindex @@ -54,7 +54,7 @@ class _AggregateReportDoc(Document): org_email = Text() org_extra_contact_info = Text() report_id = Text() - date_range = DateRange() + date_range = Date() errors = Text() published_policy = Object(_PublishedPolicy) source_ip_address = Ip() @@ -277,8 +277,8 @@ def save_aggregate_report_to_elasticsearch(aggregate_report, index_date = begin_date.strftime("%Y-%m-%d") aggregate_report["begin_date"] = begin_date aggregate_report["end_date"] = end_date - date_range = (aggregate_report["begin_date"], - aggregate_report["end_date"]) + date_range = [aggregate_report["begin_date"], + aggregate_report["end_date"]] org_name_query = Q(dict(match=dict(org_name=org_name))) report_id_query = Q(dict(match=dict(report_id=report_id))) diff --git a/requirements.txt b/requirements.txt index cb88f88..04e1e60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,8 +9,8 @@ dnspython imapclient mail-parser dateparser -elasticsearch==6.3.1 -elasticsearch-dsl==6.2.1 +elasticsearch +elasticsearch-dsl kafka-python flake8 sphinx diff --git a/setup.py b/setup.py index 1b5d3ad..95d92f8 100644 --- a/setup.py +++ b/setup.py @@ -95,8 +95,8 @@ setup( install_requires=['dnspython', 'expiringdict', 'publicsuffix', 'xmltodict', 'geoip2', 'urllib3<1.24,>=1.21.1', 'requests', 'imapclient', 'mail-parser', 'dateparser', - 'elasticsearch==6.3.1', - 'elasticsearch-dsl==6.2.1', 'kafka-python' + 'elasticsearch', + 'elasticsearch-dsl', 'kafka-python' ], entry_points={