From dedbd518e837a90d2387a8703fe7c5d1a54c9d06 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 20 Oct 2020 00:50:06 +1100 Subject: [PATCH 1/7] test --- test.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 test.txt diff --git a/test.txt b/test.txt new file mode 100644 index 0000000..9daeafb --- /dev/null +++ b/test.txt @@ -0,0 +1 @@ +test From bd7819881d7d5654480ecf06d8f93cf10bd34770 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 20 Oct 2020 00:52:01 +1100 Subject: [PATCH 2/7] delete test file --- test.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 test.txt diff --git a/test.txt b/test.txt deleted file mode 100644 index 9daeafb..0000000 --- a/test.txt +++ /dev/null @@ -1 +0,0 @@ -test From 3312387852d8842382be8dcf20bf36f3c7971b34 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 20 Oct 2020 01:14:43 +1100 Subject: [PATCH 3/7] Take 1 trying to add elastic basic auth --- parsedmarc/cli.py | 10 +++++++++- parsedmarc/elastic.py | 5 ++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index c2309f8..1a5f69e 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -219,6 +219,8 @@ def _main(): elasticsearch_ssl=True, elasticsearch_ssl_cert_path=None, elasticsearch_monthly_indexes=False, + elasticsearch_username=None, + elasticsearch_password=None, kafka_hosts=None, kafka_username=None, kafka_password=None, @@ -352,6 +354,12 @@ def _main(): if "cert_path" in elasticsearch_config: opts.elasticsearch_ssl_cert_path = elasticsearch_config[ "cert_path"] + if "user" in elasticsearch_config: + opts.elasticsearch_username = elasticsearch_config[ + "user"] + if "password" in elasticsearch_config: + opts.elasticsearch_password = elasticsearch_config[ + "password"] if "splunk_hec" in config.sections(): hec_config = config["splunk_hec"] if "url" in hec_config: @@ -488,7 +496,7 @@ def _main(): elastic.set_hosts(opts.elasticsearch_hosts, opts.elasticsearch_ssl, opts.elasticsearch_ssl_cert_path, - timeout=opts.elasticsearch_timeout) + timeout=opts.elasticsearch_timeout,elesticsearch_username,elasticsearch_password) elastic.migrate_indexes(aggregate_indexes=[es_aggregate_index], forensic_indexes=[es_forensic_index]) except elastic.ElasticsearchError as error: diff --git a/parsedmarc/elastic.py b/parsedmarc/elastic.py index b46fbb0..05d87b5 100644 --- a/parsedmarc/elastic.py +++ b/parsedmarc/elastic.py @@ -171,7 +171,7 @@ class AlreadySaved(ValueError): """Raised when a report to be saved matches an existing report""" -def set_hosts(hosts, use_ssl=False, ssl_cert_path=None, timeout=60.0): +def set_hosts(hosts, use_ssl=False, ssl_cert_path=None, timeout=60.0, username=None, password=None): """ Sets the Elasticsearch hosts to use @@ -194,6 +194,9 @@ def set_hosts(hosts, use_ssl=False, ssl_cert_path=None, timeout=60.0): conn_params['ca_certs'] = ssl_cert_path else: conn_params['verify_certs'] = False + if username: + conn_params['http_auth']=(username+":"+password) + connections.create_connection(**conn_params) From a36dc21c07fda80112060464099726b3c88dbbe2 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 20 Oct 2020 01:23:18 +1100 Subject: [PATCH 4/7] Try 2 --- parsedmarc/elastic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parsedmarc/elastic.py b/parsedmarc/elastic.py index 05d87b5..695f678 100644 --- a/parsedmarc/elastic.py +++ b/parsedmarc/elastic.py @@ -171,7 +171,7 @@ class AlreadySaved(ValueError): """Raised when a report to be saved matches an existing report""" -def set_hosts(hosts, use_ssl=False, ssl_cert_path=None, timeout=60.0, username=None, password=None): +def set_hosts(hosts, use_ssl=False, ssl_cert_path=None, timeout=60.0, username, password): """ Sets the Elasticsearch hosts to use From de02edc0a9ac0038f22fc38a3a22ca5add34c9f9 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 20 Oct 2020 01:27:45 +1100 Subject: [PATCH 5/7] take 3 --- parsedmarc/cli.py | 4 +++- parsedmarc/elastic.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index 1a5f69e..0a5d895 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -496,7 +496,9 @@ def _main(): elastic.set_hosts(opts.elasticsearch_hosts, opts.elasticsearch_ssl, opts.elasticsearch_ssl_cert_path, - timeout=opts.elasticsearch_timeout,elesticsearch_username,elasticsearch_password) + opts.elesticsearch_username, + opts.elasticsearch_password, + timeout=opts.elasticsearch_timeout) elastic.migrate_indexes(aggregate_indexes=[es_aggregate_index], forensic_indexes=[es_forensic_index]) except elastic.ElasticsearchError as error: diff --git a/parsedmarc/elastic.py b/parsedmarc/elastic.py index 695f678..6e9b654 100644 --- a/parsedmarc/elastic.py +++ b/parsedmarc/elastic.py @@ -171,7 +171,7 @@ class AlreadySaved(ValueError): """Raised when a report to be saved matches an existing report""" -def set_hosts(hosts, use_ssl=False, ssl_cert_path=None, timeout=60.0, username, password): +def set_hosts(hosts, use_ssl=False, ssl_cert_path=None, username=None, password=None, timeout=60.0): """ Sets the Elasticsearch hosts to use From 07f8a30f080b1d968ff4d533f1cf521ac4c680f9 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 20 Oct 2020 01:30:08 +1100 Subject: [PATCH 6/7] tabs and spaces :( --- parsedmarc/elastic.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/parsedmarc/elastic.py b/parsedmarc/elastic.py index 6e9b654..762940e 100644 --- a/parsedmarc/elastic.py +++ b/parsedmarc/elastic.py @@ -195,8 +195,7 @@ def set_hosts(hosts, use_ssl=False, ssl_cert_path=None, username=None, password= else: conn_params['verify_certs'] = False if username: - conn_params['http_auth']=(username+":"+password) - + conn_params['http_auth']=(username+":"+password) connections.create_connection(**conn_params) From f18322c16c9ea79c45daf61ab2df194f6f43bf29 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 20 Oct 2020 01:31:30 +1100 Subject: [PATCH 7/7] I can spell.... sometimes --- parsedmarc/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index 0a5d895..64ac175 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -496,7 +496,7 @@ def _main(): elastic.set_hosts(opts.elasticsearch_hosts, opts.elasticsearch_ssl, opts.elasticsearch_ssl_cert_path, - opts.elesticsearch_username, + opts.elasticsearch_username, opts.elasticsearch_password, timeout=opts.elasticsearch_timeout) elastic.migrate_indexes(aggregate_indexes=[es_aggregate_index],