From 83e229aeb1e563088de76c5d573bfab8a0a0ae37 Mon Sep 17 00:00:00 2001 From: atanas argirov Date: Mon, 28 Dec 2020 15:57:32 +0000 Subject: [PATCH 01/28] * added output_{json,csv}_{aggregate,forensic}_file command line args * refactored save_output() to support output_* --- parsedmarc/__init__.py | 14 +++++++++----- parsedmarc/cli.py | 14 +++++++++++++- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 4924d07..49de20b 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -1235,7 +1235,11 @@ def watch_inbox(host, username, password, callback, port=None, ssl=True, logger.warning("IMAP connection timeout. Reconnecting...") -def save_output(results, output_directory="output"): +def save_output(results, output_directory="output", \ + output_json_aggregate_file="aggregate.json", \ + output_json_forensic_file="forensic.json", \ + output_csv_aggregate_file="aggregate.csv", \ + output_csv_forensic_file="forensic.csv"): """ Save report data in the given directory @@ -1253,22 +1257,22 @@ def save_output(results, output_directory="output"): else: os.makedirs(output_directory) - with open("{0}".format(os.path.join(output_directory, "aggregate.json")), + with open("{0}".format(os.path.join(output_directory, output_json_aggregate_file)), "w", newline="\n", encoding="utf-8") as agg_json: agg_json.write(json.dumps(aggregate_reports, ensure_ascii=False, indent=2)) - with open("{0}".format(os.path.join(output_directory, "aggregate.csv")), + with open("{0}".format(os.path.join(output_directory, output_csv_aggregate_file)), "w", newline="\n", encoding="utf-8") as agg_csv: csv = parsed_aggregate_reports_to_csv(aggregate_reports) agg_csv.write(csv) - with open("{0}".format(os.path.join(output_directory, "forensic.json")), + with open("{0}".format(os.path.join(output_directory, output_json_forensic_file)), "w", newline="\n", encoding="utf-8") as for_json: for_json.write(json.dumps(forensic_reports, ensure_ascii=False, indent=2)) - with open("{0}".format(os.path.join(output_directory, "forensic.csv")), + with open("{0}".format(os.path.join(output_directory, output_csv_forensic_file)), "w", newline="\n", encoding="utf-8") as for_csv: csv = parsed_forensic_reports_to_csv(forensic_reports) for_csv.write(csv) diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index 9dad3fb..262a82b 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -82,7 +82,7 @@ def _main(): if opts.save_aggregate: for report in reports_["aggregate_reports"]: try: - if opts.elasticsearch_hosts: + opts.elasticsearch_hosts: shards = opts.elasticsearch_number_of_shards replicas = opts.elasticsearch_number_of_replicas elastic.save_aggregate_report_to_elasticsearch( @@ -160,6 +160,14 @@ def _main(): help=strip_attachment_help, action="store_true") arg_parser.add_argument("-o", "--output", help="write output files to the given directory") + arg_parser.add_argument("--output-json-aggregate-file", + help="output aggregate JSON file") + arg_parser.add_argument("--output-json-forensic-file", + help="output forensic JSON file") + arg_parser.add_argument("--output-csv-aggregate-file", + help="output aggregate CSV file") + arg_parser.add_argument("--output-csv-forensic-file", + help="output forensic CSV file") arg_parser.add_argument("-n", "--nameservers", nargs="+", help="nameservers to query") arg_parser.add_argument("-t", "--dns_timeout", @@ -188,6 +196,10 @@ def _main(): offline=args.offline, strip_attachment_payloads=args.strip_attachment_payloads, output=args.output, + output_json_aggregate_file=args.output_json_aggregate_file, + output_json_forensic_file=args.output_json_forensic_file, + output_csv_aggregate_file=args.output_csv_aggregate_file, + output_csv_forensic_file=args.output_csv_forensic_file, nameservers=args.nameservers, silent=args.silent, dns_timeout=args.dns_timeout, From 478452de203fbdcb4644b989c066124f6c98eb6a Mon Sep 17 00:00:00 2001 From: Mauro Faccenda Date: Wed, 20 Jan 2021 15:53:19 +0100 Subject: [PATCH 02/28] pass offline parameter to wait_inbox() --- parsedmarc/cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index e9119e0..68d9e89 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -670,6 +670,7 @@ def _main(): test=opts.imap_test, nameservers=opts.nameservers, dns_timeout=opts.dns_timeout, + offline=opts.offline, strip_attachment_payloads=sa) except FileExistsError as error: logger.error("{0}".format(error.__str__())) From be8395dbe341e5cca09fd7ff3630b6240f846ac3 Mon Sep 17 00:00:00 2001 From: Ola Thoresen Date: Wed, 20 Jan 2021 19:56:15 +0100 Subject: [PATCH 03/28] Detecting other IMAP-errors. Adding short sleep to avoid hammering the IMAP-server on error --- parsedmarc/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index ca4f1e7..5fe1f29 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -8,6 +8,7 @@ import shutil import xml.parsers.expat as expat import json from datetime import datetime +from time import sleep from collections import OrderedDict from io import BytesIO, StringIO from gzip import GzipFile @@ -1234,6 +1235,10 @@ def watch_inbox(host, username, password, callback, port=None, ssl=True, idle_timeout=idle_timeout) except (timeout, IMAPClientError): logger.warning("IMAP connection timeout. Reconnecting...") + sleep(5) + except Exception as e: + logger.warning("IMAP connection error. {0}. Reconnecting...".format(e)) + sleep(5) def save_output(results, output_directory="output"): From 0e2636225e7ffdb2ca7705075da665cab047a0c3 Mon Sep 17 00:00:00 2001 From: Ola Thoresen Date: Thu, 21 Jan 2021 08:24:44 +0100 Subject: [PATCH 04/28] Modifying some log-levels to INFO --- parsedmarc/__init__.py | 4 ++-- parsedmarc/cli.py | 2 ++ parsedmarc/elastic.py | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index ca4f1e7..cde57c9 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -943,7 +943,7 @@ def get_dmarc_reports_from_mbox(input_, nameservers=None, dns_timeout=2.0, input_)) for i in range(len(message_keys)): message_key = message_keys[i] - logger.debug("Processing message {0} of {1}".format( + logger.info("Processing message {0} of {1}".format( i+1, total_messages )) msg_content = mbox.get_string(message_key) @@ -1071,7 +1071,7 @@ def get_dmarc_reports_from_inbox(connection=None, reports_folder)) for i in range(len(messages)): msg_uid = messages[i] - logger.debug("Processing message {0} of {1}: UID {2}".format( + logger.info("Processing message {0} of {1}: UID {2}".format( i+1, total_messages, msg_uid )) diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index e9119e0..752eacd 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -490,6 +490,8 @@ def _main(): logger.error("You must supply input files, or an IMAP configuration") exit(1) + logger.info("Starting dmarcparse") + if opts.save_aggregate or opts.save_forensic: try: if opts.elasticsearch_hosts: diff --git a/parsedmarc/elastic.py b/parsedmarc/elastic.py index b448f28..3f2d8a2 100644 --- a/parsedmarc/elastic.py +++ b/parsedmarc/elastic.py @@ -295,7 +295,7 @@ def save_aggregate_report_to_elasticsearch(aggregate_report, Raises: AlreadySaved """ - logger.debug("Saving aggregate report to Elasticsearch") + logger.info("Saving aggregate report to Elasticsearch") aggregate_report = aggregate_report.copy() metadata = aggregate_report["report_metadata"] org_name = metadata["org_name"] @@ -423,7 +423,7 @@ def save_forensic_report_to_elasticsearch(forensic_report, AlreadySaved """ - logger.debug("Saving forensic report to Elasticsearch") + logger.info("Saving forensic report to Elasticsearch") forensic_report = forensic_report.copy() sample_date = None if forensic_report["parsed_sample"]["date"] is not None: From 76614bdc94acc796b0307372e640d3836f0a77c2 Mon Sep 17 00:00:00 2001 From: Ola Thoresen Date: Thu, 21 Jan 2021 08:34:56 +0100 Subject: [PATCH 05/28] Fixing flake-error --- parsedmarc/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 5fe1f29..809dd63 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -1237,7 +1237,8 @@ def watch_inbox(host, username, password, callback, port=None, ssl=True, logger.warning("IMAP connection timeout. Reconnecting...") sleep(5) except Exception as e: - logger.warning("IMAP connection error. {0}. Reconnecting...".format(e)) + logger.warning("IMAP connection error. {0}. " + "Reconnecting...".format(e)) sleep(5) From a00cee8ba4c493f37194e948a5de55655307fb86 Mon Sep 17 00:00:00 2001 From: Ola Thoresen Date: Fri, 22 Jan 2021 10:38:04 +0100 Subject: [PATCH 06/28] Adding a log line to see the sender of a report when it is parsed --- parsedmarc/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index cde57c9..5ac6965 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -776,6 +776,7 @@ def parse_report_email(input_, offline=False, nameservers=None, subject = None feedback_report = None sample = None + logger.info("Parsing mail from {0}".format(msg_headers["From"])) if "Subject" in msg_headers: subject = msg_headers["Subject"] for part in msg.walk(): From c853c470879ad2ba2cc56595197cb9fe9f2927d9 Mon Sep 17 00:00:00 2001 From: Ola Thoresen Date: Fri, 22 Jan 2021 15:06:35 +0100 Subject: [PATCH 07/28] Ensuring mail from is set --- parsedmarc/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 5ac6965..73c0130 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -776,7 +776,8 @@ def parse_report_email(input_, offline=False, nameservers=None, subject = None feedback_report = None sample = None - logger.info("Parsing mail from {0}".format(msg_headers["From"])) + if "From" in msg_headers: + logger.info("Parsing mail from {0}".format(msg_headers["From"])) if "Subject" in msg_headers: subject = msg_headers["Subject"] for part in msg.walk(): From bafa4861b153199db60145ae7020d18ddaa7a4bc Mon Sep 17 00:00:00 2001 From: Tom Henderson Date: Fri, 5 Feb 2021 14:27:22 +1300 Subject: [PATCH 08/28] Update docs --- README.rst | 11 +++++++++-- docs/example.ini | 4 ++++ docs/index.rst | 12 +++++++++--- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index b028445..2d297f9 100644 --- a/README.rst +++ b/README.rst @@ -128,11 +128,15 @@ For example token = HECTokenGoesHere index = email + [s3] + bucket = my-bucket + path = /parsedmarc + The full set of configuration options are: - ``general`` - - ``save_aggregate`` - bool: Save aggregate report data to the Elasticsearch and/or Splunk - - ``save_forensic`` - bool: Save forensic report data to the Elasticsearch and/or Splunk + - ``save_aggregate`` - bool: Save aggregate report data to Elasticsearch, Splunk and/or S3 + - ``save_forensic`` - bool: Save forensic report data to Elasticsearch, Splunk and/or S3 - ``strip_attachment_payloads`` - bool: Remove attachment payloads from results - ``output`` - str: Directory to place JSON and CSV files in - ``offline`` - bool: Do not use online queries for geolocation or DNS @@ -191,6 +195,9 @@ The full set of configuration options are: - ``subject`` - str: The Subject header to use in the email (Default: parsedmarc report) - ``attachment`` - str: The ZIP attachment filenames - ``message`` - str: The email message (Default: Please see the attached parsedmarc report.) +- ``s3`` + - ``bucket`` - str: The S3 bucket name + - ``path`` - int: The path to upload reports to (Default: /) .. warning:: diff --git a/docs/example.ini b/docs/example.ini index a27a670..a9a2985 100644 --- a/docs/example.ini +++ b/docs/example.ini @@ -18,3 +18,7 @@ ssl = False url = https://splunkhec.example.com token = HECTokenGoesHere index = email + +[s3] +bucket = my-bucket +path = /parsedmarc \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 449f048..7004cc4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -132,11 +132,15 @@ For example token = HECTokenGoesHere index = email + [s3] + bucket = my-bucket + path = /parsedmarc + The full set of configuration options are: - ``general`` - - ``save_aggregate`` - bool: Save aggregate report data to the Elasticsearch and/or Splunk - - ``save_forensic`` - bool: Save forensic report data to the Elasticsearch and/or Splunk + - ``save_aggregate`` - bool: Save aggregate report data to the Elasticsearch, Splunk and/or S3 + - ``save_forensic`` - bool: Save forensic report data to the Elasticsearch, Splunk and/or S3 - ``strip_attachment_payloads`` - bool: Remove attachment payloads from results - ``output`` - str: Directory to place JSON and CSV files in - ``offline`` - bool: Do not use online queries for geolocation or DNS @@ -200,7 +204,9 @@ The full set of configuration options are: - ``subject`` - str: The Subject header to use in the email (Default: parsedmarc report) - ``attachment`` - str: The ZIP attachment filenames - ``message`` - str: The email message (Default: Please see the attached parsedmarc report.) - +- ``s3`` + - ``bucket`` - str: The S3 bucket name + - ``path`` - int: The path to upload reports to (Default: /) .. warning:: From 755ee3ded70adf4c61debf09aaddc150ab757bee Mon Sep 17 00:00:00 2001 From: Tom Henderson Date: Fri, 5 Feb 2021 14:28:46 +1300 Subject: [PATCH 09/28] Add new settings for s3 --- parsedmarc/cli.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index e9119e0..a403391 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -241,6 +241,8 @@ def _main(): smtp_to=[], smtp_subject="parsedmarc report", smtp_message="Please see the attached DMARC results.", + s3_bucket=None, + s3_path=None, log_file=args.log_file, n_procs=1, chunk_size=1 @@ -469,6 +471,22 @@ def _main(): opts.smtp_attachment = smtp_config["attachment"] if "message" in smtp_config: opts.smtp_message = smtp_config["message"] + if "s3" in config.sections(): + s3_config = config["s3"] + if "bucket" in s3_config: + opts.s3_bucket = s3_config["bucket"] + else: + logger.critical("bucket setting missing from the " + "s3 config section") + exit(-1) + if "path" in s3_config: + opts.s3_path = s3_config["path"] + if opts.s3_path.startswith("/"): + opts.s3_path = opts.s3_path[1:] + if opts.s3_path.endswith("/"): + opts.s3_path = opts.s3_path[:-1] + else: + opts.s3_path = "" logging.basicConfig(level=logging.WARNING) logger.setLevel(logging.WARNING) From 291d389f69d279b249cab2b01c6bd7e7227b79e9 Mon Sep 17 00:00:00 2001 From: Tom Henderson Date: Fri, 5 Feb 2021 14:29:27 +1300 Subject: [PATCH 10/28] Add boto3 --- requirements.txt | 1 + setup.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4200a90..9ef3fab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,3 +27,4 @@ sphinx_rtd_theme>=0.4.3 wheel>=0.33.6 codecov>=2.0.15 lxml>=4.4.0 +boto3>=1.16.63 \ No newline at end of file diff --git a/setup.py b/setup.py index 6faaf91..6929893 100644 --- a/setup.py +++ b/setup.py @@ -98,7 +98,8 @@ setup( 'elasticsearch-dsl>=7.2.0,<8.0.0', 'kafka-python>=1.4.4', 'tqdm>=4.31.1', - 'lxml>=4.4.0' + 'lxml>=4.4.0', + 'boto3>=1.16.63' ], entry_points={ From a4acd5f2320ae6f518446c2a6223e7e664092301 Mon Sep 17 00:00:00 2001 From: Tom Henderson Date: Fri, 5 Feb 2021 14:30:02 +1300 Subject: [PATCH 11/28] Add S3Client --- parsedmarc/s3.py | 65 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 parsedmarc/s3.py diff --git a/parsedmarc/s3.py b/parsedmarc/s3.py new file mode 100644 index 0000000..8f64aa0 --- /dev/null +++ b/parsedmarc/s3.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- + +import logging +import json +import boto3 + +from parsedmarc.utils import human_timestamp_to_datetime + +logger = logging.getLogger("parsedmarc") + + +class S3Client(object): + """A client for a Amazon S3""" + + def __init__(self, bucket_name, bucket_path): + """ + Initializes the S3Client + Args: + bucket_name (str): The S3 Bucket + bucket_path (str): The path to save reports + """ + self.bucket_name = bucket_name + self.bucket_path = bucket_path + self.metadata_keys = [ + "org_name", + "org_email", + "report_id", + "begin_date", + "end_date", + ] + + self.s3 = boto3.resource('s3') + self.bucket = self.s3.Bucket(self.bucket_name) + + + def save_aggregate_report_to_s3(self, report): + self.save_report_to_s3(report, 'aggregate') + + + def save_forensic_report_to_s3(self, report): + self.save_report_to_s3(report, 'forensic') + + + def save_report_to_s3(self, report, report_type): + report_date = human_timestamp_to_datetime(report["report_metadata"]["begin_date"]) + report_id = report["report_metadata"]["report_id"] + object_path = "{0}/{1}/year={2}/month={3:02d}/day={4:02d}/{5}.json".format( + self.bucket_path, + report_type, + report_date.year, + report_date.month, + report_date.day, + report_id + ) + logger.debug("Saving {0} report to s3://{1}/{2}".format(report_type, self.bucket_name, object_path)) + object_metadata = { + k: v + for k, v in report["report_metadata"].items() + if k in self.metadata_keys + } + self.bucket.put_object( + Body=json.dumps(report), + Key=object_path, + Metadata=object_metadata + ) From 5f6b94583938ff3aedbf1d9e7c526e838e8c7c9c Mon Sep 17 00:00:00 2001 From: Tom Henderson Date: Fri, 5 Feb 2021 14:30:54 +1300 Subject: [PATCH 12/28] Save reports to s3 --- parsedmarc/cli.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index a403391..1087745 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -19,7 +19,7 @@ from tqdm import tqdm from parsedmarc import get_dmarc_reports_from_inbox, watch_inbox, \ parse_report_file, get_dmarc_reports_from_mbox, elastic, kafkaclient, \ splunk, save_output, email_results, ParserError, __version__, \ - InvalidDMARCReport + InvalidDMARCReport, s3 from parsedmarc.utils import is_mbox logger = logging.getLogger("parsedmarc") @@ -79,6 +79,14 @@ def _main(): ) except Exception as error_: logger.error("Kafka Error: {0}".format(error_.__str__())) + if opts.s3_bucket: + try: + s3_client = s3.S3Client( + bucket_name=opts.s3_bucket, + bucket_path=opts.s3_path, + ) + except Exception as error_: + logger.error("S3 Error: {0}".format(error_.__str__())) if opts.save_aggregate: for report in reports_["aggregate_reports"]: try: @@ -104,6 +112,11 @@ def _main(): except Exception as error_: logger.error("Kafka Error: {0}".format( error_.__str__())) + try: + if opts.s3_bucket: + s3_client.save_aggregate_report_to_s3(report) + except Exception as error_: + logger.error("S3 Error: {0}".format(error_.__str__())) if opts.hec: try: aggregate_reports_ = reports_["aggregate_reports"] @@ -138,6 +151,11 @@ def _main(): except Exception as error_: logger.error("Kafka Error: {0}".format( error_.__str__())) + try: + if opts.s3_bucket: + s3_client.save_forensic_report_to_s3(report) + except Exception as error_: + logger.error("S3 Error: {0}".format(error_.__str__())) if opts.hec: try: forensic_reports_ = reports_["forensic_reports"] From eba722cddceafd50cdcee3e7c2ed6b3aff01f387 Mon Sep 17 00:00:00 2001 From: Tom Henderson Date: Fri, 5 Feb 2021 14:38:52 +1300 Subject: [PATCH 13/28] Fix path example --- README.rst | 2 +- docs/example.ini | 2 +- docs/index.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 2d297f9..7c85ce1 100644 --- a/README.rst +++ b/README.rst @@ -130,7 +130,7 @@ For example [s3] bucket = my-bucket - path = /parsedmarc + path = parsedmarc The full set of configuration options are: diff --git a/docs/example.ini b/docs/example.ini index a9a2985..efa56b3 100644 --- a/docs/example.ini +++ b/docs/example.ini @@ -21,4 +21,4 @@ index = email [s3] bucket = my-bucket -path = /parsedmarc \ No newline at end of file +path = parsedmarc \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 7004cc4..c63ecef 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -134,7 +134,7 @@ For example [s3] bucket = my-bucket - path = /parsedmarc + path = parsedmarc The full set of configuration options are: From 85e7fd4ce6d60163e14341ed70f3da81bbe979f6 Mon Sep 17 00:00:00 2001 From: Tom Henderson Date: Fri, 5 Feb 2021 15:58:57 +1300 Subject: [PATCH 14/28] Fix flake8 errors --- parsedmarc/s3.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/parsedmarc/s3.py b/parsedmarc/s3.py index 8f64aa0..41910ed 100644 --- a/parsedmarc/s3.py +++ b/parsedmarc/s3.py @@ -28,23 +28,23 @@ class S3Client(object): "begin_date", "end_date", ] - + self.s3 = boto3.resource('s3') self.bucket = self.s3.Bucket(self.bucket_name) - def save_aggregate_report_to_s3(self, report): self.save_report_to_s3(report, 'aggregate') - def save_forensic_report_to_s3(self, report): self.save_report_to_s3(report, 'forensic') - def save_report_to_s3(self, report, report_type): - report_date = human_timestamp_to_datetime(report["report_metadata"]["begin_date"]) + report_date = human_timestamp_to_datetime( + report["report_metadata"]["begin_date"] + ) report_id = report["report_metadata"]["report_id"] - object_path = "{0}/{1}/year={2}/month={3:02d}/day={4:02d}/{5}.json".format( + path_template = "{0}/{1}/year={2}/month={3:02d}/day={4:02d}/{5}.json" + object_path = path_template.format( self.bucket_path, report_type, report_date.year, @@ -52,7 +52,10 @@ class S3Client(object): report_date.day, report_id ) - logger.debug("Saving {0} report to s3://{1}/{2}".format(report_type, self.bucket_name, object_path)) + logger.debug("Saving {0} report to s3://{1}/{2}".format( + report_type, + self.bucket_name, + object_path)) object_metadata = { k: v for k, v in report["report_metadata"].items() From 394dddd2df6c826b4702620b2c637fd2d895b59f Mon Sep 17 00:00:00 2001 From: supaeasy <59504964+supaeasy@users.noreply.github.com> Date: Fri, 5 Feb 2021 15:16:51 +0100 Subject: [PATCH 15/28] Update README.rst I struggled too long with this to not let others know. --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index b028445..3428a0c 100644 --- a/README.rst +++ b/README.rst @@ -145,7 +145,7 @@ The full set of configuration options are: - ``chunk_size`` - int: Number of files to give to each process when running in parallel. Setting this to a number larger than one can improve performance when processing thousands of files - ``imap`` - ``host`` - str: The IMAP server hostname or IP address - - ``port`` - int: The IMAP server port (Default: 993) + - ``port`` - int: The IMAP server port (Default: 993) If your Hoster publishes another port, still try 993. Otherwise Error:"wrong SSL version" - ``ssl`` - bool: Use an encrypted SSL/TLS connection (Default: True) - ``skip_certificate_verification`` - bool: Skip certificate verification (not recommended) - ``user`` - str: The IMAP user From 36c592cc5a6cf52995763c62c6087402538946f9 Mon Sep 17 00:00:00 2001 From: atanas argirov Date: Thu, 11 Feb 2021 18:22:29 +0000 Subject: [PATCH 16/28] * added defaults for arg parser --- parsedmarc/__init__.py | 1 - parsedmarc/cli.py | 17 +++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 49de20b..e27aee8 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -1234,7 +1234,6 @@ def watch_inbox(host, username, password, callback, port=None, ssl=True, except (timeout, IMAPClientError): logger.warning("IMAP connection timeout. Reconnecting...") - def save_output(results, output_directory="output", \ output_json_aggregate_file="aggregate.json", \ output_json_forensic_file="forensic.json", \ diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index 262a82b..b0d4d72 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -82,7 +82,7 @@ def _main(): if opts.save_aggregate: for report in reports_["aggregate_reports"]: try: - opts.elasticsearch_hosts: + if opts.elasticsearch_hosts: shards = opts.elasticsearch_number_of_shards replicas = opts.elasticsearch_number_of_replicas elastic.save_aggregate_report_to_elasticsearch( @@ -161,13 +161,13 @@ def _main(): arg_parser.add_argument("-o", "--output", help="write output files to the given directory") arg_parser.add_argument("--output-json-aggregate-file", - help="output aggregate JSON file") + help="output aggregate JSON file", default="aggregate.json") arg_parser.add_argument("--output-json-forensic-file", - help="output forensic JSON file") + help="output forensic JSON file", default="forensic.json") arg_parser.add_argument("--output-csv-aggregate-file", - help="output aggregate CSV file") + help="output aggregate CSV file", default="aggregate.csv") arg_parser.add_argument("--output-csv-forensic-file", - help="output forensic CSV file") + help="output forensic CSV file", default="forensic.csv") arg_parser.add_argument("-n", "--nameservers", nargs="+", help="nameservers to query") arg_parser.add_argument("-t", "--dns_timeout", @@ -191,6 +191,7 @@ def _main(): forensic_reports = [] args = arg_parser.parse_args() + opts = Namespace(file_path=args.file_path, config_file=args.config_file, offline=args.offline, @@ -631,7 +632,11 @@ def _main(): ("forensic_reports", forensic_reports)]) if opts.output: - save_output(results, output_directory=opts.output) + save_output(results, output_directory=opts.output, \ + output_json_aggregate_file=opts.output_json_aggregate_file, \ + output_json_forensic_file=opts.output_json_forensic_file, \ + output_csv_aggregate_file=opts.output_csv_aggregate_file, \ + output_csv_forensic_file=opts.output_csv_forensic_file) process_reports(results) From e51f2b0127bad34903fee4c88d44b5f4660b0913 Mon Sep 17 00:00:00 2001 From: atanas argirov Date: Fri, 12 Feb 2021 10:50:25 +0000 Subject: [PATCH 17/28] * general cleanup to meet linter rules --- parsedmarc/__init__.py | 27 ++++++++++++++++++--------- parsedmarc/cli.py | 38 +++++++++++++++++++++----------------- 2 files changed, 39 insertions(+), 26 deletions(-) diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index e27aee8..7f0c7d0 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -1234,11 +1234,12 @@ def watch_inbox(host, username, password, callback, port=None, ssl=True, except (timeout, IMAPClientError): logger.warning("IMAP connection timeout. Reconnecting...") -def save_output(results, output_directory="output", \ - output_json_aggregate_file="aggregate.json", \ - output_json_forensic_file="forensic.json", \ - output_csv_aggregate_file="aggregate.csv", \ - output_csv_forensic_file="forensic.csv"): + +def save_output(results, output_directory="output", + output_json_aggregate="aggregate.json", + output_json_forensic="forensic.json", + output_csv_aggregate="aggregate.csv", + output_csv_forensic="forensic.csv"): """ Save report data in the given directory @@ -1256,22 +1257,30 @@ def save_output(results, output_directory="output", \ else: os.makedirs(output_directory) - with open("{0}".format(os.path.join(output_directory, output_json_aggregate_file)), + with open("{0}" + .format(os.path.join(output_directory, + output_json_aggregate)), "w", newline="\n", encoding="utf-8") as agg_json: agg_json.write(json.dumps(aggregate_reports, ensure_ascii=False, indent=2)) - with open("{0}".format(os.path.join(output_directory, output_csv_aggregate_file)), + with open("{0}" + .format(os.path.join(output_directory, + output_csv_aggregate)), "w", newline="\n", encoding="utf-8") as agg_csv: csv = parsed_aggregate_reports_to_csv(aggregate_reports) agg_csv.write(csv) - with open("{0}".format(os.path.join(output_directory, output_json_forensic_file)), + with open("{0}" + .format(os.path.join(output_directory, + output_json_forensic)), "w", newline="\n", encoding="utf-8") as for_json: for_json.write(json.dumps(forensic_reports, ensure_ascii=False, indent=2)) - with open("{0}".format(os.path.join(output_directory, output_csv_forensic_file)), + with open("{0}" + .format(os.path.join(output_directory, + output_csv_forensic)), "w", newline="\n", encoding="utf-8") as for_csv: csv = parsed_forensic_reports_to_csv(forensic_reports) for_csv.write(csv) diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index b0d4d72..777dfc5 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -160,14 +160,18 @@ def _main(): help=strip_attachment_help, action="store_true") arg_parser.add_argument("-o", "--output", help="write output files to the given directory") - arg_parser.add_argument("--output-json-aggregate-file", - help="output aggregate JSON file", default="aggregate.json") - arg_parser.add_argument("--output-json-forensic-file", - help="output forensic JSON file", default="forensic.json") - arg_parser.add_argument("--output-csv-aggregate-file", - help="output aggregate CSV file", default="aggregate.csv") - arg_parser.add_argument("--output-csv-forensic-file", - help="output forensic CSV file", default="forensic.csv") + arg_parser.add_argument("--output-json-aggregate", + help="output aggregate JSON file", + default="aggregate.json") + arg_parser.add_argument("--output-json-forensic", + help="output forensic JSON file", + default="forensic.json") + arg_parser.add_argument("--output-csv-aggregate", + help="output aggregate CSV file", + default="aggregate.csv") + arg_parser.add_argument("--output-csv-forensic", + help="output forensic CSV file", + default="forensic.csv") arg_parser.add_argument("-n", "--nameservers", nargs="+", help="nameservers to query") arg_parser.add_argument("-t", "--dns_timeout", @@ -197,10 +201,10 @@ def _main(): offline=args.offline, strip_attachment_payloads=args.strip_attachment_payloads, output=args.output, - output_json_aggregate_file=args.output_json_aggregate_file, - output_json_forensic_file=args.output_json_forensic_file, - output_csv_aggregate_file=args.output_csv_aggregate_file, - output_csv_forensic_file=args.output_csv_forensic_file, + output_json_aggregate=args.output_json_aggregate, + output_json_forensic=args.output_json_forensic, + output_csv_aggregate=args.output_csv_aggregate, + output_csv_forensic=args.output_csv_forensic, nameservers=args.nameservers, silent=args.silent, dns_timeout=args.dns_timeout, @@ -632,11 +636,11 @@ def _main(): ("forensic_reports", forensic_reports)]) if opts.output: - save_output(results, output_directory=opts.output, \ - output_json_aggregate_file=opts.output_json_aggregate_file, \ - output_json_forensic_file=opts.output_json_forensic_file, \ - output_csv_aggregate_file=opts.output_csv_aggregate_file, \ - output_csv_forensic_file=opts.output_csv_forensic_file) + save_output(results, output_directory=opts.output, + output_json_aggregate=opts.output_json_aggregate, + output_json_forensic=opts.output_json_forensic, + output_csv_aggregate=opts.output_csv_aggregate, + output_csv_forensic=opts.output_csv_forensic) process_reports(results) From 609fbdce6f6a7c620ad918546b525bccac74bfb4 Mon Sep 17 00:00:00 2001 From: Owen Valentine Date: Wed, 3 Mar 2021 12:13:11 +0200 Subject: [PATCH 18/28] Typo correction: allignment -> alignment --- kibana/export.ndjson | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kibana/export.ndjson b/kibana/export.ndjson index 21a9003..d708780 100644 --- a/kibana/export.ndjson +++ b/kibana/export.ndjson @@ -8,15 +8,15 @@ {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"DMARC Passage Over Time","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"DMARC Passage Over Time\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Messages\"},\"drawLinesBetweenPoints\":true,\"mode\":\"normal\",\"show\":\"true\",\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"line\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Messages\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"date_range\",\"timeRange\":{\"from\":\"now-7d\",\"to\":\"now\",\"mode\":\"quick\"},\"useNormalizedEsInterval\":true,\"interval\":\"d\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"passed_dmarc\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Passed DMARC\"}}]}"},"id":"085eaa30-2870-11e8-b8b2-15742da3055c","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-03-21T19:33:49.735Z","version":"WzIzOCw4XQ=="} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Reporting Organizations","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}}","version":1,"visState":"{\"title\":\"Reporting Organizations\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"org_name.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"missing\",\"size\":1000,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Name\"}}]}"},"id":"620280a0-2886-11e8-b8b2-15742da3055c","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-03-21T19:33:49.735Z","version":"WzIzOSw4XQ=="} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Top 2000 Message Sources by Reverse DNS","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}}","version":1,"visState":"{\"title\":\"Top 2000 Message Sources by Reverse DNS\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_base_domain.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"none\",\"size\":2000,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Reverse DNS Base\"}}]}"},"id":"d787a580-2886-11e8-b8b2-15742da3055c","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-03-21T19:33:49.735Z","version":"WzI0MCw4XQ=="} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"SPF Allignment","uiStateJSON":"{\"vis\":{\"legendOpen\":false}}","version":1,"visState":"{\"title\":\"SPF Allignment\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"spf_aligned\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"_term\",\"customLabel\":\"SPF Alligned\"}}]}"},"id":"356caa70-28d1-11e8-b8b2-15742da3055c","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-03-21T19:33:49.735Z","version":"WzI0MSw4XQ=="} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"DKIM Alignment","uiStateJSON":"{\"vis\":{\"legendOpen\":false}}","version":1,"visState":"{\"title\":\"DKIM Alignment\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"dkim_aligned\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"_term\",\"customLabel\":\"DKIM Alligned\"}}]}"},"id":"7e26fb80-28d1-11e8-b8b2-15742da3055c","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-03-21T19:33:49.735Z","version":"WzI0Miw4XQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"SPF Alignment","uiStateJSON":"{\"vis\":{\"legendOpen\":false}}","version":1,"visState":"{\"title\":\"SPF Alignment\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"spf_aligned\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"_term\",\"customLabel\":\"SPF Aligned\"}}]}"},"id":"356caa70-28d1-11e8-b8b2-15742da3055c","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-03-21T19:33:49.735Z","version":"WzI0MSw4XQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"DKIM Alignment","uiStateJSON":"{\"vis\":{\"legendOpen\":false}}","version":1,"visState":"{\"title\":\"DKIM Alignment\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"dkim_aligned\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"_term\",\"customLabel\":\"DKIM Aligned\"}}]}"},"id":"7e26fb80-28d1-11e8-b8b2-15742da3055c","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-03-21T19:33:49.735Z","version":"WzI0Miw4XQ=="} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"DMARC Passage","uiStateJSON":"{\"vis\":{\"legendOpen\":false}}","version":1,"visState":"{\"title\":\"DMARC Passage\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"passed_dmarc\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"_term\",\"customLabel\":\"Passed DMARC\"}}]}"},"id":"93b823e0-28cf-11e8-b8b2-15742da3055c","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-03-21T19:33:49.735Z","version":"WzI0Myw4XQ=="} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Map of Message Source Countries","uiStateJSON":"{\"mapZoom\":3,\"mapCenter\":[27.68352808378776,5.537109375000001]}","version":1,"visState":"{\"title\":\"Map of Message Source Countries\",\"type\":\"region_map\",\"params\":{\"legendPosition\":\"bottomright\",\"addTooltip\":true,\"colorSchema\":\"Yellow to Red\",\"selectedLayer\":{\"attribution\":\"

Made with NaturalEarth | Elastic Maps Service

\",\"name\":\"World Countries\",\"format\":{\"type\":\"geojson\"},\"url\":\"https://vector.maps.elastic.co/blob/5659313586569216?elastic_tile_service_tos=agree&my_app_version=6.2.2\",\"fields\":[{\"name\":\"iso2\",\"description\":\"Two letter abbreviation\"},{\"name\":\"name\",\"description\":\"Country name\"},{\"name\":\"iso3\",\"description\":\"Three letter abbreviation\"}],\"created_at\":\"2017-04-26T17:12:15.978370\",\"tags\":[],\"id\":5659313586569216,\"layerId\":\"elastic_maps_service.World Countries\"},\"selectedJoinField\":{\"name\":\"iso2\",\"description\":\"Two letter abbreviation\"},\"isDisplayWarning\":true,\"wms\":{\"enabled\":false,\"options\":{\"format\":\"image/png\",\"transparent\":true},\"baseLayersAreLoaded\":{\"_c\":[],\"_s\":1,\"_d\":true,\"_v\":true,\"_h\":0,\"_n\":false},\"tmsLayers\":[{\"id\":\"road_map\",\"url\":\"https://tiles.maps.elastic.co/v2/default/{z}/{x}/{y}.png?elastic_tile_service_tos=agree&my_app_name=kibana&my_app_version=6.2.2\",\"minZoom\":0,\"maxZoom\":10,\"attribution\":\"

© OpenStreetMap contributors | Elastic Maps Service

\",\"subdomains\":[]}],\"selectedTmsLayer\":{\"id\":\"road_map\",\"url\":\"https://tiles.maps.elastic.co/v2/default/{z}/{x}/{y}.png?elastic_tile_service_tos=agree&my_app_name=kibana&my_app_version=6.2.2\",\"minZoom\":0,\"maxZoom\":10,\"attribution\":\"

© OpenStreetMap contributors | Elastic Maps Service

\",\"subdomains\":[]}},\"mapZoom\":2,\"mapCenter\":[0,0],\"outlineWeight\":1,\"showAllShapes\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"source_country.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":300,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Country\"}}]}"},"id":"895f3a70-291d-11e8-b8b2-15742da3055c","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-03-21T19:33:49.735Z","version":"WzI0NCw4XQ=="} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Message Volume by Header From","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Message Volume by Header From\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"showMetricsAtAllLevels\":false,\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":1,\"format\":{\"id\":\"number\",\"params\":{\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Messages\",\"aggType\":\"sum\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Header From\",\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"header_from.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100000,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Header From\"}}]}"},"id":"a69d0f40-2b02-11e8-8c8d-d3a0d2f2ba49","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-03-21T19:55:39.777Z","version":"WzI2NCw4XQ=="} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Top 1000 Message Source IP Addresses","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Top 1000 Message Source IP Addresses\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"showMetricsAtAllLevels\":false,\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":4,\"format\":{\"id\":\"number\",\"params\":{\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Messages\",\"aggType\":\"sum\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"IP Address\",\"aggType\":\"terms\"},{\"accessor\":1,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Reverse DNS\",\"aggType\":\"terms\"},{\"accessor\":2,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Base Domain\",\"aggType\":\"terms\"},{\"accessor\":3,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Country\",\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_ip_address.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":1000,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"IP Address\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_reverse_dns.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":1000,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"\",\"customLabel\":\"Reverse DNS\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_base_domain.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":1000,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"\",\"customLabel\":\"Base Domain\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_country.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":200,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"\",\"customLabel\":\"Country\"}}]}"},"id":"55930ba0-667f-11e8-ac01-67e661d30f69","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-03-21T19:52:29.350Z","version":"WzI2Miw4XQ=="} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Message Disposition Over Time","uiStateJSON":"{\"vis\":{\"colors\":{\"none\":\"#629E51\",\"reject\":\"#BF1B00\"}}}","version":1,"visState":"{\"title\":\"Message Disposition Over Time\",\"type\":\"line\",\"params\":{\"type\":\"line\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Messages\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"line\",\"mode\":\"normal\",\"data\":{\"label\":\"Messages\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"date_range\",\"timeRange\":{\"from\":\"now-7d\",\"to\":\"now\",\"mode\":\"quick\"},\"useNormalizedEsInterval\":true,\"interval\":\"d\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"disposition.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Disposition\"}}]}"},"id":"c9ee5ec0-67f9-11e8-ac01-67e661d30f69","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-03-21T19:33:49.735Z","version":"WzI0Nyw4XQ=="} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Message Source Countries","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{\"customLabel\":\"Messages\",\"field\":\"message_count\"},\"schema\":\"metric\",\"type\":\"sum\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customLabel\":\"Country\",\"field\":\"source_country.keyword\",\"missingBucket\":true,\"missingBucketLabel\":\"unknown\",\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"size\":400},\"schema\":\"bucket\",\"type\":\"terms\"}],\"params\":{\"perPage\":10,\"showMeticsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"title\":\"Message Source Countries\",\"type\":\"table\"}"},"id":"f4444000-7333-11e8-bfe4-d3427a6746f1","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-03-21T19:33:49.735Z","version":"WzI0OCw4XQ=="} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"SPF Alignment Details","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":5,\"direction\":\"desc\"}}}}","version":1,"visState":"{\"title\":\"SPF Alignment Details\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":5,\"direction\":\"desc\"},\"totalFunc\":\"sum\",\"showMetricsAtAllLevels\":false,\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":5,\"format\":{\"id\":\"number\",\"params\":{\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Messages\",\"aggType\":\"sum\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Header From\",\"aggType\":\"terms\"},{\"accessor\":1,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Envelope From\",\"aggType\":\"terms\"},{\"accessor\":2,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"SPF Result\",\"aggType\":\"terms\"},{\"accessor\":3,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"boolean\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"SPF Alligned\",\"aggType\":\"terms\"},{\"accessor\":4,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Reverse DNS Base\",\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"header_from.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Header From\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"envelope_from.keyword\",\"orderBy\":\"_key\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Envelope From\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"spf_results.result.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":2,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"SPF Result\"}},{\"id\":\"6\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"spf_aligned\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":2,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"SPF Alligned\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_base_domain.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"\",\"customLabel\":\"Reverse DNS Base\"}}]}"},"id":"1fad3f60-2881-11e8-b8b2-15742da3055c","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-03-21T19:50:01.908Z","version":"WzI2MSw4XQ=="} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"DKIM Alignment Details","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":6,\"direction\":\"desc\"}}}}","version":1,"visState":"{\"title\":\"DKIM Alignment Details\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"sort\":{\"columnIndex\":5,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\",\"showMetricsAtAllLevels\":false,\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":6,\"format\":{\"id\":\"number\",\"params\":{\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Messages\",\"aggType\":\"sum\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Header From\",\"aggType\":\"terms\"},{\"accessor\":1,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"DKIM Selector\",\"aggType\":\"terms\"},{\"accessor\":2,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"DKIM Domain\",\"aggType\":\"terms\"},{\"accessor\":3,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"DKIM Result\",\"aggType\":\"terms\"},{\"accessor\":4,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"boolean\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"DKIM Alligned\",\"aggType\":\"terms\"},{\"accessor\":5,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Reverse DNS Base\",\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"header_from.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Header From\"}},{\"id\":\"8\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"dkim_results.selector.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":1000,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"\",\"customLabel\":\"DKIM Selector\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"dkim_results.domain.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"\",\"customLabel\":\"DKIM Domain\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"dkim_results.result.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":2,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"\",\"customLabel\":\"DKIM Result\"}},{\"id\":\"7\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"dkim_aligned\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":2,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"DKIM Alligned\"}},{\"id\":\"6\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_base_domain.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":1000,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"\",\"customLabel\":\"Reverse DNS Base\"}}]}"},"id":"40e7a5b0-2883-11e8-b8b2-15742da3055c","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-03-21T20:39:37.003Z","version":"WzI2Niw4XQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"SPF Alignment Details","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":5,\"direction\":\"desc\"}}}}","version":1,"visState":"{\"title\":\"SPF Alignment Details\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":5,\"direction\":\"desc\"},\"totalFunc\":\"sum\",\"showMetricsAtAllLevels\":false,\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":5,\"format\":{\"id\":\"number\",\"params\":{\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Messages\",\"aggType\":\"sum\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Header From\",\"aggType\":\"terms\"},{\"accessor\":1,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Envelope From\",\"aggType\":\"terms\"},{\"accessor\":2,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"SPF Result\",\"aggType\":\"terms\"},{\"accessor\":3,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"boolean\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"SPF Aligned\",\"aggType\":\"terms\"},{\"accessor\":4,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Reverse DNS Base\",\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"header_from.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Header From\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"envelope_from.keyword\",\"orderBy\":\"_key\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Envelope From\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"spf_results.result.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":2,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"SPF Result\"}},{\"id\":\"6\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"spf_aligned\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":2,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"SPF Aligned\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_base_domain.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"\",\"customLabel\":\"Reverse DNS Base\"}}]}"},"id":"1fad3f60-2881-11e8-b8b2-15742da3055c","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-03-21T19:50:01.908Z","version":"WzI2MSw4XQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"DKIM Alignment Details","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":6,\"direction\":\"desc\"}}}}","version":1,"visState":"{\"title\":\"DKIM Alignment Details\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"sort\":{\"columnIndex\":5,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\",\"showMetricsAtAllLevels\":false,\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":6,\"format\":{\"id\":\"number\",\"params\":{\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Messages\",\"aggType\":\"sum\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Header From\",\"aggType\":\"terms\"},{\"accessor\":1,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"DKIM Selector\",\"aggType\":\"terms\"},{\"accessor\":2,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"DKIM Domain\",\"aggType\":\"terms\"},{\"accessor\":3,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"DKIM Result\",\"aggType\":\"terms\"},{\"accessor\":4,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"boolean\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"DKIM Aligned\",\"aggType\":\"terms\"},{\"accessor\":5,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Reverse DNS Base\",\"aggType\":\"terms\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"header_from.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Header From\"}},{\"id\":\"8\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"dkim_results.selector.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":1000,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"\",\"customLabel\":\"DKIM Selector\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"dkim_results.domain.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"\",\"customLabel\":\"DKIM Domain\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"dkim_results.result.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":2,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"\",\"customLabel\":\"DKIM Result\"}},{\"id\":\"7\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"dkim_aligned\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":2,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"DKIM Aligned\"}},{\"id\":\"6\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_base_domain.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":1000,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"\",\"customLabel\":\"Reverse DNS Base\"}}]}"},"id":"40e7a5b0-2883-11e8-b8b2-15742da3055c","migrationVersion":{"visualization":"7.4.2"},"references":[{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2020-03-21T20:39:37.003Z","version":"WzI2Niw4XQ=="} {"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[],\"highlightAll\":true,\"version\":true}"},"optionsJSON":"{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"embeddableConfig\":{\"vis\":{\"colors\":{\"false\":\"#E24D42\",\"true\":\"#629E51\"},\"legendOpen\":true}},\"gridData\":{\"h\":15,\"i\":\"4\",\"w\":48,\"x\":0,\"y\":13},\"panelIndex\":\"4\",\"version\":\"7.6.1\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{\"spy\":null,\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":18,\"i\":\"7\",\"w\":16,\"x\":0,\"y\":41},\"panelIndex\":\"7\",\"version\":\"7.6.1\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":18,\"i\":\"8\",\"w\":16,\"x\":16,\"y\":41},\"panelIndex\":\"8\",\"version\":\"7.6.1\",\"panelRefName\":\"panel_2\"},{\"embeddableConfig\":{\"vis\":{\"colors\":{\"false\":\"#F2C96D\",\"true\":\"#629E51\"},\"legendOpen\":false}},\"gridData\":{\"h\":13,\"i\":\"9\",\"w\":16,\"x\":0,\"y\":0},\"panelIndex\":\"9\",\"version\":\"7.6.1\",\"panelRefName\":\"panel_3\"},{\"embeddableConfig\":{\"vis\":{\"colors\":{\"false\":\"#F2C96D\",\"true\":\"#629E51\"},\"legendOpen\":false}},\"gridData\":{\"h\":13,\"i\":\"10\",\"w\":15,\"x\":16,\"y\":0},\"panelIndex\":\"10\",\"version\":\"7.6.1\",\"panelRefName\":\"panel_4\"},{\"embeddableConfig\":{\"vis\":{\"colors\":{\"false\":\"#BF1B00\",\"true\":\"#629E51\"},\"legendOpen\":false}},\"gridData\":{\"h\":13,\"i\":\"11\",\"w\":17,\"x\":31,\"y\":0},\"panelIndex\":\"11\",\"version\":\"7.6.1\",\"panelRefName\":\"panel_5\"},{\"embeddableConfig\":{\"mapCenter\":null,\"mapZoom\":null},\"gridData\":{\"h\":17,\"i\":\"12\",\"w\":36,\"x\":0,\"y\":59},\"panelIndex\":\"12\",\"version\":\"7.6.1\",\"panelRefName\":\"panel_6\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":18,\"i\":\"13\",\"w\":16,\"x\":32,\"y\":41},\"panelIndex\":\"13\",\"version\":\"7.6.1\",\"panelRefName\":\"panel_7\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":4,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":18,\"i\":\"14\",\"w\":48,\"x\":0,\"y\":76},\"panelIndex\":\"14\",\"version\":\"7.6.1\",\"panelRefName\":\"panel_8\"},{\"embeddableConfig\":{\"vis\":{\"colors\":{\"none\":\"#629E51\",\"quarantine\":\"#E5AC0E\",\"reject\":\"#BF1B00\"}}},\"gridData\":{\"h\":13,\"i\":\"15\",\"w\":48,\"x\":0,\"y\":28},\"panelIndex\":\"15\",\"version\":\"7.6.1\",\"panelRefName\":\"panel_9\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":17,\"i\":\"16\",\"w\":12,\"x\":36,\"y\":59},\"panelIndex\":\"16\",\"version\":\"7.6.1\",\"panelRefName\":\"panel_10\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":5,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":18,\"i\":\"17\",\"w\":48,\"x\":0,\"y\":94},\"panelIndex\":\"17\",\"version\":\"7.6.1\",\"panelRefName\":\"panel_11\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":6,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":19,\"i\":\"18\",\"w\":48,\"x\":0,\"y\":112},\"panelIndex\":\"18\",\"version\":\"7.6.1\",\"panelRefName\":\"panel_12\"}]","refreshInterval":{"pause":false,"value":300000},"timeFrom":"now-7d","timeRestore":true,"timeTo":"now","title":"DMARC Summary","version":1},"id":"269ba470-2871-11e8-b8b2-15742da3055c","migrationVersion":{"dashboard":"7.3.0"},"references":[{"id":"085eaa30-2870-11e8-b8b2-15742da3055c","name":"panel_0","type":"visualization"},{"id":"620280a0-2886-11e8-b8b2-15742da3055c","name":"panel_1","type":"visualization"},{"id":"d787a580-2886-11e8-b8b2-15742da3055c","name":"panel_2","type":"visualization"},{"id":"356caa70-28d1-11e8-b8b2-15742da3055c","name":"panel_3","type":"visualization"},{"id":"7e26fb80-28d1-11e8-b8b2-15742da3055c","name":"panel_4","type":"visualization"},{"id":"93b823e0-28cf-11e8-b8b2-15742da3055c","name":"panel_5","type":"visualization"},{"id":"895f3a70-291d-11e8-b8b2-15742da3055c","name":"panel_6","type":"visualization"},{"id":"a69d0f40-2b02-11e8-8c8d-d3a0d2f2ba49","name":"panel_7","type":"visualization"},{"id":"55930ba0-667f-11e8-ac01-67e661d30f69","name":"panel_8","type":"visualization"},{"id":"c9ee5ec0-67f9-11e8-ac01-67e661d30f69","name":"panel_9","type":"visualization"},{"id":"f4444000-7333-11e8-bfe4-d3427a6746f1","name":"panel_10","type":"visualization"},{"id":"1fad3f60-2881-11e8-b8b2-15742da3055c","name":"panel_11","type":"visualization"},{"id":"40e7a5b0-2883-11e8-b8b2-15742da3055c","name":"panel_12","type":"visualization"}],"type":"dashboard","updated_at":"2020-03-22T00:54:16.396Z","version":"WzI3Myw4XQ=="} {"exportedCount":21,"missingRefCount":0,"missingReferences":[]} \ No newline at end of file From 77331b55c54cb3269205295bd57d0ab680638964 Mon Sep 17 00:00:00 2001 From: supaeasy <59504964+supaeasy@users.noreply.github.com> Date: Sun, 21 Mar 2021 20:41:14 +0100 Subject: [PATCH 19/28] Update Grafana-DMARC_Reports.json - Update to current version of ES, Grafana and Grafana Plugins. - fix TimeDate Handling for Forensic Reports, was not displayed correctly - alter handling of stacking in one place (it just looked wrong) --- grafana/Grafana-DMARC_Reports.json | 146 ++++++++++++++++++----------- 1 file changed, 92 insertions(+), 54 deletions(-) diff --git a/grafana/Grafana-DMARC_Reports.json b/grafana/Grafana-DMARC_Reports.json index 6061644..6789563 100644 --- a/grafana/Grafana-DMARC_Reports.json +++ b/grafana/Grafana-DMARC_Reports.json @@ -5,19 +5,19 @@ "type": "datasource", "id": "elasticsearch", "name": "Elasticsearch", - "version": "1.0.0" + "version": "7.11.2" }, { "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "7.1.1" + "version": "7.4.5" }, { "type": "panel", "id": "grafana-piechart-panel", "name": "Pie Chart", - "version": "1.5.0" + "version": "1.6.1" }, { "type": "panel", @@ -47,7 +47,7 @@ "type": "panel", "id": "text", "name": "Text", - "version": "7.1.0" + "version": "7.4.5" } ], "annotations": { @@ -68,7 +68,7 @@ "gnetId": null, "graphTooltip": 0, "id": null, - "iteration": 1596560916058, + "iteration": 1616327630073, "links": [], "panels": [ { @@ -460,14 +460,17 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, - "pluginVersion": "7.1.1", + "pluginVersion": "7.4.5", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { @@ -604,8 +607,11 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, - "pluginVersion": "7.1.1", + "pluginVersion": "7.4.5", "pointradius": 2, "points": false, "renderer": "flot", @@ -741,8 +747,11 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, - "pluginVersion": "7.1.1", + "pluginVersion": "7.4.5", "pointradius": 2, "points": false, "renderer": "flot", @@ -880,8 +889,11 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, - "pluginVersion": "7.1.1", + "pluginVersion": "7.4.5", "pointradius": 2, "points": false, "renderer": "flot", @@ -1019,8 +1031,11 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, - "pluginVersion": "7.1.1", + "pluginVersion": "7.4.5", "pointradius": 2, "points": false, "renderer": "flot", @@ -1156,8 +1171,11 @@ "linewidth": 2, "links": [], "nullPointMode": "null", + "options": { + "alertThreshold": true + }, "percentage": false, - "pluginVersion": "7.1.1", + "pluginVersion": "7.4.5", "pointradius": 2, "points": false, "renderer": "flot", @@ -1264,7 +1282,6 @@ "value": "null" } ], - "nullValueMode": "connected", "thresholds": { "mode": "absolute", "steps": [ @@ -1299,9 +1316,10 @@ "fields": "", "values": false }, + "text": {}, "textMode": "value_and_name" }, - "pluginVersion": "7.1.1", + "pluginVersion": "7.4.5", "targets": [ { "bucketAggs": [ @@ -1345,7 +1363,8 @@ "fieldConfig": { "defaults": { "custom": { - "align": null + "align": null, + "filterable": false }, "mappings": [ { @@ -1418,7 +1437,7 @@ } ] }, - "pluginVersion": "7.1.1", + "pluginVersion": "7.4.5", "targets": [ { "bucketAggs": [ @@ -1476,7 +1495,8 @@ "fieldConfig": { "defaults": { "custom": { - "align": null + "align": null, + "filterable": false }, "mappings": [], "thresholds": { @@ -1553,7 +1573,7 @@ } ] }, - "pluginVersion": "7.1.1", + "pluginVersion": "7.4.5", "targets": [ { "bucketAggs": [ @@ -1658,7 +1678,8 @@ "fieldConfig": { "defaults": { "custom": { - "align": null + "align": null, + "filterable": false }, "mappings": [], "thresholds": { @@ -1741,7 +1762,7 @@ } ] }, - "pluginVersion": "7.1.1", + "pluginVersion": "7.4.5", "targets": [ { "bucketAggs": [ @@ -1897,7 +1918,8 @@ "fieldConfig": { "defaults": { "custom": { - "align": null + "align": null, + "filterable": false }, "mappings": [], "thresholds": { @@ -1974,7 +1996,7 @@ } ] }, - "pluginVersion": "7.1.1", + "pluginVersion": "7.4.5", "targets": [ { "bucketAggs": [ @@ -2032,7 +2054,8 @@ "fieldConfig": { "defaults": { "custom": { - "align": null + "align": null, + "filterable": false }, "mappings": [], "thresholds": { @@ -2202,7 +2225,7 @@ } ] }, - "pluginVersion": "7.1.1", + "pluginVersion": "7.4.5", "targets": [ { "bucketAggs": [ @@ -2427,7 +2450,8 @@ "fieldConfig": { "defaults": { "custom": { - "align": null + "align": null, + "filterable": false }, "mappings": [ { @@ -2610,7 +2634,7 @@ } ] }, - "pluginVersion": "7.1.1", + "pluginVersion": "7.4.5", "targets": [ { "bucketAggs": [ @@ -2784,7 +2808,8 @@ "fieldConfig": { "defaults": { "custom": { - "align": null + "align": null, + "filterable": false }, "mappings": [], "thresholds": { @@ -2904,7 +2929,7 @@ } ] }, - "pluginVersion": "7.1.1", + "pluginVersion": "7.4.5", "targets": [ { "bucketAggs": [ @@ -3007,7 +3032,8 @@ "fieldConfig": { "defaults": { "custom": { - "align": null + "align": null, + "filterable": false }, "mappings": [], "thresholds": { @@ -3116,7 +3142,7 @@ } ] }, - "pluginVersion": "7.1.1", + "pluginVersion": "7.4.5", "targets": [ { "bucketAggs": [ @@ -3234,7 +3260,8 @@ "fieldConfig": { "defaults": { "custom": { - "align": null + "align": null, + "filterable": false }, "mappings": [], "thresholds": { @@ -3347,7 +3374,7 @@ } ] }, - "pluginVersion": "7.1.1", + "pluginVersion": "7.4.5", "targets": [ { "bucketAggs": [ @@ -3496,7 +3523,8 @@ "fieldConfig": { "defaults": { "custom": { - "align": null + "align": null, + "filterable": false }, "mappings": [], "thresholds": { @@ -3517,16 +3545,12 @@ { "matcher": { "id": "byName", - "options": "Arrival_Date(UTC)" + "options": "Arrival Date (UTC)" }, "properties": [ { "id": "unit", - "value": "dateTimeAsIso" - }, - { - "id": "custom.width", - "value": 147 + "value": "dateTimeAsSystem" } ] }, @@ -3592,14 +3616,14 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "7.1.1", + "pluginVersion": "7.4.5", "targets": [ { "bucketAggs": [ { "$$hashKey": "object:340", "fake": true, - "field": "arrival_date", + "field": "Arrival Date (UTC)", "id": "6", "settings": { "interval": "auto", @@ -3744,7 +3768,7 @@ "indexByName": {}, "renameByName": { "Count": "Count", - "arrival_date": "Arrival_Date(UTC)", + "arrival_date_utc": "Arrival_Date(UTC)", "auth_failure.keyword": "AuthFailure", "authentication_results.keyword": "Auth Results", "delivery_results.keyword": "DeliveryResult", @@ -3862,7 +3886,8 @@ "fieldConfig": { "defaults": { "custom": { - "align": null + "align": null, + "filterable": false }, "mappings": [ { @@ -3942,7 +3967,7 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "7.1.1", + "pluginVersion": "7.4.5", "targets": [ { "bucketAggs": [ @@ -3999,7 +4024,8 @@ "fieldConfig": { "defaults": { "custom": { - "align": null + "align": null, + "filterable": false }, "mappings": [], "thresholds": { @@ -4094,7 +4120,7 @@ } ] }, - "pluginVersion": "7.1.1", + "pluginVersion": "7.4.5", "targets": [ { "bucketAggs": [ @@ -4193,8 +4219,8 @@ "type": "table" } ], - "refresh": false, - "schemaVersion": 26, + "refresh": "10s", + "schemaVersion": 27, "style": "dark", "tags": [ "DKIM", @@ -4207,9 +4233,11 @@ { "current": { "selected": false, - "text": "Elasticsearch-dmarc-ag", - "value": "Elasticsearch-dmarc-ag" + "text": "dmarc-ag", + "value": "dmarc-ag" }, + "description": null, + "error": null, "hide": 2, "includeAll": false, "label": "Datasource: Aggregate", @@ -4225,9 +4253,11 @@ { "current": { "selected": false, - "text": "Elasticsearch-dmarc-fo", - "value": "Elasticsearch-dmarc-fo" + "text": "dmarc-fo", + "value": "dmarc-fo" }, + "description": null, + "error": null, "hide": 2, "includeAll": false, "label": "Datasource: Forensic", @@ -4242,9 +4272,15 @@ }, { "allValue": null, - "current": {}, + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, "datasource": "$datasourceag", "definition": "{\"find\":\"terms\",\"field\":\"header_from.keyword\"}", + "description": null, + "error": null, "hide": 0, "includeAll": true, "label": "From Domain", @@ -4267,10 +4303,12 @@ "auto_count": 30, "auto_min": "10s", "current": { - "selected": true, + "selected": false, "text": "1d", "value": "1d" }, + "description": null, + "error": null, "hide": 2, "label": "Interval", "name": "interval", From 55196cb3890a8bd953641208547d4bbabf3c923e Mon Sep 17 00:00:00 2001 From: Dave Rawks Date: Mon, 19 Apr 2021 10:33:27 -0700 Subject: [PATCH 20/28] Resolve issue #233 - don't create imap folders when in test mode --- parsedmarc/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index ca4f1e7..562a0a8 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -1060,10 +1060,11 @@ def get_dmarc_reports_from_inbox(connection=None, max_retries=max_retries, initial_folder=reports_folder) - server.create_folder(archive_folder) - server.create_folder(aggregate_reports_folder) - server.create_folder(forensic_reports_folder) - server.create_folder(invalid_reports_folder) + if not test: + server.create_folder(archive_folder) + server.create_folder(aggregate_reports_folder) + server.create_folder(forensic_reports_folder) + server.create_folder(invalid_reports_folder) messages = server.search() total_messages = len(messages) From 1b61156d50b532c0a4e6ebaf2d1d6e131408a124 Mon Sep 17 00:00:00 2001 From: Dave Rawks Date: Mon, 19 Apr 2021 09:26:48 -0700 Subject: [PATCH 21/28] Resolves Issue #235 - Apply index suffix to pre-insert search * updates `save_forensic_report_to_elasticsearch` and `save_aggregate_report_to_elasticsearch` to apply suffix, if configured, to pre-insert search --- parsedmarc/elastic.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/parsedmarc/elastic.py b/parsedmarc/elastic.py index b448f28..d6c3ae0 100644 --- a/parsedmarc/elastic.py +++ b/parsedmarc/elastic.py @@ -320,7 +320,10 @@ def save_aggregate_report_to_elasticsearch(aggregate_report, begin_date_query = Q(dict(match=dict(date_range=begin_date))) end_date_query = Q(dict(match=dict(date_range=end_date))) - search = Search(index="dmarc_aggregate*") + if index_suffix is not None: + search = Search(index="dmarc_aggregate_{0}*".format(index_suffix)) + else: + search = Search(index="dmarc_aggregate*") query = org_name_query & report_id_query & domain_query query = query & begin_date_query & end_date_query search.query = query @@ -437,7 +440,10 @@ def save_forensic_report_to_elasticsearch(forensic_report, arrival_date_human = forensic_report["arrival_date_utc"] arrival_date = human_timestamp_to_datetime(arrival_date_human) - search = Search(index="dmarc_forensic*") + if index_suffix is not None: + search = Search(index="dmarc_forensic_{0}*".format(index_suffix)) + else: + search = Search(index="dmarc_forensic*") arrival_query = {"match": {"arrival_date": arrival_date}} q = Q(arrival_query) From 775a6f21819d7aaf389a75bd5f21d4e61fba5644 Mon Sep 17 00:00:00 2001 From: Silvian I Date: Mon, 31 May 2021 15:40:57 +0200 Subject: [PATCH 22/28] Fix server connection timeout while processiong large dmarc files --- parsedmarc/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 562a0a8..9873316 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -304,8 +304,12 @@ def parse_aggregate_report_xml(xml, offline=False, nameservers=None, new_report["policy_published"] = new_policy_published if type(report["record"]) == list: - for record in report["record"]: - report_record = _parse_report_record(record, + for i in range(len(report["record"])): + if i % 20 == 0 and i > 0: + logger.debug("Sending noop cmd") + server.noop() + logger.debug("Processed {0}/{1}".format(i, len(report["record"]))) + report_record = _parse_report_record(report["record"][i], offline=offline, nameservers=nameservers, dns_timeout=timeout, @@ -1039,6 +1043,7 @@ def get_dmarc_reports_from_inbox(connection=None, raise ValueError("Must supply a connection, or a username and " "password") + global server aggregate_reports = [] forensic_reports = [] aggregate_report_msg_uids = [] From 3615ad3799f058e8012b39f708aa500f51aa2cf4 Mon Sep 17 00:00:00 2001 From: Silvian I Date: Mon, 31 May 2021 15:40:57 +0200 Subject: [PATCH 23/28] Fix server connection timeout while processiong large dmarc files --- parsedmarc/__init__.py | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 562a0a8..9de72c1 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -203,7 +203,7 @@ def _parse_report_record(record, offline=False, nameservers=None, def parse_aggregate_report_xml(xml, offline=False, nameservers=None, - timeout=2.0, parallel=False): + timeout=2.0, parallel=False, server=None): """Parses a DMARC XML report string and returns a consistent OrderedDict Args: @@ -213,6 +213,7 @@ def parse_aggregate_report_xml(xml, offline=False, nameservers=None, (Cloudflare's public DNS resolvers by default) timeout (float): Sets the DNS timeout in seconds parallel (bool): Parallel processing + server (IMAPClient): Connection object Returns: OrderedDict: The parsed aggregate DMARC report @@ -304,8 +305,13 @@ def parse_aggregate_report_xml(xml, offline=False, nameservers=None, new_report["policy_published"] = new_policy_published if type(report["record"]) == list: - for record in report["record"]: - report_record = _parse_report_record(record, + for i in range(len(report["record"])): + if server is not None and i > 0 and i % 20 == 0: + logger.debug("Sending noop cmd") + server.noop() + logger.debug("Processed {0}/{1}".format( + i, len(report["record"]))) + report_record = _parse_report_record(report["record"][i], offline=offline, nameservers=nameservers, dns_timeout=timeout, @@ -385,7 +391,8 @@ def extract_xml(input_): def parse_aggregate_report_file(_input, offline=False, nameservers=None, dns_timeout=2.0, - parallel=False): + parallel=False, + server=None): """Parses a file at the given path, a file-like object. or bytes as a aggregate DMARC report @@ -396,6 +403,7 @@ def parse_aggregate_report_file(_input, offline=False, nameservers=None, (Cloudflare's public DNS resolvers by default) dns_timeout (float): Sets the DNS timeout in seconds parallel (bool): Parallel processing + server (IMAPClient): Connection object Returns: OrderedDict: The parsed DMARC aggregate report @@ -406,7 +414,8 @@ def parse_aggregate_report_file(_input, offline=False, nameservers=None, offline=offline, nameservers=nameservers, timeout=dns_timeout, - parallel=parallel) + parallel=parallel, + server=server) def parsed_aggregate_reports_to_csv_rows(reports): @@ -738,7 +747,7 @@ def parsed_forensic_reports_to_csv(reports): def parse_report_email(input_, offline=False, nameservers=None, dns_timeout=2.0, strip_attachment_payloads=False, - parallel=False): + parallel=False, server=None): """ Parses a DMARC report from an email @@ -750,6 +759,7 @@ def parse_report_email(input_, offline=False, nameservers=None, strip_attachment_payloads (bool): Remove attachment payloads from forensic report results parallel (bool): Parallel processing + server (IMAPClient): Connection object Returns: OrderedDict: @@ -813,7 +823,8 @@ def parse_report_email(input_, offline=False, nameservers=None, offline=offline, nameservers=ns, dns_timeout=dns_timeout, - parallel=parallel) + parallel=parallel, + server=server) result = OrderedDict([("report_type", "aggregate"), ("report", aggregate_report)]) return result @@ -863,7 +874,7 @@ def parse_report_email(input_, offline=False, nameservers=None, def parse_report_file(input_, nameservers=None, dns_timeout=2.0, strip_attachment_payloads=False, - offline=False, parallel=False): + offline=False, parallel=False, server=None): """Parses a DMARC aggregate or forensic file at the given path, a file-like object. or bytes @@ -876,6 +887,7 @@ def parse_report_file(input_, nameservers=None, dns_timeout=2.0, forensic report results offline (bool): Do not make online queries for geolocation or DNS parallel (bool): Parallel processing + server (IMAPClient): Connection object Returns: OrderedDict: The parsed DMARC report @@ -895,7 +907,8 @@ def parse_report_file(input_, nameservers=None, dns_timeout=2.0, offline=offline, nameservers=nameservers, dns_timeout=dns_timeout, - parallel=parallel) + parallel=parallel, + server=server) results = OrderedDict([("report_type", "aggregate"), ("report", report)]) except InvalidAggregateReport: @@ -906,7 +919,8 @@ def parse_report_file(input_, nameservers=None, dns_timeout=2.0, nameservers=nameservers, dns_timeout=dns_timeout, strip_attachment_payloads=sa, - parallel=parallel) + parallel=parallel, + server=server) except InvalidDMARCReport: raise InvalidDMARCReport("Not a valid aggregate or forensic " "report") @@ -1083,7 +1097,8 @@ def get_dmarc_reports_from_inbox(connection=None, nameservers=nameservers, dns_timeout=dns_timeout, offline=offline, - strip_attachment_payloads=sa) + strip_attachment_payloads=sa, + server=server) if parsed_email["report_type"] == "aggregate": aggregate_reports.append(parsed_email["report"]) aggregate_report_msg_uids.append(msg_uid) From 0aa7d84d0df7d6e20ddd99e7a8b3403d897d2acb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matth=C3=A4us=20Wander?= Date: Sun, 6 Jun 2021 18:41:23 +0200 Subject: [PATCH 24/28] Use UTC datetimes for Elastic. Elastic by default expects UTC. --- parsedmarc/elastic.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/parsedmarc/elastic.py b/parsedmarc/elastic.py index d6c3ae0..4eb64f0 100644 --- a/parsedmarc/elastic.py +++ b/parsedmarc/elastic.py @@ -301,10 +301,10 @@ def save_aggregate_report_to_elasticsearch(aggregate_report, org_name = metadata["org_name"] report_id = metadata["report_id"] domain = aggregate_report["policy_published"]["domain"] - begin_date = human_timestamp_to_datetime(metadata["begin_date"]) - end_date = human_timestamp_to_datetime(metadata["end_date"]) - begin_date_human = begin_date.strftime("%Y-%m-%d %H:%M:%S") - end_date_human = end_date.strftime("%Y-%m-%d %H:%M:%S") + begin_date = human_timestamp_to_datetime(metadata["begin_date"], to_utc=True) + end_date = human_timestamp_to_datetime(metadata["end_date"], to_utc=True) + begin_date_human = begin_date.strftime("%Y-%m-%d %H:%M:%SZ") + end_date_human = end_date.strftime("%Y-%m-%d %H:%M:%SZ") if monthly_indexes: index_date = begin_date.strftime("%Y-%m") else: From 4bc7b0b62c4423a79a451aeb4d00f790b2ba647b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matth=C3=A4us=20Wander?= Date: Mon, 7 Jun 2021 00:03:39 +0200 Subject: [PATCH 25/28] deduplicate over date_begin and date_end instead of date_range --- parsedmarc/elastic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parsedmarc/elastic.py b/parsedmarc/elastic.py index 4eb64f0..32cdea4 100644 --- a/parsedmarc/elastic.py +++ b/parsedmarc/elastic.py @@ -317,8 +317,8 @@ def save_aggregate_report_to_elasticsearch(aggregate_report, org_name_query = Q(dict(match_phrase=dict(org_name=org_name))) report_id_query = Q(dict(match_phrase=dict(report_id=report_id))) domain_query = Q(dict(match_phrase={"published_policy.domain": domain})) - begin_date_query = Q(dict(match=dict(date_range=begin_date))) - end_date_query = Q(dict(match=dict(date_range=end_date))) + begin_date_query = Q(dict(match=dict(date_begin=begin_date))) + end_date_query = Q(dict(match=dict(date_end=end_date))) if index_suffix is not None: search = Search(index="dmarc_aggregate_{0}*".format(index_suffix)) From ca15ff51bd1721423d8c6cd50a0a5a83a0bc526d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matth=C3=A4us=20Wander?= Date: Wed, 9 Jun 2021 14:29:04 +0200 Subject: [PATCH 26/28] handle invalid reports gracefully --- parsedmarc/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 562a0a8..c78cfc8 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -225,7 +225,8 @@ def parse_aggregate_report_xml(xml, offline=False, nameservers=None, errors.append("Invalid XML: {0}".format(e.__str__())) tree = etree.parse(BytesIO(xml.encode('utf-8')), etree.XMLParser(recover=True)) - xml = etree.tostring(tree).decode('utf-8') + s = etree.tostring(tree) + xml = '' if s is None else s.decode('utf-8') try: # Replace XML header (sometimes they are invalid) From 837ba7ef4d0e87fa67ce09b765145003eaed40d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matth=C3=A4us=20Wander?= Date: Sun, 6 Jun 2021 16:35:07 +0200 Subject: [PATCH 27/28] Added splunk installation guide --- splunk/README.rst | 73 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 68 insertions(+), 5 deletions(-) diff --git a/splunk/README.rst b/splunk/README.rst index 44a8ccf..42198f5 100644 --- a/splunk/README.rst +++ b/splunk/README.rst @@ -1,9 +1,53 @@ -================= -Splunk dashboards -================= +=================== +Splunk Installation +=================== -Setup guide ------------ +Install Splunk for use with Docker +---------------------------------- + +Download latest Splunk image:: + + docker pull splunk/splunk:latest + +Run Splunk with Docker +---------------------- + +Listen on all network interfaces:: + + docker run -d -p 8000:8000 -p 8088:8088 -e "SPLUNK_START_ARGS=--accept-license" -e "SPLUNK_PASSWORD=password1234" -e "SPLUNK_HEC_TOKEN=hec-token-1234" --name splunk splunk/splunk:latest + +Listen on localhost for use with reverse proxy with base URL `/splunk`:: + + docker run -d -p 127.0.0.1:8000:8000 -p 127.0.0.1:8088:8088 -e "SPLUNK_START_ARGS=--accept-license" -e "SPLUNK_PASSWORD=password1234" -e "SPLUNK_HEC_TOKEN=hec-token-1234" -e "SPLUNK_ROOT_ENDPOINT=/splunk" --name splunk splunk/splunk:latest + +Set up reverse proxy, e.g. Apache2:: + + ProxyPass /splunk http://127.0.0.1:8000/splunk + ProxyPassReverse /splunk http://127.0.0.1:8000/splunk + +Splunk Configuration +-------------------- + +Access web UI at http://127.0.0.1:8000 and log in with `admin:password1234`. + +Create App and Index +~~~~~~~~~~~~~~~~~~~~ + +- Settings > Data > Indexes: New Index + + - Index name: "email" + +- HEC token `hec-token-1234` should be already set up. + + - Check under Settings > Data > Data inputs: HTTP Event Collector + +- Apps > Manage Apps: Create app + + - Name: "parsedmarc" + - Folder name: "parsedmarc" + +Create Dashboards +~~~~~~~~~~~~~~~~~ 1. Navigate to the app you want to add the dashboards to, or create a new app called DMARC 2. Click Dashboards @@ -22,3 +66,22 @@ Setup guide 15. Paste the content of ''dmarc_forensic_dashboard.xml`` into the source editor 16. If the index storing the DMARC data is not named email, replace index="email" accordingly 17. Click Save + +============== +Example Config +============== + +parsedmarc.ini:: + + [splunk_hec] + url = https://127.0.0.1:8088/ + token = hec-token-1234 + index = email + skip_certificate_verification = True + +Note that `skip_certificate_verification = True` disables security checks. + +Run parsedmarc:: + + python3 -m parsedmarc.cli -c parsedmarc.ini + From ca36db5f24626f239ac7f1b239f58f68b66a343b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matth=C3=A4us=20Wander?= Date: Sun, 6 Jun 2021 16:44:40 +0200 Subject: [PATCH 28/28] Minor formatting --- splunk/README.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/splunk/README.rst b/splunk/README.rst index 42198f5..acf7bce 100644 --- a/splunk/README.rst +++ b/splunk/README.rst @@ -16,7 +16,7 @@ Listen on all network interfaces:: docker run -d -p 8000:8000 -p 8088:8088 -e "SPLUNK_START_ARGS=--accept-license" -e "SPLUNK_PASSWORD=password1234" -e "SPLUNK_HEC_TOKEN=hec-token-1234" --name splunk splunk/splunk:latest -Listen on localhost for use with reverse proxy with base URL `/splunk`:: +Listen on localhost for use with reverse proxy with base URL ``/splunk``:: docker run -d -p 127.0.0.1:8000:8000 -p 127.0.0.1:8088:8088 -e "SPLUNK_START_ARGS=--accept-license" -e "SPLUNK_PASSWORD=password1234" -e "SPLUNK_HEC_TOKEN=hec-token-1234" -e "SPLUNK_ROOT_ENDPOINT=/splunk" --name splunk splunk/splunk:latest @@ -28,7 +28,7 @@ Set up reverse proxy, e.g. Apache2:: Splunk Configuration -------------------- -Access web UI at http://127.0.0.1:8000 and log in with `admin:password1234`. +Access web UI at http://127.0.0.1:8000 and log in with ``admin:password1234``. Create App and Index ~~~~~~~~~~~~~~~~~~~~ @@ -37,7 +37,7 @@ Create App and Index - Index name: "email" -- HEC token `hec-token-1234` should be already set up. +- HEC token ``hec-token-1234`` should be already set up. - Check under Settings > Data > Data inputs: HTTP Event Collector @@ -79,7 +79,7 @@ parsedmarc.ini:: index = email skip_certificate_verification = True -Note that `skip_certificate_verification = True` disables security checks. +Note that ``skip_certificate_verification = True`` disables security checks. Run parsedmarc::