From 16a4be2205f1801afff34696d3e4683bc8cdee02 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Sun, 7 Oct 2018 12:50:02 -0400 Subject: [PATCH 1/8] 4.1.8 - Be more forgiving of weird XML --- .gitignore | 4 +- CHANGELOG.md | 5 ++ parsedmarc/__init__.py | 53 +++++++++++++----- ...ample.com!1538204542!1538463818.xml.sample | 40 +++++++++++++ ....com!1536853302!1536939702!2940.xml.sample | 34 +++++++++++ ...ample.de!1538690400!1538776800.xml.sample} | 8 +-- ...ample.com!1538784000!1538870399.xml.sample | 56 +++++++++++++++++++ setup.py | 2 +- 8 files changed, 181 insertions(+), 21 deletions(-) create mode 100644 samples/!example.com!1538204542!1538463818.xml.sample create mode 100644 samples/estadocuenta1.infonacot.gob.mx!example.com!1536853302!1536939702!2940.xml.sample rename samples/{ikea.com!xxxxxx.de!1538690400!1538776800.xml.sample => ikea.com!example.de!1538690400!1538776800.xml.sample} (87%) create mode 100644 samples/usssa.com!example.com!1538784000!1538870399.xml.sample diff --git a/.gitignore b/.gitignore index b5eaed7..89f65eb 100644 --- a/.gitignore +++ b/.gitignore @@ -106,7 +106,6 @@ ENV/ .idea/ # I/O files - output/ *.zip *.gz @@ -121,3 +120,6 @@ output/ # Data files *.dat *.mmdb + +# Temp files +tmp/ \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 46056aa..353132b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +4.1.8 +----- + +- Be more forgiving of weird XML + 4.1.7 ----- diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 1190662..0ca2bee 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -44,13 +44,14 @@ import imapclient.exceptions import dateparser import mailparser -__version__ = "4.1.7" +__version__ = "4.1.8" logger = logging.getLogger(__name__) logger.setLevel(logging.ERROR) feedback_report_regex = re.compile(r"^([\w\-]+): (.+)$", re.MULTILINE) -xml_schema_regex = re.compile(r"\s*", re.MULTILINE) +xml_header_regex = re.compile(r"^<\?xml .*$", re.MULTILINE) +xml_schema_regex = re.compile(r"<\/?xs:schema.>", re.MULTILINE) MAGIC_ZIP = b"\x50\x4B\x03\x04" MAGIC_GZIP = b"\x1F\x8B" @@ -424,16 +425,19 @@ def _parse_report_record(record, nameservers=None, timeout=2.0): new_record["auth_results"]["spf"].append(new_result) if "envelope_from" not in new_record["identifiers"]: - envelope_from = new_record["auth_results"]["spf"][-1]["domain"] + envelope_from = None + if len(auth_results["spf"]) > 0: + envelope_from = new_record["auth_results"]["spf"][-1]["domain"] if envelope_from is not None: envelope_from = str(envelope_from).lower() new_record["identifiers"]["envelope_from"] = envelope_from elif new_record["identifiers"]["envelope_from"] is None: - envelope_from = new_record["auth_results"]["spf"][-1]["domain"] - if envelope_from is not None: - envelope_from = str(envelope_from).lower() - new_record["identifiers"]["envelope_from"] = envelope_from + if len(auth_results["spf"]) > 0: + envelope_from = new_record["auth_results"]["spf"][-1]["domain"] + if envelope_from is not None: + envelope_from = str(envelope_from).lower() + new_record["identifiers"]["envelope_from"] = envelope_from envelope_to = None if "envelope_to" in new_record["identifiers"]: @@ -457,9 +461,20 @@ def parse_aggregate_report_xml(xml, nameservers=None, timeout=2.0): Returns: OrderedDict: The parsed aggregate DMARC report """ + errors = [] + try: + xmltodict.parse(xml)["feedback"] + except Exception as e: + errors.append(e.__str__()) + + try: + # Replace XML header (sometimes they are invalid) + xml = xml_header_regex.sub("", xml) + # Remove invalid schema tags - xml = xml_schema_regex.sub("", xml) + xml = xml_schema_regex.sub('', xml) + report = xmltodict.parse(xml)["feedback"] report_metadata = report["report_metadata"] schema = "draft" @@ -467,7 +482,13 @@ def parse_aggregate_report_xml(xml, nameservers=None, timeout=2.0): schema = report["version"] new_report = OrderedDict([("xml_schema", schema)]) new_report_metadata = OrderedDict() - org_name = _get_base_domain(report_metadata["org_name"]) + if report_metadata["org_name"] is None: + if report_metadata["email"] is not None: + report_metadata["org_name"] = report_metadata[ + "email"].split("@")[-1] + org_name = report_metadata["org_name"] + if org_name is not None: + org_name = _get_base_domain(org_name) new_report_metadata["org_name"] = org_name new_report_metadata["org_email"] = report_metadata["email"] extra = None @@ -484,7 +505,6 @@ def parse_aggregate_report_xml(xml, nameservers=None, timeout=2.0): date_range["end"] = _timestamp_to_human(date_range["end"]) new_report_metadata["begin_date"] = date_range["begin"] new_report_metadata["end_date"] = date_range["end"] - errors = [] if "error" in report["report_metadata"]: if type(report["report_metadata"]["error"]) != list: errors = [report["report_metadata"]["error"]] @@ -526,13 +546,16 @@ def parse_aggregate_report_xml(xml, nameservers=None, timeout=2.0): if type(report["record"]) == list: for record in report["record"]: - records.append(_parse_report_record(record, - nameservers=nameservers, - timeout=timeout)) + report_record = _parse_report_record(record, + nameservers=nameservers, + timeout=timeout) + records.append(report_record) else: - records.append(_parse_report_record(report["record"], - nameservers=nameservers)) + report_record = _parse_report_record(report["record"], + nameservers=nameservers, + timeout=timeout) + records.append(report_record) new_report["records"] = records diff --git a/samples/!example.com!1538204542!1538463818.xml.sample b/samples/!example.com!1538204542!1538463818.xml.sample new file mode 100644 index 0000000..b68f81a --- /dev/null +++ b/samples/!example.com!1538204542!1538463818.xml.sample @@ -0,0 +1,40 @@ + + + + + administrator@accurateplastics.com + example.com:1538463741 + + 1538413632 + 1538413632 + + + + example.com + r + r +

none

+ reject + 100 + + + + 12.20.127.122 + 1 + + none + fail + fail + + + + example.com + + + + + none + + + + diff --git a/samples/estadocuenta1.infonacot.gob.mx!example.com!1536853302!1536939702!2940.xml.sample b/samples/estadocuenta1.infonacot.gob.mx!example.com!1536853302!1536939702!2940.xml.sample new file mode 100644 index 0000000..795e9d9 --- /dev/null +++ b/samples/estadocuenta1.infonacot.gob.mx!example.com!1536853302!1536939702!2940.xml.sample @@ -0,0 +1,34 @@ + + + + 2940 + XYZ Corporation + admin@estadocuenta1.infonacot.gob.mx + http://estadocuenta1.infonacot.gob.mx + + 1536853302 + 1536939702 + + + + example.com +

none

+ + + + 148.243.137.254 + 1 + + none + fail + fail + + + + estadocuenta1.infonacot.gob.mx + example.com + + + + + diff --git a/samples/ikea.com!xxxxxx.de!1538690400!1538776800.xml.sample b/samples/ikea.com!example.de!1538690400!1538776800.xml.sample similarity index 87% rename from samples/ikea.com!xxxxxx.de!1538690400!1538776800.xml.sample rename to samples/ikea.com!example.de!1538690400!1538776800.xml.sample index ac3f8b0..1f0c3d1 100644 --- a/samples/ikea.com!xxxxxx.de!1538690400!1538776800.xml.sample +++ b/samples/ikea.com!example.de!1538690400!1538776800.xml.sample @@ -10,7 +10,7 @@ - xxxxxx.de + example.de r r

none

@@ -29,12 +29,12 @@ - xxxxxx.de - xxxxxx.de + example.de + example.de - xxxxxx.de + example.de pass diff --git a/samples/usssa.com!example.com!1538784000!1538870399.xml.sample b/samples/usssa.com!example.com!1538784000!1538870399.xml.sample new file mode 100644 index 0000000..3593b4c --- /dev/null +++ b/samples/usssa.com!example.com!1538784000!1538870399.xml.sample @@ -0,0 +1,56 @@ + + + 1.0 + + usssa.com + postmaster@usssa.com + 8953b4d4a4ee4218b6ac0e2cb2667ee1 + + 1538784000 + 1538870399 + + + + example.com + r + r +

none

+ none + 100 + 0 + + + + 12.20.127.40 + 1 + + none + fail + fail + + + + + example.com + + + + + + + 199.230.200.36 + 1 + + none + fail + fail + + + + + example.com + + + + + diff --git a/setup.py b/setup.py index c2da17e..47fba1c 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ from setuptools import setup from codecs import open from os import path -__version__ = "4.1.7" +__version__ = "4.1.8" description = "A Python package and CLI for parsing aggregate and " \ "forensic DMARC reports" From 37d40c01ba0f01ed0e711ffc4935e0acc02ae2d8 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Sun, 7 Oct 2018 17:54:25 -0400 Subject: [PATCH 2/8] Still parse if spf is missing from auth_results --- parsedmarc/__init__.py | 35 ++++++++------- ...ample.com!1536105600!1536191999.xml.sample | 44 +++++++++++++++++++ 2 files changed, 64 insertions(+), 15 deletions(-) create mode 100644 samples/addisonfoods.com!example.com!1536105600!1536191999.xml.sample diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 0ca2bee..411bb11 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -393,23 +393,28 @@ def _parse_report_record(record, nameservers=None, timeout=2.0): new_record["auth_results"] = OrderedDict([("dkim", []), ("spf", [])]) if record["auth_results"] is not None: auth_results = record["auth_results"].copy() + if "spf" not in auth_results: + auth_results["spf"] = [] + if "dkim" not in auth_results: + auth_results["dkim"] = [] else: auth_results = new_record["auth_results"].copy() - if "dkim" in auth_results: - if type(auth_results["dkim"]) != list: - auth_results["dkim"] = [auth_results["dkim"]] - for result in auth_results["dkim"]: - if "domain" in result and result["domain"] is not None: - new_result = OrderedDict([("domain", result["domain"])]) - if "selector" in result and result["selector"] is not None: - new_result["selector"] = result["selector"] - else: - new_result["selector"] = "none" - if "result" in result and result["result"] is not None: - new_result["result"] = result["result"] - else: - new_result["result"] = "none" - new_record["auth_results"]["dkim"].append(new_result) + + if type(auth_results["dkim"]) != list: + auth_results["dkim"] = [auth_results["dkim"]] + for result in auth_results["dkim"]: + if "domain" in result and result["domain"] is not None: + new_result = OrderedDict([("domain", result["domain"])]) + if "selector" in result and result["selector"] is not None: + new_result["selector"] = result["selector"] + else: + new_result["selector"] = "none" + if "result" in result and result["result"] is not None: + new_result["result"] = result["result"] + else: + new_result["result"] = "none" + new_record["auth_results"]["dkim"].append(new_result) + if type(auth_results["spf"]) != list: auth_results["spf"] = [auth_results["spf"]] for result in auth_results["spf"]: diff --git a/samples/addisonfoods.com!example.com!1536105600!1536191999.xml.sample b/samples/addisonfoods.com!example.com!1536105600!1536191999.xml.sample new file mode 100644 index 0000000..8b7b0fe --- /dev/null +++ b/samples/addisonfoods.com!example.com!1536105600!1536191999.xml.sample @@ -0,0 +1,44 @@ + + + 1.0 + + addisonfoods.com + postmaster@addisonfoods.com + 3ceb5548498640beaeb47327e202b0b9 + + 1536105600 + 1536191999 + + + + example.com + r + r +

none

+ none + 100 + 0 + + + + 109.203.100.17 + 1 + + none + fail + fail + + + + example.com + example.com + + + + toptierhighticket.club + default + pass + + + + From 5b5e65d48b885d55a532311ac3200b6a4c7cff69 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Mon, 8 Oct 2018 07:07:43 -0400 Subject: [PATCH 3/8] Update docs/index.rst --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 22b8fee..72ae0f6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -390,7 +390,7 @@ To set up visual dashboards of DMARC data, install Elasticsearch and Kibana. wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - echo "deb https://artifacts.elastic.co/packages/6.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-6.x.list sudo apt-get update - sudo apt-get install -y elasticsearch kibana + sudo apt-get install -y default-jre-headless elasticsearch kibana sudo systemctl daemon-reload sudo systemctl enable elasticsearch.service sudo systemctl enable kibana.service From f1fa8709c2b75aa55e4edc77f554d6438d8a8bd0 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Mon, 8 Oct 2018 09:09:11 -0400 Subject: [PATCH 4/8] 4.1.9 - Workaround for forensic/ruf reports that are missing `Arrival-Date` --- CHANGELOG.md | 5 +++++ parsedmarc/__init__.py | 23 +++++++++++++---------- setup.py | 2 +- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 353132b..b192d4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +4.1.9 +----- + +- Workaround for forensic/ruf reports that are missing `Arrival-Date` + 4.1.8 ----- diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 411bb11..740adcb 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -44,7 +44,7 @@ import imapclient.exceptions import dateparser import mailparser -__version__ = "4.1.8" +__version__ = "4.1.9" logger = logging.getLogger(__name__) logger.setLevel(logging.ERROR) @@ -754,7 +754,7 @@ def parsed_aggregate_reports_to_csv(reports): def parse_forensic_report(feedback_report, sample, sample_headers_only, - nameservers=None, timeout=2.0): + msg_date, nameservers=None, timeout=2.0): """ Converts a DMARC forensic report and sample to a ``OrderedDict`` @@ -762,12 +762,13 @@ def parse_forensic_report(feedback_report, sample, sample_headers_only, feedback_report (str): A message's feedback report as a string sample (str): The RFC 822 headers or RFC 822 message sample sample_headers_only (bool): Set true if the sample is only headers + msg_date (str): The message's date header nameservers (list): A list of one or more nameservers to use (Cloudflare's public DNS resolvers by default) timeout (float): Sets the DNS timeout in seconds Returns: - OrderedDict: An parsed report and sample + OrderedDict: A parsed report and sample """ def convert_address(original_address): @@ -805,14 +806,14 @@ def parse_forensic_report(feedback_report, sample, sample_headers_only, for report_value in report_values: key = report_value[0].lower().replace("-", "_") parsed_report[key] = report_value[1] - if key == "arrival_date": - arrival_utc = dateparser.parse(parsed_report["arrival_date"], - settings={"TO_TIMEZONE": "UTC"}) - arrival_utc = arrival_utc.strftime("%Y-%m-%d %H:%M:%S") - parsed_report["arrival_date_utc"] = arrival_utc - if "arrival_date_utc" not in parsed_report: - raise InvalidForensicReport("Missing Arrival-Date") + if "arrival_date" not in parsed_report: + parsed_report["arrival_date"] = msg_date + + arrival_utc = dateparser.parse(parsed_report["arrival_date"], + settings={"TO_TIMEZONE": "UTC"}) + arrival_utc = arrival_utc.strftime("%Y-%m-%d %H:%M:%S") + parsed_report["arrival_date_utc"] = arrival_utc ip_address = parsed_report["source_ip"] parsed_report["source"] = _get_ip_address_info(ip_address, @@ -1040,6 +1041,7 @@ def parse_report_email(input_, nameservers=None, timeout=2.0): sample = None if "subject" in msg: subject = decode_header(msg["subject"]) + date = decode_header(msg["date"]) for part in msg.walk(): content_type = part.get_content_type() payload = part.get_payload() @@ -1067,6 +1069,7 @@ def parse_report_email(input_, nameservers=None, timeout=2.0): forensic_report = parse_forensic_report(feedback_report, sample, sample_headers_only, + date, nameservers=nameservers, timeout=timeout) diff --git a/setup.py b/setup.py index 47fba1c..2ff8ee6 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ from setuptools import setup from codecs import open from os import path -__version__ = "4.1.8" +__version__ = "4.1.9" description = "A Python package and CLI for parsing aggregate and " \ "forensic DMARC reports" From 66af8e60901ba50003d1ff1bf3f34c9c45ff889f Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Mon, 8 Oct 2018 09:31:53 -0400 Subject: [PATCH 5/8] Workaround for forensic/ruf reports that are missing Reported-Domain --- CHANGELOG.md | 3 ++- parsedmarc/__init__.py | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b192d4e..b31dc26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,8 @@ 4.1.9 ----- -- Workaround for forensic/ruf reports that are missing `Arrival-Date` +- Workaround for forensic/ruf reports that are missing `Arrival-Date` and/or +`Reported-Domain` 4.1.8 ----- diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 740adcb..03c98e9 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -856,7 +856,11 @@ def parse_forensic_report(feedback_report, sample, sample_headers_only, if "date_utc" in received: received["date_utc"] = received["date_utc"].replace("T", " ") - parsed_sample["from"] = convert_address(parsed_sample["from"][0]) + msg_from = convert_address(parsed_sample["from"][0]) + parsed_sample["from"] = msg_from + if "reported_domain" not in parsed_report: + domain = msg_from["address"].split("@")[-1].lower() + parsed_report["reported_domain"] = domain if "reply_to" in parsed_sample: parsed_sample["reply_to"] = list(map(lambda x: convert_address(x), From f235149863fbd3480b983eeb9d8211bff97bac3a Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Mon, 8 Oct 2018 14:22:43 -0400 Subject: [PATCH 6/8] 4.2.0 - Save each aggregate report record as a separate Splunk event --- CHANGELOG.md | 5 +++++ parsedmarc/__init__.py | 2 +- parsedmarc/splunk.py | 11 ++++++----- setup.py | 2 +- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b31dc26..1176ee2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +4.2.0 +------ + +- Save each aggregate report record as a separate Splunk event + 4.1.9 ----- diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 03c98e9..cc075e3 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -44,7 +44,7 @@ import imapclient.exceptions import dateparser import mailparser -__version__ = "4.1.9" +__version__ = "4.2.0" logger = logging.getLogger(__name__) logger.setLevel(logging.ERROR) diff --git a/parsedmarc/splunk.py b/parsedmarc/splunk.py index 96189b9..8448d15 100644 --- a/parsedmarc/splunk.py +++ b/parsedmarc/splunk.py @@ -93,11 +93,12 @@ class HECClient(object): new_report["spf_results"] = record["auth_results"][ "spf"] - data["sourcetype"] = "dmarc:aggregate" - timestamp = human_timestamp_to_timestamp(new_report["begin_date"]) - data["time"] = timestamp - data["event"] = new_report.copy() - json_str += "{0}\n".format(json.dumps(data)) + data["sourcetype"] = "dmarc:aggregate" + timestamp = human_timestamp_to_timestamp( + new_report["begin_date"]) + data["time"] = timestamp + data["event"] = new_report.copy() + json_str += "{0}\n".format(json.dumps(data)) try: response = self.session.post(self.url, data=json_str).json() except Exception as e: diff --git a/setup.py b/setup.py index 2ff8ee6..fe80350 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ from setuptools import setup from codecs import open from os import path -__version__ = "4.1.9" +__version__ = "4.2.0" description = "A Python package and CLI for parsing aggregate and " \ "forensic DMARC reports" From 20433cd2b6aaccf85bea0226050663c112c00b30 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Tue, 9 Oct 2018 10:34:42 -0400 Subject: [PATCH 7/8] Logging and packaging fixes --- CHANGELOG.md | 4 ++++ parsedmarc/__init__.py | 8 +++----- parsedmarc/cli.py | 12 ++++++++---- parsedmarc/splunk.py | 6 ++++++ requirements.txt | 1 + setup.py | 4 ++-- 6 files changed, 24 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1176ee2..46db8a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ------ - Save each aggregate report record as a separate Splunk event +- Fix IAMP delete action +- Suppress Splunk SSL validation warnings +- Change default logging level to `INFO` + 4.1.9 ----- diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index cc075e3..aeaa626 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -47,7 +47,7 @@ import mailparser __version__ = "4.2.0" logger = logging.getLogger(__name__) -logger.setLevel(logging.ERROR) +logger.setLevel(logging.INFO) feedback_report_regex = re.compile(r"^([\w\-]+): (.+)$", re.MULTILINE) xml_header_regex = re.compile(r"^<\?xml .*$", re.MULTILINE) @@ -1249,10 +1249,8 @@ def get_dmarc_reports_from_inbox(host=None, if type(msg_uids) == str: msg_uids = [msg_uids] - for chunk in chunks(msg_uids, 100): - server.add_flags(chunk, [imapclient.DELETED]) - - server.expunge() + server.delete_messages(msg_uids, silent=True) + server.expunge(msg_uids) def move_messages(msg_uids, folder): if type(msg_uids) == str: diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index fcf3d62..8405689 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -83,7 +83,7 @@ def _main(): arg_parser.add_argument("--imap-port", default=None, help="IMAP port") arg_parser.add_argument("--imap-no-ssl", action="store_true", default=False, - help="Do not use SSL when connecting to IMAP") + help="Do not use SSL/TLS when connecting to IMAP") arg_parser.add_argument("-r", "--reports-folder", default="INBOX", help="The IMAP folder containing the reports\n" "Default: INBOX") @@ -157,7 +157,7 @@ def _main(): help="Do not move or delete IMAP messages", action="store_true", default=False) arg_parser.add_argument("-s", "--silent", action="store_true", - help="Only print errors") + help="Only print errors and warnings") arg_parser.add_argument("--debug", action="store_true", help="Print debugging information") arg_parser.add_argument("-v", "--version", action="version", @@ -168,8 +168,12 @@ def _main(): args = arg_parser.parse_args() - logging.basicConfig(level=logging.ERROR) - logger.setLevel(logging.ERROR) + logging.basicConfig(level=logging.INFO) + logger.setLevel(logging.INFO) + + if args.silent: + logging.basicConfig(level=logging.WARNING) + logger.setLevel(logging.WARNING) if args.debug: logging.basicConfig(level=logging.DEBUG) logger.setLevel(logging.DEBUG) diff --git a/parsedmarc/splunk.py b/parsedmarc/splunk.py index 8448d15..714f2d4 100644 --- a/parsedmarc/splunk.py +++ b/parsedmarc/splunk.py @@ -1,11 +1,14 @@ from urllib.parse import urlparse import socket import json +import urllib3 import requests from parsedmarc import __version__, human_timestamp_to_timestamp +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + class SplunkError(RuntimeError): """Raised when a Splunk API error occurs""" @@ -136,3 +139,6 @@ class HECClient(object): raise SplunkError(e.__str__()) if response["code"] != 0: raise SplunkError(response["text"]) + + + diff --git a/requirements.txt b/requirements.txt index b10e457..aff41e8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ dnspython +urllib3 requests publicsuffix xmltodict diff --git a/setup.py b/setup.py index fe80350..b94a56c 100644 --- a/setup.py +++ b/setup.py @@ -92,8 +92,8 @@ setup( # requirements files see: # https://packaging.python.org/en/latest/requirements.html install_requires=['dnspython', 'publicsuffix', 'xmltodict', 'geoip2', - 'dnspython', 'imapclient', 'mail-parser', 'dateparser', - 'elasticsearch>=6.3.0,<7.0.0', + 'urllib3', 'requests', 'imapclient', 'mail-parser', + 'dateparser', 'elasticsearch>=6.3.0,<7.0.0', 'elasticsearch-dsl>=6.2.1,<7.0.0' ], From e2ec3bc2dad1e59f20bb682b94e85900f44446b7 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Tue, 9 Oct 2018 10:55:40 -0400 Subject: [PATCH 8/8] Change default logging level to `WARNING` --- CHANGELOG.md | 2 +- parsedmarc/__init__.py | 2 +- parsedmarc/cli.py | 7 ++----- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46db8a1..1a716a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ - Save each aggregate report record as a separate Splunk event - Fix IAMP delete action - Suppress Splunk SSL validation warnings -- Change default logging level to `INFO` +- Change default logging level to `WARNING` 4.1.9 diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index aeaa626..7e65715 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -47,7 +47,7 @@ import mailparser __version__ = "4.2.0" logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) +logger.setLevel(logging.WARNING) feedback_report_regex = re.compile(r"^([\w\-]+): (.+)$", re.MULTILINE) xml_header_regex = re.compile(r"^<\?xml .*$", re.MULTILINE) diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index 8405689..7be74b5 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -168,12 +168,9 @@ def _main(): args = arg_parser.parse_args() - logging.basicConfig(level=logging.INFO) - logger.setLevel(logging.INFO) + logging.basicConfig(level=logging.WARNING) + logger.setLevel(logging.WARNING) - if args.silent: - logging.basicConfig(level=logging.WARNING) - logger.setLevel(logging.WARNING) if args.debug: logging.basicConfig(level=logging.DEBUG) logger.setLevel(logging.DEBUG)