diff --git a/_modules/index.html b/_modules/index.html index 43ab976..11b1df5 100644 --- a/_modules/index.html +++ b/_modules/index.html @@ -5,14 +5,14 @@ - Overview: module code — parsedmarc 10.1.1 documentation + Overview: module code — parsedmarc 10.2.0 documentation - + diff --git a/_modules/parsedmarc.html b/_modules/parsedmarc.html index 7cef393..bc058ed 100644 --- a/_modules/parsedmarc.html +++ b/_modules/parsedmarc.html @@ -5,14 +5,14 @@ - parsedmarc — parsedmarc 10.1.1 documentation + parsedmarc — parsedmarc 10.2.0 documentation - + @@ -89,11 +89,13 @@ import email import email.utils import json +import logging import mailbox import os import re import shutil import tempfile +import traceback import xml.parsers.expat as expat import zipfile import zlib @@ -240,6 +242,26 @@ InvalidForensicReport = InvalidFailureReport +def _exc_origin(error: BaseException) -> str: + """Returns a ``" (raised at <file>:<line>)"`` suffix pointing at where an + unexpected exception actually originated, but only when the parsedmarc + logger is at ``DEBUG`` level. Returns ``""`` otherwise, so normal output is + unchanged. + + The deepest traceback frame is the line that raised, which is the useful + breadcrumb when a catch-all ``except Exception`` turns an unforeseen error + into a generic ``Invalid*Report`` -- without it, the message says *what* + failed but not *where*. + """ + if not logger.isEnabledFor(logging.DEBUG): + return "" + frames = traceback.extract_tb(error.__traceback__) + if not frames: + return "" + last = frames[-1] + return " (raised at {0}:{1})".format(last.filename, last.lineno) + + def _text(value: Any) -> Optional[str]: """Unwrap a possibly-langAttrString value parsed by xmltodict. @@ -606,7 +628,9 @@ new_result["human_result"] = _text(result.get("human_result")) new_record["auth_results"]["spf"].append(new_result) - if "envelope_from" not in new_record["identifiers"]: + # Backfill envelope_from from the last SPF result's domain when the + # reporter omitted the identifier or sent it empty. + if new_record["identifiers"].get("envelope_from") is None: envelope_from = None if len(auth_results["spf"]) > 0: spf_result = auth_results["spf"][-1] @@ -616,13 +640,6 @@ envelope_from = str(envelope_from).lower() new_record["identifiers"]["envelope_from"] = envelope_from - elif new_record["identifiers"]["envelope_from"] is None: - if len(auth_results["spf"]) > 0: - envelope_from = new_record["auth_results"]["spf"][-1]["domain"] - if envelope_from is not None: - envelope_from = str(envelope_from).lower() - new_record["identifiers"]["envelope_from"] = envelope_from - envelope_to = None if "envelope_to" in new_record["identifiers"]: envelope_to = new_record["identifiers"]["envelope_to"] @@ -664,9 +681,11 @@ return new_failure_details except KeyError as e: - raise InvalidSMTPTLSReport(f"Missing required failure details field: {e}") + raise InvalidSMTPTLSReport( + f"Missing required failure details field: {e}" + ) from e except Exception as e: - raise InvalidSMTPTLSReport(str(e)) + raise InvalidSMTPTLSReport(str(e) + _exc_origin(e)) from e def _parse_smtp_tls_report_policy(policy: dict[str, Any]): @@ -704,9 +723,9 @@ return new_policy except KeyError as e: - raise InvalidSMTPTLSReport(f"Missing required policy field: {e}") + raise InvalidSMTPTLSReport(f"Missing required policy field: {e}") from e except Exception as e: - raise InvalidSMTPTLSReport(str(e)) + raise InvalidSMTPTLSReport(str(e) + _exc_origin(e)) from e
@@ -748,9 +767,9 @@ return new_report except KeyError as e: - raise InvalidSMTPTLSReport(f"Missing required field: {e}") + raise InvalidSMTPTLSReport(f"Missing required field: {e}") from e except Exception as e: - raise InvalidSMTPTLSReport(str(e))
+ raise InvalidSMTPTLSReport(str(e) + _exc_origin(e)) from e @@ -1133,15 +1152,21 @@ return cast(AggregateReport, new_report) except expat.ExpatError as error: - raise InvalidAggregateReport("Invalid XML: {0}".format(error.__str__())) + raise InvalidAggregateReport( + "Invalid XML: {0}".format(error.__str__()) + ) from error except KeyError as error: - raise InvalidAggregateReport("Missing field: {0}".format(error.__str__())) - except AttributeError: - raise InvalidAggregateReport("Report missing required section") + raise InvalidAggregateReport( + "Missing field: {0}".format(error.__str__()) + ) from error + except AttributeError as error: + raise InvalidAggregateReport("Report missing required section") from error except Exception as error: - raise InvalidAggregateReport("Unexpected error: {0}".format(error.__str__())) + raise InvalidAggregateReport( + "Unexpected error: {0}{1}".format(error.__str__(), _exc_origin(error)) + ) from error @@ -1222,10 +1247,10 @@ else: raise ParserError("Not a valid zip, gzip, json, or xml file") - except UnicodeDecodeError: - raise ParserError("File objects must be opened in binary (rb) mode") except Exception as error: - raise ParserError("Invalid archive file: {0}".format(error.__str__())) + raise ParserError( + "Invalid archive file: {0}{1}".format(error.__str__(), _exc_origin(error)) + ) from error finally: if file_object: try: @@ -1292,7 +1317,7 @@ try: xml = extract_report(_input) except Exception as e: - raise InvalidAggregateReport(e) + raise InvalidAggregateReport(str(e) + _exc_origin(e)) from e return parse_aggregate_report_xml( xml, @@ -1682,10 +1707,14 @@ return cast(FailureReport, parsed_report) except KeyError as error: - raise InvalidFailureReport("Missing value: {0}".format(error.__str__())) + raise InvalidFailureReport( + "Missing value: {0}".format(error.__str__()) + ) from error except Exception as error: - raise InvalidFailureReport("Unexpected error: {0}".format(error.__str__())) + raise InvalidFailureReport( + "Unexpected error: {0}{1}".format(error.__str__(), _exc_origin(error)) + ) from error @@ -1862,7 +1891,7 @@ msg = email.message_from_string(input_str) except Exception as e: - raise ParserError(e.__str__()) + raise ParserError(e.__str__() + _exc_origin(e)) from e subject = None feedback_report = None smtp_tls_report = None @@ -1919,10 +1948,10 @@ fields["received-date"], fields["sender-ip-address"] ) except Exception as e: - error = 'Unable to parse message with subject "{0}": {1}'.format( - subject, e + error = 'Unable to parse message with subject "{0}": {1}{2}'.format( + subject, e, _exc_origin(e) ) - raise InvalidDMARCReport(error) + raise InvalidDMARCReport(error) from e sample = parts[1].lstrip() logger.debug(sample) @@ -1961,17 +1990,18 @@ except (TypeError, ValueError, binascii.Error): pass - except InvalidDMARCReport: - error = 'Message with subject "{0}" is not a valid DMARC report'.format( - subject + except InvalidDMARCReport as e: + error = ( + 'Message with subject "{0}" is not a valid ' + "DMARC report: {1}".format(subject, e) ) - raise ParserError(error) + raise ParserError(error) from e except Exception as e: - error = 'Unable to parse message with subject "{0}": {1}'.format( - subject, e + error = 'Unable to parse message with subject "{0}": {1}{2}'.format( + subject, e, _exc_origin(e) ) - raise ParserError(error) + raise ParserError(error) from e if feedback_report and sample: try: @@ -1995,9 +2025,7 @@ "is not a valid " "failure DMARC report: {1}".format(subject, e) ) - raise InvalidFailureReport(error) - except Exception as e: - raise InvalidFailureReport(e.__str__()) + raise InvalidFailureReport(error) from e result = {"report_type": "failure", "report": failure_report} return result @@ -2010,6 +2038,54 @@ +# An RFC 5322 header field name (printable ASCII excluding the colon) followed +# by a colon at the start of a line, or an mbox "From " separator. +_email_header_regex = re.compile(r"^(From |[\x21-\x39\x3b-\x7e]+:)") + + +def _looks_like_email(text: str) -> bool: + """Returns True if the first line looks like an email header. + + Callers pass already-``lstrip()``-ed text, so the first line is the first + meaningful line. + """ + first_line = text.split("\n", 1)[0] + return _email_header_regex.match(first_line) is not None + + +def _describe_parse_failure( + content: Union[str, bytes], + aggregate_error: InvalidAggregateReport, + smtp_tls_error: InvalidSMTPTLSReport, + email_error: InvalidDMARCReport, +) -> str: + """Builds a human-readable reason for a parse_report_file failure. + + parse_report_file tries the aggregate XML, SMTP TLS JSON, and report-email + parsers in turn; when all three reject the input, only the parser for the + format the content actually resembles produced a meaningful error. The + other two are noise -- a malformed aggregate report is also "not JSON" and + "not an email". Sniff the leading non-whitespace byte to surface the single + relevant reason. + """ + if isinstance(content, (bytes, bytearray, memoryview)): + sniff = bytes(content)[:512].decode("utf-8", errors="replace") + else: + sniff = content[:512] + sniff = sniff.lstrip() + + if sniff.startswith("<"): + return "Invalid aggregate report: {0}".format(aggregate_error) + if sniff.startswith("{"): + return "Invalid SMTP TLS report: {0}".format(smtp_tls_error) + if _looks_like_email(sniff): + return "Invalid report email: {0}".format(email_error) + return ( + "Not a recognized report format (not a DMARC aggregate XML report, " + "an SMTP TLS JSON report, or a DMARC report email)" + ) + +
[docs] def parse_report_file( @@ -2067,6 +2143,10 @@ results: Optional[ParsedReport] = None + # parse_report_file tries the three report formats in turn. When all three + # reject the input, keep each format's specific error so the final message + # can explain *why* the file is invalid instead of a bare "Not a valid + # report" (see _describe_parse_failure). try: report = parse_aggregate_report_file( content, @@ -2082,11 +2162,11 @@ normalize_timespan_threshold_hours=normalize_timespan_threshold_hours, ) results = {"report_type": "aggregate", "report": report} - except InvalidAggregateReport: + except InvalidAggregateReport as aggregate_error: try: report = parse_smtp_tls_report_json(content) results = {"report_type": "smtp_tls", "report": report} - except InvalidSMTPTLSReport: + except InvalidSMTPTLSReport as smtp_tls_error: try: results = parse_report_email( content, @@ -2102,8 +2182,12 @@ keep_alive=keep_alive, normalize_timespan_threshold_hours=normalize_timespan_threshold_hours, ) - except InvalidDMARCReport: - raise ParserError("Not a valid report") + except InvalidDMARCReport as email_error: + raise ParserError( + _describe_parse_failure( + content, aggregate_error, smtp_tls_error, email_error + ) + ) from email_error if results is None: raise ParserError("Not a valid report") diff --git a/_modules/parsedmarc/elastic.html b/_modules/parsedmarc/elastic.html index 13834c1..878fc13 100644 --- a/_modules/parsedmarc/elastic.html +++ b/_modules/parsedmarc/elastic.html @@ -5,14 +5,14 @@ - parsedmarc.elastic — parsedmarc 10.1.1 documentation + parsedmarc.elastic — parsedmarc 10.2.0 documentation - + diff --git a/_modules/parsedmarc/opensearch.html b/_modules/parsedmarc/opensearch.html index d191b92..8bb458a 100644 --- a/_modules/parsedmarc/opensearch.html +++ b/_modules/parsedmarc/opensearch.html @@ -5,14 +5,14 @@ - parsedmarc.opensearch — parsedmarc 10.1.1 documentation + parsedmarc.opensearch — parsedmarc 10.2.0 documentation - + diff --git a/_modules/parsedmarc/splunk.html b/_modules/parsedmarc/splunk.html index 016d742..24d7d3c 100644 --- a/_modules/parsedmarc/splunk.html +++ b/_modules/parsedmarc/splunk.html @@ -5,14 +5,14 @@ - parsedmarc.splunk — parsedmarc 10.1.1 documentation + parsedmarc.splunk — parsedmarc 10.2.0 documentation - + diff --git a/_modules/parsedmarc/types.html b/_modules/parsedmarc/types.html index 186308c..a320fa8 100644 --- a/_modules/parsedmarc/types.html +++ b/_modules/parsedmarc/types.html @@ -5,14 +5,14 @@ - parsedmarc.types — parsedmarc 10.1.1 documentation + parsedmarc.types — parsedmarc 10.2.0 documentation - + diff --git a/_modules/parsedmarc/utils.html b/_modules/parsedmarc/utils.html index e1cc801..1120492 100644 --- a/_modules/parsedmarc/utils.html +++ b/_modules/parsedmarc/utils.html @@ -5,14 +5,14 @@ - parsedmarc.utils — parsedmarc 10.1.1 documentation + parsedmarc.utils — parsedmarc 10.2.0 documentation - + diff --git a/_static/documentation_options.js b/_static/documentation_options.js index 37b65ed..5c223cd 100644 --- a/_static/documentation_options.js +++ b/_static/documentation_options.js @@ -1,5 +1,5 @@ const DOCUMENTATION_OPTIONS = { - VERSION: '10.1.1', + VERSION: '10.2.0', LANGUAGE: 'en', COLLAPSE_INDEX: false, BUILDER: 'html', diff --git a/api.html b/api.html index 2c49455..c71d73e 100644 --- a/api.html +++ b/api.html @@ -6,14 +6,14 @@ - API reference — parsedmarc 10.1.1 documentation + API reference — parsedmarc 10.2.0 documentation - + diff --git a/contributing.html b/contributing.html index facb901..04fd8b9 100644 --- a/contributing.html +++ b/contributing.html @@ -6,14 +6,14 @@ - Contributing to parsedmarc — parsedmarc 10.1.1 documentation + Contributing to parsedmarc — parsedmarc 10.2.0 documentation - + diff --git a/davmail.html b/davmail.html index 2c5632c..568ed11 100644 --- a/davmail.html +++ b/davmail.html @@ -6,14 +6,14 @@ - Accessing an inbox using OWA/EWS — parsedmarc 10.1.1 documentation + Accessing an inbox using OWA/EWS — parsedmarc 10.2.0 documentation - + diff --git a/dmarc.html b/dmarc.html index 8bb8773..17e7311 100644 --- a/dmarc.html +++ b/dmarc.html @@ -6,14 +6,14 @@ - Understanding DMARC — parsedmarc 10.1.1 documentation + Understanding DMARC — parsedmarc 10.2.0 documentation - + diff --git a/elasticsearch.html b/elasticsearch.html index 25d50a7..7f95b97 100644 --- a/elasticsearch.html +++ b/elasticsearch.html @@ -6,14 +6,14 @@ - Elasticsearch and Kibana — parsedmarc 10.1.1 documentation + Elasticsearch and Kibana — parsedmarc 10.2.0 documentation - + diff --git a/genindex.html b/genindex.html index 568b07b..70a3f5a 100644 --- a/genindex.html +++ b/genindex.html @@ -5,14 +5,14 @@ - Index — parsedmarc 10.1.1 documentation + Index — parsedmarc 10.2.0 documentation - + diff --git a/index.html b/index.html index 163cbb0..46ba26c 100644 --- a/index.html +++ b/index.html @@ -6,14 +6,14 @@ - parsedmarc documentation - Open source DMARC report analyzer and visualizer — parsedmarc 10.1.1 documentation + parsedmarc documentation - Open source DMARC report analyzer and visualizer — parsedmarc 10.2.0 documentation - + diff --git a/installation.html b/installation.html index a39c36d..c10a8a0 100644 --- a/installation.html +++ b/installation.html @@ -6,14 +6,14 @@ - Installation — parsedmarc 10.1.1 documentation + Installation — parsedmarc 10.2.0 documentation - + diff --git a/kibana.html b/kibana.html index cc3b4a2..dca3507 100644 --- a/kibana.html +++ b/kibana.html @@ -6,14 +6,14 @@ - Using the Kibana dashboards — parsedmarc 10.1.1 documentation + Using the Kibana dashboards — parsedmarc 10.2.0 documentation - + diff --git a/mailing-lists.html b/mailing-lists.html index 7e49e86..d2e6492 100644 --- a/mailing-lists.html +++ b/mailing-lists.html @@ -6,14 +6,14 @@ - What about mailing lists? — parsedmarc 10.1.1 documentation + What about mailing lists? — parsedmarc 10.2.0 documentation - + diff --git a/objects.inv b/objects.inv index f93a5d1..fa74b7a 100644 Binary files a/objects.inv and b/objects.inv differ diff --git a/opensearch.html b/opensearch.html index a641c4f..da6b136 100644 --- a/opensearch.html +++ b/opensearch.html @@ -6,14 +6,14 @@ - OpenSearch and Grafana — parsedmarc 10.1.1 documentation + OpenSearch and Grafana — parsedmarc 10.2.0 documentation - + diff --git a/output.html b/output.html index 9133901..4e9b766 100644 --- a/output.html +++ b/output.html @@ -6,14 +6,14 @@ - Sample outputs — parsedmarc 10.1.1 documentation + Sample outputs — parsedmarc 10.2.0 documentation - + diff --git a/py-modindex.html b/py-modindex.html index b170d33..1f40ad4 100644 --- a/py-modindex.html +++ b/py-modindex.html @@ -5,14 +5,14 @@ - Python Module Index — parsedmarc 10.1.1 documentation + Python Module Index — parsedmarc 10.2.0 documentation - + diff --git a/search.html b/search.html index 9dc16df..151d832 100644 --- a/search.html +++ b/search.html @@ -5,7 +5,7 @@ - Search — parsedmarc 10.1.1 documentation + Search — parsedmarc 10.2.0 documentation @@ -13,7 +13,7 @@ - + diff --git a/splunk.html b/splunk.html index 83a7b0a..e16284f 100644 --- a/splunk.html +++ b/splunk.html @@ -6,14 +6,14 @@ - Splunk — parsedmarc 10.1.1 documentation + Splunk — parsedmarc 10.2.0 documentation - + diff --git a/usage.html b/usage.html index d1c1391..7931dd5 100644 --- a/usage.html +++ b/usage.html @@ -6,14 +6,14 @@ - Using parsedmarc — parsedmarc 10.1.1 documentation + Using parsedmarc — parsedmarc 10.2.0 documentation - +