mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-07-07 01:05:09 +00:00
Merge pull request #122 from michaeldavie/handle_bad_data
Handle invalid aggregate reports
This commit is contained in:
Sean Whalen
GitHub
+18
-6
@@ -24,6 +24,7 @@ import mailbox
|
||||
import mailparser
|
||||
from expiringdict import ExpiringDict
|
||||
import xmltodict
|
||||
from lxml import etree
|
||||
from mailsuite.imap import IMAPClient
|
||||
from mailsuite.smtp import send_email
|
||||
|
||||
@@ -133,7 +134,10 @@ def _parse_report_record(record, offline=False, nameservers=None,
|
||||
new_record["policy_evaluated"] = new_policy_evaluated
|
||||
new_record["identifiers"] = record["identifiers"].copy()
|
||||
new_record["auth_results"] = OrderedDict([("dkim", []), ("spf", [])])
|
||||
lowered_from = new_record["identifiers"]["header_from"].lower()
|
||||
if type(new_record["identifiers"]["header_from"]) == str:
|
||||
lowered_from = new_record["identifiers"]["header_from"].lower()
|
||||
else:
|
||||
lowered_from = ''
|
||||
new_record["identifiers"]["header_from"] = lowered_from
|
||||
if record["auth_results"] is not None:
|
||||
auth_results = record["auth_results"].copy()
|
||||
@@ -215,10 +219,13 @@ def parse_aggregate_report_xml(xml, offline=False, nameservers=None,
|
||||
"""
|
||||
errors = []
|
||||
|
||||
# Parse XML and recover from errors
|
||||
try:
|
||||
xmltodict.parse(xml)["feedback"]
|
||||
except Exception as e:
|
||||
errors.append(e.__str__())
|
||||
tree = etree.parse(BytesIO(xml.encode('utf-8')),
|
||||
etree.XMLParser(recover=True))
|
||||
xml = etree.tostring(tree).decode('utf-8')
|
||||
|
||||
try:
|
||||
# Replace XML header (sometimes they are invalid)
|
||||
@@ -355,11 +362,11 @@ def extract_xml(input_):
|
||||
file_object.seek(0)
|
||||
if header.startswith(MAGIC_ZIP):
|
||||
_zip = zipfile.ZipFile(file_object)
|
||||
xml = _zip.open(_zip.namelist()[0]).read().decode()
|
||||
xml = _zip.open(_zip.namelist()[0]).read().decode(errors='ignore')
|
||||
elif header.startswith(MAGIC_GZIP):
|
||||
xml = GzipFile(fileobj=file_object).read().decode()
|
||||
xml = GzipFile(fileobj=file_object).read().decode(errors='ignore')
|
||||
elif header.startswith(MAGIC_XML):
|
||||
xml = file_object.read().decode()
|
||||
xml = file_object.read().decode(errors='ignore')
|
||||
else:
|
||||
file_object.close()
|
||||
raise InvalidAggregateReport("Not a valid zip, gzip, or xml file")
|
||||
@@ -419,7 +426,7 @@ def parsed_aggregate_reports_to_csv_rows(reports):
|
||||
return str(obj).lower()
|
||||
|
||||
if type(reports) == OrderedDict:
|
||||
reports = [reports]
|
||||
reports = [reports['report']]
|
||||
|
||||
rows = []
|
||||
|
||||
@@ -497,6 +504,11 @@ def parsed_aggregate_reports_to_csv_rows(reports):
|
||||
row["spf_results"] = ",".join(map(to_str, dkim_results))
|
||||
rows.append(row)
|
||||
|
||||
for r in rows:
|
||||
for k, v in r.items():
|
||||
if type(v) is not str:
|
||||
r[k] = ''
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
|
||||
@@ -29,3 +29,4 @@ sphinx>=1.0.5
|
||||
sphinx_rtd_theme>=0.4.3
|
||||
wheel>=0.33.6
|
||||
codecov>=2.0.15
|
||||
lxml>=4.4.1
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
<?xml version="1.0" encoding="UTF-8" ?>
|
||||
<feedback>
|
||||
<report_metadata>
|
||||
<org_name></org_name>
|
||||
<email>administrator@accurateplastics.com</email>
|
||||
<report_id>example.com:1538463741</report_id>
|
||||
<date_range>
|
||||
<begin>1538413632</begin>
|
||||
<end>1538413632</end>
|
||||
</date_range>
|
||||
</report_metadata>
|
||||
<policy_published>
|
||||
<domain>example.com</domain>
|
||||
<adkim>r</adkim>
|
||||
<aspf>r</aspf>
|
||||
<p>none</p>
|
||||
<sp>reject</sp>
|
||||
<pct>100</pct>
|
||||
</policy_published>
|
||||
<record>
|
||||
<row>
|
||||
<source_ip>12.20.127.122</source_ip>
|
||||
<count>1</count>
|
||||
<policy_evaluated>
|
||||
<disposition>none</disposition>
|
||||
<dkim>fail</dkim>
|
||||
<spf>fail</spf>
|
||||
</policy_evaluated>
|
||||
</row>
|
||||
<identifiers>
|
||||
<header_from>bad_byte‘</header_from>
|
||||
</identifiers>
|
||||
<auth_results>
|
||||
<spf>
|
||||
<domain></domain>
|
||||
<result>none</result>
|
||||
</spf>
|
||||
</auth_results>
|
||||
</record>
|
||||
</feedback>
|
||||
@@ -0,0 +1,40 @@
|
||||
<?xml version="1.0" encoding="UTF-8" ?>
|
||||
<feedback>
|
||||
<report_metadata>
|
||||
<org_name>veeam.com</org_name>
|
||||
<email><bad-xml@bad-xml.net></email>
|
||||
<report_id>sonexushealth.com:1530233361</report_id>
|
||||
<date_range>
|
||||
<begin>1530133200</begin>
|
||||
<end>1530219600</end>
|
||||
</date_range>
|
||||
</report_metadata>
|
||||
<policy_published>
|
||||
<domain>example.com</domain>
|
||||
<adkim>r</adkim>
|
||||
<aspf>r</aspf>
|
||||
<p>none</p>
|
||||
<sp>none</sp>
|
||||
<pct>100</pct>
|
||||
</policy_published>
|
||||
<record>
|
||||
<row>
|
||||
<source_ip>199.230.200.36</source_ip>
|
||||
<count>1</count>
|
||||
<policy_evaluated>
|
||||
<disposition>none</disposition>
|
||||
<dkim>fail</dkim>
|
||||
<spf>fail</spf>
|
||||
</policy_evaluated>
|
||||
</row>
|
||||
<identifiers>
|
||||
<header_from>bad<xml.net</header_from>
|
||||
</identifiers>
|
||||
<auth_results>
|
||||
<spf>
|
||||
<domain></domain>
|
||||
<result>none</result>
|
||||
</spf>
|
||||
</auth_results>
|
||||
</record>
|
||||
</feedback>
|
||||
Reference in New Issue
Block a user