diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py
index 559b59a..fc1c50b 100644
--- a/parsedmarc/__init__.py
+++ b/parsedmarc/__init__.py
@@ -24,6 +24,7 @@ import mailbox
import mailparser
from expiringdict import ExpiringDict
import xmltodict
+from lxml import etree
from mailsuite.imap import IMAPClient
from mailsuite.smtp import send_email
@@ -133,7 +134,10 @@ def _parse_report_record(record, offline=False, nameservers=None,
new_record["policy_evaluated"] = new_policy_evaluated
new_record["identifiers"] = record["identifiers"].copy()
new_record["auth_results"] = OrderedDict([("dkim", []), ("spf", [])])
- lowered_from = new_record["identifiers"]["header_from"].lower()
+ if type(new_record["identifiers"]["header_from"]) == str:
+ lowered_from = new_record["identifiers"]["header_from"].lower()
+ else:
+ lowered_from = ''
new_record["identifiers"]["header_from"] = lowered_from
if record["auth_results"] is not None:
auth_results = record["auth_results"].copy()
@@ -215,10 +219,13 @@ def parse_aggregate_report_xml(xml, offline=False, nameservers=None,
"""
errors = []
+ # Parse XML and recover from errors
try:
xmltodict.parse(xml)["feedback"]
except Exception as e:
- errors.append(e.__str__())
+ tree = etree.parse(BytesIO(xml.encode('utf-8')),
+ etree.XMLParser(recover=True))
+ xml = etree.tostring(tree).decode('utf-8')
try:
# Replace XML header (sometimes they are invalid)
@@ -355,11 +362,11 @@ def extract_xml(input_):
file_object.seek(0)
if header.startswith(MAGIC_ZIP):
_zip = zipfile.ZipFile(file_object)
- xml = _zip.open(_zip.namelist()[0]).read().decode()
+ xml = _zip.open(_zip.namelist()[0]).read().decode(errors='ignore')
elif header.startswith(MAGIC_GZIP):
- xml = GzipFile(fileobj=file_object).read().decode()
+ xml = GzipFile(fileobj=file_object).read().decode(errors='ignore')
elif header.startswith(MAGIC_XML):
- xml = file_object.read().decode()
+ xml = file_object.read().decode(errors='ignore')
else:
file_object.close()
raise InvalidAggregateReport("Not a valid zip, gzip, or xml file")
@@ -419,7 +426,7 @@ def parsed_aggregate_reports_to_csv_rows(reports):
return str(obj).lower()
if type(reports) == OrderedDict:
- reports = [reports]
+ reports = [reports['report']]
rows = []
@@ -497,6 +504,11 @@ def parsed_aggregate_reports_to_csv_rows(reports):
row["spf_results"] = ",".join(map(to_str, dkim_results))
rows.append(row)
+ for r in rows:
+ for k, v in r.items():
+ if type(v) is not str:
+ r[k] = ''
+
return rows
diff --git a/requirements.txt b/requirements.txt
index 0680ca2..478c64d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -29,3 +29,4 @@ sphinx>=1.0.5
sphinx_rtd_theme>=0.4.3
wheel>=0.33.6
codecov>=2.0.15
+lxml>=4.4.1
diff --git a/samples/aggregate/invalid_utf_8.xml b/samples/aggregate/invalid_utf_8.xml
new file mode 100644
index 0000000..e2346f3
--- /dev/null
+++ b/samples/aggregate/invalid_utf_8.xml
@@ -0,0 +1,40 @@
+
+
+
+
+ administrator@accurateplastics.com
+ example.com:1538463741
+
+ 1538413632
+ 1538413632
+
+
+
+ example.com
+ r
+ r
+ none
+ reject
+ 100
+
+
+
+ 12.20.127.122
+ 1
+
+ none
+ fail
+ fail
+
+
+
+ bad_byte‘
+
+
+
+
+ none
+
+
+
+
diff --git a/samples/aggregate/invalid_xml.xml b/samples/aggregate/invalid_xml.xml
new file mode 100644
index 0000000..fdbeabc
--- /dev/null
+++ b/samples/aggregate/invalid_xml.xml
@@ -0,0 +1,40 @@
+
+
+
+ veeam.com
+
+ sonexushealth.com:1530233361
+
+ 1530133200
+ 1530219600
+
+
+
+ example.com
+ r
+ r
+ none
+ none
+ 100
+
+
+
+ 199.230.200.36
+ 1
+
+ none
+ fail
+ fail
+
+
+
+ bad
+
+
+
+
+ none
+
+
+
+
diff --git a/setup.py b/setup.py
index 4657876..a277a64 100644
--- a/setup.py
+++ b/setup.py
@@ -104,7 +104,7 @@ setup(
'elasticsearch>=6.3.1,<7.0.0',
'elasticsearch-dsl>=6.3.1,<7.0.0',
'kafka-python>=1.4.4',
- 'tqdm>=4.31.1'
+ 'tqdm>=4.31.1', 'lxml>=4.4.1'
],
entry_points={