mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-10-29 05:35:44 +00:00
Ensure aggregate report output always has an xml_schema value
Ensure that the JSON or CSV output from a parsed DMARC aggregate report always has an xml_schema value (falling back to "draft" when <version> is missing, empty, or attribute-wrapped), so that the reports can be detected by Google SecOps and other output parsers. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Sean WhalenandClaude Fable 5 committed
1 parent
8935e733cf
commit
7ce8b63688
5 files changed
+84
-8
No files matched your search
@@ -34,6 +34,9 @@
|
||||
`policy_strings` / `mx_host_patterns` from an earlier policy were reused for
|
||||
a later policy that did not define them, because the row template dict was
|
||||
built once per report instead of once per policy.
|
||||
- Ensure that the JSON or CSV output from a parsed DMARC aggregate report
|
||||
always has an `xml_schema` value, so that the reports can be detected by
|
||||
Google SecOps and other output parsers.
|
||||
|
||||
## 10.2.1
|
||||
|
||||
|
||||
@@ -57,7 +57,8 @@ The `or` fallbacks matter: text-format failure reports have no `Feedback-Type`
|
||||
field (parsedmarc emits no `feedback_type` key for them, but always computes
|
||||
`arrival_date_utc`), and SMTP TLS failure-detail rows from parsedmarc versions
|
||||
older than this parser lack `policy_type` (every RFC 8460 failure detail has a
|
||||
`result_type`).
|
||||
`result_type`). `xml_schema` needs no fallback: parsedmarc always sets it on
|
||||
aggregate rows so that output parsers like this one can detect them.
|
||||
|
||||
`EMAIL_TRANSACTION` and `GENERIC_EVENT` are both valid `metadata.event_type`
|
||||
values. Note that **`GENERIC_EVENT` events only appear in raw-log and UDM
|
||||
|
||||
@@ -241,12 +241,12 @@ filter {
|
||||
# policy_type alone is not enough for streams from parsedmarc versions
|
||||
# that omitted policy_domain/policy_type on SMTP TLS failure-detail rows;
|
||||
# result_type is required in every RFC 8460 failure detail.
|
||||
# xml_schema is aggregate-only and parsedmarc defaults it to "draft" when
|
||||
# the report omits <version> (parsedmarc/__init__.py), so it survives a
|
||||
# missing version. It is preferred over: header_from (can be empty when a
|
||||
# record carries no identifiers), adkim (a defaulted policy field), domain
|
||||
# (a generic name), and dmarc_aligned (a boolean that only becomes testable
|
||||
# after the convert in step 1b -- detection should not depend on that).
|
||||
# xml_schema is aggregate-only and parsedmarc always sets it on aggregate
|
||||
# rows, so every aggregate row carries a usable value. It is preferred
|
||||
# over: header_from (can be empty when a record carries no identifiers),
|
||||
# adkim (a defaulted policy field), domain (a generic name), and
|
||||
# dmarc_aligned (a boolean that only becomes testable after the convert
|
||||
# in step 1b -- detection should not depend on that).
|
||||
# ---------------------------------------------------------------------------
|
||||
if [feedback_type] != "" or [arrival_date_utc] != "" {
|
||||
mutate { replace => { "report_type" => "failure" } }
|
||||
|
||||
@@ -848,9 +848,14 @@ def parse_aggregate_report_xml(
|
||||
report_metadata["email"],
|
||||
)
|
||||
report_metadata["email"] = unwrapped
|
||||
# Always give xml_schema a non-empty value, even when <version> is
|
||||
# missing, empty, or attribute-wrapped, so that output parsers
|
||||
# (e.g. Google SecOps) can detect aggregate rows by it.
|
||||
schema = "draft"
|
||||
if "version" in report:
|
||||
schema = report["version"]
|
||||
version = _text(report["version"])
|
||||
if isinstance(version, str) and version.strip():
|
||||
schema = version.strip()
|
||||
new_report: dict[str, Any] = {
|
||||
"xml_schema": schema,
|
||||
"xml_namespace": xml_namespace,
|
||||
|
||||
@@ -1497,6 +1497,73 @@ class Test(unittest.TestCase):
|
||||
self.assertIsNone(pp["testing"])
|
||||
self.assertIsNone(pp["discovery_method"])
|
||||
|
||||
def testEmptyVersionElementFallsBackToDraftSchema(self):
|
||||
"""An empty <version/> must not blank out xml_schema.
|
||||
|
||||
The <feedback> root element is what identifies an aggregate report;
|
||||
<version> is optional metadata (RFC 7489 Appendix C) that reporters
|
||||
omit or leave empty in the wild. xmltodict parses an empty
|
||||
<version/> as None, which previously became xml_schema=None and an
|
||||
empty string in the flat rows — breaking downstream consumers that
|
||||
detect aggregate rows by a non-empty xml_schema (the Google SecOps
|
||||
parser's report-type cascade).
|
||||
"""
|
||||
xml = """<?xml version="1.0"?>
|
||||
<feedback>
|
||||
<version/>
|
||||
<report_metadata>
|
||||
<org_name>TestOrg</org_name>
|
||||
<email>test@example.com</email>
|
||||
<report_id>test-empty-version</report_id>
|
||||
<date_range><begin>1704067200</begin><end>1704153599</end></date_range>
|
||||
</report_metadata>
|
||||
<policy_published>
|
||||
<domain>example.com</domain>
|
||||
<p>none</p>
|
||||
</policy_published>
|
||||
<record>
|
||||
<row>
|
||||
<source_ip>192.0.2.1</source_ip>
|
||||
<count>1</count>
|
||||
<policy_evaluated><disposition>none</disposition><dkim>pass</dkim><spf>pass</spf></policy_evaluated>
|
||||
</row>
|
||||
<identifiers><header_from>example.com</header_from></identifiers>
|
||||
<auth_results><spf><domain>example.com</domain><result>pass</result></spf></auth_results>
|
||||
</record>
|
||||
</feedback>"""
|
||||
report = parsedmarc.parse_aggregate_report_xml(xml, offline=True)
|
||||
self.assertEqual(report["xml_schema"], "draft")
|
||||
rows = parsedmarc.parsed_aggregate_reports_to_csv_rows(report)
|
||||
self.assertEqual(rows[0]["xml_schema"], "draft")
|
||||
|
||||
def testWhitespaceVersionElementFallsBackToDraftSchema(self):
|
||||
"""A whitespace-only <version> value also falls back to 'draft'."""
|
||||
xml = """<?xml version="1.0"?>
|
||||
<feedback>
|
||||
<version> </version>
|
||||
<report_metadata>
|
||||
<org_name>TestOrg</org_name>
|
||||
<email>test@example.com</email>
|
||||
<report_id>test-ws-version</report_id>
|
||||
<date_range><begin>1704067200</begin><end>1704153599</end></date_range>
|
||||
</report_metadata>
|
||||
<policy_published>
|
||||
<domain>example.com</domain>
|
||||
<p>none</p>
|
||||
</policy_published>
|
||||
<record>
|
||||
<row>
|
||||
<source_ip>192.0.2.1</source_ip>
|
||||
<count>1</count>
|
||||
<policy_evaluated><disposition>none</disposition><dkim>pass</dkim><spf>pass</spf></policy_evaluated>
|
||||
</row>
|
||||
<identifiers><header_from>example.com</header_from></identifiers>
|
||||
<auth_results><spf><domain>example.com</domain><result>pass</result></spf></auth_results>
|
||||
</record>
|
||||
</feedback>"""
|
||||
report = parsedmarc.parse_aggregate_report_xml(xml, offline=True)
|
||||
self.assertEqual(report["xml_schema"], "draft")
|
||||
|
||||
def testMagicXmlTagDetection(self):
|
||||
"""XML without declaration (starting with '<') is extracted"""
|
||||
xml_no_decl = b"<feedback><report_metadata><org_name>T</org_name><email>a@b.com</email><report_id>r1</report_id><date_range><begin>1704067200</begin><end>1704153599</end></date_range></report_metadata><policy_published><domain>example.com</domain><p>none</p></policy_published><record><row><source_ip>192.0.2.1</source_ip><count>1</count><policy_evaluated><disposition>none</disposition><dkim>pass</dkim><spf>pass</spf></policy_evaluated></row><identifiers><header_from>example.com</header_from></identifiers><auth_results><spf><domain>example.com</domain><result>pass</result></spf></auth_results></record></feedback>"
|
||||
|
||||
Reference in new issue
Block a user