diff --git a/CHANGELOG.md b/CHANGELOG.md index 61521713..12195c81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,17 +34,6 @@ `policy_strings` / `mx_host_patterns` from an earlier policy were reused for a later policy that did not define them, because the row template dict was built once per report instead of once per policy. -- **The `xml_schema` field of a parsed aggregate report can no longer be - empty.** parsedmarc reports the schema revision of an aggregate report in a - synthesized `xml_schema` field (surfaced in the JSON/CSV output for every - record), defaulting to `"draft"` when the report omits the optional - `` element (RFC 7489 Appendix C). That default did not cover an - empty `` or a whitespace-only value, which produced - `xml_schema=None` — an empty string in the flat JSON/CSV output — so - consumers that identify aggregate records by a non-empty `xml_schema` - (such as the Google SecOps parser's report-type detection) dropped those - records. The `"draft"` fallback now applies unless `` contains - non-empty text. ## 10.2.1 diff --git a/google_secops_parser/README.md b/google_secops_parser/README.md index 92af8338..ff4d4ed6 100644 --- a/google_secops_parser/README.md +++ b/google_secops_parser/README.md @@ -49,7 +49,7 @@ detects them by a field unique to each and maps them as follows: | parsedmarc report | Detected by | UDM `metadata.event_type` | | --- | --- | --- | -| DMARC aggregate | `xml_schema` (the serialized marker of the `` XML root) | `EMAIL_TRANSACTION` | +| DMARC aggregate | `xml_schema` | `EMAIL_TRANSACTION` | | DMARC failure | `feedback_type` or `arrival_date_utc` | `EMAIL_TRANSACTION` | | SMTP TLS (RFC 8460) | `policy_type` or `result_type` | `GENERIC_EVENT` | @@ -59,14 +59,6 @@ field (parsedmarc emits no `feedback_type` key for them, but always computes older than this parser lack `policy_type` (every RFC 8460 failure detail has a `result_type`). -`xml_schema` needs no fallback because it is not something the report -supplies. What actually identifies an aggregate report is its `` -XML root element — but the raw XML never reaches SecOps: parsedmarc parses -it and only the flattened JSON rows go over syslog. `xml_schema` is the -field parsedmarc synthesizes on every row that came from a `` -document ("draft" whenever `` is missing, empty, or whitespace), so -testing it is one-to-one equivalent to testing for the `` root. - `EMAIL_TRANSACTION` and `GENERIC_EVENT` are both valid `metadata.event_type` values. Note that **`GENERIC_EVENT` events only appear in raw-log and UDM search**, not in the curated SecOps views — that is the documented behaviour for diff --git a/google_secops_parser/parsedmarc.conf b/google_secops_parser/parsedmarc.conf index 0ea3065b..e9f8c70c 100644 --- a/google_secops_parser/parsedmarc.conf +++ b/google_secops_parser/parsedmarc.conf @@ -241,12 +241,9 @@ filter { # policy_type alone is not enough for streams from parsedmarc versions # that omitted policy_domain/policy_type on SMTP TLS failure-detail rows; # result_type is required in every RFC 8460 failure detail. - # xml_schema is aggregate-only and guaranteed non-empty: the - # XML root element is what identifies an aggregate report, and parsedmarc - # falls back to "draft" whenever is missing, empty, or - # whitespace (parsedmarc/__init__.py), so every JSON line serialized - # from an aggregate report carries a usable value. It is preferred - # over: header_from (can be empty when a + # xml_schema is aggregate-only and parsedmarc defaults it to "draft" when + # the report omits (parsedmarc/__init__.py), so it survives a + # missing version. It is preferred over: header_from (can be empty when a # record carries no identifiers), adkim (a defaulted policy field), domain # (a generic name), and dmarc_aligned (a boolean that only becomes testable # after the convert in step 1b -- detection should not depend on that). diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index ee8d0819..d964f5a1 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -848,20 +848,9 @@ def parse_aggregate_report_xml( report_metadata["email"], ) report_metadata["email"] = unwrapped - # The root element (anchored above) is what identifies an - # aggregate report; is only metadata about the schema - # revision. Reports in the wild omit entirely or send an - # empty (which xmltodict parses as None) or one carrying - # attributes (a dict). Only a non-empty text value may override the - # "draft" default: xml_schema must never be empty, because consumers - # of the flattened JSON/CSV output (e.g. the Google SecOps parser's - # report-type detection) rely on it being non-empty on every line - # serialized from an aggregate report. schema = "draft" if "version" in report: - version = _text(report["version"]) - if isinstance(version, str) and version.strip(): - schema = version.strip() + schema = report["version"] new_report: dict[str, Any] = { "xml_schema": schema, "xml_namespace": xml_namespace, diff --git a/tests/test_init.py b/tests/test_init.py index 48751f36..6ea50912 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -1497,73 +1497,6 @@ class Test(unittest.TestCase): self.assertIsNone(pp["testing"]) self.assertIsNone(pp["discovery_method"]) - def testEmptyVersionElementFallsBackToDraftSchema(self): - """An empty must not blank out xml_schema. - - The root element is what identifies an aggregate report; - is optional metadata (RFC 7489 Appendix C) that reporters - omit or leave empty in the wild. xmltodict parses an empty - as None, which previously became xml_schema=None and an - empty string in the flat rows — breaking downstream consumers that - detect aggregate rows by a non-empty xml_schema (the Google SecOps - parser's report-type cascade). - """ - xml = """ - - - - TestOrg - test@example.com - test-empty-version - 17040672001704153599 - - - example.com -

none

- - - - 192.0.2.1 - 1 - nonepasspass - - example.com - example.compass - - """ - report = parsedmarc.parse_aggregate_report_xml(xml, offline=True) - self.assertEqual(report["xml_schema"], "draft") - rows = parsedmarc.parsed_aggregate_reports_to_csv_rows(report) - self.assertEqual(rows[0]["xml_schema"], "draft") - - def testWhitespaceVersionElementFallsBackToDraftSchema(self): - """A whitespace-only value also falls back to 'draft'.""" - xml = """ - - - - TestOrg - test@example.com - test-ws-version - 17040672001704153599 - - - example.com -

none

- - - - 192.0.2.1 - 1 - nonepasspass - - example.com - example.compass - - """ - report = parsedmarc.parse_aggregate_report_xml(xml, offline=True) - self.assertEqual(report["xml_schema"], "draft") - def testMagicXmlTagDetection(self): """XML without declaration (starting with '<') is extracted""" xml_no_decl = b"Ta@b.comr117040672001704153599example.com

none

192.0.2.11nonepasspassexample.comexample.compass"