From da43efa4bf7f7a385746e802f4b67bace2dd7a8f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 9 Jan 2026 18:54:30 +0000 Subject: [PATCH] Move DMARC dimensions to detection_fields for Chronicle dashboard support Co-authored-by: seanthegeek <44679+seanthegeek@users.noreply.github.com> --- docs/source/google_secops.md | 71 +++++++------ parsedmarc/google_secops.py | 201 ++++++++++++++++++++--------------- tests.py | 11 +- 3 files changed, 159 insertions(+), 124 deletions(-) diff --git a/docs/source/google_secops.md b/docs/source/google_secops.md index a8001cb..603b139 100644 --- a/docs/source/google_secops.md +++ b/docs/source/google_secops.md @@ -47,8 +47,12 @@ Each event includes: - **metadata**: Event timestamp, type, product name, and vendor - **principal**: Source IP address, location (country), and hostname (reverse DNS) - **target**: Domain name (from DMARC policy) -- **security_result**: Severity level, description, and detection fields -- **additional.fields**: Extended metadata including report details, counts, and authentication results +- **security_result**: Severity level, description, and detection fields for dashboarding + - **detection_fields**: Key DMARC dimensions for filtering and grouping (e.g., `dmarc.disposition`, `dmarc.pass`, `dmarc.header_from`, `dmarc.report_org`) + - All dashboard-relevant fields use `dmarc.*` or `smtp_tls.*` prefixes for easy identification +- **additional.fields** (optional): Low-value context fields (e.g., detailed auth results) not typically used for dashboarding + +**Design Rationale**: DMARC dimensions are placed in `security_result[].detection_fields` rather than `additional.fields` because Chronicle dashboards, stats searches, and aggregations work best with UDM label arrays. The `additional.fields` is a protobuf Struct intended for opaque context and is not reliably queryable for dashboard operations. ### Severity Heuristics @@ -78,25 +82,30 @@ Each event includes: }, "security_result": [{ "severity": "LOW", - "description": "DMARC fail; disposition=none", + "description": "DMARC fail; SPF=pass; DKIM=pass; SPF not aligned; DKIM not aligned; disposition=none", "detection_fields": [ - {"key": "dmarc_disposition", "value": "none"}, - {"key": "dmarc_policy", "value": "none"}, - {"key": "dmarc_pass", "value": false}, - {"key": "spf_aligned", "value": false}, - {"key": "dkim_aligned", "value": false} + {"key": "dmarc.disposition", "value": "none"}, + {"key": "dmarc.policy", "value": "none"}, + {"key": "dmarc.pass", "value": false}, + {"key": "dmarc.spf_aligned", "value": false}, + {"key": "dmarc.dkim_aligned", "value": false}, + {"key": "dmarc.header_from", "value": "example.com"}, + {"key": "dmarc.envelope_from", "value": "example.com"}, + {"key": "dmarc.report_org", "value": "example.net"}, + {"key": "dmarc.report_id", "value": "b043f0e264cf4ea995e93765242f6dfb"}, + {"key": "dmarc.report_begin", "value": "2018-06-19 00:00:00"}, + {"key": "dmarc.report_end", "value": "2018-06-19 23:59:59"}, + {"key": "dmarc.row_count", "value": 1}, + {"key": "dmarc.spf_result", "value": "pass"}, + {"key": "dmarc.dkim_result", "value": "pass"} ] }], "additional": { "fields": [ - {"key": "report_org", "value": "example.net"}, - {"key": "report_id", "value": "b043f0e264cf4ea995e93765242f6dfb"}, - {"key": "report_begin", "value": "2018-06-19 00:00:00"}, - {"key": "report_end", "value": "2018-06-19 23:59:59"}, - {"key": "message_count", "value": 1}, - {"key": "interval_begin", "value": "2018-06-19 00:00:00"}, - {"key": "interval_end", "value": "2018-06-19 23:59:59"}, - {"key": "envelope_from", "value": "example.com"} + {"key": "spf_0_domain", "value": "example.com"}, + {"key": "spf_0_result", "value": "pass"}, + {"key": "dkim_0_domain", "value": "example.com"}, + {"key": "dkim_0_result", "value": "pass"} ] } } @@ -123,12 +132,12 @@ Each event includes: "severity": "MEDIUM", "description": "DMARC forensic report: authentication failure (dmarc)", "detection_fields": [ - {"key": "auth_failure", "value": "dmarc"} + {"key": "dmarc.auth_failure", "value": "dmarc"}, + {"key": "dmarc.reported_domain", "value": "example.com"} ] }], "additional": { "fields": [ - {"key": "arrival_date", "value": "2019-04-30 02:09:00"}, {"key": "feedback_type", "value": "auth-failure"}, {"key": "message_id", "value": "<01010101010101010101010101010101@ABAB01MS0016.someserver.loc>"}, {"key": "authentication_results", "value": "dmarc=fail (p=none; dis=none) header.from=example.com"}, @@ -156,17 +165,16 @@ Each event includes: }, "security_result": [{ "severity": "LOW", - "description": "SMTP TLS failure: certificate-expired" - }], - "additional": { - "fields": [ - {"key": "organization_name", "value": "Company-X"}, - {"key": "report_begin", "value": "2016-04-01T00:00:00Z"}, - {"key": "report_end", "value": "2016-04-01T23:59:59Z"}, - {"key": "result_type", "value": "certificate-expired"}, - {"key": "failed_session_count", "value": 100} + "description": "SMTP TLS failure: certificate-expired", + "detection_fields": [ + {"key": "smtp_tls.policy_domain", "value": "company-y.example"}, + {"key": "smtp_tls.result_type", "value": "certificate-expired"}, + {"key": "smtp_tls.failed_session_count", "value": 100}, + {"key": "smtp_tls.report_org", "value": "Company-X"}, + {"key": "smtp_tls.report_begin", "value": "2016-04-01T00:00:00Z"}, + {"key": "smtp_tls.report_end", "value": "2016-04-01T23:59:59Z"} ] - }, + }], "principal": { "ip": ["2001:db8:abcd:0012::1"] } @@ -206,7 +214,7 @@ rule dmarc_aggregate_failures { events: $e.metadata.product_name = "parsedmarc" $e.event_type = "DMARC_AGGREGATE" - $e.security_result.detection_fields.key = "dmarc_pass" + $e.security_result.detection_fields.key = "dmarc.pass" $e.security_result.detection_fields.value = false condition: @@ -243,7 +251,7 @@ rule repeated_dmarc_failures { events: $e.metadata.product_name = "parsedmarc" $e.event_type = "DMARC_AGGREGATE" - $e.security_result.detection_fields.key = "dmarc_pass" + $e.security_result.detection_fields.key = "dmarc.pass" $e.security_result.detection_fields.value = false $e.principal.ip = $source_ip @@ -266,8 +274,7 @@ rule dmarc_forensic_failures { events: $e.metadata.product_name = "parsedmarc" $e.event_type = "DMARC_FORENSIC" - $e.additional.fields.key = "auth_failure" - $e.additional.fields.value = "dmarc" + $e.security_result.detection_fields.key = "dmarc.auth_failure" condition: $e diff --git a/parsedmarc/google_secops.py b/parsedmarc/google_secops.py index 6c26040..d022349 100644 --- a/parsedmarc/google_secops.py +++ b/parsedmarc/google_secops.py @@ -164,7 +164,6 @@ class GoogleSecOpsClient: count = record["count"] interval_begin = record["interval_begin"] - interval_end = record["interval_end"] # Get auth results spf_results = record["auth_results"].get("spf", []) @@ -204,73 +203,85 @@ class GoogleSecOpsClient: disposition, ), "detection_fields": [ - {"key": "dmarc_disposition", "value": disposition}, - {"key": "dmarc_policy", "value": policy_published["p"]}, - {"key": "dmarc_pass", "value": dmarc_pass}, - {"key": "spf_aligned", "value": spf_aligned}, - {"key": "dkim_aligned", "value": dkim_aligned}, + {"key": "dmarc.disposition", "value": disposition}, + {"key": "dmarc.policy", "value": policy_published["p"]}, + {"key": "dmarc.pass", "value": dmarc_pass}, + {"key": "dmarc.spf_aligned", "value": spf_aligned}, + {"key": "dmarc.dkim_aligned", "value": dkim_aligned}, + {"key": "dmarc.header_from", "value": header_from}, + {"key": "dmarc.envelope_from", "value": envelope_from}, + {"key": "dmarc.report_org", "value": report_metadata["org_name"]}, + {"key": "dmarc.report_id", "value": report_metadata["report_id"]}, + {"key": "dmarc.report_begin", "value": report_metadata["begin_date"]}, + {"key": "dmarc.report_end", "value": report_metadata["end_date"]}, + {"key": "dmarc.row_count", "value": count}, ], } ], - "additional": { - "fields": [ - {"key": "report_org", "value": report_metadata["org_name"]}, - {"key": "report_id", "value": report_metadata["report_id"]}, - {"key": "report_begin", "value": report_metadata["begin_date"]}, - {"key": "report_end", "value": report_metadata["end_date"]}, - {"key": "message_count", "value": count}, - {"key": "interval_begin", "value": interval_begin}, - {"key": "interval_end", "value": interval_end}, - {"key": "envelope_from", "value": envelope_from}, - ] - }, } - # Add optional fields + # Add optional fields to detection_fields + if spf_result: + event["security_result"][0]["detection_fields"].append( + {"key": "dmarc.spf_result", "value": spf_result} + ) + if dkim_result: + event["security_result"][0]["detection_fields"].append( + {"key": "dmarc.dkim_result", "value": dkim_result} + ) + + # Add optional context fields (low-value, not for dashboarding) + additional_context = [] + + if source_base_domain: + additional_context.append( + {"key": "source_base_domain", "value": source_base_domain} + ) + + if source_name: + additional_context.append( + {"key": "source_name", "value": source_name} + ) + + if self.static_environment: + additional_context.append( + {"key": "environment", "value": self.static_environment} + ) + + # Add SPF auth results to context + if spf_results: + for idx, spf in enumerate(spf_results): + additional_context.append( + {"key": f"spf_{idx}_domain", "value": spf.get("domain", "")} + ) + additional_context.append( + {"key": f"spf_{idx}_result", "value": spf.get("result", "")} + ) + + # Add DKIM auth results to context + if dkim_results: + for idx, dkim in enumerate(dkim_results): + additional_context.append( + {"key": f"dkim_{idx}_domain", "value": dkim.get("domain", "")} + ) + additional_context.append( + {"key": f"dkim_{idx}_result", "value": dkim.get("result", "")} + ) + + # Only add additional section if there's context to include + if additional_context: + event["additional"] = {"fields": additional_context} + + # Add optional UDM fields if source_country: event["principal"]["location"] = {"country_or_region": source_country} if source_reverse_dns: event["principal"]["hostname"] = source_reverse_dns - if source_base_domain: - event["additional"]["fields"].append( - {"key": "source_base_domain", "value": source_base_domain} - ) - - if source_name: - event["additional"]["fields"].append( - {"key": "source_name", "value": source_name} - ) - if self.static_observer_name: event["metadata"]["product_deployment_id"] = self.static_observer_name - if self.static_environment: - event["additional"]["fields"].append( - {"key": "environment", "value": self.static_environment} - ) - - # Add SPF results - if spf_results: - for idx, spf in enumerate(spf_results): - event["additional"]["fields"].append( - {"key": f"spf_{idx}_domain", "value": spf.get("domain", "")} - ) - event["additional"]["fields"].append( - {"key": f"spf_{idx}_result", "value": spf.get("result", "")} - ) - - # Add DKIM results - if dkim_results: - for idx, dkim in enumerate(dkim_results): - event["additional"]["fields"].append( - {"key": f"dkim_{idx}_domain", "value": dkim.get("domain", "")} - ) - event["additional"]["fields"].append( - {"key": f"dkim_{idx}_result", "value": dkim.get("result", "")} - ) - events.append(json.dumps(event, ensure_ascii=False)) except Exception as e: @@ -349,45 +360,38 @@ class GoogleSecOpsClient: "severity": severity, "description": description, "detection_fields": [ - {"key": "auth_failure", "value": auth_failure_str}, + {"key": "dmarc.auth_failure", "value": auth_failure_str}, + {"key": "dmarc.reported_domain", "value": reported_domain}, ], } ], - "additional": { - "fields": [ - {"key": "arrival_date", "value": arrival_date}, - {"key": "feedback_type", "value": forensic_report.get("feedback_type", "")}, - ] - }, } - # Add optional fields - if source_country: - event["principal"]["location"] = {"country_or_region": source_country} + # Add optional context fields (low-value, not for dashboarding) + additional_context = [] - if source_reverse_dns: - event["principal"]["hostname"] = source_reverse_dns + if forensic_report.get("feedback_type"): + additional_context.append( + {"key": "feedback_type", "value": forensic_report["feedback_type"]} + ) if forensic_report.get("message_id"): - event["additional"]["fields"].append( + additional_context.append( {"key": "message_id", "value": forensic_report["message_id"]} ) if forensic_report.get("authentication_results"): - event["additional"]["fields"].append( + additional_context.append( {"key": "authentication_results", "value": forensic_report["authentication_results"]} ) if forensic_report.get("delivery_result"): - event["additional"]["fields"].append( + additional_context.append( {"key": "delivery_result", "value": forensic_report["delivery_result"]} ) - if self.static_observer_name: - event["metadata"]["product_deployment_id"] = self.static_observer_name - if self.static_environment: - event["additional"]["fields"].append( + additional_context.append( {"key": "environment", "value": self.static_environment} ) @@ -396,10 +400,24 @@ class GoogleSecOpsClient: sample = forensic_report["sample"] if len(sample) > self.ruf_payload_max_bytes: sample = sample[:self.ruf_payload_max_bytes] + "... [truncated]" - event["additional"]["fields"].append( + additional_context.append( {"key": "message_sample", "value": sample} ) + # Only add additional section if there's context to include + if additional_context: + event["additional"] = {"fields": additional_context} + + # Add optional UDM fields + if source_country: + event["principal"]["location"] = {"country_or_region": source_country} + + if source_reverse_dns: + event["principal"]["hostname"] = source_reverse_dns + + if self.static_observer_name: + event["metadata"]["product_deployment_id"] = self.static_observer_name + events.append(json.dumps(event, ensure_ascii=False)) except Exception as e: @@ -466,30 +484,37 @@ class GoogleSecOpsClient: { "severity": "LOW", "description": f"SMTP TLS failure: {failure.get('result_type', 'unknown')}", + "detection_fields": [ + {"key": "smtp_tls.policy_domain", "value": policy_domain}, + {"key": "smtp_tls.result_type", "value": failure.get("result_type", "")}, + {"key": "smtp_tls.failed_session_count", "value": failure.get("failed_session_count", 0)}, + {"key": "smtp_tls.report_org", "value": organization_name}, + {"key": "smtp_tls.report_begin", "value": begin_date}, + {"key": "smtp_tls.report_end", "value": end_date}, + ], } ], - "additional": { - "fields": [ - {"key": "organization_name", "value": organization_name}, - {"key": "report_begin", "value": begin_date}, - {"key": "report_end", "value": end_date}, - {"key": "result_type", "value": failure.get("result_type", "")}, - {"key": "failed_session_count", "value": failure.get("failed_session_count", 0)}, - ] - }, } + # Add optional context fields (low-value, not for dashboarding) + additional_context = [] + + if self.static_environment: + additional_context.append( + {"key": "environment", "value": self.static_environment} + ) + + # Only add additional section if there's context to include + if additional_context: + event["additional"] = {"fields": additional_context} + + # Add optional UDM fields if failure.get("sending_mta_ip"): event["principal"] = {"ip": [failure["sending_mta_ip"]]} if self.static_observer_name: event["metadata"]["product_deployment_id"] = self.static_observer_name - if self.static_environment: - event["additional"]["fields"].append( - {"key": "environment", "value": self.static_environment} - ) - events.append(json.dumps(event, ensure_ascii=False)) except Exception as e: diff --git a/tests.py b/tests.py index d8b3087..4984c6e 100755 --- a/tests.py +++ b/tests.py @@ -305,12 +305,15 @@ class Test(unittest.TestCase): assert "metadata" in event_dict assert "target" in event_dict assert "security_result" in event_dict - assert "additional" in event_dict - # Verify failed_session_count is an integer not a string - for field in event_dict["additional"]["fields"]: - if field["key"] == "failed_session_count": + # Verify failed_session_count is in detection_fields as an integer + found_count = False + for field in event_dict["security_result"][0]["detection_fields"]: + if field["key"] == "smtp_tls.failed_session_count": assert isinstance(field["value"], int), "failed_session_count should be an integer" + found_count = True + break + assert found_count, "failed_session_count should be in detection_fields" print("Passed!")