From 61266bc12d8923011d60541478a7dc723ced389e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 4 Jun 2026 00:07:29 +0000 Subject: [PATCH] =?UTF-8?q?Align=20Google=20SecOps=20parser=20with=20parse?= =?UTF-8?q?dmarc=2010+:=20rename=20forensic=E2=86=92failure,=20add=20new?= =?UTF-8?q?=20fields?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- google_secops_parser/README.md | 15 +++-- google_secops_parser/parsedmarc.conf | 96 ++++++++++++++++++++++------ 2 files changed, 88 insertions(+), 23 deletions(-) diff --git a/google_secops_parser/README.md b/google_secops_parser/README.md index 8e9677a8..857a2dd8 100644 --- a/google_secops_parser/README.md +++ b/google_secops_parser/README.md @@ -4,14 +4,14 @@ A [Google Security Operations (Chronicle)](https://cloud.google.com/security/pro ## Overview -parsedmarc sends DMARC aggregate reports, forensic reports, and SMTP TLS reports as JSON-formatted syslog messages. This parser transforms those JSON events into Google SecOps UDM events for threat detection and investigation. +parsedmarc sends DMARC aggregate reports, failure reports, and SMTP TLS reports as JSON-formatted syslog messages. This parser transforms those JSON events into Google SecOps UDM events for threat detection and investigation. ### Supported Report Types | Report Type | UDM Event Type | Description | |---|---|---| | DMARC Aggregate | `EMAIL_TRANSACTION` | Aggregate DMARC authentication results from reporting organizations | -| DMARC Forensic | `EMAIL_TRANSACTION` | Individual email authentication failure reports | +| DMARC Failure | `EMAIL_TRANSACTION` | Individual email authentication failure reports | | SMTP TLS | `GENERIC_EVENT` | SMTP TLS session success/failure reports (RFC 8460) | ## UDM Field Mappings @@ -35,10 +35,15 @@ parsedmarc sends DMARC aggregate reports, forensic reports, and SMTP TLS reports | `org_name` | `additional.fields` | Reporting organization name | | `count` | `additional.fields` | Number of messages | | `p`, `sp`, `pct` | `additional.fields` | DMARC policy settings | +| `np` | `additional.fields` | Non-existent domain policy | +| `fo` | `additional.fields` | Failure reporting options | +| `testing` | `additional.fields` | Whether policy is in testing mode | +| `discovery_method` | `additional.fields` | How the DMARC policy was discovered | +| `normalized_timespan` | `additional.fields` | Whether report timespan was normalized | | `dkim_domains`, `dkim_results` | `additional.fields` | DKIM authentication details | | `spf_domains`, `spf_results` | `additional.fields` | SPF authentication details | -### DMARC Forensic Reports +### DMARC Failure Reports | parsedmarc Field | UDM Field | Notes | |---|---|---| @@ -108,10 +113,10 @@ parsedmarc sends DMARC aggregate reports, forensic reports, and SMTP TLS reports ### Aggregate Report ```json -{"xml_schema": "1.0", "org_name": "Example Inc", "org_email": "noreply@example.net", "report_id": "abc123", "begin_date": "2024-01-01 00:00:00", "end_date": "2024-01-01 23:59:59", "domain": "example.com", "adkim": "r", "aspf": "r", "p": "reject", "sp": "reject", "pct": "100", "fo": "0", "source_ip_address": "203.0.113.1", "source_country": "United States", "source_reverse_dns": "mail.example.org", "source_base_domain": "example.org", "count": 42, "spf_aligned": true, "dkim_aligned": true, "dmarc_aligned": true, "disposition": "none", "header_from": "example.com", "envelope_from": "example.com", "envelope_to": null, "dkim_domains": "example.com", "dkim_selectors": "selector1", "dkim_results": "pass", "spf_domains": "example.com", "spf_scopes": "mfrom", "spf_results": "pass"} +{"xml_schema": "1.0", "org_name": "Example Inc", "org_email": "noreply@example.net", "report_id": "abc123", "begin_date": "2024-01-01 00:00:00", "end_date": "2024-01-01 23:59:59", "normalized_timespan": false, "domain": "example.com", "adkim": "r", "aspf": "r", "p": "reject", "sp": "reject", "pct": "100", "fo": "0", "np": "reject", "testing": false, "discovery_method": null, "source_ip_address": "203.0.113.1", "source_country": "United States", "source_reverse_dns": "mail.example.org", "source_base_domain": "example.org", "count": 42, "spf_aligned": true, "dkim_aligned": true, "dmarc_aligned": true, "disposition": "none", "header_from": "example.com", "envelope_from": "example.com", "envelope_to": null, "dkim_domains": "example.com", "dkim_selectors": "selector1", "dkim_results": "pass", "spf_domains": "example.com", "spf_scopes": "mfrom", "spf_results": "pass"} ``` -### Forensic Report +### Failure Report ```json {"feedback_type": "auth-failure", "user_agent": "Lua/1.0", "version": "1.0", "original_mail_from": "sender@example.com", "original_rcpt_to": "recipient@example.org", "arrival_date": "Mon, 01 Jan 2024 12:00:00 +0000", "arrival_date_utc": "2024-01-01 12:00:00", "source_ip_address": "198.51.100.1", "source_country": "Germany", "source_reverse_dns": "mail.example.com", "source_base_domain": "example.com", "subject": "Test Email", "message_id": "", "authentication_results": "dmarc=fail (p=reject; dis=reject) header.from=example.com", "dkim_domain": "example.com", "delivery_result": "reject", "auth_failure": "dmarc", "reported_domain": "example.com", "authentication_mechanisms": "dmarc"} diff --git a/google_secops_parser/parsedmarc.conf b/google_secops_parser/parsedmarc.conf index 5e6e4e24..251e935b 100644 --- a/google_secops_parser/parsedmarc.conf +++ b/google_secops_parser/parsedmarc.conf @@ -8,7 +8,7 @@ filter { # # parsedmarc outputs three types of reports as single-line JSON via syslog: # 1. DMARC Aggregate Reports (contains "header_from" and "dmarc_aligned") - # 2. DMARC Forensic Reports (contains "feedback_type") + # 2. DMARC Failure Reports (contains "feedback_type") # 3. SMTP TLS Reports (contains "policy_type") # # UDM Reference: @@ -48,11 +48,11 @@ filter { } } - # Forensic reports contain "feedback_type" + # Failure reports contain "feedback_type" if [feedback_type] { mutate { replace => { - "report_type" => "forensic" + "report_type" => "failure" } } } @@ -365,6 +365,66 @@ filter { } } + if [np] { + mutate { + replace => { + "np_label.key" => "dmarc_np_policy" + "np_label.value.string_value" => "%{np}" + } + merge => { + "event.idm.read_only_udm.additional.fields" => "np_label" + } + } + } + + if [fo] { + mutate { + replace => { + "fo_label.key" => "dmarc_fo" + "fo_label.value.string_value" => "%{fo}" + } + merge => { + "event.idm.read_only_udm.additional.fields" => "fo_label" + } + } + } + + if [testing] { + mutate { + replace => { + "testing_label.key" => "dmarc_testing" + "testing_label.value.string_value" => "%{testing}" + } + merge => { + "event.idm.read_only_udm.additional.fields" => "testing_label" + } + } + } + + if [discovery_method] { + mutate { + replace => { + "discovery_method_label.key" => "discovery_method" + "discovery_method_label.value.string_value" => "%{discovery_method}" + } + merge => { + "event.idm.read_only_udm.additional.fields" => "discovery_method_label" + } + } + } + + if [normalized_timespan] { + mutate { + replace => { + "normalized_timespan_label.key" => "normalized_timespan" + "normalized_timespan_label.value.string_value" => "%{normalized_timespan}" + } + merge => { + "event.idm.read_only_udm.additional.fields" => "normalized_timespan_label" + } + } + } + # -- alignment detail fields -- if [dmarc_aligned] { mutate { @@ -515,9 +575,9 @@ filter { } # ========================================================================== - # DMARC Forensic Report Parsing + # DMARC Failure Report Parsing # ========================================================================== - if [report_type] == "forensic" { + if [report_type] == "failure" { mutate { replace => { @@ -548,7 +608,7 @@ filter { merge => { "event.idm.read_only_udm.principal.ip" => "source_ip_address" } - on_error => "merge_forensic_ip_error" + on_error => "merge_failure_ip_error" } } @@ -642,14 +702,14 @@ filter { # -- security result -- mutate { replace => { - "security_result_forensic.summary" => "DMARC Forensic Report" + "security_result_failure.summary" => "DMARC Failure Report" } } if [auth_failure] { mutate { replace => { - "security_result_forensic.description" => "auth_failure=%{auth_failure}" + "security_result_failure.description" => "auth_failure=%{auth_failure}" } } } @@ -670,40 +730,40 @@ filter { if [auth_failure] == "dmarc" { mutate { replace => { - "security_result_forensic.category" => "POLICY_VIOLATION" - "security_result_forensic.action" => "ALLOW" + "security_result_failure.category" => "POLICY_VIOLATION" + "security_result_failure.action" => "ALLOW" } } } else if [auth_failure] == "spf" { mutate { replace => { - "security_result_forensic.category" => "POLICY_VIOLATION" - "security_result_forensic.action" => "ALLOW" + "security_result_failure.category" => "POLICY_VIOLATION" + "security_result_failure.action" => "ALLOW" } } } else if [auth_failure] == "dkim" { mutate { replace => { - "security_result_forensic.category" => "POLICY_VIOLATION" - "security_result_forensic.action" => "ALLOW" + "security_result_failure.category" => "POLICY_VIOLATION" + "security_result_failure.action" => "ALLOW" } } } else { mutate { replace => { - "security_result_forensic.category" => "POLICY_VIOLATION" - "security_result_forensic.action" => "UNKNOWN_ACTION" + "security_result_failure.category" => "POLICY_VIOLATION" + "security_result_failure.action" => "UNKNOWN_ACTION" } } } mutate { merge => { - "event.idm.read_only_udm.security_result" => "security_result_forensic" + "event.idm.read_only_udm.security_result" => "security_result_failure" } } - # -- additional forensic fields -- + # -- additional failure report fields -- if [feedback_type] { mutate { replace => {