mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-04-04 04:38:53 +00:00
Move DMARC dimensions to detection_fields for Chronicle dashboard support
Co-authored-by: seanthegeek <44679+seanthegeek@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
@@ -47,8 +47,12 @@ Each event includes:
|
||||
- **metadata**: Event timestamp, type, product name, and vendor
|
||||
- **principal**: Source IP address, location (country), and hostname (reverse DNS)
|
||||
- **target**: Domain name (from DMARC policy)
|
||||
- **security_result**: Severity level, description, and detection fields
|
||||
- **additional.fields**: Extended metadata including report details, counts, and authentication results
|
||||
- **security_result**: Severity level, description, and detection fields for dashboarding
|
||||
- **detection_fields**: Key DMARC dimensions for filtering and grouping (e.g., `dmarc.disposition`, `dmarc.pass`, `dmarc.header_from`, `dmarc.report_org`)
|
||||
- All dashboard-relevant fields use `dmarc.*` or `smtp_tls.*` prefixes for easy identification
|
||||
- **additional.fields** (optional): Low-value context fields (e.g., detailed auth results) not typically used for dashboarding
|
||||
|
||||
**Design Rationale**: DMARC dimensions are placed in `security_result[].detection_fields` rather than `additional.fields` because Chronicle dashboards, stats searches, and aggregations work best with UDM label arrays. The `additional.fields` is a protobuf Struct intended for opaque context and is not reliably queryable for dashboard operations.
|
||||
|
||||
### Severity Heuristics
|
||||
|
||||
@@ -78,25 +82,30 @@ Each event includes:
|
||||
},
|
||||
"security_result": [{
|
||||
"severity": "LOW",
|
||||
"description": "DMARC fail; disposition=none",
|
||||
"description": "DMARC fail; SPF=pass; DKIM=pass; SPF not aligned; DKIM not aligned; disposition=none",
|
||||
"detection_fields": [
|
||||
{"key": "dmarc_disposition", "value": "none"},
|
||||
{"key": "dmarc_policy", "value": "none"},
|
||||
{"key": "dmarc_pass", "value": false},
|
||||
{"key": "spf_aligned", "value": false},
|
||||
{"key": "dkim_aligned", "value": false}
|
||||
{"key": "dmarc.disposition", "value": "none"},
|
||||
{"key": "dmarc.policy", "value": "none"},
|
||||
{"key": "dmarc.pass", "value": false},
|
||||
{"key": "dmarc.spf_aligned", "value": false},
|
||||
{"key": "dmarc.dkim_aligned", "value": false},
|
||||
{"key": "dmarc.header_from", "value": "example.com"},
|
||||
{"key": "dmarc.envelope_from", "value": "example.com"},
|
||||
{"key": "dmarc.report_org", "value": "example.net"},
|
||||
{"key": "dmarc.report_id", "value": "b043f0e264cf4ea995e93765242f6dfb"},
|
||||
{"key": "dmarc.report_begin", "value": "2018-06-19 00:00:00"},
|
||||
{"key": "dmarc.report_end", "value": "2018-06-19 23:59:59"},
|
||||
{"key": "dmarc.row_count", "value": 1},
|
||||
{"key": "dmarc.spf_result", "value": "pass"},
|
||||
{"key": "dmarc.dkim_result", "value": "pass"}
|
||||
]
|
||||
}],
|
||||
"additional": {
|
||||
"fields": [
|
||||
{"key": "report_org", "value": "example.net"},
|
||||
{"key": "report_id", "value": "b043f0e264cf4ea995e93765242f6dfb"},
|
||||
{"key": "report_begin", "value": "2018-06-19 00:00:00"},
|
||||
{"key": "report_end", "value": "2018-06-19 23:59:59"},
|
||||
{"key": "message_count", "value": 1},
|
||||
{"key": "interval_begin", "value": "2018-06-19 00:00:00"},
|
||||
{"key": "interval_end", "value": "2018-06-19 23:59:59"},
|
||||
{"key": "envelope_from", "value": "example.com"}
|
||||
{"key": "spf_0_domain", "value": "example.com"},
|
||||
{"key": "spf_0_result", "value": "pass"},
|
||||
{"key": "dkim_0_domain", "value": "example.com"},
|
||||
{"key": "dkim_0_result", "value": "pass"}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -123,12 +132,12 @@ Each event includes:
|
||||
"severity": "MEDIUM",
|
||||
"description": "DMARC forensic report: authentication failure (dmarc)",
|
||||
"detection_fields": [
|
||||
{"key": "auth_failure", "value": "dmarc"}
|
||||
{"key": "dmarc.auth_failure", "value": "dmarc"},
|
||||
{"key": "dmarc.reported_domain", "value": "example.com"}
|
||||
]
|
||||
}],
|
||||
"additional": {
|
||||
"fields": [
|
||||
{"key": "arrival_date", "value": "2019-04-30 02:09:00"},
|
||||
{"key": "feedback_type", "value": "auth-failure"},
|
||||
{"key": "message_id", "value": "<01010101010101010101010101010101@ABAB01MS0016.someserver.loc>"},
|
||||
{"key": "authentication_results", "value": "dmarc=fail (p=none; dis=none) header.from=example.com"},
|
||||
@@ -156,17 +165,16 @@ Each event includes:
|
||||
},
|
||||
"security_result": [{
|
||||
"severity": "LOW",
|
||||
"description": "SMTP TLS failure: certificate-expired"
|
||||
}],
|
||||
"additional": {
|
||||
"fields": [
|
||||
{"key": "organization_name", "value": "Company-X"},
|
||||
{"key": "report_begin", "value": "2016-04-01T00:00:00Z"},
|
||||
{"key": "report_end", "value": "2016-04-01T23:59:59Z"},
|
||||
{"key": "result_type", "value": "certificate-expired"},
|
||||
{"key": "failed_session_count", "value": 100}
|
||||
"description": "SMTP TLS failure: certificate-expired",
|
||||
"detection_fields": [
|
||||
{"key": "smtp_tls.policy_domain", "value": "company-y.example"},
|
||||
{"key": "smtp_tls.result_type", "value": "certificate-expired"},
|
||||
{"key": "smtp_tls.failed_session_count", "value": 100},
|
||||
{"key": "smtp_tls.report_org", "value": "Company-X"},
|
||||
{"key": "smtp_tls.report_begin", "value": "2016-04-01T00:00:00Z"},
|
||||
{"key": "smtp_tls.report_end", "value": "2016-04-01T23:59:59Z"}
|
||||
]
|
||||
},
|
||||
}],
|
||||
"principal": {
|
||||
"ip": ["2001:db8:abcd:0012::1"]
|
||||
}
|
||||
@@ -206,7 +214,7 @@ rule dmarc_aggregate_failures {
|
||||
events:
|
||||
$e.metadata.product_name = "parsedmarc"
|
||||
$e.event_type = "DMARC_AGGREGATE"
|
||||
$e.security_result.detection_fields.key = "dmarc_pass"
|
||||
$e.security_result.detection_fields.key = "dmarc.pass"
|
||||
$e.security_result.detection_fields.value = false
|
||||
|
||||
condition:
|
||||
@@ -243,7 +251,7 @@ rule repeated_dmarc_failures {
|
||||
events:
|
||||
$e.metadata.product_name = "parsedmarc"
|
||||
$e.event_type = "DMARC_AGGREGATE"
|
||||
$e.security_result.detection_fields.key = "dmarc_pass"
|
||||
$e.security_result.detection_fields.key = "dmarc.pass"
|
||||
$e.security_result.detection_fields.value = false
|
||||
$e.principal.ip = $source_ip
|
||||
|
||||
@@ -266,8 +274,7 @@ rule dmarc_forensic_failures {
|
||||
events:
|
||||
$e.metadata.product_name = "parsedmarc"
|
||||
$e.event_type = "DMARC_FORENSIC"
|
||||
$e.additional.fields.key = "auth_failure"
|
||||
$e.additional.fields.value = "dmarc"
|
||||
$e.security_result.detection_fields.key = "dmarc.auth_failure"
|
||||
|
||||
condition:
|
||||
$e
|
||||
|
||||
@@ -164,7 +164,6 @@ class GoogleSecOpsClient:
|
||||
|
||||
count = record["count"]
|
||||
interval_begin = record["interval_begin"]
|
||||
interval_end = record["interval_end"]
|
||||
|
||||
# Get auth results
|
||||
spf_results = record["auth_results"].get("spf", [])
|
||||
@@ -204,73 +203,85 @@ class GoogleSecOpsClient:
|
||||
disposition,
|
||||
),
|
||||
"detection_fields": [
|
||||
{"key": "dmarc_disposition", "value": disposition},
|
||||
{"key": "dmarc_policy", "value": policy_published["p"]},
|
||||
{"key": "dmarc_pass", "value": dmarc_pass},
|
||||
{"key": "spf_aligned", "value": spf_aligned},
|
||||
{"key": "dkim_aligned", "value": dkim_aligned},
|
||||
{"key": "dmarc.disposition", "value": disposition},
|
||||
{"key": "dmarc.policy", "value": policy_published["p"]},
|
||||
{"key": "dmarc.pass", "value": dmarc_pass},
|
||||
{"key": "dmarc.spf_aligned", "value": spf_aligned},
|
||||
{"key": "dmarc.dkim_aligned", "value": dkim_aligned},
|
||||
{"key": "dmarc.header_from", "value": header_from},
|
||||
{"key": "dmarc.envelope_from", "value": envelope_from},
|
||||
{"key": "dmarc.report_org", "value": report_metadata["org_name"]},
|
||||
{"key": "dmarc.report_id", "value": report_metadata["report_id"]},
|
||||
{"key": "dmarc.report_begin", "value": report_metadata["begin_date"]},
|
||||
{"key": "dmarc.report_end", "value": report_metadata["end_date"]},
|
||||
{"key": "dmarc.row_count", "value": count},
|
||||
],
|
||||
}
|
||||
],
|
||||
"additional": {
|
||||
"fields": [
|
||||
{"key": "report_org", "value": report_metadata["org_name"]},
|
||||
{"key": "report_id", "value": report_metadata["report_id"]},
|
||||
{"key": "report_begin", "value": report_metadata["begin_date"]},
|
||||
{"key": "report_end", "value": report_metadata["end_date"]},
|
||||
{"key": "message_count", "value": count},
|
||||
{"key": "interval_begin", "value": interval_begin},
|
||||
{"key": "interval_end", "value": interval_end},
|
||||
{"key": "envelope_from", "value": envelope_from},
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
# Add optional fields
|
||||
# Add optional fields to detection_fields
|
||||
if spf_result:
|
||||
event["security_result"][0]["detection_fields"].append(
|
||||
{"key": "dmarc.spf_result", "value": spf_result}
|
||||
)
|
||||
if dkim_result:
|
||||
event["security_result"][0]["detection_fields"].append(
|
||||
{"key": "dmarc.dkim_result", "value": dkim_result}
|
||||
)
|
||||
|
||||
# Add optional context fields (low-value, not for dashboarding)
|
||||
additional_context = []
|
||||
|
||||
if source_base_domain:
|
||||
additional_context.append(
|
||||
{"key": "source_base_domain", "value": source_base_domain}
|
||||
)
|
||||
|
||||
if source_name:
|
||||
additional_context.append(
|
||||
{"key": "source_name", "value": source_name}
|
||||
)
|
||||
|
||||
if self.static_environment:
|
||||
additional_context.append(
|
||||
{"key": "environment", "value": self.static_environment}
|
||||
)
|
||||
|
||||
# Add SPF auth results to context
|
||||
if spf_results:
|
||||
for idx, spf in enumerate(spf_results):
|
||||
additional_context.append(
|
||||
{"key": f"spf_{idx}_domain", "value": spf.get("domain", "")}
|
||||
)
|
||||
additional_context.append(
|
||||
{"key": f"spf_{idx}_result", "value": spf.get("result", "")}
|
||||
)
|
||||
|
||||
# Add DKIM auth results to context
|
||||
if dkim_results:
|
||||
for idx, dkim in enumerate(dkim_results):
|
||||
additional_context.append(
|
||||
{"key": f"dkim_{idx}_domain", "value": dkim.get("domain", "")}
|
||||
)
|
||||
additional_context.append(
|
||||
{"key": f"dkim_{idx}_result", "value": dkim.get("result", "")}
|
||||
)
|
||||
|
||||
# Only add additional section if there's context to include
|
||||
if additional_context:
|
||||
event["additional"] = {"fields": additional_context}
|
||||
|
||||
# Add optional UDM fields
|
||||
if source_country:
|
||||
event["principal"]["location"] = {"country_or_region": source_country}
|
||||
|
||||
if source_reverse_dns:
|
||||
event["principal"]["hostname"] = source_reverse_dns
|
||||
|
||||
if source_base_domain:
|
||||
event["additional"]["fields"].append(
|
||||
{"key": "source_base_domain", "value": source_base_domain}
|
||||
)
|
||||
|
||||
if source_name:
|
||||
event["additional"]["fields"].append(
|
||||
{"key": "source_name", "value": source_name}
|
||||
)
|
||||
|
||||
if self.static_observer_name:
|
||||
event["metadata"]["product_deployment_id"] = self.static_observer_name
|
||||
|
||||
if self.static_environment:
|
||||
event["additional"]["fields"].append(
|
||||
{"key": "environment", "value": self.static_environment}
|
||||
)
|
||||
|
||||
# Add SPF results
|
||||
if spf_results:
|
||||
for idx, spf in enumerate(spf_results):
|
||||
event["additional"]["fields"].append(
|
||||
{"key": f"spf_{idx}_domain", "value": spf.get("domain", "")}
|
||||
)
|
||||
event["additional"]["fields"].append(
|
||||
{"key": f"spf_{idx}_result", "value": spf.get("result", "")}
|
||||
)
|
||||
|
||||
# Add DKIM results
|
||||
if dkim_results:
|
||||
for idx, dkim in enumerate(dkim_results):
|
||||
event["additional"]["fields"].append(
|
||||
{"key": f"dkim_{idx}_domain", "value": dkim.get("domain", "")}
|
||||
)
|
||||
event["additional"]["fields"].append(
|
||||
{"key": f"dkim_{idx}_result", "value": dkim.get("result", "")}
|
||||
)
|
||||
|
||||
events.append(json.dumps(event, ensure_ascii=False))
|
||||
|
||||
except Exception as e:
|
||||
@@ -349,45 +360,38 @@ class GoogleSecOpsClient:
|
||||
"severity": severity,
|
||||
"description": description,
|
||||
"detection_fields": [
|
||||
{"key": "auth_failure", "value": auth_failure_str},
|
||||
{"key": "dmarc.auth_failure", "value": auth_failure_str},
|
||||
{"key": "dmarc.reported_domain", "value": reported_domain},
|
||||
],
|
||||
}
|
||||
],
|
||||
"additional": {
|
||||
"fields": [
|
||||
{"key": "arrival_date", "value": arrival_date},
|
||||
{"key": "feedback_type", "value": forensic_report.get("feedback_type", "")},
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
# Add optional fields
|
||||
if source_country:
|
||||
event["principal"]["location"] = {"country_or_region": source_country}
|
||||
# Add optional context fields (low-value, not for dashboarding)
|
||||
additional_context = []
|
||||
|
||||
if source_reverse_dns:
|
||||
event["principal"]["hostname"] = source_reverse_dns
|
||||
if forensic_report.get("feedback_type"):
|
||||
additional_context.append(
|
||||
{"key": "feedback_type", "value": forensic_report["feedback_type"]}
|
||||
)
|
||||
|
||||
if forensic_report.get("message_id"):
|
||||
event["additional"]["fields"].append(
|
||||
additional_context.append(
|
||||
{"key": "message_id", "value": forensic_report["message_id"]}
|
||||
)
|
||||
|
||||
if forensic_report.get("authentication_results"):
|
||||
event["additional"]["fields"].append(
|
||||
additional_context.append(
|
||||
{"key": "authentication_results", "value": forensic_report["authentication_results"]}
|
||||
)
|
||||
|
||||
if forensic_report.get("delivery_result"):
|
||||
event["additional"]["fields"].append(
|
||||
additional_context.append(
|
||||
{"key": "delivery_result", "value": forensic_report["delivery_result"]}
|
||||
)
|
||||
|
||||
if self.static_observer_name:
|
||||
event["metadata"]["product_deployment_id"] = self.static_observer_name
|
||||
|
||||
if self.static_environment:
|
||||
event["additional"]["fields"].append(
|
||||
additional_context.append(
|
||||
{"key": "environment", "value": self.static_environment}
|
||||
)
|
||||
|
||||
@@ -396,10 +400,24 @@ class GoogleSecOpsClient:
|
||||
sample = forensic_report["sample"]
|
||||
if len(sample) > self.ruf_payload_max_bytes:
|
||||
sample = sample[:self.ruf_payload_max_bytes] + "... [truncated]"
|
||||
event["additional"]["fields"].append(
|
||||
additional_context.append(
|
||||
{"key": "message_sample", "value": sample}
|
||||
)
|
||||
|
||||
# Only add additional section if there's context to include
|
||||
if additional_context:
|
||||
event["additional"] = {"fields": additional_context}
|
||||
|
||||
# Add optional UDM fields
|
||||
if source_country:
|
||||
event["principal"]["location"] = {"country_or_region": source_country}
|
||||
|
||||
if source_reverse_dns:
|
||||
event["principal"]["hostname"] = source_reverse_dns
|
||||
|
||||
if self.static_observer_name:
|
||||
event["metadata"]["product_deployment_id"] = self.static_observer_name
|
||||
|
||||
events.append(json.dumps(event, ensure_ascii=False))
|
||||
|
||||
except Exception as e:
|
||||
@@ -466,30 +484,37 @@ class GoogleSecOpsClient:
|
||||
{
|
||||
"severity": "LOW",
|
||||
"description": f"SMTP TLS failure: {failure.get('result_type', 'unknown')}",
|
||||
"detection_fields": [
|
||||
{"key": "smtp_tls.policy_domain", "value": policy_domain},
|
||||
{"key": "smtp_tls.result_type", "value": failure.get("result_type", "")},
|
||||
{"key": "smtp_tls.failed_session_count", "value": failure.get("failed_session_count", 0)},
|
||||
{"key": "smtp_tls.report_org", "value": organization_name},
|
||||
{"key": "smtp_tls.report_begin", "value": begin_date},
|
||||
{"key": "smtp_tls.report_end", "value": end_date},
|
||||
],
|
||||
}
|
||||
],
|
||||
"additional": {
|
||||
"fields": [
|
||||
{"key": "organization_name", "value": organization_name},
|
||||
{"key": "report_begin", "value": begin_date},
|
||||
{"key": "report_end", "value": end_date},
|
||||
{"key": "result_type", "value": failure.get("result_type", "")},
|
||||
{"key": "failed_session_count", "value": failure.get("failed_session_count", 0)},
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
# Add optional context fields (low-value, not for dashboarding)
|
||||
additional_context = []
|
||||
|
||||
if self.static_environment:
|
||||
additional_context.append(
|
||||
{"key": "environment", "value": self.static_environment}
|
||||
)
|
||||
|
||||
# Only add additional section if there's context to include
|
||||
if additional_context:
|
||||
event["additional"] = {"fields": additional_context}
|
||||
|
||||
# Add optional UDM fields
|
||||
if failure.get("sending_mta_ip"):
|
||||
event["principal"] = {"ip": [failure["sending_mta_ip"]]}
|
||||
|
||||
if self.static_observer_name:
|
||||
event["metadata"]["product_deployment_id"] = self.static_observer_name
|
||||
|
||||
if self.static_environment:
|
||||
event["additional"]["fields"].append(
|
||||
{"key": "environment", "value": self.static_environment}
|
||||
)
|
||||
|
||||
events.append(json.dumps(event, ensure_ascii=False))
|
||||
|
||||
except Exception as e:
|
||||
|
||||
11
tests.py
11
tests.py
@@ -305,12 +305,15 @@ class Test(unittest.TestCase):
|
||||
assert "metadata" in event_dict
|
||||
assert "target" in event_dict
|
||||
assert "security_result" in event_dict
|
||||
assert "additional" in event_dict
|
||||
|
||||
# Verify failed_session_count is an integer not a string
|
||||
for field in event_dict["additional"]["fields"]:
|
||||
if field["key"] == "failed_session_count":
|
||||
# Verify failed_session_count is in detection_fields as an integer
|
||||
found_count = False
|
||||
for field in event_dict["security_result"][0]["detection_fields"]:
|
||||
if field["key"] == "smtp_tls.failed_session_count":
|
||||
assert isinstance(field["value"], int), "failed_session_count should be an integer"
|
||||
found_count = True
|
||||
break
|
||||
assert found_count, "failed_session_count should be in detection_fields"
|
||||
|
||||
print("Passed!")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user