From 576c68ed6749b407696189d5a98d362f74dd97af Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Thu, 23 Apr 2026 02:26:30 -0400 Subject: [PATCH] =?UTF-8?q?Add=20DMARCbis=20report=20support;=20rename=20f?= =?UTF-8?q?orensic=E2=86=92failure=20project-wide?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebased on top of master @ 2cda5bf (9.9.0), which added the ASN source attribution work (#712, #713, #714, #715). Individual Copilot iteration commits squashed into this single commit — the per-commit history on the feature branch was iterative (add tests, fix lint, move field, revert, etc.) and not worth preserving; GitHub squash- merges PRs anyway. New fields from the DMARCbis XSD, plumbed through types, parsing, CSV output, and the Elasticsearch / OpenSearch mappings: - ``np`` — non-existent subdomain policy (``none`` / ``quarantine`` / ``reject``) - ``testing`` — testing mode flag (``n`` / ``y``), replaces RFC 7489 ``pct`` - ``discovery_method`` — policy discovery method (``psl`` / ``treewalk``) - ``generator`` — report generator software identifier (metadata) - ``human_result`` — optional descriptive text on DKIM / SPF results RFC 7489 reports parse with ``None`` for DMARCbis-only fields. Forensic reports have been renamed to failure reports throughout the project to reflect the proper naming since RFC 7489. - Core: ``types.py``, ``__init__.py`` — ``ForensicReport`` → ``FailureReport``, ``parse_forensic_report`` → ``parse_failure_report``, report type ``"failure"``. - Output modules: ``elastic.py``, ``opensearch.py``, ``splunk.py``, ``kafkaclient.py``, ``syslog.py``, ``gelf.py``, ``webhook.py``, ``loganalytics.py``, ``s3.py``. - CLI: ``cli.py`` — args, config keys, index names (``dmarc_failure``). - Docs + dashboards: all markdown, Grafana JSON, Kibana NDJSON, Splunk XML. Backward compatibility preserved: old function / type names remain as aliases (``parse_forensic_report = parse_failure_report``, ``ForensicReport = FailureReport``, etc.), CLI accepts both the old (``save_forensic``, ``forensic_topic``) and new (``save_failure``, ``failure_topic``) config keys, and updated dashboards query both old and new index / sourcetype names so data from before and after the rename appears together. Merge conflicts resolved in ``parsedmarc/constants.py`` (took bis's 10.0.0 bump), ``parsedmarc/__init__.py`` (combined bis's "failure" wording with master's IPinfo MMDB mention), ``parsedmarc/elastic.py`` and ``parsedmarc/opensearch.py`` (kept master's ``source_asn`` / ``source_asn_name`` / ``source_asn_domain`` on the failure doc path while renaming ``forensic_report`` → ``failure_report``), and ``CHANGELOG.md`` (10.0.0 entry now sits above the 9.9.0 entry). All 324 tests pass; ``ruff check`` / ``ruff format --check`` clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- .claude/settings.json | 1 + .github/workflows/python-tests.yml | 5 +- AGENTS.md | 16 +- CHANGELOG.md | 41 +- CLAUDE.md | 4 +- docs/source/elasticsearch.md | 2 +- docs/source/example.ini | 2 +- docs/source/index.md | 2 +- docs/source/kibana.md | 8 +- docs/source/output.md | 8 +- docs/source/splunk.md | 4 +- docs/source/usage.md | 48 +- grafana/Grafana-DMARC_Reports.json | 4 +- kibana/export.ndjson | 2 +- parsedmarc/__init__.py | 201 +- parsedmarc/cli.py | 132 +- parsedmarc/constants.py | 2 +- parsedmarc/elastic.py | 185 +- parsedmarc/gelf.py | 16 +- parsedmarc/kafkaclient.py | 30 +- parsedmarc/loganalytics.py | 36 +- parsedmarc/opensearch.py | 187 +- parsedmarc/s3.py | 8 +- parsedmarc/splunk.py | 24 +- parsedmarc/syslog.py | 10 +- parsedmarc/types.py | 34 +- parsedmarc/utils.py | 2 +- parsedmarc/webhook.py | 16 +- samples/aggregate/dmarcbis-draft-sample.xml | 48 + ....net!example.com!1700000000!1700086399.xml | 77 + ...=sharepoint@domain.de, ip=10.10.10.10).eml | 0 ...se DMARC Failure Report] Rent Reminder.eml | 0 .../dmarc_ruf_report_linkedin.crlf.eml | 0 .../dmarc_ruf_report_linkedin.eml | 0 splunk/README.rst | 4 +- ...hboard.xml => dmarc_failure_dashboard.xml} | 12 +- tests.py | 2588 ++++++++++++++++- 37 files changed, 3285 insertions(+), 474 deletions(-) create mode 100644 samples/aggregate/dmarcbis-draft-sample.xml create mode 100644 samples/aggregate/dmarcbis-example.net!example.com!1700000000!1700086399.xml rename samples/{forensic => failure}/DMARC Failure Report for domain.de (mail-from=sharepoint@domain.de, ip=10.10.10.10).eml (100%) rename samples/{forensic => failure}/[Netease DMARC Failure Report] Rent Reminder.eml (100%) rename samples/{forensic => failure}/dmarc_ruf_report_linkedin.crlf.eml (100%) rename samples/{forensic => failure}/dmarc_ruf_report_linkedin.eml (100%) rename splunk/{dmarc_forensic_dashboard.xml => dmarc_failure_dashboard.xml} (85%) diff --git a/.claude/settings.json b/.claude/settings.json index fc59110..07e7473 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -1,6 +1,7 @@ { "permissions": { "allow": [ + "Bash(git fetch:*)", "Bash(python -c \"import py_compile; py_compile.compile\\(''parsedmarc/cli.py'', doraise=True\\)\")", "Bash(ruff check:*)", "Bash(ruff format:*)", diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 54ae5db..39e1c26 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -78,7 +78,10 @@ jobs: run: | pip install -e . parsedmarc --debug -c ci.ini samples/aggregate/* - parsedmarc --debug -c ci.ini samples/forensic/* + parsedmarc --debug -c ci.ini samples/failure/* + - name: Test building packages + run: | + hatch build - name: Upload coverage to Codecov uses: codecov/codecov-action@v5 with: diff --git a/AGENTS.md b/AGENTS.md index 4cf8dfb..b6a7592 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,7 @@ This file provides guidance to AI agents when working with code in this reposito ## Project Overview -parsedmarc is a Python module and CLI utility for parsing DMARC aggregate (RUA), forensic (RUF), and SMTP TLS reports. It reads reports from IMAP, Microsoft Graph, Gmail API, Maildir, mbox files, or direct file paths, and outputs to JSON/CSV, Elasticsearch, OpenSearch, Splunk, Kafka, S3, Azure Log Analytics, syslog, or webhooks. +parsedmarc is a Python module and CLI utility for parsing DMARC aggregate (RUA), failure/forensic (RUF), and SMTP TLS reports. It supports both RFC 7489 and DMARCbis (draft-ietf-dmarc-dmarcbis-41, draft-ietf-dmarc-aggregate-reporting-32, draft-ietf-dmarc-failure-reporting-24) report formats. It reads reports from IMAP, Microsoft Graph, Gmail API, Maildir, mbox files, or direct file paths, and outputs to JSON/CSV, Elasticsearch, OpenSearch, Splunk, Kafka, S3, Azure Log Analytics, syslog, or webhooks. ## Common Commands @@ -24,7 +24,7 @@ ruff format . # Test CLI with sample reports parsedmarc --debug -c ci.ini samples/aggregate/* -parsedmarc --debug -c ci.ini samples/forensic/* +parsedmarc --debug -c ci.ini samples/failure/* # Build docs cd docs && make html @@ -41,16 +41,20 @@ To skip DNS lookups during testing, set `GITHUB_ACTIONS=true`. ### Key modules -- `parsedmarc/__init__.py` — Core parsing logic. Main functions: `parse_report_file()`, `parse_report_email()`, `parse_aggregate_report_xml()`, `parse_forensic_report()`, `parse_smtp_tls_report_json()`, `get_dmarc_reports_from_mailbox()`, `watch_inbox()` -- `parsedmarc/cli.py` — CLI entry point (`_main`), config file parsing (`_load_config` + `_parse_config`), output orchestration. Supports configuration via INI files, `PARSEDMARC_{SECTION}_{KEY}` environment variables, or both (env vars override file values). -- `parsedmarc/types.py` — TypedDict definitions for all report types (`AggregateReport`, `ForensicReport`, `SMTPTLSReport`, `ParsingResults`) +- `parsedmarc/__init__.py` — Core parsing logic. Main functions: `parse_report_file()`, `parse_report_email()`, `parse_aggregate_report_xml()`, `parse_failure_report()`, `parse_smtp_tls_report_json()`, `get_dmarc_reports_from_mailbox()`, `watch_inbox()`. Legacy aliases (`parse_forensic_report`, etc.) are preserved for backward compatibility. +- `parsedmarc/cli.py` — CLI entry point (`_main`), config file parsing (`_load_config` + `_parse_config`), output orchestration. Supports configuration via INI files, `PARSEDMARC_{SECTION}_{KEY}` environment variables, or both (env vars override file values). Accepts both old (`save_forensic`, `forensic_topic`) and new (`save_failure`, `failure_topic`) config keys. +- `parsedmarc/types.py` — TypedDict definitions for all report types (`AggregateReport`, `FailureReport`, `SMTPTLSReport`, `ParsingResults`). Legacy alias `ForensicReport = FailureReport` preserved. - `parsedmarc/utils.py` — IP/DNS/GeoIP enrichment, base64 decoding, compression handling - `parsedmarc/mail/` — Polymorphic mail connections: `IMAPConnection`, `GmailConnection`, `MSGraphConnection`, `MaildirConnection` - `parsedmarc/{elastic,opensearch,splunk,kafkaclient,loganalytics,syslog,s3,webhook,gelf}.py` — Output integrations ### Report type system -`ReportType = Literal["aggregate", "forensic", "smtp_tls"]`. Exception hierarchy: `ParserError` → `InvalidDMARCReport` → `InvalidAggregateReport`/`InvalidForensicReport`, and `InvalidSMTPTLSReport`. +`ReportType = Literal["aggregate", "failure", "smtp_tls"]`. Exception hierarchy: `ParserError` → `InvalidDMARCReport` → `InvalidAggregateReport`/`InvalidFailureReport`, and `InvalidSMTPTLSReport`. Legacy alias `InvalidForensicReport = InvalidFailureReport` preserved. + +### DMARCbis support + +Aggregate reports support both RFC 7489 and DMARCbis formats. DMARCbis adds fields: `np` (non-existent subdomain policy), `testing` (replaces `pct`), `discovery_method` (`psl`/`treewalk`), `generator` (report metadata), and `human_result` (DKIM/SPF auth results). `pct` and `fo` default to `None` when absent (DMARCbis drops these). Namespaced XML is handled automatically. ### Configuration diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a9d982..2c52199 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,42 @@ # Changelog +## 10.0.0 + +### Enhancements + +#### Support for DMARCbis reports + +New fields from the XSD schema, added to types, parsing, CSV output, and Elasticsearch/OpenSearch mappings: + +- `np` — non-existent subdomain policy (`none`/`quarantine`/`reject`) +- `testing` — testing mode flag (`n`/`y`), replaces RFC7489 `pct` +- `discovery_method` — policy discovery method (`psl`/`treewalk`) +- `generator` — report generator software identifier (metadata) +- `human_result` — optional descriptive text on DKIM/SPF auth results + +Backwards compatibility to RFC7489 is maintained. + +### Breaking changes + +#### Forensic reports have been renamed to failure reports + +Forensic reports have been renamed to failure reports throughout the project to reflect the proper naming of the reports since RFC7489. + +- **Core**: `types.py`, `__init__.py` — `ForensicReport`→`FailureReport`, `parse_forensic_report`→`parse_failure_report`, report type `"failure"` +- **Output modules**: `elastic.py`, `opensearch.py`, `splunk.py`, `kafkaclient.py`, `syslog.py`, `gelf.py`, `webhook.py`, `loganalytics.py`, `s3.py` +- **CLI**: `cli.py` — args, config keys, index names (`dmarc_failure`) +- **Docs & dashboards**: all markdown, Grafana JSON, Kibana NDJSON, Splunk XML + +##### Backward compatibility + +- Old function/type names preserved as aliases: `parse_forensic_report = parse_failure_report`, `ForensicReport = FailureReport`, etc. +- CLI config accepts both old (`save_forensic`, `forensic_topic`) and new keys (`save_failure`, `failure_topic`) +- RFC 7489 reports parse with `None` for DMARCbis-only fields +- **Updated dashboards with queries are backward compatible**: queries match data indexed under both old (`dmarc_forensic*` / `dmarc:forensic`) and new (`dmarc_failure*` / `dmarc:failure`) names, so dashboards show data from before and after the rename: + - **Kibana**: Index pattern uses `dmarc_f*` to match both `dmarc_forensic*` and `dmarc_failure*` + - **Splunk**: Base search queries `(sourcetype="dmarc:failure" OR sourcetype="dmarc:forensic")` + - **Elasticsearch/OpenSearch**: Duplicate-check searches query across both `dmarc_failure*` and `dmarc_forensic*` index patterns + ## 9.10.1 ### Fixed @@ -12,7 +49,7 @@ - Renamed `[general] ip_db_url` to `ipinfo_url` to reflect what it actually overrides (the bundled IPinfo Lite MMDB download URL). The old name is still accepted as a deprecated alias and logs a warning on use; the env-var equivalent is now `PARSEDMARC_GENERAL_IPINFO_URL`, with `PARSEDMARC_GENERAL_IP_DB_URL` also still honored. - Added an optional IPinfo Lite REST API path for country + ASN lookups, so deployments that want the freshest data can query the API directly instead of waiting for the next MMDB release. Configure `[general] ipinfo_api_token` (or `PARSEDMARC_GENERAL_IPINFO_API_TOKEN`) and every IP lookup hits `https://api.ipinfo.io/lite/` first. At startup the `https://ipinfo.io/me` account endpoint is hit once to validate the token and log the plan, month-to-date usage, and remaining quota at info level (e.g. `IPinfo API configured — plan: Lite, usage: 12345/50000 this month, 37655 remaining`). An invalid token exits the process with a fatal error. Rate-limit (HTTP 429) and quota-exhausted (HTTP 402) responses put the API in a cooldown (honoring `Retry-After`, with a 5-minute / 1-hour default) and fall through to the bundled/cached MMDB; the first event is logged once at warning level and recovery is logged once at info level when the next lookup succeeds. Transient network errors fall through per-request without triggering a cooldown. The API token is never logged. -- Renamed the ASN name and domain fields to match the IPinfo Lite MMDB's native schema: `asn_name` → `as_name` and `asn_domain` → `as_domain` on every source record (JSON output), and `source_asn_name` → `source_as_name` / `source_asn_domain` → `source_as_domain` in CSV output (aggregate + forensic) and the Elasticsearch / OpenSearch / Splunk integrations. The integer `asn` / `source_asn` field is unchanged. The emitted order is `asn`, `as_name`, `as_domain`. +- Renamed the ASN name and domain fields to match the IPinfo Lite MMDB's native schema: `asn_name` → `as_name` and `asn_domain` → `as_domain` on every source record (JSON output), and `source_asn_name` → `source_as_name` / `source_asn_domain` → `source_as_domain` in CSV output (aggregate + failure) and the Elasticsearch / OpenSearch / Splunk integrations. The integer `asn` / `source_asn` field is unchanged. The emitted order is `asn`, `as_name`, `as_domain`. ### Upgrade notes @@ -192,7 +229,7 @@ ### Fixed -- `get_index_prefix()` crashed on forensic reports with `TypeError` due to `report()` instead of `report[]` dict access. +- `get_index_prefix()` crashed on failure reports with `TypeError` due to `report()` instead of `report[]` dict access. - Missing `exit(1)` after IMAP user/password validation failure allowed execution to continue with `None` credentials. ## 9.2.1 diff --git a/CLAUDE.md b/CLAUDE.md index 07926f1..078c29c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,3 +1,5 @@ -# CLAUD.md +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. @AGENTS.md diff --git a/docs/source/elasticsearch.md b/docs/source/elasticsearch.md index 1ed9676..ac5a6d5 100644 --- a/docs/source/elasticsearch.md +++ b/docs/source/elasticsearch.md @@ -214,7 +214,7 @@ Kibana index patterns with versions that match the upgraded indexes: 1. Login in to Kibana, and click on Management 2. Under Kibana, click on Saved Objects -3. Check the checkboxes for the `dmarc_aggregate` and `dmarc_forensic` +3. Check the checkboxes for the `dmarc_aggregate` and `dmarc_failure` index patterns 4. Click Delete 5. Click Delete on the conformation message diff --git a/docs/source/example.ini b/docs/source/example.ini index faeb5f3..12bd64d 100644 --- a/docs/source/example.ini +++ b/docs/source/example.ini @@ -2,7 +2,7 @@ [general] save_aggregate = True -save_forensic = True +save_failure = True [imap] host = imap.example.com diff --git a/docs/source/index.md b/docs/source/index.md index 3d86fef..540439d 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -30,7 +30,7 @@ and Valimail. ## Features - Parses draft and 1.0 standard aggregate/rua DMARC reports -- Parses forensic/failure/ruf DMARC reports +- Parses failure/ruf DMARC reports - Parses reports from SMTP TLS Reporting - Can parse reports from an inbox over IMAP, Microsoft Graph, or Gmail API - Transparently handles gzip or zip compressed reports diff --git a/docs/source/kibana.md b/docs/source/kibana.md index bf2cf1a..6d67e4e 100644 --- a/docs/source/kibana.md +++ b/docs/source/kibana.md @@ -74,14 +74,14 @@ the DMARC Summary dashboard. To view failures only, use the pie chart. Any other filters work the same way. You can also add your own custom temporary filters by clicking on Add Filter at the upper right of the page. -## DMARC Forensic Samples +## DMARC Failure Samples -The DMARC Forensic Samples dashboard contains information on DMARC forensic -reports (also known as failure reports or ruf reports). These reports contain +The DMARC Failure Samples dashboard contains information on DMARC failure +reports (also known as ruf reports). These reports contain samples of emails that have failed to pass DMARC. :::{note} -Most recipients do not send forensic/failure/ruf reports at all to avoid +Most recipients do not send failure/ruf reports at all to avoid privacy leaks. Some recipients (notably Chinese webmail services) will only supply the headers of sample emails. Very few provide the entire email. ::: diff --git a/docs/source/output.md b/docs/source/output.md index 095193b..6a6b2bb 100644 --- a/docs/source/output.md +++ b/docs/source/output.md @@ -99,12 +99,12 @@ draft,acme.com,noreply-dmarc-support@acme.com,http://acme.com/dmarc/support,9391 ``` -## Sample forensic report output +## Sample failure report output Thanks to GitHub user [xennn](https://github.com/xennn) for the anonymized -[forensic report email sample](). +[failure report email sample](). -### JSON forensic report +### JSON failure report ```json { @@ -198,7 +198,7 @@ Thanks to GitHub user [xennn](https://github.com/xennn) for the anonymized } ``` -### CSV forensic report +### CSV failure report ```text feedback_type,user_agent,version,original_envelope_id,original_mail_from,original_rcpt_to,arrival_date,arrival_date_utc,subject,message_id,authentication_results,dkim_domain,source_ip_address,source_country,source_reverse_dns,source_base_domain,source_name,source_type,source_asn,source_as_name,source_as_domain,delivery_result,auth_failure,reported_domain,authentication_mechanisms,sample_headers_only diff --git a/docs/source/splunk.md b/docs/source/splunk.md index f21d24b..c884ef8 100644 --- a/docs/source/splunk.md +++ b/docs/source/splunk.md @@ -1,10 +1,10 @@ # Splunk Starting in version 4.3.0 `parsedmarc` supports sending aggregate and/or -forensic DMARC data to a Splunk [HTTP Event collector (HEC)]. +failure DMARC data to a Splunk [HTTP Event collector (HEC)]. The project repository contains [XML files] for premade Splunk -dashboards for aggregate and forensic DMARC reports. +dashboards for aggregate and failure DMARC reports. Copy and paste the contents of each file into a separate Splunk dashboard XML editor. diff --git a/docs/source/usage.md b/docs/source/usage.md index cd9cd57..59e7f81 100644 --- a/docs/source/usage.md +++ b/docs/source/usage.md @@ -4,9 +4,9 @@ ```text usage: parsedmarc [-h] [-c CONFIG_FILE] [--strip-attachment-payloads] [-o OUTPUT] - [--aggregate-json-filename AGGREGATE_JSON_FILENAME] [--forensic-json-filename FORENSIC_JSON_FILENAME] + [--aggregate-json-filename AGGREGATE_JSON_FILENAME] [--failure-json-filename FAILURE_JSON_FILENAME] [--smtp-tls-json-filename SMTP_TLS_JSON_FILENAME] [--aggregate-csv-filename AGGREGATE_CSV_FILENAME] - [--forensic-csv-filename FORENSIC_CSV_FILENAME] [--smtp-tls-csv-filename SMTP_TLS_CSV_FILENAME] + [--failure-csv-filename FAILURE_CSV_FILENAME] [--smtp-tls-csv-filename SMTP_TLS_CSV_FILENAME] [-n NAMESERVERS [NAMESERVERS ...]] [-t DNS_TIMEOUT] [--offline] [-s] [-w] [--verbose] [--debug] [--log-file LOG_FILE] [--no-prettify-json] [-v] [file_path ...] @@ -14,26 +14,26 @@ usage: parsedmarc [-h] [-c CONFIG_FILE] [--strip-attachment-payloads] [-o OUTPUT Parses DMARC reports positional arguments: - file_path one or more paths to aggregate or forensic report files, emails, or mbox files' + file_path one or more paths to aggregate or failure report files, emails, or mbox files' options: -h, --help show this help message and exit -c CONFIG_FILE, --config-file CONFIG_FILE a path to a configuration file (--silent implied) --strip-attachment-payloads - remove attachment payloads from forensic report output + remove attachment payloads from failure report output -o OUTPUT, --output OUTPUT write output files to the given directory --aggregate-json-filename AGGREGATE_JSON_FILENAME filename for the aggregate JSON output file - --forensic-json-filename FORENSIC_JSON_FILENAME - filename for the forensic JSON output file + --failure-json-filename FAILURE_JSON_FILENAME + filename for the failure JSON output file --smtp-tls-json-filename SMTP_TLS_JSON_FILENAME filename for the SMTP TLS JSON output file --aggregate-csv-filename AGGREGATE_CSV_FILENAME filename for the aggregate CSV output file - --forensic-csv-filename FORENSIC_CSV_FILENAME - filename for the forensic CSV output file + --failure-csv-filename FAILURE_CSV_FILENAME + filename for the failure CSV output file --smtp-tls-csv-filename SMTP_TLS_CSV_FILENAME filename for the SMTP TLS CSV output file -n NAMESERVERS [NAMESERVERS ...], --nameservers NAMESERVERS [NAMESERVERS ...] @@ -70,7 +70,7 @@ For example [general] save_aggregate = True -save_forensic = True +save_failure = True [imap] host = imap.example.com @@ -109,7 +109,7 @@ mode = tcp [webhook] aggregate_url = https://aggregate_url.example.com -forensic_url = https://forensic_url.example.com +failure_url = https://failure_url.example.com smtp_tls_url = https://smtp_tls_url.example.com timeout = 60 ``` @@ -119,7 +119,7 @@ The full set of configuration options are: - `general` - `save_aggregate` - bool: Save aggregate report data to Elasticsearch, Splunk and/or S3 - - `save_forensic` - bool: Save forensic report data to + - `save_failure` - bool: Save failure report data to Elasticsearch, Splunk and/or S3 - `save_smtp_tls` - bool: Save SMTP-STS report data to Elasticsearch, Splunk and/or S3 @@ -130,7 +130,7 @@ The full set of configuration options are: - `output` - str: Directory to place JSON and CSV files in. This is required if you set either of the JSON output file options. - `aggregate_json_filename` - str: filename for the aggregate JSON output file - - `forensic_json_filename` - str: filename for the forensic + - `failure_json_filename` - str: filename for the failure JSON output file - `ip_db_path` - str: An optional custom path to a MMDB file from IPinfo, MaxMind, or DBIP @@ -340,7 +340,7 @@ The full set of configuration options are: - `skip_certificate_verification` - bool: Skip certificate verification (not recommended) - `aggregate_topic` - str: The Kafka topic for aggregate reports - - `forensic_topic` - str: The Kafka topic for forensic reports + - `failure_topic` - str: The Kafka topic for failure reports - `smtp` - `host` - str: The SMTP hostname - `port` - int: The SMTP port (Default: `25`) @@ -458,7 +458,7 @@ The full set of configuration options are: - `dce` - str: The Data Collection Endpoint (DCE). Example: `https://{DCE-NAME}.{REGION}.ingest.monitor.azure.com`. - `dcr_immutable_id` - str: The immutable ID of the Data Collection Rule (DCR) - `dcr_aggregate_stream` - str: The stream name for aggregate reports in the DCR - - `dcr_forensic_stream` - str: The stream name for the forensic reports in the DCR + - `dcr_failure_stream` - str: The stream name for the failure reports in the DCR - `dcr_smtp_tls_stream` - str: The stream name for the SMTP TLS reports in the DCR :::{note} @@ -475,7 +475,7 @@ The full set of configuration options are: - `webhook` - Post the individual reports to a webhook url with the report as the JSON body - `aggregate_url` - str: URL of the webhook which should receive the aggregate reports - - `forensic_url` - str: URL of the webhook which should receive the forensic reports + - `failure_url` - str: URL of the webhook which should receive the failure reports - `smtp_tls_url` - str: URL of the webhook which should receive the smtp_tls reports - `timeout` - int: Interval in which the webhook call should timeout @@ -490,26 +490,26 @@ blocks DNS requests to outside resolvers. ::: :::{note} -`save_aggregate` and `save_forensic` are separate options -because you may not want to save forensic reports -(also known as failure reports) to your Elasticsearch instance, +`save_aggregate` and `save_failure` are separate options +because you may not want to save failure reports +(formerly known as forensic reports) to your Elasticsearch instance, particularly if you are in a highly-regulated industry that handles sensitive data, such as healthcare or finance. If your legitimate outgoing email fails DMARC, it is possible -that email may appear later in a forensic report. +that email may appear later in a failure report. -Forensic reports contain the original headers of an email that +Failure reports contain the original headers of an email that failed a DMARC check, and sometimes may also include the full message body, depending on the policy of the reporting organization. -Most reporting organizations do not send forensic reports of any +Most reporting organizations do not send failure reports of any kind for privacy reasons. While aggregate DMARC reports are sent -at least daily, it is normal to receive very few forensic reports. +at least daily, it is normal to receive very few failure reports. -An alternative approach is to still collect forensic/failure/ruf +An alternative approach is to still collect failure/ruf reports in your DMARC inbox, but run `parsedmarc` with -```save_forensic = True``` manually on a separate IMAP folder (using +```save_failure = True``` manually on a separate IMAP folder (using the ```reports_folder``` option), after you have manually moved known samples you want to save to that folder (e.g. malicious samples and non-sensitive legitimate samples). diff --git a/grafana/Grafana-DMARC_Reports.json b/grafana/Grafana-DMARC_Reports.json index cb3a967..2bcbc34 100644 --- a/grafana/Grafana-DMARC_Reports.json +++ b/grafana/Grafana-DMARC_Reports.json @@ -83,7 +83,7 @@ "id": 28, "panels": [ { - "content": "# DMARC Summary\r\nAs the name suggests, this dashboard is the best place to start reviewing your aggregate DMARC data.\r\n\r\nAcross the top of the dashboard, three pie charts display the percentage of alignment pass/fail for SPF, DKIM, and DMARC. Clicking on any chart segment will filter for that value.\r\n\r\n***Note***\r\nMessages should not be considered malicious just because they failed to pass DMARC; especially if you have just started collecting data. It may be a legitimate service that needs SPF and DKIM configured correctly.\r\n\r\nStart by filtering the results to only show failed DKIM alignment. While DMARC passes if a message passes SPF or DKIM alignment, only DKIM alignment remains valid when a message is forwarded without changing the from address, which is often caused by a mailbox forwarding rule. This is because DKIM signatures are part of the message headers, whereas SPF relies on SMTP session headers.\r\n\r\nUnderneath the pie charts. you can see graphs of DMARC passage and message disposition over time.\r\n\r\nUnder the graphs you will find the most useful data tables on the dashboard. On the left, there is a list of organizations that are sending you DMARC reports. In the center, there is a list of sending servers grouped by the base domain in their reverse DNS. On the right, there is a list of email from domains, sorted by message volume.\r\n\r\nBy hovering your mouse over a data table value and using the magnifying glass icons, you can filter on or filter out different values. Start by looking at the Message Sources by Reverse DNS table. Find a sender that you recognize, such as an email marketing service, hover over it, and click on the plus (+) magnifying glass icon, to add a filter that only shows results for that sender. Now, look at the Message From Header table to the right. That shows you the domains that a sender is sending as, which might tell you which brand/business is using a particular service. With that information, you can contact them and have them set up DKIM.\r\n\r\n***Note***\r\nIf you have a lot of B2C customers, you may see a high volume of emails as your domains coming from consumer email services, such as Google/Gmail and Yahoo! This occurs when customers have mailbox rules in place that forward emails from an old account to a new account, which is why DKIM authentication is so important, as mentioned earlier. Similar patterns may be observed with businesses who send from reverse DNS addressees of parent, subsidiary, and outdated brands.\r\n\r\n***Note***\r\nYou can add your own custom temporary filters by clicking on Add Filter at the upper right of the page.\r\n\r\n# DMARC Forensic Samples\r\nThe DMARC Forensic Samples section contains information on DMARC forensic reports (also known as failure reports or ruf reports). These reports contain samples of emails that have failed to pass DMARC.\r\n\r\n***Note***\r\nMost recipients do not send forensic/failure/ruf reports at all to avoid privacy leaks. Some recipients (notably Chinese webmail services) will only supply the headers of sample emails. Very few provide the entire email.\r\n\r\n# DMARC Alignment Guide\r\nDMARC ensures that SPF and DKIM authentication mechanisms actually authenticate against the same domain that the end user sees.\r\n\r\nA message passes a DMARC check by passing DKIM or SPF, **as long as the related indicators are also in alignment.**\r\n\r\n| \t| DKIM \t| SPF \t|\r\n|-----------\t|--------------------------------------------------------------------------------------------------------------------------------------------------\t|----------------------------------------------------------------------------------------------------------------\t|\r\n| **Passing** \t| The signature in the DKIM header is validated using a public key that is published as a DNS record of the domain name specified in the signature \t| The mail server's IP address is listed in the SPF record of the domain in the SMTP envelope's mail from header \t|\r\n| **Alignment** \t| The signing domain aligns with the domain in the message's from header \t| The domain in the SMTP envelope's mail from header aligns with the domain in the message's from header \t|\r\n\r\n\r\n# Further Reading\r\n[Demystifying DMARC: A guide to preventing email spoofing](https://seanthegeek.net/459/demystifying-dmarc/amp/)\r\n\r\n[DMARC Manual](https://menainfosec.com/wp-content/uploads/2017/12/DMARC_Service_Manual.pdf)\r\n\r\n[What is “External Destination Verification”?](https://dmarcian.com/what-is-external-destination-verification/)", + "content": "# DMARC Summary\r\nAs the name suggests, this dashboard is the best place to start reviewing your aggregate DMARC data.\r\n\r\nAcross the top of the dashboard, three pie charts display the percentage of alignment pass/fail for SPF, DKIM, and DMARC. Clicking on any chart segment will filter for that value.\r\n\r\n***Note***\r\nMessages should not be considered malicious just because they failed to pass DMARC; especially if you have just started collecting data. It may be a legitimate service that needs SPF and DKIM configured correctly.\r\n\r\nStart by filtering the results to only show failed DKIM alignment. While DMARC passes if a message passes SPF or DKIM alignment, only DKIM alignment remains valid when a message is forwarded without changing the from address, which is often caused by a mailbox forwarding rule. This is because DKIM signatures are part of the message headers, whereas SPF relies on SMTP session headers.\r\n\r\nUnderneath the pie charts. you can see graphs of DMARC passage and message disposition over time.\r\n\r\nUnder the graphs you will find the most useful data tables on the dashboard. On the left, there is a list of organizations that are sending you DMARC reports. In the center, there is a list of sending servers grouped by the base domain in their reverse DNS. On the right, there is a list of email from domains, sorted by message volume.\r\n\r\nBy hovering your mouse over a data table value and using the magnifying glass icons, you can filter on or filter out different values. Start by looking at the Message Sources by Reverse DNS table. Find a sender that you recognize, such as an email marketing service, hover over it, and click on the plus (+) magnifying glass icon, to add a filter that only shows results for that sender. Now, look at the Message From Header table to the right. That shows you the domains that a sender is sending as, which might tell you which brand/business is using a particular service. With that information, you can contact them and have them set up DKIM.\r\n\r\n***Note***\r\nIf you have a lot of B2C customers, you may see a high volume of emails as your domains coming from consumer email services, such as Google/Gmail and Yahoo! This occurs when customers have mailbox rules in place that forward emails from an old account to a new account, which is why DKIM authentication is so important, as mentioned earlier. Similar patterns may be observed with businesses who send from reverse DNS addressees of parent, subsidiary, and outdated brands.\r\n\r\n***Note***\r\nYou can add your own custom temporary filters by clicking on Add Filter at the upper right of the page.\r\n\r\n# DMARC Failure Samples\r\nThe DMARC Failure Samples section contains information on DMARC failure reports (also known as ruf reports). These reports contain samples of emails that have failed to pass DMARC.\r\n\r\n***Note***\r\nMost recipients do not send failure/ruf reports at all to avoid privacy leaks. Some recipients (notably Chinese webmail services) will only supply the headers of sample emails. Very few provide the entire email.\r\n\r\n# DMARC Alignment Guide\r\nDMARC ensures that SPF and DKIM authentication mechanisms actually authenticate against the same domain that the end user sees.\r\n\r\nA message passes a DMARC check by passing DKIM or SPF, **as long as the related indicators are also in alignment.**\r\n\r\n| \t| DKIM \t| SPF \t|\r\n|-----------\t|--------------------------------------------------------------------------------------------------------------------------------------------------\t|----------------------------------------------------------------------------------------------------------------\t|\r\n| **Passing** \t| The signature in the DKIM header is validated using a public key that is published as a DNS record of the domain name specified in the signature \t| The mail server's IP address is listed in the SPF record of the domain in the SMTP envelope's mail from header \t|\r\n| **Alignment** \t| The signing domain aligns with the domain in the message's from header \t| The domain in the SMTP envelope's mail from header aligns with the domain in the message's from header \t|\r\n\r\n\r\n# Further Reading\r\n[Demystifying DMARC: A guide to preventing email spoofing](https://seanthegeek.net/459/demystifying-dmarc/amp/)\r\n\r\n[DMARC Manual](https://menainfosec.com/wp-content/uploads/2017/12/DMARC_Service_Manual.pdf)\r\n\r\n[What is “External Destination Verification”?](https://dmarcian.com/what-is-external-destination-verification/)", "datasource": null, "fieldConfig": { "defaults": { @@ -101,7 +101,7 @@ "links": [], "mode": "markdown", "options": { - "content": "# DMARC Summary\r\nAs the name suggests, this dashboard is the best place to start reviewing your aggregate DMARC data.\r\n\r\nAcross the top of the dashboard, three pie charts display the percentage of alignment pass/fail for SPF, DKIM, and DMARC. Clicking on any chart segment will filter for that value.\r\n\r\n***Note***\r\nMessages should not be considered malicious just because they failed to pass DMARC; especially if you have just started collecting data. It may be a legitimate service that needs SPF and DKIM configured correctly.\r\n\r\nStart by filtering the results to only show failed DKIM alignment. While DMARC passes if a message passes SPF or DKIM alignment, only DKIM alignment remains valid when a message is forwarded without changing the from address, which is often caused by a mailbox forwarding rule. This is because DKIM signatures are part of the message headers, whereas SPF relies on SMTP session headers.\r\n\r\nUnderneath the pie charts. you can see graphs of DMARC passage and message disposition over time.\r\n\r\nUnder the graphs you will find the most useful data tables on the dashboard. On the left, there is a list of organizations that are sending you DMARC reports. In the center, there is a list of sending servers grouped by the base domain in their reverse DNS. On the right, there is a list of email from domains, sorted by message volume.\r\n\r\nBy hovering your mouse over a data table value and using the magnifying glass icons, you can filter on or filter out different values. Start by looking at the Message Sources by Reverse DNS table. Find a sender that you recognize, such as an email marketing service, hover over it, and click on the plus (+) magnifying glass icon, to add a filter that only shows results for that sender. Now, look at the Message From Header table to the right. That shows you the domains that a sender is sending as, which might tell you which brand/business is using a particular service. With that information, you can contact them and have them set up DKIM.\r\n\r\n***Note***\r\nIf you have a lot of B2C customers, you may see a high volume of emails as your domains coming from consumer email services, such as Google/Gmail and Yahoo! This occurs when customers have mailbox rules in place that forward emails from an old account to a new account, which is why DKIM authentication is so important, as mentioned earlier. Similar patterns may be observed with businesses who send from reverse DNS addressees of parent, subsidiary, and outdated brands.\r\n\r\n***Note***\r\nYou can add your own custom temporary filters by clicking on Add Filter at the upper right of the page.\r\n\r\n# DMARC Forensic Samples\r\nThe DMARC Forensic Samples section contains information on DMARC forensic reports (also known as failure reports or ruf reports). These reports contain samples of emails that have failed to pass DMARC.\r\n\r\n***Note***\r\nMost recipients do not send forensic/failure/ruf reports at all to avoid privacy leaks. Some recipients (notably Chinese webmail services) will only supply the headers of sample emails. Very few provide the entire email.\r\n\r\n# DMARC Alignment Guide\r\nDMARC ensures that SPF and DKIM authentication mechanisms actually authenticate against the same domain that the end user sees.\r\n\r\nA message passes a DMARC check by passing DKIM or SPF, **as long as the related indicators are also in alignment.**\r\n\r\n| \t| DKIM \t| SPF \t|\r\n|-----------\t|--------------------------------------------------------------------------------------------------------------------------------------------------\t|----------------------------------------------------------------------------------------------------------------\t|\r\n| **Passing** \t| The signature in the DKIM header is validated using a public key that is published as a DNS record of the domain name specified in the signature \t| The mail server's IP address is listed in the SPF record of the domain in the SMTP envelope's mail from header \t|\r\n| **Alignment** \t| The signing domain aligns with the domain in the message's from header \t| The domain in the SMTP envelope's mail from header aligns with the domain in the message's from header \t|\r\n\r\n\r\n# Further Reading\r\n[Demystifying DMARC: A guide to preventing email spoofing](https://seanthegeek.net/459/demystifying-dmarc/amp/)\r\n\r\n[DMARC Manual](https://menainfosec.com/wp-content/uploads/2017/12/DMARC_Service_Manual.pdf)\r\n\r\n[What is “External Destination Verification”?](https://dmarcian.com/what-is-external-destination-verification/)", + "content": "# DMARC Summary\r\nAs the name suggests, this dashboard is the best place to start reviewing your aggregate DMARC data.\r\n\r\nAcross the top of the dashboard, three pie charts display the percentage of alignment pass/fail for SPF, DKIM, and DMARC. Clicking on any chart segment will filter for that value.\r\n\r\n***Note***\r\nMessages should not be considered malicious just because they failed to pass DMARC; especially if you have just started collecting data. It may be a legitimate service that needs SPF and DKIM configured correctly.\r\n\r\nStart by filtering the results to only show failed DKIM alignment. While DMARC passes if a message passes SPF or DKIM alignment, only DKIM alignment remains valid when a message is forwarded without changing the from address, which is often caused by a mailbox forwarding rule. This is because DKIM signatures are part of the message headers, whereas SPF relies on SMTP session headers.\r\n\r\nUnderneath the pie charts. you can see graphs of DMARC passage and message disposition over time.\r\n\r\nUnder the graphs you will find the most useful data tables on the dashboard. On the left, there is a list of organizations that are sending you DMARC reports. In the center, there is a list of sending servers grouped by the base domain in their reverse DNS. On the right, there is a list of email from domains, sorted by message volume.\r\n\r\nBy hovering your mouse over a data table value and using the magnifying glass icons, you can filter on or filter out different values. Start by looking at the Message Sources by Reverse DNS table. Find a sender that you recognize, such as an email marketing service, hover over it, and click on the plus (+) magnifying glass icon, to add a filter that only shows results for that sender. Now, look at the Message From Header table to the right. That shows you the domains that a sender is sending as, which might tell you which brand/business is using a particular service. With that information, you can contact them and have them set up DKIM.\r\n\r\n***Note***\r\nIf you have a lot of B2C customers, you may see a high volume of emails as your domains coming from consumer email services, such as Google/Gmail and Yahoo! This occurs when customers have mailbox rules in place that forward emails from an old account to a new account, which is why DKIM authentication is so important, as mentioned earlier. Similar patterns may be observed with businesses who send from reverse DNS addressees of parent, subsidiary, and outdated brands.\r\n\r\n***Note***\r\nYou can add your own custom temporary filters by clicking on Add Filter at the upper right of the page.\r\n\r\n# DMARC Failure Samples\r\nThe DMARC Failure Samples section contains information on DMARC failure reports (also known as ruf reports). These reports contain samples of emails that have failed to pass DMARC.\r\n\r\n***Note***\r\nMost recipients do not send failure/ruf reports at all to avoid privacy leaks. Some recipients (notably Chinese webmail services) will only supply the headers of sample emails. Very few provide the entire email.\r\n\r\n# DMARC Alignment Guide\r\nDMARC ensures that SPF and DKIM authentication mechanisms actually authenticate against the same domain that the end user sees.\r\n\r\nA message passes a DMARC check by passing DKIM or SPF, **as long as the related indicators are also in alignment.**\r\n\r\n| \t| DKIM \t| SPF \t|\r\n|-----------\t|--------------------------------------------------------------------------------------------------------------------------------------------------\t|----------------------------------------------------------------------------------------------------------------\t|\r\n| **Passing** \t| The signature in the DKIM header is validated using a public key that is published as a DNS record of the domain name specified in the signature \t| The mail server's IP address is listed in the SPF record of the domain in the SMTP envelope's mail from header \t|\r\n| **Alignment** \t| The signing domain aligns with the domain in the message's from header \t| The domain in the SMTP envelope's mail from header aligns with the domain in the message's from header \t|\r\n\r\n\r\n# Further Reading\r\n[Demystifying DMARC: A guide to preventing email spoofing](https://seanthegeek.net/459/demystifying-dmarc/amp/)\r\n\r\n[DMARC Manual](https://menainfosec.com/wp-content/uploads/2017/12/DMARC_Service_Manual.pdf)\r\n\r\n[What is “External Destination Verification”?](https://dmarcian.com/what-is-external-destination-verification/)", "mode": "markdown" }, "pluginVersion": "7.1.0", diff --git a/kibana/export.ndjson b/kibana/export.ndjson index dfa898a..9b7015a 100644 --- a/kibana/export.ndjson +++ b/kibana/export.ndjson @@ -13,7 +13,7 @@ {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"DKIM Alignment Details","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":6,\"direction\":\"desc\"}}}}","version":1,"visState":"{\"title\":\"DKIM Alignment Details\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"sort\":{\"columnIndex\":5,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\",\"showMetricsAtAllLevels\":false,\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":6,\"format\":{\"id\":\"number\",\"params\":{\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Messages\",\"aggType\":\"sum\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Header From\",\"aggType\":\"terms\"},{\"accessor\":1,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"DKIM Selector\",\"aggType\":\"terms\"},{\"accessor\":2,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"DKIM Domain\",\"aggType\":\"terms\"},{\"accessor\":3,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"DKIM Result\",\"aggType\":\"terms\"},{\"accessor\":4,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"boolean\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"DKIM Aligned\",\"aggType\":\"terms\"},{\"accessor\":5,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Reverse DNS Base\",\"aggType\":\"terms\"}]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"header_from.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Header From\"}},{\"id\":\"8\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"dkim_results.selector.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":1000,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"\",\"customLabel\":\"DKIM Selector\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"dkim_results.domain.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"\",\"customLabel\":\"DKIM Domain\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"dkim_results.result.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":2,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"\",\"customLabel\":\"DKIM Result\"}},{\"id\":\"7\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"dkim_aligned\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":2,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"DKIM Aligned\"}},{\"id\":\"6\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_base_domain.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":1000,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"\",\"customLabel\":\"Reverse DNS Base\"}}]}"},"coreMigrationVersion":"8.8.0","created_at":"2025-12-01T16:43:47.976Z","id":"40e7a5b0-2883-11e8-b8b2-15742da3055c","managed":false,"references":[{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","typeMigrationVersion":"8.5.0","updated_at":"2025-12-01T16:43:47.976Z","version":"WzE2MywxXQ=="} {"attributes":{"description":"","layerListJSON":"[{\"sourceDescriptor\":{\"type\":\"EMS_TMS\",\"isAutoSelect\":true,\"lightModeDefault\":\"road_map_desaturated\"},\"id\":\"5a631c7c-17d0-4b75-aa4f-93bc9ecca9c7\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"alpha\":1,\"visible\":true,\"style\":{\"type\":\"EMS_VECTOR_TILE\",\"color\":\"\"},\"includeInFitToBounds\":true,\"type\":\"EMS_VECTOR_TILE\",\"locale\":\"autoselect\"},{\"joins\":[{\"leftField\":\"iso2\",\"right\":{\"type\":\"ES_TERM_SOURCE\",\"id\":\"384092bd-b96f-4644-9d3d-b0bf09601d93\",\"term\":\"source_country.keyword\",\"metrics\":[{\"type\":\"count\"}],\"applyGlobalQuery\":true,\"applyGlobalTime\":true,\"applyForceRefresh\":true,\"indexPatternRefName\":\"layer_1_join_0_index_pattern\"}}],\"sourceDescriptor\":{\"type\":\"EMS_FILE\",\"id\":\"world_countries\",\"tooltipProperties\":[\"name\"]},\"style\":{\"type\":\"VECTOR\",\"properties\":{\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"marker\"}},\"fillColor\":{\"type\":\"DYNAMIC\",\"options\":{\"color\":\"Yellow to Red\",\"colorCategory\":\"palette_0\",\"field\":{\"name\":\"__kbnjoin__count__384092bd-b96f-4644-9d3d-b0bf09601d93\",\"origin\":\"join\"},\"fieldMetaOptions\":{\"isEnabled\":true,\"sigma\":3},\"type\":\"ORDINAL\"}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#3d3d3d\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"STATIC\",\"options\":{\"size\":6}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"labelText\":{\"type\":\"DYNAMIC\",\"options\":{\"field\":{\"name\":\"__kbnjoin__count__384092bd-b96f-4644-9d3d-b0bf09601d93\",\"origin\":\"join\"}}},\"labelColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#000000\"}},\"labelSize\":{\"type\":\"STATIC\",\"options\":{\"size\":14}},\"labelZoomRange\":{\"options\":{\"useLayerZoomRange\":true,\"minZoom\":0,\"maxZoom\":24}},\"labelBorderColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"labelBorderSize\":{\"options\":{\"size\":\"SMALL\"}},\"labelPosition\":{\"options\":{\"position\":\"CENTER\"}}},\"isTimeAware\":true},\"id\":\"30b6be32-3531-45dc-a2bd-1426e2485eb0\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"alpha\":0.75,\"visible\":true,\"includeInFitToBounds\":true,\"type\":\"GEOJSON_VECTOR\",\"disableTooltips\":false}]","mapStateJSON":"{\"adHocDataViews\":[],\"zoom\":1.94,\"center\":{\"lon\":-1.46337,\"lat\":30.78428},\"timeFilters\":{\"from\":\"now-1M\",\"to\":\"now\"},\"refreshConfig\":{\"isPaused\":true,\"interval\":60000},\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filters\":[],\"settings\":{\"autoFitToDataBounds\":false,\"backgroundColor\":\"#ffffff\",\"customIcons\":[],\"disableInteractive\":false,\"disableTooltipControl\":false,\"hideToolbarOverlay\":false,\"hideLayerControl\":false,\"hideViewControl\":false,\"initialLocation\":\"LAST_SAVED_LOCATION\",\"fixedLocation\":{\"lat\":27.68352808378776,\"lon\":5.537109375000001,\"zoom\":3},\"browserLocation\":{\"zoom\":2},\"keydownScrollZoom\":false,\"maxZoom\":24,\"minZoom\":0,\"showScaleControl\":false,\"showSpatialFilters\":true,\"showTimesliderToggleButton\":true,\"spatialFiltersAlpa\":0.3,\"spatialFiltersFillColor\":\"#DA8B45\",\"spatialFiltersLineColor\":\"#DA8B45\"}}","title":"Map of Message Source Countries","uiStateJSON":"{\"isLayerTOCOpen\":true,\"openTOCDetails\":[\"660f7aea-6c50-41bb-bffe-4d08bbb5f8e5\"]}"},"coreMigrationVersion":"8.8.0","created_at":"2025-12-01T16:43:47.976Z","id":"1a892210-8ae6-11ee-b032-c5c657f8d4c9","managed":false,"references":[{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"layer_1_join_0_index_pattern","type":"index-pattern"}],"type":"map","typeMigrationVersion":"8.4.0","updated_at":"2025-12-01T16:43:47.976Z","version":"WzE2NCwxXQ=="} {"attributes":{"controlGroupInput":{"chainingSystem":"HIERARCHICAL","controlStyle":"oneLine","ignoreParentSettingsJSON":"{\"ignoreFilters\":false,\"ignoreQuery\":false,\"ignoreTimerange\":false,\"ignoreValidations\":false}","panelsJSON":"{}","showApplySelections":false},"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"language\":\"kuery\",\"query\":\"\"}}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"type\":\"visualization\",\"title\":\"DMARC Passage Over Time\",\"panelRefName\":\"panel_4\",\"embeddableConfig\":{\"description\":\"\",\"enhancements\":{\"dynamicActions\":{\"events\":[]}}},\"panelIndex\":\"4\",\"gridData\":{\"i\":\"4\",\"y\":13,\"x\":0,\"w\":48,\"h\":15}},{\"type\":\"visualization\",\"title\":\"Reporting Organizations\",\"panelRefName\":\"panel_7\",\"embeddableConfig\":{\"description\":\"\",\"enhancements\":{\"dynamicActions\":{\"events\":[]}},\"uiState\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}}},\"panelIndex\":\"7\",\"gridData\":{\"i\":\"7\",\"y\":55,\"x\":0,\"w\":16,\"h\":18}},{\"type\":\"visualization\",\"title\":\"Top 2000 Message Sources by Reverse DNS\",\"panelRefName\":\"panel_8\",\"embeddableConfig\":{\"description\":\"\",\"enhancements\":{\"dynamicActions\":{\"events\":[]}},\"uiState\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}}},\"panelIndex\":\"8\",\"gridData\":{\"i\":\"8\",\"y\":55,\"x\":16,\"w\":16,\"h\":18}},{\"type\":\"visualization\",\"title\":\"SPF Alignment\",\"panelRefName\":\"panel_9\",\"embeddableConfig\":{\"description\":\"\",\"enhancements\":{\"dynamicActions\":{\"events\":[]}},\"uiState\":{\"vis\":{\"legendOpen\":false}}},\"panelIndex\":\"9\",\"gridData\":{\"i\":\"9\",\"y\":0,\"x\":0,\"w\":16,\"h\":13}},{\"type\":\"visualization\",\"title\":\"DKIM Alignment\",\"panelRefName\":\"panel_10\",\"embeddableConfig\":{\"description\":\"\",\"enhancements\":{\"dynamicActions\":{\"events\":[]}},\"uiState\":{\"vis\":{\"legendOpen\":false}}},\"panelIndex\":\"10\",\"gridData\":{\"i\":\"10\",\"y\":0,\"x\":16,\"w\":15,\"h\":13}},{\"type\":\"visualization\",\"title\":\"DMARC Passage\",\"panelRefName\":\"panel_11\",\"embeddableConfig\":{\"description\":\"\",\"enhancements\":{\"dynamicActions\":{\"events\":[]}},\"uiState\":{\"vis\":{\"legendOpen\":false}}},\"panelIndex\":\"11\",\"gridData\":{\"i\":\"11\",\"y\":0,\"x\":31,\"w\":17,\"h\":13}},{\"type\":\"visualization\",\"title\":\"Message Volume by Header From\",\"panelRefName\":\"panel_13\",\"embeddableConfig\":{\"description\":\"\",\"enhancements\":{\"dynamicActions\":{\"events\":[]}},\"uiState\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}},\"panelIndex\":\"13\",\"gridData\":{\"i\":\"13\",\"y\":55,\"x\":32,\"w\":16,\"h\":18}},{\"type\":\"visualization\",\"title\":\"Top 1000 Message Source IP Addresses\",\"panelRefName\":\"panel_14\",\"embeddableConfig\":{\"description\":\"\",\"enhancements\":{\"dynamicActions\":{\"events\":[]}},\"uiState\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}},\"panelIndex\":\"14\",\"gridData\":{\"i\":\"14\",\"y\":108,\"x\":0,\"w\":48,\"h\":18}},{\"type\":\"visualization\",\"title\":\"Message Disposition Over Time\",\"panelRefName\":\"panel_15\",\"embeddableConfig\":{\"description\":\"\",\"enhancements\":{\"dynamicActions\":{\"events\":[]}}},\"panelIndex\":\"15\",\"gridData\":{\"i\":\"15\",\"y\":42,\"x\":0,\"w\":48,\"h\":13}},{\"type\":\"visualization\",\"title\":\"Message Source Countries\",\"panelRefName\":\"panel_16\",\"embeddableConfig\":{\"description\":\"\",\"enhancements\":{\"dynamicActions\":{\"events\":[]}},\"uiState\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}},\"panelIndex\":\"16\",\"gridData\":{\"i\":\"16\",\"y\":91,\"x\":36,\"w\":12,\"h\":17}},{\"type\":\"visualization\",\"title\":\"SPF Alignment Details\",\"panelRefName\":\"panel_17\",\"embeddableConfig\":{\"description\":\"\",\"enhancements\":{\"dynamicActions\":{\"events\":[]}},\"uiState\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":5,\"direction\":\"desc\"}}}}},\"panelIndex\":\"17\",\"gridData\":{\"i\":\"17\",\"y\":126,\"x\":0,\"w\":48,\"h\":16}},{\"type\":\"visualization\",\"title\":\"DKIM Alignment Details\",\"panelRefName\":\"panel_18\",\"embeddableConfig\":{\"description\":\"\",\"enhancements\":{\"dynamicActions\":{\"events\":[]}},\"uiState\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":6,\"direction\":\"desc\"},\"colWidth\":[{\"colIndex\":6,\"width\":269.42857142857144}]}}}},\"panelIndex\":\"18\",\"gridData\":{\"i\":\"18\",\"y\":142,\"x\":0,\"w\":48,\"h\":19}},{\"type\":\"visualization\",\"title\":\"SPF Alignment Over Time\",\"embeddableConfig\":{\"hidePanelTitles\":false,\"enhancements\":{\"dynamicActions\":{\"events\":[]}},\"savedVis\":{\"title\":\"\",\"description\":\"\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100,\"filter\":true},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Messages\"},\"drawLinesBetweenPoints\":true,\"mode\":\"normal\",\"show\":\"true\",\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"line\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Messages\"},\"type\":\"value\"}],\"palette\":{\"type\":\"palette\",\"name\":\"status\"},\"isVislibVis\":true,\"detailedTooltip\":true,\"fittingFunction\":\"zero\",\"truncateLegend\":true,\"maxLegendLines\":1,\"labels\":{},\"radiusRatio\":9,\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"},\"legendSize\":\"auto\"},\"uiState\":{\"vis\":{\"colors\":{\"false\":\"#E24D42\",\"true\":\"#629E51\"},\"legendOpen\":true}},\"data\":{\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\",\"emptyAsNull\":false},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"date_begin\",\"timeRange\":{\"from\":\"2025-11-03T14:17:56.531Z\",\"to\":\"2025-12-03T14:17:56.531Z\"},\"useNormalizedEsInterval\":true,\"extendToTimeRange\":false,\"scaleMetricValues\":true,\"interval\":\"d\",\"used_interval\":\"1d\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Date\"},\"schema\":\"segment\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"spf_aligned\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":5,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"includeIsRegex\":true,\"excludeIsRegex\":true},\"schema\":\"group\"}],\"searchSource\":{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}}}},\"panelIndex\":\"ba73361d-48e3-4b81-b460-f39bbbf529d7\",\"gridData\":{\"i\":\"ba73361d-48e3-4b81-b460-f39bbbf529d7\",\"y\":28,\"x\":0,\"w\":24,\"h\":14}},{\"type\":\"visualization\",\"title\":\"DKIM Alignment Over Time\",\"embeddableConfig\":{\"hidePanelTitles\":false,\"enhancements\":{\"dynamicActions\":{\"events\":[]}},\"savedVis\":{\"title\":\"\",\"description\":\"\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100,\"filter\":true},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Messages\"},\"drawLinesBetweenPoints\":true,\"mode\":\"normal\",\"show\":\"true\",\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"line\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Messages\"},\"type\":\"value\"}],\"palette\":{\"type\":\"palette\",\"name\":\"status\"},\"isVislibVis\":true,\"detailedTooltip\":true,\"fittingFunction\":\"zero\",\"truncateLegend\":true,\"maxLegendLines\":1,\"labels\":{},\"radiusRatio\":9,\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"},\"legendSize\":\"auto\"},\"uiState\":{\"vis\":{\"colors\":{\"false\":\"#E24D42\",\"true\":\"#629E51\"},\"legendOpen\":true}},\"data\":{\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\",\"emptyAsNull\":false},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"date_begin\",\"timeRange\":{\"from\":\"2025-11-03T14:17:56.531Z\",\"to\":\"2025-12-03T14:17:56.531Z\"},\"useNormalizedEsInterval\":true,\"extendToTimeRange\":false,\"scaleMetricValues\":true,\"interval\":\"d\",\"used_interval\":\"1d\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Date\"},\"schema\":\"segment\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"dkim_aligned\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":5,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"includeIsRegex\":true,\"excludeIsRegex\":true},\"schema\":\"group\"}],\"searchSource\":{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}}}},\"panelIndex\":\"4cd04d8b-94d5-42c6-8003-46412bc96e60\",\"gridData\":{\"i\":\"4cd04d8b-94d5-42c6-8003-46412bc96e60\",\"y\":28,\"x\":24,\"w\":24,\"h\":14}},{\"type\":\"map\",\"panelRefName\":\"panel_83ea58e8-fe32-4775-aabe-2b2bb0acf28e\",\"embeddableConfig\":{\"mapCenter\":{\"lon\":-9.66797,\"lat\":37.99616,\"zoom\":1},\"mapBuffer\":{\"minLon\":-360,\"minLat\":-85.05113,\"maxLon\":180,\"maxLat\":85.05113},\"isLayerTOCOpen\":true,\"openTOCDetails\":[\"660f7aea-6c50-41bb-bffe-4d08bbb5f8e5\"],\"hiddenLayers\":[],\"enhancements\":{\"dynamicActions\":{\"events\":[]}}},\"panelIndex\":\"83ea58e8-fe32-4775-aabe-2b2bb0acf28e\",\"gridData\":{\"i\":\"83ea58e8-fe32-4775-aabe-2b2bb0acf28e\",\"y\":91,\"x\":0,\"w\":36,\"h\":17}},{\"type\":\"visualization\",\"title\":\"Top 1000 Message Sources by Name\",\"embeddableConfig\":{\"description\":\"\",\"enhancements\":{\"dynamicActions\":{\"events\":[]}},\"savedVis\":{\"title\":\"Top 1000 Message Source IP Addresses\",\"description\":\"\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"showMetricsAtAllLevels\":false,\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":4,\"format\":{\"id\":\"number\",\"params\":{\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Messages\",\"aggType\":\"sum\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"IP Address\",\"aggType\":\"terms\"},{\"accessor\":1,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Reverse DNS\",\"aggType\":\"terms\"},{\"accessor\":2,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Base Domain\",\"aggType\":\"terms\"},{\"accessor\":3,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Country\",\"aggType\":\"terms\"}]},\"showToolbar\":true,\"autoFitRowToContent\":false},\"uiState\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"data\":{\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\",\"emptyAsNull\":false},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"source_name.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":1000,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"None\",\"includeIsRegex\":true,\"excludeIsRegex\":true,\"customLabel\":\"Source Name\"},\"schema\":\"bucket\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"source_type.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":1000,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"None\",\"includeIsRegex\":true,\"excludeIsRegex\":true,\"customLabel\":\"Source Type\"},\"schema\":\"bucket\"}],\"searchSource\":{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}}}},\"panelIndex\":\"91942803-88e9-4f4e-8b54-3e3f6137b07e\",\"gridData\":{\"i\":\"91942803-88e9-4f4e-8b54-3e3f6137b07e\",\"y\":73,\"x\":0,\"w\":48,\"h\":18}}]","refreshInterval":{"pause":true,"value":300000},"timeFrom":"now-1M","timeRestore":true,"timeTo":"now","title":"DMARC Summary","version":3},"coreMigrationVersion":"8.8.0","created_at":"2025-12-01T16:43:47.976Z","id":"269ba470-2871-11e8-b8b2-15742da3055c","managed":false,"references":[{"id":"085eaa30-2870-11e8-b8b2-15742da3055c","name":"4:panel_4","type":"visualization"},{"id":"620280a0-2886-11e8-b8b2-15742da3055c","name":"7:panel_7","type":"visualization"},{"id":"d787a580-2886-11e8-b8b2-15742da3055c","name":"8:panel_8","type":"visualization"},{"id":"356caa70-28d1-11e8-b8b2-15742da3055c","name":"9:panel_9","type":"visualization"},{"id":"7e26fb80-28d1-11e8-b8b2-15742da3055c","name":"10:panel_10","type":"visualization"},{"id":"93b823e0-28cf-11e8-b8b2-15742da3055c","name":"11:panel_11","type":"visualization"},{"id":"a69d0f40-2b02-11e8-8c8d-d3a0d2f2ba49","name":"13:panel_13","type":"visualization"},{"id":"55930ba0-667f-11e8-ac01-67e661d30f69","name":"14:panel_14","type":"visualization"},{"id":"c9ee5ec0-67f9-11e8-ac01-67e661d30f69","name":"15:panel_15","type":"visualization"},{"id":"f4444000-7333-11e8-bfe4-d3427a6746f1","name":"16:panel_16","type":"visualization"},{"id":"1fad3f60-2881-11e8-b8b2-15742da3055c","name":"17:panel_17","type":"visualization"},{"id":"40e7a5b0-2883-11e8-b8b2-15742da3055c","name":"18:panel_18","type":"visualization"},{"id":"1a892210-8ae6-11ee-b032-c5c657f8d4c9","name":"83ea58e8-fe32-4775-aabe-2b2bb0acf28e:panel_83ea58e8-fe32-4775-aabe-2b2bb0acf28e","type":"map"},{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"4:kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"7:kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"8:kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"9:kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"10:kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"11:kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"13:kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"14:kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"15:kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"16:kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"17:kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"18:kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"ba73361d-48e3-4b81-b460-f39bbbf529d7:kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"4cd04d8b-94d5-42c6-8003-46412bc96e60:kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"79544470-313a-11e8-a742-83431eb55d58","name":"91942803-88e9-4f4e-8b54-3e3f6137b07e:kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"dashboard","typeMigrationVersion":"10.2.0","updated_at":"2025-12-03T14:19:30.111Z","version":"WzIzMywxXQ=="} -{"attributes":{"fields":"[{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"arrival_date\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"auth_failure\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"auth_failure.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"authentication_mechanisms\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"authentication_mechanisms.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"authentication_results\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"authentication_results.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"delivery_results\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"delivery_results.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dkim_domain\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"dkim_domain.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"domain\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"domain.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"feedback_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"feedback_type.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"original_envelope_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"original_envelope_id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"original_mail_from\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"original_mail_from.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"original_rcpt_to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"original_rcpt_to.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.attachments.content_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.attachments.content_type.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.attachments.filename\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.attachments.filename.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.bcc.address\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.bcc.address.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.bcc.display_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.bcc.display_name.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.body\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.body.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.cc.address\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.cc.address.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.cc.display_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.cc.display_name.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.date\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.filename_safe_subject\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.filename_safe_subject.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.accept-language\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.accept-language.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.aeizauksle\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.aeizauksle.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.agsmwgdbea\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.agsmwgdbea.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.aqr\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.aqr.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.arc-authentication-results\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.arc-authentication-results.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.arc-message-signature\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.arc-message-signature.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.arc-seal\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.arc-seal.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.artemrn\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.artemrn.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.authentication-results\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.authentication-results.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.auto-submitted\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.auto-submitted.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.bbnxrpnk\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.bbnxrpnk.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.bcc\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.bcc.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.bevxwgma\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.bevxwgma.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.bhwbzv\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.bhwbzv.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.bswwwimaubpc\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.bswwwimaubpc.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.cc\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.cc.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.chefmdeoj\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.chefmdeoj.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.cnyme\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.cnyme.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.content-class\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.content-class.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.content-disposition\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.content-disposition.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.content-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.content-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.content-language\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.content-language.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.content-transfer-encoding\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.content-transfer-encoding.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.content-type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.content-type.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.cswfghb\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.cswfghb.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.cxovzdvcstkw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.cxovzdvcstkw.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.czlrnwzpcl\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.czlrnwzpcl.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.date\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.date.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.dedczqooptf\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.dedczqooptf.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.delivered-to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.delivered-to.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.disposition-notification-to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.disposition-notification-to.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.dkim-signature\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.dkim-signature.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.dozuevmishp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.dozuevmishp.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.enavpeeiwo\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.enavpeeiwo.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.errors-to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.errors-to.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.eszikys\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.eszikys.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.eziino\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.eziino.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.fbukiyt\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.fbukiyt.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.feedback-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.feedback-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.frkgbxehnav\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.frkgbxehnav.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.from\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.from.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.ftywy\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.ftywy.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.gjhqo\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.gjhqo.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.gsxsra\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.gsxsra.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.hpbzb\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.hpbzb.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.icfvtf\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.icfvtf.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.ieuv\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.ieuv.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.ileikg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.ileikg.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.importance\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.importance.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.in-reply-to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.in-reply-to.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.jbnedvuvo\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.jbnedvuvo.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.jcmzfrlwa\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.jcmzfrlwa.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.jgv\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.jgv.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.kgjj\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.kgjj.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.kjzobyp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.kjzobyp.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.lebkujharxf\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.lebkujharxf.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.licymhr\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.licymhr.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.list-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.list-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.list-unsubscribe\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.list-unsubscribe-post\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.list-unsubscribe-post.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.list-unsubscribe.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.lpm\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.lpm.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.mailformat\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.mailformat.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.mailpriority\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.mailpriority.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.message-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.message-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.mime-version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.mime-version.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.mmnuvwdjp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.mmnuvwdjp.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.muxriulgy\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.muxriulgy.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.mzhabgdjjvo\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.mzhabgdjjvo.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.oawgps\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.oawgps.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.ocfyueo\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.ocfyueo.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.ohcqkqdu\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.ohcqkqdu.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.otxibplvzlda\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.otxibplvzlda.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.pmdtjhu\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.pmdtjhu.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.pqergj\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.pqergj.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.precedence\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.precedence.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.priority\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.priority.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.qrc\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.qrc.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.raxvz\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.raxvz.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.received\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.received-spf\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.received-spf.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.received.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.references\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.references.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.reply-to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.reply-to.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.resent-date\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.resent-date.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.resent-from\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.resent-from.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.resent-message-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.resent-message-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.resent-to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.resent-to.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.return-path\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.return-path.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.rhztphrqb\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.rhztphrqb.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.rlhkosbgp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.rlhkosbgp.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.sender\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.sender.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.spamdiagnosticmetadata\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.spamdiagnosticmetadata.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.spamdiagnosticoutput\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.spamdiagnosticoutput.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.subject\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.subject.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.thgvrqgzmgpx\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.thgvrqgzmgpx.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.thread-index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.thread-index.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.thread-topic\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.thread-topic.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.to.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.uelc\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.uelc.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.user-agent\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.user-agent.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.uvbl\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.uvbl.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.vefvzft\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.vefvzft.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.veqzyyga\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.veqzyyga.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.vgne\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.vgne.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.vgsxfhl\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.vgsxfhl.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.vojn\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.vojn.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.vzxhmfnlb\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.vzxhmfnlb.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.wlvry\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.wlvry.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.wurakhfltt\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.wurakhfltt.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.wvvlzninv\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.wvvlzninv.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x--mailscanner\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x--mailscanner-from\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x--mailscanner-from.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x--mailscanner-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x--mailscanner-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x--mailscanner-information\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x--mailscanner-information.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x--mailscanner.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-antiabuse\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-antiabuse.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-authenticated-sender\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-authenticated-sender.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-authentication-warning\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-authentication-warning.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-authority-reason\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-authority-reason.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-auto-response-suppress\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-auto-response-suppress.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-bwhitelist\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-bwhitelist.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-callingtelephonenumber\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-callingtelephonenumber.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-campaign-activity-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-campaign-activity-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-channel-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-channel-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-cm-forward-sender\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-cm-forward-sender.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-cm-senderinfo\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-cm-senderinfo.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-cm-transid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-cm-transid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-cmae-envelope\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-cmae-envelope.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-coremail-antispam\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-coremail-antispam.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-csa-complaints\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-csa-complaints.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ctct-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ctct-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-email-count\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-email-count.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-exchange-antispam-report-cfa-test\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-exchange-antispam-report-cfa-test.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-exchange-antispam-report-test\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-exchange-antispam-report-test.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-exim-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-exim-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-faxnumberofpages\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-faxnumberofpages.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-feedback-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-feedback-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-filter-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-filter-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-forefront-prvs\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-forefront-prvs.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-forwarded-for\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-forwarded-for.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-forwarded-to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-forwarded-to.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-get-message-sender-via\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-get-message-sender-via.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-gm-message-state\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-gm-message-state.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-google-dkim-signature\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-google-dkim-signature.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-google-smtp-source\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-google-smtp-source.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-has-attach\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-has-attach.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ironport-av\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ironport-av.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-job\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-job.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-kk-mid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-kk-mid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ld-processed\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ld-processed.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-linkedin-fe\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-linkedin-fe.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-local-domain\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-local-domain.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-mailer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-mailer.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-microsoft-antispam\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-microsoft-antispam-message-info\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-microsoft-antispam-message-info.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-microsoft-antispam-prvs\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-microsoft-antispam-prvs.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-microsoft-antispam.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-mimeole\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-mimeole.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-exchange-antispam-srfa-diagnostics\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-exchange-antispam-srfa-diagnostics.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-exchange-crosstenant-fromentityheader\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-exchange-crosstenant-fromentityheader.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-exchange-crosstenant-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-exchange-crosstenant-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-exchange-crosstenant-network-message-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-exchange-crosstenant-network-message-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-exchange-crosstenant-originalarrivaltime\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-exchange-crosstenant-originalarrivaltime.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-exchange-generated-message-source\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-exchange-generated-message-source.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-exchange-inbox-rules-loop\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-exchange-inbox-rules-loop.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-exchange-messagesentrepresentingtype\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-exchange-messagesentrepresentingtype.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-exchange-organization-recordreviewcfmtype\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-exchange-organization-recordreviewcfmtype.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-exchange-parent-message-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-exchange-parent-message-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-exchange-senderadcheck\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-exchange-senderadcheck.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-exchange-transport-crosstenantheadersstamped\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-exchange-transport-crosstenantheadersstamped.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-has-attach\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-has-attach.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-office365-filtering-correlation-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-office365-filtering-correlation-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-publictraffictype\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-publictraffictype.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-tnef-correlator\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-tnef-correlator.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-traffictypediagnostic\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-traffictypediagnostic.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-msmail-priority\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-msmail-priority.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-onpremexternalip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-onpremexternalip.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-original-authentication-results\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-original-authentication-results.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-originalarrivaltime\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-originalarrivaltime.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-originating-ip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-originating-ip.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-originatororg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-originatororg.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-outgoing-spam-status\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-outgoing-spam-status.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-php-originating-script\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-php-originating-script.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-php-script\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-php-script.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-priority\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-priority.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-qq-auto-fwd\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-qq-auto-fwd.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-qq-csender\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-qq-csender.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-qq-feat\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-qq-feat.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-qq-inner-pending\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-qq-inner-pending.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-qq-mailinfo\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-qq-mailinfo.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-qq-mid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-qq-mid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-qq-orgsender\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-qq-orgsender.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-qq-sendsize\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-qq-sendsize.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-qumid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-qumid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-received\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-received.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-recommended-action\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-recommended-action.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-report-abuse-to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-report-abuse-to.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-return-path-hint\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-return-path-hint.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-roving-campaignid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-roving-campaignid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-roving-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-roving-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-sg-eid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-sg-eid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-source\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-source-args\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-source-args.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-source-auth\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-source-auth.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-source-cap\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-source-cap.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-source-dir\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-source-dir.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-source-ip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-source-ip.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-source-l\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-source-l.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-source-sender\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-source-sender.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-source.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-spam-status\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-spam-status.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-spamexperts-domain\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-spamexperts-domain.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-spamexperts-outgoing-class\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-spamexperts-outgoing-class.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-spamexperts-outgoing-evidence\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-spamexperts-outgoing-evidence.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-spamexperts-username\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-spamexperts-username.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-test-mailing\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-test-mailing.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-umail-auto_fwd\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-umail-auto_fwd.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-voicemessageduration\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-voicemessageduration.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-wm-delivered\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-wm-delivered.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x_spam_cmae\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x_spam_cmae.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.xcvzpzex\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.xcvzpzex.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.xsusak\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.xsusak.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.xxbfqpuxvur\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.xxbfqpuxvur.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.yin\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.yin.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.yrofbecepmnf\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.yrofbecepmnf.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.zlbpkomebaac\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.zlbpkomebaac.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers_only\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.raw.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.reply_to.address\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.reply_to.address.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.reply_to.display_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.reply_to.display_name.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.subject\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.subject.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.to.address\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.to.address.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.to.display_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.to.display_name.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source_base_domain\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source_base_domain.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source_country\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source_country.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source_ip_address\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source_ip_address.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source_reverse_dns\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source_reverse_dns.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"user_agent\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"user_agent.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"version.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]","timeFieldName":"arrival_date","title":"dmarc_forensic*"},"coreMigrationVersion":"8.8.0","created_at":"2025-12-01T16:43:47.976Z","id":"c49bf720-313a-11e8-a742-83431eb55d58","managed":false,"references":[],"type":"index-pattern","typeMigrationVersion":"8.0.0","updated_at":"2025-12-01T16:43:47.976Z","version":"WzE2NiwxXQ=="} +{"attributes":{"fields":"[{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"arrival_date\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"auth_failure\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"auth_failure.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"authentication_mechanisms\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"authentication_mechanisms.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"authentication_results\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"authentication_results.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"delivery_results\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"delivery_results.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"dkim_domain\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"dkim_domain.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"domain\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"domain.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"feedback_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"feedback_type.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"original_envelope_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"original_envelope_id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"original_mail_from\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"original_mail_from.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"original_rcpt_to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"original_rcpt_to.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.attachments.content_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.attachments.content_type.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.attachments.filename\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.attachments.filename.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.bcc.address\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.bcc.address.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.bcc.display_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.bcc.display_name.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.body\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.body.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.cc.address\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.cc.address.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.cc.display_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.cc.display_name.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.date\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.filename_safe_subject\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.filename_safe_subject.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.accept-language\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.accept-language.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.aeizauksle\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.aeizauksle.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.agsmwgdbea\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.agsmwgdbea.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.aqr\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.aqr.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.arc-authentication-results\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.arc-authentication-results.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.arc-message-signature\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.arc-message-signature.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.arc-seal\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.arc-seal.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.artemrn\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.artemrn.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.authentication-results\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.authentication-results.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.auto-submitted\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.auto-submitted.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.bbnxrpnk\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.bbnxrpnk.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.bcc\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.bcc.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.bevxwgma\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.bevxwgma.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.bhwbzv\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.bhwbzv.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.bswwwimaubpc\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.bswwwimaubpc.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.cc\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.cc.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.chefmdeoj\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.chefmdeoj.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.cnyme\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.cnyme.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.content-class\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.content-class.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.content-disposition\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.content-disposition.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.content-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.content-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.content-language\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.content-language.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.content-transfer-encoding\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.content-transfer-encoding.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.content-type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.content-type.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.cswfghb\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.cswfghb.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.cxovzdvcstkw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.cxovzdvcstkw.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.czlrnwzpcl\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.czlrnwzpcl.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.date\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.date.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.dedczqooptf\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.dedczqooptf.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.delivered-to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.delivered-to.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.disposition-notification-to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.disposition-notification-to.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.dkim-signature\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.dkim-signature.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.dozuevmishp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.dozuevmishp.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.enavpeeiwo\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.enavpeeiwo.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.errors-to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.errors-to.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.eszikys\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.eszikys.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.eziino\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.eziino.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.fbukiyt\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.fbukiyt.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.feedback-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.feedback-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.frkgbxehnav\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.frkgbxehnav.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.from\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.from.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.ftywy\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.ftywy.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.gjhqo\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.gjhqo.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.gsxsra\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.gsxsra.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.hpbzb\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.hpbzb.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.icfvtf\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.icfvtf.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.ieuv\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.ieuv.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.ileikg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.ileikg.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.importance\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.importance.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.in-reply-to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.in-reply-to.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.jbnedvuvo\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.jbnedvuvo.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.jcmzfrlwa\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.jcmzfrlwa.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.jgv\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.jgv.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.kgjj\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.kgjj.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.kjzobyp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.kjzobyp.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.lebkujharxf\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.lebkujharxf.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.licymhr\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.licymhr.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.list-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.list-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.list-unsubscribe\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.list-unsubscribe-post\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.list-unsubscribe-post.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.list-unsubscribe.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.lpm\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.lpm.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.mailformat\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.mailformat.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.mailpriority\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.mailpriority.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.message-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.message-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.mime-version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.mime-version.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.mmnuvwdjp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.mmnuvwdjp.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.muxriulgy\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.muxriulgy.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.mzhabgdjjvo\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.mzhabgdjjvo.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.oawgps\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.oawgps.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.ocfyueo\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.ocfyueo.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.ohcqkqdu\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.ohcqkqdu.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.otxibplvzlda\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.otxibplvzlda.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.pmdtjhu\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.pmdtjhu.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.pqergj\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.pqergj.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.precedence\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.precedence.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.priority\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.priority.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.qrc\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.qrc.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.raxvz\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.raxvz.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.received\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.received-spf\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.received-spf.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.received.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.references\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.references.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.reply-to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.reply-to.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.resent-date\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.resent-date.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.resent-from\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.resent-from.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.resent-message-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.resent-message-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.resent-to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.resent-to.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.return-path\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.return-path.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.rhztphrqb\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.rhztphrqb.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.rlhkosbgp\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.rlhkosbgp.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.sender\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.sender.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.spamdiagnosticmetadata\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.spamdiagnosticmetadata.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.spamdiagnosticoutput\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.spamdiagnosticoutput.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.subject\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.subject.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.thgvrqgzmgpx\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.thgvrqgzmgpx.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.thread-index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.thread-index.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.thread-topic\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.thread-topic.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.to.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.uelc\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.uelc.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.user-agent\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.user-agent.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.uvbl\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.uvbl.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.vefvzft\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.vefvzft.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.veqzyyga\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.veqzyyga.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.vgne\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.vgne.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.vgsxfhl\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.vgsxfhl.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.vojn\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.vojn.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.vzxhmfnlb\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.vzxhmfnlb.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.wlvry\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.wlvry.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.wurakhfltt\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.wurakhfltt.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.wvvlzninv\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.wvvlzninv.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x--mailscanner\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x--mailscanner-from\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x--mailscanner-from.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x--mailscanner-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x--mailscanner-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x--mailscanner-information\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x--mailscanner-information.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x--mailscanner.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-antiabuse\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-antiabuse.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-authenticated-sender\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-authenticated-sender.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-authentication-warning\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-authentication-warning.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-authority-reason\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-authority-reason.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-auto-response-suppress\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-auto-response-suppress.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-bwhitelist\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-bwhitelist.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-callingtelephonenumber\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-callingtelephonenumber.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-campaign-activity-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-campaign-activity-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-channel-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-channel-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-cm-forward-sender\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-cm-forward-sender.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-cm-senderinfo\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-cm-senderinfo.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-cm-transid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-cm-transid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-cmae-envelope\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-cmae-envelope.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-coremail-antispam\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-coremail-antispam.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-csa-complaints\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-csa-complaints.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ctct-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ctct-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-email-count\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-email-count.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-exchange-antispam-report-cfa-test\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-exchange-antispam-report-cfa-test.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-exchange-antispam-report-test\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-exchange-antispam-report-test.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-exim-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-exim-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-faxnumberofpages\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-faxnumberofpages.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-feedback-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-feedback-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-filter-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-filter-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-forefront-prvs\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-forefront-prvs.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-forwarded-for\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-forwarded-for.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-forwarded-to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-forwarded-to.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-get-message-sender-via\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-get-message-sender-via.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-gm-message-state\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-gm-message-state.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-google-dkim-signature\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-google-dkim-signature.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-google-smtp-source\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-google-smtp-source.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-has-attach\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-has-attach.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ironport-av\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ironport-av.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-job\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-job.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-kk-mid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-kk-mid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ld-processed\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ld-processed.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-linkedin-fe\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-linkedin-fe.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-local-domain\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-local-domain.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-mailer\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-mailer.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-microsoft-antispam\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-microsoft-antispam-message-info\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-microsoft-antispam-message-info.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-microsoft-antispam-prvs\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-microsoft-antispam-prvs.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-microsoft-antispam.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-mimeole\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-mimeole.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-exchange-antispam-srfa-diagnostics\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-exchange-antispam-srfa-diagnostics.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-exchange-crosstenant-fromentityheader\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-exchange-crosstenant-fromentityheader.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-exchange-crosstenant-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-exchange-crosstenant-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-exchange-crosstenant-network-message-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-exchange-crosstenant-network-message-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-exchange-crosstenant-originalarrivaltime\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-exchange-crosstenant-originalarrivaltime.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-exchange-generated-message-source\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-exchange-generated-message-source.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-exchange-inbox-rules-loop\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-exchange-inbox-rules-loop.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-exchange-messagesentrepresentingtype\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-exchange-messagesentrepresentingtype.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-exchange-organization-recordreviewcfmtype\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-exchange-organization-recordreviewcfmtype.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-exchange-parent-message-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-exchange-parent-message-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-exchange-senderadcheck\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-exchange-senderadcheck.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-exchange-transport-crosstenantheadersstamped\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-exchange-transport-crosstenantheadersstamped.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-has-attach\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-has-attach.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-office365-filtering-correlation-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-office365-filtering-correlation-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-publictraffictype\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-publictraffictype.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-tnef-correlator\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-tnef-correlator.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-ms-traffictypediagnostic\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-ms-traffictypediagnostic.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-msmail-priority\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-msmail-priority.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-onpremexternalip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-onpremexternalip.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-original-authentication-results\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-original-authentication-results.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-originalarrivaltime\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-originalarrivaltime.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-originating-ip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-originating-ip.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-originatororg\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-originatororg.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-outgoing-spam-status\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-outgoing-spam-status.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-php-originating-script\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-php-originating-script.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-php-script\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-php-script.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-priority\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-priority.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-qq-auto-fwd\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-qq-auto-fwd.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-qq-csender\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-qq-csender.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-qq-feat\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-qq-feat.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-qq-inner-pending\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-qq-inner-pending.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-qq-mailinfo\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-qq-mailinfo.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-qq-mid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-qq-mid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-qq-orgsender\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-qq-orgsender.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-qq-sendsize\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-qq-sendsize.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-qumid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-qumid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-received\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-received.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-recommended-action\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-recommended-action.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-report-abuse-to\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-report-abuse-to.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-return-path-hint\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-return-path-hint.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-roving-campaignid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-roving-campaignid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-roving-id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-roving-id.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-sg-eid\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-sg-eid.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-source\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-source-args\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-source-args.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-source-auth\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-source-auth.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-source-cap\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-source-cap.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-source-dir\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-source-dir.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-source-ip\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-source-ip.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-source-l\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-source-l.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-source-sender\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-source-sender.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-source.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-spam-status\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-spam-status.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-spamexperts-domain\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-spamexperts-domain.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-spamexperts-outgoing-class\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-spamexperts-outgoing-class.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-spamexperts-outgoing-evidence\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-spamexperts-outgoing-evidence.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-spamexperts-username\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-spamexperts-username.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-test-mailing\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-test-mailing.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-umail-auto_fwd\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-umail-auto_fwd.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-voicemessageduration\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-voicemessageduration.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x-wm-delivered\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x-wm-delivered.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.x_spam_cmae\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.x_spam_cmae.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.xcvzpzex\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.xcvzpzex.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.xsusak\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.xsusak.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.xxbfqpuxvur\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.xxbfqpuxvur.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.yin\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.yin.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.yrofbecepmnf\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.yrofbecepmnf.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers.zlbpkomebaac\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.headers.zlbpkomebaac.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.headers_only\",\"type\":\"boolean\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.raw.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.reply_to.address\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.reply_to.address.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.reply_to.display_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.reply_to.display_name.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.subject\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.subject.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.to.address\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.to.address.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"sample.to.display_name\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"sample.to.display_name.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source_base_domain\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source_base_domain.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source_country\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source_country.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source_ip_address\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source_ip_address.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"source_reverse_dns\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"source_reverse_dns.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"user_agent\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"user_agent.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"version.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]","timeFieldName":"arrival_date","title":"dmarc_f*"},"coreMigrationVersion":"8.8.0","created_at":"2025-12-01T16:43:47.976Z","id":"c49bf720-313a-11e8-a742-83431eb55d58","managed":false,"references":[],"type":"index-pattern","typeMigrationVersion":"8.0.0","updated_at":"2025-12-01T16:43:47.976Z","version":"WzE2NiwxXQ=="} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Forensic Samples","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Forensic Samples\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"arrival_date\",\"orderBy\":\"_key\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Arrival Date\"},\"schema\":\"bucket\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"sample.headers.from.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"\",\"customLabel\":\"From\"},\"schema\":\"bucket\"},{\"id\":\"7\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"sample.headers.sender.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"\",\"customLabel\":\"Sender\"},\"schema\":\"bucket\"},{\"id\":\"8\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"sample.headers.reply-to.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":5,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Reply-To\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"sample.headers.to.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"\",\"customLabel\":\"To\"},\"schema\":\"bucket\"},{\"id\":\"6\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"sample.subject.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"\",\"customLabel\":\"Subject\"},\"schema\":\"bucket\"}],\"params\":{\"dimensions\":{\"buckets\":[{\"accessor\":0,\"aggType\":\"terms\",\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"date\",\"missingBucketLabel\":\"Missing\",\"otherBucketLabel\":\"Other\",\"parsedUrl\":{\"basePath\":\"\",\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\"}}},\"label\":\"Arrival Date\",\"params\":{}},{\"accessor\":1,\"aggType\":\"terms\",\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"missingBucketLabel\":\"\",\"otherBucketLabel\":\"Other\",\"parsedUrl\":{\"basePath\":\"\",\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\"}}},\"label\":\"From\",\"params\":{}},{\"accessor\":2,\"aggType\":\"terms\",\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"missingBucketLabel\":\"\",\"otherBucketLabel\":\"Other\",\"parsedUrl\":{\"basePath\":\"\",\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\"}}},\"label\":\"Sender\",\"params\":{}},{\"accessor\":3,\"aggType\":\"terms\",\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"missingBucketLabel\":\"\",\"otherBucketLabel\":\"Other\",\"parsedUrl\":{\"basePath\":\"\",\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\"}}},\"label\":\"To\",\"params\":{}},{\"accessor\":4,\"aggType\":\"terms\",\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"missingBucketLabel\":\"\",\"otherBucketLabel\":\"Other\",\"parsedUrl\":{\"basePath\":\"\",\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\"}}},\"label\":\"Reply To\",\"params\":{}},{\"accessor\":5,\"aggType\":\"terms\",\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"missingBucketLabel\":\"\",\"otherBucketLabel\":\"Other\",\"parsedUrl\":{\"basePath\":\"\",\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\"}}},\"label\":\"Subject\",\"params\":{}}],\"metrics\":[{\"accessor\":6,\"aggType\":\"count\",\"format\":{\"id\":\"number\"},\"label\":\"Count\",\"params\":{}}]},\"perPage\":10,\"percentageCol\":\"\",\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showToolbar\":true,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"}}"},"coreMigrationVersion":"8.8.0","created_at":"2025-12-01T16:43:47.976Z","id":"def63400-295b-11e8-b8b2-15742da3055c","managed":false,"references":[{"id":"c49bf720-313a-11e8-a742-83431eb55d58","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","typeMigrationVersion":"8.5.0","updated_at":"2025-12-01T16:43:47.976Z","version":"WzE2NywxXQ=="} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Top 1000 Forensic Sample Source IP Addresses","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Top 1000 Forensic Sample Source IP Addresses\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"showMetricsAtAllLevels\":false,\"percentageCol\":\"\",\"dimensions\":{\"metrics\":[{\"accessor\":4,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}],\"buckets\":[{\"accessor\":0,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"Missing\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"IP Address\",\"aggType\":\"terms\"},{\"accessor\":1,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Reverse DNS\",\"aggType\":\"terms\"},{\"accessor\":2,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Base Domain\",\"aggType\":\"terms\"},{\"accessor\":3,\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"otherBucketLabel\":\"Other\",\"missingBucketLabel\":\"\",\"parsedUrl\":{\"origin\":\"https://secopselk.cardinalhealth.net\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}},\"params\":{},\"label\":\"Country\",\"aggType\":\"terms\"}]},\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_ip_address.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":1000,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"IP Address\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_reverse_dns.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":1000,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"\",\"customLabel\":\"Reverse DNS\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_base_domain.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":1000,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"\",\"customLabel\":\"Base Domain\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_country.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":200,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"\",\"customLabel\":\"Country\"}}]}"},"coreMigrationVersion":"8.8.0","created_at":"2025-12-01T16:43:47.976Z","id":"316ef4e0-295e-11e8-b8b2-15742da3055c","managed":false,"references":[{"id":"c49bf720-313a-11e8-a742-83431eb55d58","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","typeMigrationVersion":"8.5.0","updated_at":"2025-12-01T16:43:47.976Z","version":"WzE2OCwxXQ=="} {"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"DMARC Forensic Sample Source Countries","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"DMARC Forensic Sample Source Countries\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"showToolbar\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_country.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"unknown\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Country\"}}]}"},"coreMigrationVersion":"8.8.0","created_at":"2025-12-01T16:43:47.976Z","id":"ae61a330-7337-11e8-bfe4-d3427a6746f1","managed":false,"references":[{"id":"c49bf720-313a-11e8-a742-83431eb55d58","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","typeMigrationVersion":"8.5.0","updated_at":"2025-12-01T16:43:47.976Z","version":"WzE2OSwxXQ=="} diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index ce48c19..aefdc8d 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -52,7 +52,8 @@ from parsedmarc.mail import ( ) from parsedmarc.types import ( AggregateReport, - ForensicReport, + FailureReport, + ForensicReport as ForensicReport, ParsedReport, ParsingResults, SMTPTLSReport, @@ -77,6 +78,7 @@ text_report_regex = re.compile(r"\s*([a-zA-Z\s]+):\s(.+)", re.MULTILINE) MAGIC_ZIP = b"\x50\x4b\x03\x04" MAGIC_GZIP = b"\x1f\x8b" MAGIC_XML = b"\x3c\x3f\x78\x6d\x6c\x20" +MAGIC_XML_TAG = b"\x3c" # '<' - XML starting with an element tag (no declaration) MAGIC_JSON = b"\7b" EMAIL_SAMPLE_CONTENT_TYPES = ( @@ -111,8 +113,12 @@ class InvalidAggregateReport(InvalidDMARCReport): """Raised when an invalid DMARC aggregate report is encountered""" -class InvalidForensicReport(InvalidDMARCReport): - """Raised when an invalid DMARC forensic report is encountered""" +class InvalidFailureReport(InvalidDMARCReport): + """Raised when an invalid DMARC failure report is encountered""" + + +# Backward-compatible alias +InvalidForensicReport = InvalidFailureReport def _bucket_interval_by_day( @@ -356,8 +362,6 @@ def _parse_report_record( } if "disposition" in policy_evaluated: new_policy_evaluated["disposition"] = policy_evaluated["disposition"] - if new_policy_evaluated["disposition"].strip().lower() == "pass": - new_policy_evaluated["disposition"] = "none" if "dkim" in policy_evaluated: new_policy_evaluated["dkim"] = policy_evaluated["dkim"] if "spf" in policy_evaluated: @@ -418,6 +422,7 @@ def _parse_report_record( new_result["result"] = result["result"] else: new_result["result"] = "none" + new_result["human_result"] = result.get("human_result", None) new_record["auth_results"]["dkim"].append(new_result) if not isinstance(auth_results["spf"], list): @@ -433,6 +438,7 @@ def _parse_report_record( new_result["result"] = result["result"] else: new_result["result"] = "none" + new_result["human_result"] = result.get("human_result", None) new_record["auth_results"]["spf"].append(new_result) if "envelope_from" not in new_record["identifiers"]: @@ -788,6 +794,10 @@ def parse_aggregate_report_xml( else: errors = report["report_metadata"]["error"] new_report_metadata["errors"] = errors + generator = None + if "generator" in report_metadata: + generator = report_metadata["generator"] + new_report_metadata["generator"] = generator new_report["report_metadata"] = new_report_metadata records = [] policy_published = report["policy_published"] @@ -811,16 +821,39 @@ def parse_aggregate_report_xml( if policy_published["sp"] is not None: sp = policy_published["sp"] new_policy_published["sp"] = sp - pct = "100" + pct = None if "pct" in policy_published: if policy_published["pct"] is not None: pct = policy_published["pct"] new_policy_published["pct"] = pct - fo = "0" + fo = None if "fo" in policy_published: if policy_published["fo"] is not None: fo = policy_published["fo"] new_policy_published["fo"] = fo + np_ = None + if "np" in policy_published: + if policy_published["np"] is not None: + np_ = policy_published["np"] + if np_ not in ("none", "quarantine", "reject"): + logger.warning("Invalid np value: {0}".format(np_)) + new_policy_published["np"] = np_ + testing = None + if "testing" in policy_published: + if policy_published["testing"] is not None: + testing = policy_published["testing"] + if testing not in ("n", "y"): + logger.warning("Invalid testing value: {0}".format(testing)) + new_policy_published["testing"] = testing + discovery_method = None + if "discovery_method" in policy_published: + if policy_published["discovery_method"] is not None: + discovery_method = policy_published["discovery_method"] + if discovery_method not in ("psl", "treewalk"): + logger.warning( + "Invalid discovery_method value: {0}".format(discovery_method) + ) + new_policy_published["discovery_method"] = discovery_method new_report["policy_published"] = new_policy_published if type(report["record"]) is list: @@ -955,6 +988,7 @@ def extract_report(content: Union[bytes, str, BinaryIO]) -> str: ) elif ( header[: len(MAGIC_XML)] == MAGIC_XML + or header[: len(MAGIC_XML_TAG)] == MAGIC_XML_TAG or header[: len(MAGIC_JSON)] == MAGIC_JSON ): report = file_object.read().decode(errors="ignore") @@ -1084,6 +1118,9 @@ def parsed_aggregate_reports_to_csv_rows( sp = report["policy_published"]["sp"] pct = report["policy_published"]["pct"] fo = report["policy_published"]["fo"] + np_ = report["policy_published"].get("np", None) + testing = report["policy_published"].get("testing", None) + discovery_method = report["policy_published"].get("discovery_method", None) report_dict: dict[str, Any] = dict( xml_schema=xml_schema, @@ -1100,8 +1137,11 @@ def parsed_aggregate_reports_to_csv_rows( aspf=aspf, p=p, sp=sp, + np=np_, pct=pct, fo=fo, + testing=testing, + discovery_method=discovery_method, ) for record in report["records"]: @@ -1200,8 +1240,11 @@ def parsed_aggregate_reports_to_csv( "aspf", "p", "sp", + "np", "pct", "fo", + "testing", + "discovery_method", "source_ip_address", "source_country", "source_reverse_dns", @@ -1242,7 +1285,7 @@ def parsed_aggregate_reports_to_csv( return csv_file_object.getvalue() -def parse_forensic_report( +def parse_failure_report( feedback_report: str, sample: str, msg_date: datetime, @@ -1256,9 +1299,9 @@ def parse_forensic_report( dns_timeout: float = DEFAULT_DNS_TIMEOUT, dns_retries: int = DEFAULT_DNS_MAX_RETRIES, strip_attachment_payloads: bool = False, -) -> ForensicReport: +) -> FailureReport: """ - Converts a DMARC forensic report and sample to a dict + Converts a DMARC failure report and sample to a dict Args: feedback_report (str): A message's feedback report as a string @@ -1275,7 +1318,7 @@ def parse_forensic_report( dns_retries (int): Number of times to retry DNS queries on timeout or other transient errors strip_attachment_payloads (bool): Remove attachment payloads from - forensic report results + failure report results Returns: dict: A parsed report and sample @@ -1291,7 +1334,7 @@ def parse_forensic_report( if "arrival_date" not in parsed_report: if msg_date is None: - raise InvalidForensicReport("Forensic sample is not a valid email") + raise InvalidFailureReport("Failure sample is not a valid email") parsed_report["arrival_date"] = msg_date.isoformat() if "version" not in parsed_report: @@ -1378,27 +1421,27 @@ def parse_forensic_report( parsed_report["sample"] = sample parsed_report["parsed_sample"] = parsed_sample - return cast(ForensicReport, parsed_report) + return cast(FailureReport, parsed_report) except KeyError as error: - raise InvalidForensicReport("Missing value: {0}".format(error.__str__())) + raise InvalidFailureReport("Missing value: {0}".format(error.__str__())) except Exception as error: - raise InvalidForensicReport("Unexpected error: {0}".format(error.__str__())) + raise InvalidFailureReport("Unexpected error: {0}".format(error.__str__())) -def parsed_forensic_reports_to_csv_rows( - reports: Union[ForensicReport, list[ForensicReport]], +def parsed_failure_reports_to_csv_rows( + reports: Union[FailureReport, list[FailureReport]], ) -> list[dict[str, Any]]: """ - Converts one or more parsed forensic reports to a list of dicts in flat CSV + Converts one or more parsed failure reports to a list of dicts in flat CSV format Args: - reports: A parsed forensic report or list of parsed forensic reports + reports: A parsed failure report or list of parsed failure reports Returns: - list: Parsed forensic report data as a list of dicts in flat CSV format + list: Parsed failure report data as a list of dicts in flat CSV format """ if isinstance(reports, dict): reports = [reports] @@ -1428,18 +1471,18 @@ def parsed_forensic_reports_to_csv_rows( return rows -def parsed_forensic_reports_to_csv( - reports: Union[ForensicReport, list[ForensicReport]], +def parsed_failure_reports_to_csv( + reports: Union[FailureReport, list[FailureReport]], ) -> str: """ - Converts one or more parsed forensic reports to flat CSV format, including + Converts one or more parsed failure reports to flat CSV format, including headers Args: - reports: A parsed forensic report or list of parsed forensic reports + reports: A parsed failure report or list of parsed failure reports Returns: - str: Parsed forensic report data in flat CSV format, including headers + str: Parsed failure report data in flat CSV format, including headers """ fields = [ "feedback_type", @@ -1474,7 +1517,7 @@ def parsed_forensic_reports_to_csv( csv_writer = DictWriter(csv_file, fieldnames=fields) csv_writer.writeheader() - rows = parsed_forensic_reports_to_csv_rows(reports) + rows = parsed_failure_reports_to_csv_rows(reports) for row in rows: new_row: dict[str, Any] = {} @@ -1515,13 +1558,13 @@ def parse_report_email( dns_retries (int): Number of times to retry DNS queries on timeout or other transient errors strip_attachment_payloads (bool): Remove attachment payloads from - forensic report results + failure report results keep_alive (callable): keep alive function normalize_timespan_threshold_hours (float): Normalize timespans beyond this Returns: dict: - * ``report_type``: ``aggregate`` or ``forensic`` + * ``report_type``: ``aggregate`` or ``failure`` * ``report``: The parsed report """ result: Optional[ParsedReport] = None @@ -1665,7 +1708,7 @@ def parse_report_email( if feedback_report and sample: try: - forensic_report = parse_forensic_report( + failure_report = parse_failure_report( feedback_report, sample, msg_date, @@ -1679,17 +1722,17 @@ def parse_report_email( dns_retries=dns_retries, strip_attachment_payloads=strip_attachment_payloads, ) - except InvalidForensicReport as e: + except InvalidFailureReport as e: error = ( 'Message with subject "{0}" ' "is not a valid " - "forensic DMARC report: {1}".format(subject, e) + "failure DMARC report: {1}".format(subject, e) ) - raise InvalidForensicReport(error) + raise InvalidFailureReport(error) except Exception as e: - raise InvalidForensicReport(e.__str__()) + raise InvalidFailureReport(e.__str__()) - result = {"report_type": "forensic", "report": forensic_report} + result = {"report_type": "failure", "report": failure_report} return result if result is None: @@ -1714,7 +1757,7 @@ def parse_report_file( keep_alive: Optional[Callable] = None, normalize_timespan_threshold_hours: float = 24, ) -> ParsedReport: - """Parses a DMARC aggregate or forensic file at the given path, a + """Parses a DMARC aggregate or failure file at the given path, a file-like object. or bytes Args: @@ -1726,7 +1769,7 @@ def parse_report_file( dns_retries (int): Number of times to retry DNS queries on timeout or other transient errors strip_attachment_payloads (bool): Remove attachment payloads from - forensic report results + failure report results ip_db_path (str): Path to a MMDB file from IPinfo, MaxMind, or DBIP always_use_local_files (bool): Do not download files reverse_dns_map_path (str): Path to a reverse DNS map @@ -1822,7 +1865,7 @@ def get_dmarc_reports_from_mbox( dns_retries (int): Number of times to retry DNS queries on timeout or other transient errors strip_attachment_payloads (bool): Remove attachment payloads from - forensic report results + failure report results always_use_local_files (bool): Do not download files reverse_dns_map_path (str): Path to a reverse DNS map file reverse_dns_map_url (str): URL to a reverse DNS map file @@ -1831,11 +1874,11 @@ def get_dmarc_reports_from_mbox( normalize_timespan_threshold_hours (float): Normalize timespans beyond this Returns: - dict: Lists of ``aggregate_reports``, ``forensic_reports``, and ``smtp_tls_reports`` + dict: Lists of ``aggregate_reports``, ``failure_reports``, and ``smtp_tls_reports`` """ aggregate_reports: list[AggregateReport] = [] - forensic_reports: list[ForensicReport] = [] + failure_reports: list[FailureReport] = [] smtp_tls_reports: list[SMTPTLSReport] = [] try: mbox = mailbox.mbox(input_) @@ -1873,8 +1916,8 @@ def get_dmarc_reports_from_mbox( "Skipping duplicate aggregate report " f"from {report_org} with ID: {report_id}" ) - elif parsed_email["report_type"] == "forensic": - forensic_reports.append(parsed_email["report"]) + elif parsed_email["report_type"] == "failure": + failure_reports.append(parsed_email["report"]) elif parsed_email["report_type"] == "smtp_tls": smtp_tls_reports.append(parsed_email["report"]) except InvalidDMARCReport as error: @@ -1883,7 +1926,7 @@ def get_dmarc_reports_from_mbox( raise InvalidDMARCReport("Mailbox {0} does not exist".format(input_)) return { "aggregate_reports": aggregate_reports, - "forensic_reports": forensic_reports, + "failure_reports": failure_reports, "smtp_tls_reports": smtp_tls_reports, } @@ -1929,7 +1972,7 @@ def get_dmarc_reports_from_mailbox( dns_retries (int): Number of times to retry DNS queries on timeout or other transient errors strip_attachment_payloads (bool): Remove attachment payloads from - forensic report results + failure report results results (dict): Results from the previous run batch_size (int): Number of messages to read and process before saving (use 0 for no limit) @@ -1940,7 +1983,7 @@ def get_dmarc_reports_from_mailbox( normalize_timespan_threshold_hours (float): Normalize timespans beyond this Returns: - dict: Lists of ``aggregate_reports``, ``forensic_reports``, and ``smtp_tls_reports`` + dict: Lists of ``aggregate_reports``, ``failure_reports``, and ``smtp_tls_reports`` """ if delete and test: raise ValueError("delete and test options are mutually exclusive") @@ -1952,25 +1995,25 @@ def get_dmarc_reports_from_mailbox( current_time: Optional[Union[datetime, date, str]] = None aggregate_reports: list[AggregateReport] = [] - forensic_reports: list[ForensicReport] = [] + failure_reports: list[FailureReport] = [] smtp_tls_reports: list[SMTPTLSReport] = [] aggregate_report_msg_uids = [] - forensic_report_msg_uids = [] + failure_report_msg_uids = [] smtp_tls_msg_uids = [] aggregate_reports_folder = "{0}/Aggregate".format(archive_folder) - forensic_reports_folder = "{0}/Forensic".format(archive_folder) + failure_reports_folder = "{0}/Forensic".format(archive_folder) smtp_tls_reports_folder = "{0}/SMTP-TLS".format(archive_folder) invalid_reports_folder = "{0}/Invalid".format(archive_folder) if results: aggregate_reports = results["aggregate_reports"].copy() - forensic_reports = results["forensic_reports"].copy() + failure_reports = results["failure_reports"].copy() smtp_tls_reports = results["smtp_tls_reports"].copy() if not test and create_folders: connection.create_folder(archive_folder) connection.create_folder(aggregate_reports_folder) - connection.create_folder(forensic_reports_folder) + connection.create_folder(failure_reports_folder) connection.create_folder(smtp_tls_reports_folder) connection.create_folder(invalid_reports_folder) @@ -2073,9 +2116,9 @@ def get_dmarc_reports_from_mailbox( f"Skipping duplicate aggregate report with ID: {report_id}" ) aggregate_report_msg_uids.append(message_id) - elif parsed_email["report_type"] == "forensic": - forensic_reports.append(parsed_email["report"]) - forensic_report_msg_uids.append(message_id) + elif parsed_email["report_type"] == "failure": + failure_reports.append(parsed_email["report"]) + failure_report_msg_uids.append(message_id) elif parsed_email["report_type"] == "smtp_tls": smtp_tls_reports.append(parsed_email["report"]) smtp_tls_msg_uids.append(message_id) @@ -2102,7 +2145,7 @@ def get_dmarc_reports_from_mailbox( if not test: if delete: processed_messages = ( - aggregate_report_msg_uids + forensic_report_msg_uids + smtp_tls_msg_uids + aggregate_report_msg_uids + failure_report_msg_uids + smtp_tls_msg_uids ) number_of_processed_msgs = len(processed_messages) @@ -2142,24 +2185,24 @@ def get_dmarc_reports_from_mailbox( message = "Error moving message UID" e = "{0} {1}: {2}".format(message, msg_uid, e) logger.error("Mailbox error: {0}".format(e)) - if len(forensic_report_msg_uids) > 0: - message = "Moving forensic report messages from" + if len(failure_report_msg_uids) > 0: + message = "Moving failure report messages from" logger.debug( "{0} {1} to {2}".format( - message, reports_folder, forensic_reports_folder + message, reports_folder, failure_reports_folder ) ) - number_of_forensic_msgs = len(forensic_report_msg_uids) - for i in range(number_of_forensic_msgs): - msg_uid = forensic_report_msg_uids[i] + number_of_failure_msgs = len(failure_report_msg_uids) + for i in range(number_of_failure_msgs): + msg_uid = failure_report_msg_uids[i] message = "Moving message" logger.debug( "{0} {1} of {2}: UID {3}".format( - message, i + 1, number_of_forensic_msgs, msg_uid + message, i + 1, number_of_failure_msgs, msg_uid ) ) try: - connection.move_message(msg_uid, forensic_reports_folder) + connection.move_message(msg_uid, failure_reports_folder) except Exception as e: e = "Error moving message UID {0}: {1}".format(msg_uid, e) logger.error("Mailbox error: {0}".format(e)) @@ -2186,7 +2229,7 @@ def get_dmarc_reports_from_mailbox( logger.error("Mailbox error: {0}".format(e)) results = { "aggregate_reports": aggregate_reports, - "forensic_reports": forensic_reports, + "failure_reports": failure_reports, "smtp_tls_reports": smtp_tls_reports, } @@ -2272,7 +2315,7 @@ def watch_inbox( dns_retries (int): Number of times to retry DNS queries on timeout or other transient errors strip_attachment_payloads (bool): Replace attachment payloads in - forensic report samples with None + failure report samples with None batch_size (int): Number of messages to read and process before saving since: Search for messages since certain time normalize_timespan_threshold_hours (float): Normalize timespans beyond this @@ -2317,7 +2360,7 @@ def append_json( filename: str, reports: Union[ Sequence[AggregateReport], - Sequence[ForensicReport], + Sequence[FailureReport], Sequence[SMTPTLSReport], ], ) -> None: @@ -2360,10 +2403,10 @@ def save_output( *, output_directory: str = "output", aggregate_json_filename: str = "aggregate.json", - forensic_json_filename: str = "forensic.json", + failure_json_filename: str = "failure.json", smtp_tls_json_filename: str = "smtp_tls.json", aggregate_csv_filename: str = "aggregate.csv", - forensic_csv_filename: str = "forensic.csv", + failure_csv_filename: str = "failure.csv", smtp_tls_csv_filename: str = "smtp_tls.csv", ): """ @@ -2373,15 +2416,15 @@ def save_output( results: Parsing results output_directory (str): The path to the directory to save in aggregate_json_filename (str): Filename for the aggregate JSON file - forensic_json_filename (str): Filename for the forensic JSON file + failure_json_filename (str): Filename for the failure JSON file smtp_tls_json_filename (str): Filename for the SMTP TLS JSON file aggregate_csv_filename (str): Filename for the aggregate CSV file - forensic_csv_filename (str): Filename for the forensic CSV file + failure_csv_filename (str): Filename for the failure CSV file smtp_tls_csv_filename (str): Filename for the SMTP TLS CSV file """ aggregate_reports = results["aggregate_reports"] - forensic_reports = results["forensic_reports"] + failure_reports = results["failure_reports"] smtp_tls_reports = results["smtp_tls_reports"] output_directory = os.path.expanduser(output_directory) @@ -2400,13 +2443,11 @@ def save_output( parsed_aggregate_reports_to_csv(aggregate_reports), ) - append_json( - os.path.join(output_directory, forensic_json_filename), forensic_reports - ) + append_json(os.path.join(output_directory, failure_json_filename), failure_reports) append_csv( - os.path.join(output_directory, forensic_csv_filename), - parsed_forensic_reports_to_csv(forensic_reports), + os.path.join(output_directory, failure_csv_filename), + parsed_failure_reports_to_csv(failure_reports), ) append_json( @@ -2423,10 +2464,10 @@ def save_output( os.makedirs(samples_directory) sample_filenames = [] - for forensic_report in forensic_reports: - sample = forensic_report["sample"] + for failure_report in failure_reports: + sample = failure_report["sample"] message_count = 0 - parsed_sample = forensic_report["parsed_sample"] + parsed_sample = failure_report["parsed_sample"] subject = ( parsed_sample.get("filename_safe_subject") or parsed_sample.get("subject") @@ -2560,3 +2601,9 @@ def email_results( attachments=attachments, plain_message=message, ) + + +# Backward-compatible aliases +parse_forensic_report = parse_failure_report +parsed_forensic_reports_to_csv_rows = parsed_failure_reports_to_csv_rows +parsed_forensic_reports_to_csv = parsed_failure_reports_to_csv diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index 25dccf8..823bd74 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -335,14 +335,18 @@ def _parse_config(config: ConfigParser, opts): opts.output = _expand_path(general_config["output"]) if "aggregate_json_filename" in general_config: opts.aggregate_json_filename = general_config["aggregate_json_filename"] - if "forensic_json_filename" in general_config: - opts.forensic_json_filename = general_config["forensic_json_filename"] + if "failure_json_filename" in general_config: + opts.failure_json_filename = general_config["failure_json_filename"] + elif "forensic_json_filename" in general_config: + opts.failure_json_filename = general_config["forensic_json_filename"] if "smtp_tls_json_filename" in general_config: opts.smtp_tls_json_filename = general_config["smtp_tls_json_filename"] if "aggregate_csv_filename" in general_config: opts.aggregate_csv_filename = general_config["aggregate_csv_filename"] - if "forensic_csv_filename" in general_config: - opts.forensic_csv_filename = general_config["forensic_csv_filename"] + if "failure_csv_filename" in general_config: + opts.failure_csv_filename = general_config["failure_csv_filename"] + elif "forensic_csv_filename" in general_config: + opts.failure_csv_filename = general_config["forensic_csv_filename"] if "smtp_tls_csv_filename" in general_config: opts.smtp_tls_csv_filename = general_config["smtp_tls_csv_filename"] if "dns_timeout" in general_config: @@ -377,8 +381,10 @@ def _parse_config(config: ConfigParser, opts): ) if "save_aggregate" in general_config: opts.save_aggregate = bool(general_config.getboolean("save_aggregate")) - if "save_forensic" in general_config: - opts.save_forensic = bool(general_config.getboolean("save_forensic")) + if "save_failure" in general_config: + opts.save_failure = bool(general_config.getboolean("save_failure")) + elif "save_forensic" in general_config: + opts.save_failure = bool(general_config.getboolean("save_forensic")) if "save_smtp_tls" in general_config: opts.save_smtp_tls = bool(general_config.getboolean("save_smtp_tls")) if "debug" in general_config: @@ -772,11 +778,13 @@ def _parse_config(config: ConfigParser, opts): raise ConfigurationError( "aggregate_topic setting missing from the kafka config section" ) - if "forensic_topic" in kafka_config: - opts.kafka_forensic_topic = kafka_config["forensic_topic"] + if "failure_topic" in kafka_config: + opts.kafka_failure_topic = kafka_config["failure_topic"] + elif "forensic_topic" in kafka_config: + opts.kafka_failure_topic = kafka_config["forensic_topic"] else: raise ConfigurationError( - "forensic_topic setting missing from the kafka config section" + "failure_topic setting missing from the kafka config section" ) if "smtp_tls_topic" in kafka_config: opts.kafka_smtp_tls_topic = kafka_config["smtp_tls_topic"] @@ -940,7 +948,9 @@ def _parse_config(config: ConfigParser, opts): opts.la_dce = log_analytics_config.get("dce") opts.la_dcr_immutable_id = log_analytics_config.get("dcr_immutable_id") opts.la_dcr_aggregate_stream = log_analytics_config.get("dcr_aggregate_stream") - opts.la_dcr_forensic_stream = log_analytics_config.get("dcr_forensic_stream") + opts.la_dcr_failure_stream = log_analytics_config.get( + "dcr_failure_stream" + ) or log_analytics_config.get("dcr_forensic_stream") opts.la_dcr_smtp_tls_stream = log_analytics_config.get("dcr_smtp_tls_stream") if "gelf" in config.sections(): @@ -968,8 +978,10 @@ def _parse_config(config: ConfigParser, opts): webhook_config = config["webhook"] if "aggregate_url" in webhook_config: opts.webhook_aggregate_url = webhook_config["aggregate_url"] - if "forensic_url" in webhook_config: - opts.webhook_forensic_url = webhook_config["forensic_url"] + if "failure_url" in webhook_config: + opts.webhook_failure_url = webhook_config["failure_url"] + elif "forensic_url" in webhook_config: + opts.webhook_failure_url = webhook_config["forensic_url"] if "smtp_tls_url" in webhook_config: opts.webhook_smtp_tls_url = webhook_config["smtp_tls_url"] if "timeout" in webhook_config: @@ -1112,13 +1124,13 @@ def _init_output_clients(opts): try: if ( opts.webhook_aggregate_url - or opts.webhook_forensic_url + or opts.webhook_failure_url or opts.webhook_smtp_tls_url ): logger.debug("Initializing webhook client") clients["webhook_client"] = webhook.WebhookClient( aggregate_url=opts.webhook_aggregate_url, - forensic_url=opts.webhook_forensic_url, + failure_url=opts.webhook_failure_url, smtp_tls_url=opts.webhook_smtp_tls_url, timeout=opts.webhook_timeout, ) @@ -1130,7 +1142,7 @@ def _init_output_clients(opts): # step fails. Initialise them last so that all other clients are created # successfully first; this minimizes the window for partial-init problems # during config reload. - if opts.save_aggregate or opts.save_forensic or opts.save_smtp_tls: + if opts.save_aggregate or opts.save_failure or opts.save_smtp_tls: try: if opts.elasticsearch_hosts: logger.debug( @@ -1139,17 +1151,17 @@ def _init_output_clients(opts): opts.elasticsearch_ssl, ) es_aggregate_index = "dmarc_aggregate" - es_forensic_index = "dmarc_forensic" + es_failure_index = "dmarc_failure" es_smtp_tls_index = "smtp_tls" if opts.elasticsearch_index_suffix: suffix = opts.elasticsearch_index_suffix es_aggregate_index = "{0}_{1}".format(es_aggregate_index, suffix) - es_forensic_index = "{0}_{1}".format(es_forensic_index, suffix) + es_failure_index = "{0}_{1}".format(es_failure_index, suffix) es_smtp_tls_index = "{0}_{1}".format(es_smtp_tls_index, suffix) if opts.elasticsearch_index_prefix: prefix = opts.elasticsearch_index_prefix es_aggregate_index = "{0}{1}".format(prefix, es_aggregate_index) - es_forensic_index = "{0}{1}".format(prefix, es_forensic_index) + es_failure_index = "{0}{1}".format(prefix, es_failure_index) es_smtp_tls_index = "{0}{1}".format(prefix, es_smtp_tls_index) elastic_timeout_value = ( float(opts.elasticsearch_timeout) @@ -1168,7 +1180,7 @@ def _init_output_clients(opts): ) elastic.migrate_indexes( aggregate_indexes=[es_aggregate_index], - forensic_indexes=[es_forensic_index], + failure_indexes=[es_failure_index], ) clients["elasticsearch"] = _ElasticsearchHandle() except Exception as e: @@ -1182,17 +1194,17 @@ def _init_output_clients(opts): opts.opensearch_ssl, ) os_aggregate_index = "dmarc_aggregate" - os_forensic_index = "dmarc_forensic" + os_failure_index = "dmarc_failure" os_smtp_tls_index = "smtp_tls" if opts.opensearch_index_suffix: suffix = opts.opensearch_index_suffix os_aggregate_index = "{0}_{1}".format(os_aggregate_index, suffix) - os_forensic_index = "{0}_{1}".format(os_forensic_index, suffix) + os_failure_index = "{0}_{1}".format(os_failure_index, suffix) os_smtp_tls_index = "{0}_{1}".format(os_smtp_tls_index, suffix) if opts.opensearch_index_prefix: prefix = opts.opensearch_index_prefix os_aggregate_index = "{0}{1}".format(prefix, os_aggregate_index) - os_forensic_index = "{0}{1}".format(prefix, os_forensic_index) + os_failure_index = "{0}{1}".format(prefix, os_failure_index) os_smtp_tls_index = "{0}{1}".format(prefix, os_smtp_tls_index) opensearch_timeout_value = ( float(opts.opensearch_timeout) @@ -1214,7 +1226,7 @@ def _init_output_clients(opts): ) opensearch.migrate_indexes( aggregate_indexes=[os_aggregate_index], - forensic_indexes=[os_forensic_index], + failure_indexes=[os_failure_index], ) clients["opensearch"] = _OpenSearchHandle() except Exception as e: @@ -1306,10 +1318,10 @@ def _main(): reports_, output_directory=opts.output, aggregate_json_filename=opts.aggregate_json_filename, - forensic_json_filename=opts.forensic_json_filename, + failure_json_filename=opts.failure_json_filename, smtp_tls_json_filename=opts.smtp_tls_json_filename, aggregate_csv_filename=opts.aggregate_csv_filename, - forensic_csv_filename=opts.forensic_csv_filename, + failure_csv_filename=opts.failure_csv_filename, smtp_tls_csv_filename=opts.smtp_tls_csv_filename, ) @@ -1321,7 +1333,7 @@ def _main(): webhook_client = clients.get("webhook_client") kafka_aggregate_topic = opts.kafka_aggregate_topic - kafka_forensic_topic = opts.kafka_forensic_topic + kafka_failure_topic = opts.kafka_failure_topic kafka_smtp_tls_topic = opts.kafka_smtp_tls_topic if opts.save_aggregate: @@ -1409,13 +1421,13 @@ def _main(): except splunk.SplunkError as e: log_output_error("Splunk HEC", e.__str__()) - if opts.save_forensic: - for report in reports_["forensic_reports"]: + if opts.save_failure: + for report in reports_["failure_reports"]: try: shards = opts.elasticsearch_number_of_shards replicas = opts.elasticsearch_number_of_replicas if opts.elasticsearch_hosts: - elastic.save_forensic_report_to_elasticsearch( + elastic.save_failure_report_to_elasticsearch( report, index_suffix=opts.elasticsearch_index_suffix, index_prefix=opts.elasticsearch_index_prefix @@ -1435,7 +1447,7 @@ def _main(): shards = opts.opensearch_number_of_shards replicas = opts.opensearch_number_of_replicas if opts.opensearch_hosts: - opensearch.save_forensic_report_to_opensearch( + opensearch.save_failure_report_to_opensearch( report, index_suffix=opts.opensearch_index_suffix, index_prefix=opts.opensearch_index_prefix @@ -1453,34 +1465,34 @@ def _main(): try: if kafka_client: - kafka_client.save_forensic_reports_to_kafka( - report, kafka_forensic_topic + kafka_client.save_failure_reports_to_kafka( + report, kafka_failure_topic ) except Exception as error_: log_output_error("Kafka", error_.__str__()) try: if s3_client: - s3_client.save_forensic_report_to_s3(report) + s3_client.save_failure_report_to_s3(report) except Exception as error_: log_output_error("S3", error_.__str__()) try: if syslog_client: - syslog_client.save_forensic_report_to_syslog(report) + syslog_client.save_failure_report_to_syslog(report) except Exception as error_: log_output_error("Syslog", error_.__str__()) try: if gelf_client: - gelf_client.save_forensic_report_to_gelf(report) + gelf_client.save_failure_report_to_gelf(report) except Exception as error_: log_output_error("GELF", error_.__str__()) try: - if opts.webhook_forensic_url and webhook_client: + if opts.webhook_failure_url and webhook_client: indent_value = 2 if opts.prettify_json else None - webhook_client.save_forensic_report_to_webhook( + webhook_client.save_failure_report_to_webhook( json.dumps(report, ensure_ascii=False, indent=indent_value) ) except Exception as error_: @@ -1488,9 +1500,9 @@ def _main(): if hec_client: try: - forensic_reports_ = reports_["forensic_reports"] - if len(forensic_reports_) > 0: - hec_client.save_forensic_reports_to_splunk(forensic_reports_) + failure_reports_ = reports_["failure_reports"] + if len(failure_reports_) > 0: + hec_client.save_failure_reports_to_splunk(failure_reports_) except splunk.SplunkError as e: log_output_error("Splunk HEC", e.__str__()) @@ -1588,13 +1600,13 @@ def _main(): dce=opts.la_dce, dcr_immutable_id=opts.la_dcr_immutable_id, dcr_aggregate_stream=opts.la_dcr_aggregate_stream, - dcr_forensic_stream=opts.la_dcr_forensic_stream, + dcr_failure_stream=opts.la_dcr_failure_stream, dcr_smtp_tls_stream=opts.la_dcr_smtp_tls_stream, ) la_client.publish_results( reports_, opts.save_aggregate, - opts.save_forensic, + opts.save_failure, opts.save_smtp_tls, ) except loganalytics.LogAnalyticsException as e: @@ -1634,9 +1646,9 @@ def _main(): default="aggregate.json", ) arg_parser.add_argument( - "--forensic-json-filename", - help="filename for the forensic JSON output file", - default="forensic.json", + "--failure-json-filename", + help="filename for the failure JSON output file", + default="failure.json", ) arg_parser.add_argument( "--smtp-tls-json-filename", @@ -1649,9 +1661,9 @@ def _main(): default="aggregate.csv", ) arg_parser.add_argument( - "--forensic-csv-filename", - help="filename for the forensic CSV output file", - default="forensic.csv", + "--failure-csv-filename", + help="filename for the failure CSV output file", + default="failure.csv", ) arg_parser.add_argument( "--smtp-tls-csv-filename", @@ -1706,7 +1718,7 @@ def _main(): arg_parser.add_argument("-v", "--version", action="version", version=__version__) aggregate_reports = [] - forensic_reports = [] + failure_reports = [] smtp_tls_reports = [] args = arg_parser.parse_args() @@ -1719,8 +1731,8 @@ def _main(): output=args.output, aggregate_csv_filename=args.aggregate_csv_filename, aggregate_json_filename=args.aggregate_json_filename, - forensic_csv_filename=args.forensic_csv_filename, - forensic_json_filename=args.forensic_json_filename, + failure_csv_filename=args.failure_csv_filename, + failure_json_filename=args.failure_json_filename, smtp_tls_json_filename=args.smtp_tls_json_filename, smtp_tls_csv_filename=args.smtp_tls_csv_filename, nameservers=args.nameservers, @@ -1733,7 +1745,7 @@ def _main(): verbose=args.verbose, prettify_json=args.prettify_json, save_aggregate=False, - save_forensic=False, + save_failure=False, save_smtp_tls=False, mailbox_reports_folder="INBOX", mailbox_archive_folder="Archive", @@ -1799,7 +1811,7 @@ def _main(): kafka_username=None, kafka_password=None, kafka_aggregate_topic=None, - kafka_forensic_topic=None, + kafka_failure_topic=None, kafka_smtp_tls_topic=None, kafka_ssl=False, kafka_skip_certificate_verification=False, @@ -1854,13 +1866,13 @@ def _main(): la_dce=None, la_dcr_immutable_id=None, la_dcr_aggregate_stream=None, - la_dcr_forensic_stream=None, + la_dcr_failure_stream=None, la_dcr_smtp_tls_stream=None, gelf_host=None, gelf_port=None, gelf_mode=None, webhook_aggregate_url=None, - webhook_forensic_url=None, + webhook_failure_url=None, webhook_smtp_tls_url=None, webhook_timeout=60, normalize_timespan_threshold_hours=24.0, @@ -2062,8 +2074,8 @@ def _main(): "Skipping duplicate aggregate report " f"from {report_org} with ID: {report_id}" ) - elif result[0]["report_type"] == "forensic": - forensic_reports.append(result[0]["report"]) + elif result[0]["report_type"] == "failure": + failure_reports.append(result[0]["report"]) elif result[0]["report_type"] == "smtp_tls": smtp_tls_reports.append(result[0]["report"]) @@ -2088,7 +2100,7 @@ def _main(): normalize_timespan_threshold_hours=normalize_timespan_threshold_hours_value, ) aggregate_reports += reports["aggregate_reports"] - forensic_reports += reports["forensic_reports"] + failure_reports += reports["failure_reports"] smtp_tls_reports += reports["smtp_tls_reports"] mailbox_connection = None @@ -2229,7 +2241,7 @@ def _main(): ) aggregate_reports += reports["aggregate_reports"] - forensic_reports += reports["forensic_reports"] + failure_reports += reports["failure_reports"] smtp_tls_reports += reports["smtp_tls_reports"] except Exception: @@ -2238,7 +2250,7 @@ def _main(): parsing_results: ParsingResults = { "aggregate_reports": aggregate_reports, - "forensic_reports": forensic_reports, + "failure_reports": failure_reports, "smtp_tls_reports": smtp_tls_reports, } diff --git a/parsedmarc/constants.py b/parsedmarc/constants.py index d5bbebf..c053484 100644 --- a/parsedmarc/constants.py +++ b/parsedmarc/constants.py @@ -1,4 +1,4 @@ -__version__ = "9.10.1" +__version__ = "10.0.0" USER_AGENT = f"parsedmarc/{__version__}" diff --git a/parsedmarc/elastic.py b/parsedmarc/elastic.py index bec69cb..872504e 100644 --- a/parsedmarc/elastic.py +++ b/parsedmarc/elastic.py @@ -13,6 +13,7 @@ from elasticsearch_dsl import ( InnerDoc, Integer, Ip, + Keyword, Nested, Object, Search, @@ -21,7 +22,7 @@ from elasticsearch_dsl import ( ) from elasticsearch_dsl.search import Q -from parsedmarc import InvalidForensicReport +from parsedmarc import InvalidFailureReport from parsedmarc.log import logger from parsedmarc.utils import human_timestamp_to_datetime @@ -43,18 +44,23 @@ class _PublishedPolicy(InnerDoc): sp = Text() pct = Integer() fo = Text() + np = Keyword() + testing = Keyword() + discovery_method = Keyword() class _DKIMResult(InnerDoc): domain = Text() selector = Text() result = Text() + human_result = Text() class _SPFResult(InnerDoc): domain = Text() scope = Text() results = Text() + human_result = Text() class _AggregateReportDoc(Document): @@ -93,17 +99,45 @@ class _AggregateReportDoc(Document): envelope_to = Text() dkim_results = Nested(_DKIMResult) spf_results = Nested(_SPFResult) + np = Keyword() + testing = Keyword() + discovery_method = Keyword() + generator = Text() def add_policy_override(self, type_: str, comment: str): self.policy_overrides.append(_PolicyOverride(type=type_, comment=comment)) # pyright: ignore[reportCallIssue] - def add_dkim_result(self, domain: str, selector: str, result: _DKIMResult): + def add_dkim_result( + self, + domain: str, + selector: str, + result: _DKIMResult, + human_result: str = None, + ): self.dkim_results.append( - _DKIMResult(domain=domain, selector=selector, result=result) + _DKIMResult( + domain=domain, + selector=selector, + result=result, + human_result=human_result, + ) ) # pyright: ignore[reportCallIssue] - def add_spf_result(self, domain: str, scope: str, result: _SPFResult): - self.spf_results.append(_SPFResult(domain=domain, scope=scope, result=result)) # pyright: ignore[reportCallIssue] + def add_spf_result( + self, + domain: str, + scope: str, + result: _SPFResult, + human_result: str = None, + ): + self.spf_results.append( + _SPFResult( + domain=domain, + scope=scope, + result=result, + human_result=human_result, + ) + ) # pyright: ignore[reportCallIssue] def save(self, **kwargs): # pyright: ignore[reportIncompatibleMethodOverride] self.passed_dmarc = False @@ -123,7 +157,7 @@ class _EmailAttachmentDoc(Document): sha256 = Text() -class _ForensicSampleDoc(InnerDoc): +class _FailureSampleDoc(InnerDoc): raw = Text() headers = Object() headers_only = Boolean() @@ -160,9 +194,9 @@ class _ForensicSampleDoc(InnerDoc): ) # pyright: ignore[reportCallIssue] -class _ForensicReportDoc(Document): +class _FailureReportDoc(Document): class Index: - name = "dmarc_forensic" + name = "dmarc_failure" feedback_type = Text() user_agent = Text() @@ -183,7 +217,7 @@ class _ForensicReportDoc(Document): source_auth_failures = Text() dkim_domain = Text() original_rcpt_to = Text() - sample = Object(_ForensicSampleDoc) + sample = Object(_FailureSampleDoc) class _SMTPTLSFailureDetailsDoc(InnerDoc): @@ -336,20 +370,20 @@ def create_indexes(names: list[str], settings: Optional[dict[str, Any]] = None): def migrate_indexes( aggregate_indexes: Optional[list[str]] = None, - forensic_indexes: Optional[list[str]] = None, + failure_indexes: Optional[list[str]] = None, ): """ Updates index mappings Args: aggregate_indexes (list): A list of aggregate index names - forensic_indexes (list): A list of forensic index names + failure_indexes (list): A list of failure index names """ version = 2 if aggregate_indexes is None: aggregate_indexes = [] - if forensic_indexes is None: - forensic_indexes = [] + if failure_indexes is None: + failure_indexes = [] for aggregate_index_name in aggregate_indexes: if not Index(aggregate_index_name).exists(): continue @@ -379,7 +413,7 @@ def migrate_indexes( reindex(connections.get_connection(), aggregate_index_name, new_index_name) # pyright: ignore[reportArgumentType] Index(aggregate_index_name).delete() - for forensic_index in forensic_indexes: + for failure_index in failure_indexes: pass @@ -395,7 +429,7 @@ def save_aggregate_report_to_elasticsearch( Saves a parsed DMARC aggregate report to Elasticsearch Args: - aggregate_report (dict): A parsed forensic report + aggregate_report (dict): A parsed aggregate report index_suffix (str): The suffix of the name of the index to save to index_prefix (str): The prefix of the name of the index to save to monthly_indexes (bool): Use monthly indexes instead of daily indexes @@ -463,6 +497,9 @@ def save_aggregate_report_to_elasticsearch( sp=aggregate_report["policy_published"]["sp"], pct=aggregate_report["policy_published"]["pct"], fo=aggregate_report["policy_published"]["fo"], + np=aggregate_report["policy_published"].get("np"), + testing=aggregate_report["policy_published"].get("testing"), + discovery_method=aggregate_report["policy_published"].get("discovery_method"), ) for record in aggregate_report["records"]: @@ -507,6 +544,12 @@ def save_aggregate_report_to_elasticsearch( header_from=record["identifiers"]["header_from"], envelope_from=record["identifiers"]["envelope_from"], envelope_to=record["identifiers"]["envelope_to"], + np=aggregate_report["policy_published"].get("np"), + testing=aggregate_report["policy_published"].get("testing"), + discovery_method=aggregate_report["policy_published"].get( + "discovery_method" + ), + generator=metadata.get("generator"), ) for override in record["policy_evaluated"]["policy_override_reasons"]: @@ -519,6 +562,7 @@ def save_aggregate_report_to_elasticsearch( domain=dkim_result["domain"], selector=dkim_result["selector"], result=dkim_result["result"], + human_result=dkim_result.get("human_result"), ) for spf_result in record["auth_results"]["spf"]: @@ -526,6 +570,7 @@ def save_aggregate_report_to_elasticsearch( domain=spf_result["domain"], scope=spf_result["scope"], result=spf_result["result"], + human_result=spf_result.get("human_result"), ) index = "dmarc_aggregate" @@ -547,8 +592,8 @@ def save_aggregate_report_to_elasticsearch( raise ElasticsearchError("Elasticsearch error: {0}".format(e.__str__())) -def save_forensic_report_to_elasticsearch( - forensic_report: dict[str, Any], +def save_failure_report_to_elasticsearch( + failure_report: dict[str, Any], index_suffix: Optional[Any] = None, index_prefix: Optional[str] = None, monthly_indexes: Optional[bool] = False, @@ -556,10 +601,10 @@ def save_forensic_report_to_elasticsearch( number_of_replicas: int = 0, ): """ - Saves a parsed DMARC forensic report to Elasticsearch + Saves a parsed DMARC failure report to Elasticsearch Args: - forensic_report (dict): A parsed forensic report + failure_report (dict): A parsed failure report index_suffix (str): The suffix of the name of the index to save to index_prefix (str): The prefix of the name of the index to save to monthly_indexes (bool): Use monthly indexes instead of daily @@ -572,26 +617,28 @@ def save_forensic_report_to_elasticsearch( AlreadySaved """ - logger.info("Saving forensic report to Elasticsearch") - forensic_report = forensic_report.copy() + logger.info("Saving failure report to Elasticsearch") + failure_report = failure_report.copy() sample_date = None - if forensic_report["parsed_sample"]["date"] is not None: - sample_date = forensic_report["parsed_sample"]["date"] + if failure_report["parsed_sample"]["date"] is not None: + sample_date = failure_report["parsed_sample"]["date"] sample_date = human_timestamp_to_datetime(sample_date) - original_headers = forensic_report["parsed_sample"]["headers"] + original_headers = failure_report["parsed_sample"]["headers"] headers: dict[str, Any] = {} for original_header in original_headers: headers[original_header.lower()] = original_headers[original_header] - arrival_date = human_timestamp_to_datetime(forensic_report["arrival_date_utc"]) + arrival_date = human_timestamp_to_datetime(failure_report["arrival_date_utc"]) arrival_date_epoch_milliseconds = int(arrival_date.timestamp() * 1000) if index_suffix is not None: - search_index = "dmarc_forensic_{0}*".format(index_suffix) + search_index = "dmarc_failure_{0}*,dmarc_forensic_{0}*".format(index_suffix) else: - search_index = "dmarc_forensic*" + search_index = "dmarc_failure*,dmarc_forensic*" if index_prefix is not None: - search_index = "{0}{1}".format(index_prefix, search_index) + search_index = ",".join( + "{0}{1}".format(index_prefix, part) for part in search_index.split(",") + ) search = Search(index=search_index) q = Q(dict(match=dict(arrival_date=arrival_date_epoch_milliseconds))) # pyright: ignore[reportArgumentType] @@ -632,67 +679,67 @@ def save_forensic_report_to_elasticsearch( if len(existing) > 0: raise AlreadySaved( - "A forensic sample to {0} from {1} " + "A failure sample to {0} from {1} " "with a subject of {2} and arrival date of {3} " "already exists in " "Elasticsearch".format( - to_, from_, subject, forensic_report["arrival_date_utc"] + to_, from_, subject, failure_report["arrival_date_utc"] ) ) - parsed_sample = forensic_report["parsed_sample"] - sample = _ForensicSampleDoc( - raw=forensic_report["sample"], + parsed_sample = failure_report["parsed_sample"] + sample = _FailureSampleDoc( + raw=failure_report["sample"], headers=headers, - headers_only=forensic_report["sample_headers_only"], + headers_only=failure_report["sample_headers_only"], date=sample_date, - subject=forensic_report["parsed_sample"]["subject"], + subject=failure_report["parsed_sample"]["subject"], filename_safe_subject=parsed_sample["filename_safe_subject"], - body=forensic_report["parsed_sample"]["body"], + body=failure_report["parsed_sample"]["body"], ) - for address in forensic_report["parsed_sample"]["to"]: + for address in failure_report["parsed_sample"]["to"]: sample.add_to(display_name=address["display_name"], address=address["address"]) - for address in forensic_report["parsed_sample"]["reply_to"]: + for address in failure_report["parsed_sample"]["reply_to"]: sample.add_reply_to( display_name=address["display_name"], address=address["address"] ) - for address in forensic_report["parsed_sample"]["cc"]: + for address in failure_report["parsed_sample"]["cc"]: sample.add_cc(display_name=address["display_name"], address=address["address"]) - for address in forensic_report["parsed_sample"]["bcc"]: + for address in failure_report["parsed_sample"]["bcc"]: sample.add_bcc(display_name=address["display_name"], address=address["address"]) - for attachment in forensic_report["parsed_sample"]["attachments"]: + for attachment in failure_report["parsed_sample"]["attachments"]: sample.add_attachment( filename=attachment["filename"], content_type=attachment["mail_content_type"], sha256=attachment["sha256"], ) try: - forensic_doc = _ForensicReportDoc( - feedback_type=forensic_report["feedback_type"], - user_agent=forensic_report["user_agent"], - version=forensic_report["version"], - original_mail_from=forensic_report["original_mail_from"], + failure_doc = _FailureReportDoc( + feedback_type=failure_report["feedback_type"], + user_agent=failure_report["user_agent"], + version=failure_report["version"], + original_mail_from=failure_report["original_mail_from"], arrival_date=arrival_date_epoch_milliseconds, - domain=forensic_report["reported_domain"], - original_envelope_id=forensic_report["original_envelope_id"], - authentication_results=forensic_report["authentication_results"], - delivery_results=forensic_report["delivery_result"], - source_ip_address=forensic_report["source"]["ip_address"], - source_country=forensic_report["source"]["country"], - source_reverse_dns=forensic_report["source"]["reverse_dns"], - source_base_domain=forensic_report["source"]["base_domain"], - source_asn=forensic_report["source"]["asn"], - source_as_name=forensic_report["source"]["as_name"], - source_as_domain=forensic_report["source"]["as_domain"], - authentication_mechanisms=forensic_report["authentication_mechanisms"], - auth_failure=forensic_report["auth_failure"], - dkim_domain=forensic_report["dkim_domain"], - original_rcpt_to=forensic_report["original_rcpt_to"], + domain=failure_report["reported_domain"], + original_envelope_id=failure_report["original_envelope_id"], + authentication_results=failure_report["authentication_results"], + delivery_results=failure_report["delivery_result"], + source_ip_address=failure_report["source"]["ip_address"], + source_country=failure_report["source"]["country"], + source_reverse_dns=failure_report["source"]["reverse_dns"], + source_base_domain=failure_report["source"]["base_domain"], + source_asn=failure_report["source"]["asn"], + source_as_name=failure_report["source"]["as_name"], + source_as_domain=failure_report["source"]["as_domain"], + authentication_mechanisms=failure_report["authentication_mechanisms"], + auth_failure=failure_report["auth_failure"], + dkim_domain=failure_report["dkim_domain"], + original_rcpt_to=failure_report["original_rcpt_to"], sample=sample, ) - index = "dmarc_forensic" + index = "dmarc_failure" if index_suffix: index = "{0}_{1}".format(index, index_suffix) if index_prefix: @@ -706,14 +753,14 @@ def save_forensic_report_to_elasticsearch( number_of_shards=number_of_shards, number_of_replicas=number_of_replicas ) create_indexes([index], index_settings) - forensic_doc.meta.index = index # pyright: ignore[reportAttributeAccessIssue, reportOptionalMemberAccess] + failure_doc.meta.index = index # pyright: ignore[reportAttributeAccessIssue, reportOptionalMemberAccess] try: - forensic_doc.save() + failure_doc.save() except Exception as e: raise ElasticsearchError("Elasticsearch error: {0}".format(e.__str__())) except KeyError as e: - raise InvalidForensicReport( - "Forensic report missing required field: {0}".format(e.__str__()) + raise InvalidFailureReport( + "Failure report missing required field: {0}".format(e.__str__()) ) @@ -867,3 +914,9 @@ def save_smtp_tls_report_to_elasticsearch( smtp_tls_doc.save() except Exception as e: raise ElasticsearchError("Elasticsearch error: {0}".format(e.__str__())) + + +# Backward-compatible aliases +_ForensicSampleDoc = _FailureSampleDoc +_ForensicReportDoc = _FailureReportDoc +save_forensic_report_to_elasticsearch = save_failure_report_to_elasticsearch diff --git a/parsedmarc/gelf.py b/parsedmarc/gelf.py index 2ac5a5a..a0dfd93 100644 --- a/parsedmarc/gelf.py +++ b/parsedmarc/gelf.py @@ -9,10 +9,12 @@ from pygelf import GelfTcpHandler, GelfTlsHandler, GelfUdpHandler from parsedmarc import ( parsed_aggregate_reports_to_csv_rows, - parsed_forensic_reports_to_csv_rows, + parsed_failure_reports_to_csv_rows, parsed_smtp_tls_reports_to_csv_rows, ) -from parsedmarc.types import AggregateReport, ForensicReport, SMTPTLSReport +from typing import Any + +from parsedmarc.types import AggregateReport, SMTPTLSReport log_context_data = threading.local() @@ -57,11 +59,11 @@ class GelfClient(object): log_context_data.parsedmarc = None - def save_forensic_report_to_gelf(self, forensic_reports: list[ForensicReport]): - rows = parsed_forensic_reports_to_csv_rows(forensic_reports) + def save_failure_report_to_gelf(self, failure_reports: list[dict[str, Any]]): + rows = parsed_failure_reports_to_csv_rows(failure_reports) for row in rows: log_context_data.parsedmarc = row - self.logger.info("parsedmarc forensic report") + self.logger.info("parsedmarc failure report") def save_smtp_tls_report_to_gelf(self, smtp_tls_reports: SMTPTLSReport): rows = parsed_smtp_tls_reports_to_csv_rows(smtp_tls_reports) @@ -73,3 +75,7 @@ class GelfClient(object): """Remove and close the GELF handler, releasing its connection.""" self.logger.removeHandler(self.handler) self.handler.close() + + +# Backward-compatible aliases +GelfClient.save_forensic_report_to_gelf = GelfClient.save_failure_report_to_gelf diff --git a/parsedmarc/kafkaclient.py b/parsedmarc/kafkaclient.py index 227e102..b15dfb5 100644 --- a/parsedmarc/kafkaclient.py +++ b/parsedmarc/kafkaclient.py @@ -143,31 +143,31 @@ class KafkaClient(object): except Exception as e: raise KafkaError("Kafka error: {0}".format(e.__str__())) - def save_forensic_reports_to_kafka( + def save_failure_reports_to_kafka( self, - forensic_reports: Union[dict[str, Any], list[dict[str, Any]]], - forensic_topic: str, + failure_reports: Union[dict[str, Any], list[dict[str, Any]]], + failure_topic: str, ): """ - Saves forensic DMARC reports to Kafka, sends individual + Saves failure DMARC reports to Kafka, sends individual records (slices) since Kafka requires messages to be <= 1MB by default. Args: - forensic_reports (list): A list of forensic report dicts + failure_reports (list): A list of failure report dicts to save to Kafka - forensic_topic (str): The name of the Kafka topic + failure_topic (str): The name of the Kafka topic """ - if isinstance(forensic_reports, dict): - forensic_reports = [forensic_reports] + if isinstance(failure_reports, dict): + failure_reports = [failure_reports] - if len(forensic_reports) < 1: + if len(failure_reports) < 1: return try: - logger.debug("Saving forensic reports to Kafka") - self.producer.send(forensic_topic, forensic_reports) + logger.debug("Saving failure reports to Kafka") + self.producer.send(failure_topic, failure_reports) except UnknownTopicOrPartitionError: raise KafkaError("Kafka error: Unknown topic or partition on broker") except Exception as e: @@ -188,7 +188,7 @@ class KafkaClient(object): by default. Args: - smtp_tls_reports (list): A list of forensic report dicts + smtp_tls_reports (list): A list of SMTP TLS report dicts to save to Kafka smtp_tls_topic (str): The name of the Kafka topic @@ -200,7 +200,7 @@ class KafkaClient(object): return try: - logger.debug("Saving forensic reports to Kafka") + logger.debug("Saving SMTP TLS reports to Kafka") self.producer.send(smtp_tls_topic, smtp_tls_reports) except UnknownTopicOrPartitionError: raise KafkaError("Kafka error: Unknown topic or partition on broker") @@ -210,3 +210,7 @@ class KafkaClient(object): self.producer.flush() except Exception as e: raise KafkaError("Kafka error: {0}".format(e.__str__())) + + +# Backward-compatible aliases +KafkaClient.save_forensic_reports_to_kafka = KafkaClient.save_failure_reports_to_kafka diff --git a/parsedmarc/loganalytics.py b/parsedmarc/loganalytics.py index 10a941b..079d316 100644 --- a/parsedmarc/loganalytics.py +++ b/parsedmarc/loganalytics.py @@ -38,9 +38,9 @@ class LogAnalyticsConfig: The Stream name where the Aggregate DMARC reports need to be pushed. - dcr_forensic_stream (str): + dcr_failure_stream (str): The Stream name where - the Forensic DMARC reports + the Failure DMARC reports need to be pushed. dcr_smtp_tls_stream (str): The Stream name where @@ -56,7 +56,7 @@ class LogAnalyticsConfig: dce: str, dcr_immutable_id: str, dcr_aggregate_stream: str, - dcr_forensic_stream: str, + dcr_failure_stream: str, dcr_smtp_tls_stream: str, ): self.client_id = client_id @@ -65,7 +65,7 @@ class LogAnalyticsConfig: self.dce = dce self.dcr_immutable_id = dcr_immutable_id self.dcr_aggregate_stream = dcr_aggregate_stream - self.dcr_forensic_stream = dcr_forensic_stream + self.dcr_failure_stream = dcr_failure_stream self.dcr_smtp_tls_stream = dcr_smtp_tls_stream @@ -84,7 +84,7 @@ class LogAnalyticsClient(object): dce: str, dcr_immutable_id: str, dcr_aggregate_stream: str, - dcr_forensic_stream: str, + dcr_failure_stream: str, dcr_smtp_tls_stream: str, ): self.conf = LogAnalyticsConfig( @@ -94,7 +94,7 @@ class LogAnalyticsClient(object): dce=dce, dcr_immutable_id=dcr_immutable_id, dcr_aggregate_stream=dcr_aggregate_stream, - dcr_forensic_stream=dcr_forensic_stream, + dcr_failure_stream=dcr_failure_stream, dcr_smtp_tls_stream=dcr_smtp_tls_stream, ) if ( @@ -135,7 +135,7 @@ class LogAnalyticsClient(object): self, results: dict[str, Any], save_aggregate: bool, - save_forensic: bool, + save_failure: bool, save_smtp_tls: bool, ): """ @@ -146,13 +146,13 @@ class LogAnalyticsClient(object): Args: results (list): - The DMARC reports (Aggregate & Forensic) + The DMARC reports (Aggregate & Failure) save_aggregate (bool): Whether Aggregate reports can be saved into Log Analytics - save_forensic (bool): - Whether Forensic reports can be saved into Log Analytics + save_failure (bool): + Whether Failure reports can be saved into Log Analytics save_smtp_tls (bool): - Whether Forensic reports can be saved into Log Analytics + Whether Failure reports can be saved into Log Analytics """ conf = self.conf credential = ClientSecretCredential( @@ -173,16 +173,16 @@ class LogAnalyticsClient(object): ) logger.info("Successfully pushed aggregate reports.") if ( - results["forensic_reports"] - and conf.dcr_forensic_stream - and len(results["forensic_reports"]) > 0 - and save_forensic + results["failure_reports"] + and conf.dcr_failure_stream + and len(results["failure_reports"]) > 0 + and save_failure ): - logger.info("Publishing forensic reports.") + logger.info("Publishing failure reports.") self.publish_json( - results["forensic_reports"], logs_client, conf.dcr_forensic_stream + results["failure_reports"], logs_client, conf.dcr_failure_stream ) - logger.info("Successfully pushed forensic reports.") + logger.info("Successfully pushed failure reports.") if ( results["smtp_tls_reports"] and conf.dcr_smtp_tls_stream diff --git a/parsedmarc/opensearch.py b/parsedmarc/opensearch.py index f3826bf..741ae6c 100644 --- a/parsedmarc/opensearch.py +++ b/parsedmarc/opensearch.py @@ -14,6 +14,7 @@ from opensearchpy import ( InnerDoc, Integer, Ip, + Keyword, Nested, Object, Q, @@ -24,7 +25,7 @@ from opensearchpy import ( ) from opensearchpy.helpers import reindex -from parsedmarc import InvalidForensicReport +from parsedmarc import InvalidFailureReport from parsedmarc.log import logger from parsedmarc.utils import human_timestamp_to_datetime @@ -46,18 +47,23 @@ class _PublishedPolicy(InnerDoc): sp = Text() pct = Integer() fo = Text() + np = Keyword() + testing = Keyword() + discovery_method = Keyword() class _DKIMResult(InnerDoc): domain = Text() selector = Text() result = Text() + human_result = Text() class _SPFResult(InnerDoc): domain = Text() scope = Text() results = Text() + human_result = Text() class _AggregateReportDoc(Document): @@ -96,17 +102,45 @@ class _AggregateReportDoc(Document): envelope_to = Text() dkim_results = Nested(_DKIMResult) spf_results = Nested(_SPFResult) + np = Keyword() + testing = Keyword() + discovery_method = Keyword() + generator = Text() def add_policy_override(self, type_: str, comment: str): self.policy_overrides.append(_PolicyOverride(type=type_, comment=comment)) - def add_dkim_result(self, domain: str, selector: str, result: _DKIMResult): + def add_dkim_result( + self, + domain: str, + selector: str, + result: _DKIMResult, + human_result: str = None, + ): self.dkim_results.append( - _DKIMResult(domain=domain, selector=selector, result=result) + _DKIMResult( + domain=domain, + selector=selector, + result=result, + human_result=human_result, + ) ) - def add_spf_result(self, domain: str, scope: str, result: _SPFResult): - self.spf_results.append(_SPFResult(domain=domain, scope=scope, result=result)) + def add_spf_result( + self, + domain: str, + scope: str, + result: _SPFResult, + human_result: str = None, + ): + self.spf_results.append( + _SPFResult( + domain=domain, + scope=scope, + result=result, + human_result=human_result, + ) + ) def save(self, **kwargs): # pyright: ignore[reportIncompatibleMethodOverride] self.passed_dmarc = False @@ -126,7 +160,7 @@ class _EmailAttachmentDoc(Document): sha256 = Text() -class _ForensicSampleDoc(InnerDoc): +class _FailureSampleDoc(InnerDoc): raw = Text() headers = Object() headers_only = Boolean() @@ -163,9 +197,9 @@ class _ForensicSampleDoc(InnerDoc): ) -class _ForensicReportDoc(Document): +class _FailureReportDoc(Document): class Index: - name = "dmarc_forensic" + name = "dmarc_failure" feedback_type = Text() user_agent = Text() @@ -186,7 +220,7 @@ class _ForensicReportDoc(Document): source_auth_failures = Text() dkim_domain = Text() original_rcpt_to = Text() - sample = Object(_ForensicSampleDoc) + sample = Object(_FailureSampleDoc) class _SMTPTLSFailureDetailsDoc(InnerDoc): @@ -366,20 +400,20 @@ def create_indexes(names: list[str], settings: Optional[dict[str, Any]] = None): def migrate_indexes( aggregate_indexes: Optional[list[str]] = None, - forensic_indexes: Optional[list[str]] = None, + failure_indexes: Optional[list[str]] = None, ): """ Updates index mappings Args: aggregate_indexes (list): A list of aggregate index names - forensic_indexes (list): A list of forensic index names + failure_indexes (list): A list of failure index names """ version = 2 if aggregate_indexes is None: aggregate_indexes = [] - if forensic_indexes is None: - forensic_indexes = [] + if failure_indexes is None: + failure_indexes = [] for aggregate_index_name in aggregate_indexes: if not Index(aggregate_index_name).exists(): continue @@ -409,7 +443,7 @@ def migrate_indexes( reindex(connections.get_connection(), aggregate_index_name, new_index_name) Index(aggregate_index_name).delete() - for forensic_index in forensic_indexes: + for failure_index in failure_indexes: pass @@ -425,7 +459,7 @@ def save_aggregate_report_to_opensearch( Saves a parsed DMARC aggregate report to OpenSearch Args: - aggregate_report (dict): A parsed forensic report + aggregate_report (dict): A parsed aggregate report index_suffix (str): The suffix of the name of the index to save to index_prefix (str): The prefix of the name of the index to save to monthly_indexes (bool): Use monthly indexes instead of daily indexes @@ -493,6 +527,9 @@ def save_aggregate_report_to_opensearch( sp=aggregate_report["policy_published"]["sp"], pct=aggregate_report["policy_published"]["pct"], fo=aggregate_report["policy_published"]["fo"], + np=aggregate_report["policy_published"].get("np"), + testing=aggregate_report["policy_published"].get("testing"), + discovery_method=aggregate_report["policy_published"].get("discovery_method"), ) for record in aggregate_report["records"]: @@ -537,6 +574,12 @@ def save_aggregate_report_to_opensearch( header_from=record["identifiers"]["header_from"], envelope_from=record["identifiers"]["envelope_from"], envelope_to=record["identifiers"]["envelope_to"], + np=aggregate_report["policy_published"].get("np"), + testing=aggregate_report["policy_published"].get("testing"), + discovery_method=aggregate_report["policy_published"].get( + "discovery_method" + ), + generator=metadata.get("generator"), ) for override in record["policy_evaluated"]["policy_override_reasons"]: @@ -549,6 +592,7 @@ def save_aggregate_report_to_opensearch( domain=dkim_result["domain"], selector=dkim_result["selector"], result=dkim_result["result"], + human_result=dkim_result.get("human_result"), ) for spf_result in record["auth_results"]["spf"]: @@ -556,6 +600,7 @@ def save_aggregate_report_to_opensearch( domain=spf_result["domain"], scope=spf_result["scope"], result=spf_result["result"], + human_result=spf_result.get("human_result"), ) index = "dmarc_aggregate" @@ -577,8 +622,8 @@ def save_aggregate_report_to_opensearch( raise OpenSearchError("OpenSearch error: {0}".format(e.__str__())) -def save_forensic_report_to_opensearch( - forensic_report: dict[str, Any], +def save_failure_report_to_opensearch( + failure_report: dict[str, Any], index_suffix: Optional[str] = None, index_prefix: Optional[str] = None, monthly_indexes: bool = False, @@ -586,10 +631,10 @@ def save_forensic_report_to_opensearch( number_of_replicas: int = 0, ): """ - Saves a parsed DMARC forensic report to OpenSearch + Saves a parsed DMARC failure report to OpenSearch Args: - forensic_report (dict): A parsed forensic report + failure_report (dict): A parsed failure report index_suffix (str): The suffix of the name of the index to save to index_prefix (str): The prefix of the name of the index to save to monthly_indexes (bool): Use monthly indexes instead of daily @@ -602,26 +647,28 @@ def save_forensic_report_to_opensearch( AlreadySaved """ - logger.info("Saving forensic report to OpenSearch") - forensic_report = forensic_report.copy() + logger.info("Saving failure report to OpenSearch") + failure_report = failure_report.copy() sample_date = None - if forensic_report["parsed_sample"]["date"] is not None: - sample_date = forensic_report["parsed_sample"]["date"] + if failure_report["parsed_sample"]["date"] is not None: + sample_date = failure_report["parsed_sample"]["date"] sample_date = human_timestamp_to_datetime(sample_date) - original_headers = forensic_report["parsed_sample"]["headers"] + original_headers = failure_report["parsed_sample"]["headers"] headers: dict[str, Any] = {} for original_header in original_headers: headers[original_header.lower()] = original_headers[original_header] - arrival_date = human_timestamp_to_datetime(forensic_report["arrival_date_utc"]) + arrival_date = human_timestamp_to_datetime(failure_report["arrival_date_utc"]) arrival_date_epoch_milliseconds = int(arrival_date.timestamp() * 1000) if index_suffix is not None: - search_index = "dmarc_forensic_{0}*".format(index_suffix) + search_index = "dmarc_failure_{0}*,dmarc_forensic_{0}*".format(index_suffix) else: - search_index = "dmarc_forensic*" + search_index = "dmarc_failure*,dmarc_forensic*" if index_prefix is not None: - search_index = "{0}{1}".format(index_prefix, search_index) + search_index = ",".join( + "{0}{1}".format(index_prefix, part) for part in search_index.split(",") + ) search = Search(index=search_index) q = Q(dict(match=dict(arrival_date=arrival_date_epoch_milliseconds))) @@ -662,67 +709,65 @@ def save_forensic_report_to_opensearch( if len(existing) > 0: raise AlreadySaved( - "A forensic sample to {0} from {1} " + "A failure sample to {0} from {1} " "with a subject of {2} and arrival date of {3} " "already exists in " - "OpenSearch".format( - to_, from_, subject, forensic_report["arrival_date_utc"] - ) + "OpenSearch".format(to_, from_, subject, failure_report["arrival_date_utc"]) ) - parsed_sample = forensic_report["parsed_sample"] - sample = _ForensicSampleDoc( - raw=forensic_report["sample"], + parsed_sample = failure_report["parsed_sample"] + sample = _FailureSampleDoc( + raw=failure_report["sample"], headers=headers, - headers_only=forensic_report["sample_headers_only"], + headers_only=failure_report["sample_headers_only"], date=sample_date, - subject=forensic_report["parsed_sample"]["subject"], + subject=failure_report["parsed_sample"]["subject"], filename_safe_subject=parsed_sample["filename_safe_subject"], - body=forensic_report["parsed_sample"]["body"], + body=failure_report["parsed_sample"]["body"], ) - for address in forensic_report["parsed_sample"]["to"]: + for address in failure_report["parsed_sample"]["to"]: sample.add_to(display_name=address["display_name"], address=address["address"]) - for address in forensic_report["parsed_sample"]["reply_to"]: + for address in failure_report["parsed_sample"]["reply_to"]: sample.add_reply_to( display_name=address["display_name"], address=address["address"] ) - for address in forensic_report["parsed_sample"]["cc"]: + for address in failure_report["parsed_sample"]["cc"]: sample.add_cc(display_name=address["display_name"], address=address["address"]) - for address in forensic_report["parsed_sample"]["bcc"]: + for address in failure_report["parsed_sample"]["bcc"]: sample.add_bcc(display_name=address["display_name"], address=address["address"]) - for attachment in forensic_report["parsed_sample"]["attachments"]: + for attachment in failure_report["parsed_sample"]["attachments"]: sample.add_attachment( filename=attachment["filename"], content_type=attachment["mail_content_type"], sha256=attachment["sha256"], ) try: - forensic_doc = _ForensicReportDoc( - feedback_type=forensic_report["feedback_type"], - user_agent=forensic_report["user_agent"], - version=forensic_report["version"], - original_mail_from=forensic_report["original_mail_from"], + failure_doc = _FailureReportDoc( + feedback_type=failure_report["feedback_type"], + user_agent=failure_report["user_agent"], + version=failure_report["version"], + original_mail_from=failure_report["original_mail_from"], arrival_date=arrival_date_epoch_milliseconds, - domain=forensic_report["reported_domain"], - original_envelope_id=forensic_report["original_envelope_id"], - authentication_results=forensic_report["authentication_results"], - delivery_results=forensic_report["delivery_result"], - source_ip_address=forensic_report["source"]["ip_address"], - source_country=forensic_report["source"]["country"], - source_reverse_dns=forensic_report["source"]["reverse_dns"], - source_base_domain=forensic_report["source"]["base_domain"], - source_asn=forensic_report["source"]["asn"], - source_as_name=forensic_report["source"]["as_name"], - source_as_domain=forensic_report["source"]["as_domain"], - authentication_mechanisms=forensic_report["authentication_mechanisms"], - auth_failure=forensic_report["auth_failure"], - dkim_domain=forensic_report["dkim_domain"], - original_rcpt_to=forensic_report["original_rcpt_to"], + domain=failure_report["reported_domain"], + original_envelope_id=failure_report["original_envelope_id"], + authentication_results=failure_report["authentication_results"], + delivery_results=failure_report["delivery_result"], + source_ip_address=failure_report["source"]["ip_address"], + source_country=failure_report["source"]["country"], + source_reverse_dns=failure_report["source"]["reverse_dns"], + source_base_domain=failure_report["source"]["base_domain"], + source_asn=failure_report["source"]["asn"], + source_as_name=failure_report["source"]["as_name"], + source_as_domain=failure_report["source"]["as_domain"], + authentication_mechanisms=failure_report["authentication_mechanisms"], + auth_failure=failure_report["auth_failure"], + dkim_domain=failure_report["dkim_domain"], + original_rcpt_to=failure_report["original_rcpt_to"], sample=sample, ) - index = "dmarc_forensic" + index = "dmarc_failure" if index_suffix: index = "{0}_{1}".format(index, index_suffix) if index_prefix: @@ -736,14 +781,14 @@ def save_forensic_report_to_opensearch( number_of_shards=number_of_shards, number_of_replicas=number_of_replicas ) create_indexes([index], index_settings) - forensic_doc.meta.index = index + failure_doc.meta.index = index try: - forensic_doc.save() + failure_doc.save() except Exception as e: raise OpenSearchError("OpenSearch error: {0}".format(e.__str__())) except KeyError as e: - raise InvalidForensicReport( - "Forensic report missing required field: {0}".format(e.__str__()) + raise InvalidFailureReport( + "Failure report missing required field: {0}".format(e.__str__()) ) @@ -897,3 +942,9 @@ def save_smtp_tls_report_to_opensearch( smtp_tls_doc.save() except Exception as e: raise OpenSearchError("OpenSearch error: {0}".format(e.__str__())) + + +# Backward-compatible aliases +_ForensicSampleDoc = _FailureSampleDoc +_ForensicReportDoc = _FailureReportDoc +save_forensic_report_to_opensearch = save_failure_report_to_opensearch diff --git a/parsedmarc/s3.py b/parsedmarc/s3.py index 99e03b3..09cd8c4 100644 --- a/parsedmarc/s3.py +++ b/parsedmarc/s3.py @@ -56,8 +56,8 @@ class S3Client(object): def save_aggregate_report_to_s3(self, report: dict[str, Any]): self.save_report_to_s3(report, "aggregate") - def save_forensic_report_to_s3(self, report: dict[str, Any]): - self.save_report_to_s3(report, "forensic") + def save_failure_report_to_s3(self, report: dict[str, Any]): + self.save_report_to_s3(report, "failure") def save_smtp_tls_report_to_s3(self, report: dict[str, Any]): self.save_report_to_s3(report, "smtp_tls") @@ -101,3 +101,7 @@ class S3Client(object): self.s3.meta.client.close() except Exception: pass + + +# Backward-compatible aliases +S3Client.save_forensic_report_to_s3 = S3Client.save_failure_report_to_s3 diff --git a/parsedmarc/splunk.py b/parsedmarc/splunk.py index 7e3754f..eaffad8 100644 --- a/parsedmarc/splunk.py +++ b/parsedmarc/splunk.py @@ -139,28 +139,28 @@ class HECClient(object): if response["code"] != 0: raise SplunkError(response["text"]) - def save_forensic_reports_to_splunk( + def save_failure_reports_to_splunk( self, - forensic_reports: Union[list[dict[str, Any]], dict[str, Any]], + failure_reports: Union[list[dict[str, Any]], dict[str, Any]], ): """ - Saves forensic DMARC reports to Splunk + Saves failure DMARC reports to Splunk Args: - forensic_reports (list): A list of forensic report dictionaries + failure_reports (list): A list of failure report dictionaries to save in Splunk """ - logger.debug("Saving forensic reports to Splunk") - if isinstance(forensic_reports, dict): - forensic_reports = [forensic_reports] + logger.debug("Saving failure reports to Splunk") + if isinstance(failure_reports, dict): + failure_reports = [failure_reports] - if len(forensic_reports) < 1: + if len(failure_reports) < 1: return json_str = "" - for report in forensic_reports: + for report in failure_reports: data = self._common_data.copy() - data["sourcetype"] = "dmarc:forensic" + data["sourcetype"] = "dmarc:failure" timestamp = human_timestamp_to_unix_timestamp(report["arrival_date_utc"]) data["time"] = timestamp data["event"] = report.copy() @@ -220,3 +220,7 @@ class HECClient(object): def close(self): """Close the underlying HTTP session.""" self.session.close() + + +# Backward-compatible aliases +HECClient.save_forensic_reports_to_splunk = HECClient.save_failure_reports_to_splunk diff --git a/parsedmarc/syslog.py b/parsedmarc/syslog.py index ec8e757..883987a 100644 --- a/parsedmarc/syslog.py +++ b/parsedmarc/syslog.py @@ -13,7 +13,7 @@ from typing import Any, Optional from parsedmarc import ( parsed_aggregate_reports_to_csv_rows, - parsed_forensic_reports_to_csv_rows, + parsed_failure_reports_to_csv_rows, parsed_smtp_tls_reports_to_csv_rows, ) @@ -170,8 +170,8 @@ class SyslogClient(object): for row in rows: self.logger.info(json.dumps(row)) - def save_forensic_report_to_syslog(self, forensic_reports: list[dict[str, Any]]): - rows = parsed_forensic_reports_to_csv_rows(forensic_reports) + def save_failure_report_to_syslog(self, failure_reports: list[dict[str, Any]]): + rows = parsed_failure_reports_to_csv_rows(failure_reports) for row in rows: self.logger.info(json.dumps(row)) @@ -184,3 +184,7 @@ class SyslogClient(object): """Remove and close the syslog handler, releasing its socket.""" self.logger.removeHandler(self.log_handler) self.log_handler.close() + + +# Backward-compatible aliases +SyslogClient.save_forensic_report_to_syslog = SyslogClient.save_failure_report_to_syslog diff --git a/parsedmarc/types.py b/parsedmarc/types.py index 6a7c325..7e07bc6 100644 --- a/parsedmarc/types.py +++ b/parsedmarc/types.py @@ -8,7 +8,7 @@ from typing import Any, Dict, List, Literal, Optional, TypedDict, Union # For optional keys, use total=False TypedDicts. -ReportType = Literal["aggregate", "forensic", "smtp_tls"] +ReportType = Literal["aggregate", "failure", "smtp_tls"] class AggregateReportMetadata(TypedDict): @@ -21,6 +21,7 @@ class AggregateReportMetadata(TypedDict): timespan_requires_normalization: bool original_timespan_seconds: int errors: List[str] + generator: Optional[str] class AggregatePolicyPublished(TypedDict): @@ -29,8 +30,11 @@ class AggregatePolicyPublished(TypedDict): aspf: str p: str sp: str - pct: str - fo: str + pct: Optional[str] + fo: Optional[str] + np: Optional[str] + testing: Optional[str] + discovery_method: Optional[str] class IPSourceInfo(TypedDict): @@ -66,12 +70,14 @@ class AggregateAuthResultDKIM(TypedDict): domain: str result: str selector: str + human_result: Optional[str] class AggregateAuthResultSPF(TypedDict): domain: str result: str scope: str + human_result: Optional[str] class AggregateAuthResults(TypedDict): @@ -122,7 +128,7 @@ ParsedEmail = TypedDict( "ParsedEmail", { # This is a lightly-specified version of mailsuite/mailparser JSON. - # It focuses on the fields parsedmarc uses in forensic handling. + # It focuses on the fields parsedmarc uses in failure report handling. "headers": Dict[str, Any], "subject": Optional[str], "filename_safe_subject": Optional[str], @@ -141,7 +147,7 @@ ParsedEmail = TypedDict( ) -class ForensicReport(TypedDict): +class FailureReport(TypedDict): feedback_type: Optional[str] user_agent: Optional[str] version: Optional[str] @@ -162,6 +168,10 @@ class ForensicReport(TypedDict): parsed_sample: ParsedEmail +# Backward-compatible alias +ForensicReport = FailureReport + + class SMTPTLSFailureDetails(TypedDict): result_type: str failed_session_count: int @@ -204,9 +214,13 @@ class AggregateParsedReport(TypedDict): report: AggregateReport -class ForensicParsedReport(TypedDict): - report_type: Literal["forensic"] - report: ForensicReport +class FailureParsedReport(TypedDict): + report_type: Literal["failure"] + report: FailureReport + + +# Backward-compatible alias +ForensicParsedReport = FailureParsedReport class SMTPTLSParsedReport(TypedDict): @@ -214,10 +228,10 @@ class SMTPTLSParsedReport(TypedDict): report: SMTPTLSReport -ParsedReport = Union[AggregateParsedReport, ForensicParsedReport, SMTPTLSParsedReport] +ParsedReport = Union[AggregateParsedReport, FailureParsedReport, SMTPTLSParsedReport] class ParsingResults(TypedDict): aggregate_reports: List[AggregateReport] - forensic_reports: List[ForensicReport] + failure_reports: List[FailureReport] smtp_tls_reports: List[SMTPTLSReport] diff --git a/parsedmarc/utils.py b/parsedmarc/utils.py index 4e880ff..35d1631 100644 --- a/parsedmarc/utils.py +++ b/parsedmarc/utils.py @@ -57,7 +57,7 @@ _RETRYABLE_DNS_ERRORS = ( parenthesis_regex = re.compile(r"\s*\(.*\)\s*") -null_file = open(os.devnull, "w") +null_file = subprocess.DEVNULL mailparser_logger = logging.getLogger("mailparser") mailparser_logger.setLevel(logging.CRITICAL) psl = publicsuffixlist.PublicSuffixList() diff --git a/parsedmarc/webhook.py b/parsedmarc/webhook.py index 9b6f66f..c122a72 100644 --- a/parsedmarc/webhook.py +++ b/parsedmarc/webhook.py @@ -16,7 +16,7 @@ class WebhookClient(object): def __init__( self, aggregate_url: str, - forensic_url: str, + failure_url: str, smtp_tls_url: str, timeout: Optional[int] = 60, ): @@ -24,12 +24,12 @@ class WebhookClient(object): Initializes the WebhookClient Args: aggregate_url (str): The aggregate report webhook url - forensic_url (str): The forensic report webhook url + failure_url (str): The failure report webhook url smtp_tls_url (str): The smtp_tls report webhook url timeout (int): The timeout to use when calling the webhooks """ self.aggregate_url = aggregate_url - self.forensic_url = forensic_url + self.failure_url = failure_url self.smtp_tls_url = smtp_tls_url self.timeout = timeout self.session = requests.Session() @@ -38,9 +38,9 @@ class WebhookClient(object): "Content-Type": "application/json", } - def save_forensic_report_to_webhook(self, report: str): + def save_failure_report_to_webhook(self, report: str): try: - self._send_to_webhook(self.forensic_url, report) + self._send_to_webhook(self.failure_url, report) except Exception as error_: logger.error("Webhook Error: {0}".format(error_.__str__())) @@ -67,3 +67,9 @@ class WebhookClient(object): def close(self): """Close the underlying HTTP session.""" self.session.close() + + +# Backward-compatible aliases +WebhookClient.save_forensic_report_to_webhook = ( + WebhookClient.save_failure_report_to_webhook +) diff --git a/samples/aggregate/dmarcbis-draft-sample.xml b/samples/aggregate/dmarcbis-draft-sample.xml new file mode 100644 index 0000000..b75408c --- /dev/null +++ b/samples/aggregate/dmarcbis-draft-sample.xml @@ -0,0 +1,48 @@ + + 1.0 + + Sample Reporter + report_sender@example-reporter.com + ... + 3v98abbp8ya9n3va8yr8oa3ya + + 302832000 + 302918399 + + Example DMARC Aggregate Reporter v1.2 + + + example.com +

quarantine

+ none + none + n + treewalk + + + + 192.0.2.123 + 123 + + pass + pass + fail + + + + example.com + example.com + + + + example.com + pass + abc123 + + + example.com + fail + + + + diff --git a/samples/aggregate/dmarcbis-example.net!example.com!1700000000!1700086399.xml b/samples/aggregate/dmarcbis-example.net!example.com!1700000000!1700086399.xml new file mode 100644 index 0000000..4a1baa9 --- /dev/null +++ b/samples/aggregate/dmarcbis-example.net!example.com!1700000000!1700086399.xml @@ -0,0 +1,77 @@ + + + 2.0 + + example.net + postmaster@example.net + dmarcbis-test-report-001 + + 1700000000 + 1700086399 + + + + example.com + s + s +

reject

+ quarantine + reject + y + treewalk + 1 + + + + 198.51.100.1 + 5 + + none + pass + pass + + + + example.com + example.com + + + + example.com + selector1 + pass + + + example.com + mfrom + pass + + + + + + 203.0.113.10 + 2 + + reject + fail + fail + + other + sender not authorized + + + + + spoofed.example.com + example.com + + + + spoofed.example.com + mfrom + fail + + + + diff --git a/samples/forensic/DMARC Failure Report for domain.de (mail-from=sharepoint@domain.de, ip=10.10.10.10).eml b/samples/failure/DMARC Failure Report for domain.de (mail-from=sharepoint@domain.de, ip=10.10.10.10).eml similarity index 100% rename from samples/forensic/DMARC Failure Report for domain.de (mail-from=sharepoint@domain.de, ip=10.10.10.10).eml rename to samples/failure/DMARC Failure Report for domain.de (mail-from=sharepoint@domain.de, ip=10.10.10.10).eml diff --git a/samples/forensic/[Netease DMARC Failure Report] Rent Reminder.eml b/samples/failure/[Netease DMARC Failure Report] Rent Reminder.eml similarity index 100% rename from samples/forensic/[Netease DMARC Failure Report] Rent Reminder.eml rename to samples/failure/[Netease DMARC Failure Report] Rent Reminder.eml diff --git a/samples/forensic/dmarc_ruf_report_linkedin.crlf.eml b/samples/failure/dmarc_ruf_report_linkedin.crlf.eml similarity index 100% rename from samples/forensic/dmarc_ruf_report_linkedin.crlf.eml rename to samples/failure/dmarc_ruf_report_linkedin.crlf.eml diff --git a/samples/forensic/dmarc_ruf_report_linkedin.eml b/samples/failure/dmarc_ruf_report_linkedin.eml similarity index 100% rename from samples/forensic/dmarc_ruf_report_linkedin.eml rename to samples/failure/dmarc_ruf_report_linkedin.eml diff --git a/splunk/README.rst b/splunk/README.rst index acf7bce..631bb64 100644 --- a/splunk/README.rst +++ b/splunk/README.rst @@ -60,10 +60,10 @@ Create Dashboards 9. Click Save 10. Click Dashboards 11. Click Create New Dashboard -12. Use a descriptive title, such as "Forensic DMARC Data" +12. Use a descriptive title, such as "Failure DMARC Data" 13. Click Create Dashboard 14. Click on the Source button -15. Paste the content of ''dmarc_forensic_dashboard.xml`` into the source editor +15. Paste the content of ''dmarc_failure_dashboard.xml`` into the source editor 16. If the index storing the DMARC data is not named email, replace index="email" accordingly 17. Click Save diff --git a/splunk/dmarc_forensic_dashboard.xml b/splunk/dmarc_failure_dashboard.xml similarity index 85% rename from splunk/dmarc_forensic_dashboard.xml rename to splunk/dmarc_failure_dashboard.xml index be3b290..5854d4a 100644 --- a/splunk/dmarc_forensic_dashboard.xml +++ b/splunk/dmarc_failure_dashboard.xml @@ -1,8 +1,8 @@
- + - index="email" sourcetype="dmarc:forensic" parsed_sample.headers.From=$header_from$ parsed_sample.headers.To=$header_to$ parsed_sample.headers.Subject=$header_subject$ source.ip_address=$source_ip_address$ source.reverse_dns=$source_reverse_dns$ source.country=$source_country$ + index="email" (sourcetype="dmarc:failure" OR sourcetype="dmarc:forensic") parsed_sample.headers.From=$header_from$ parsed_sample.headers.To=$header_to$ parsed_sample.headers.Subject=$header_subject$ source.ip_address=$source_ip_address$ source.reverse_dns=$source_reverse_dns$ source.country=$source_country$ | table * $time_range.earliest$ @@ -43,7 +43,7 @@ - Forensic samples + Failure samples | table arrival_date_utc authentication_results parsed_sample.headers.From,parsed_sample.headers.To,parsed_sample.headers.Subject | sort -arrival_date_utc @@ -59,7 +59,7 @@ - Forensic samples by country + Failure samples by country | iplocation source.ip_address| stats count by Country | geom geo_countries featureIdField="Country" @@ -72,7 +72,7 @@ - Forensic samples by IP address + Failure samples by IP address
| iplocation source.ip_address | stats count by source.ip_address,source.reverse_dns | sort -count @@ -85,7 +85,7 @@
- Forensic samples by country ISO code + Failure samples by country ISO code | stats count by source.country | sort - count diff --git a/tests.py b/tests.py index 54f34cb..5107746 100755 --- a/tests.py +++ b/tests.py @@ -12,20 +12,28 @@ import tempfile import unittest from base64 import urlsafe_b64encode from configparser import ConfigParser +from datetime import datetime, timedelta, timezone +from io import BytesIO from glob import glob from pathlib import Path from tempfile import NamedTemporaryFile, TemporaryDirectory -from typing import cast +from typing import BinaryIO, cast from types import SimpleNamespace from unittest.mock import MagicMock, patch +from expiringdict import ExpiringDict from lxml import etree # type: ignore[import-untyped] from googleapiclient.errors import HttpError from httplib2 import Response from imapclient.exceptions import IMAPClientError +import dns.exception +import requests + import parsedmarc import parsedmarc.cli +import parsedmarc.webhook +from parsedmarc.types import AggregateReport, FailureReport, SMTPTLSReport from parsedmarc.mail.gmail import GmailConnection from parsedmarc.mail.gmail import _get_creds from parsedmarc.mail.graph import MSGraphConnection @@ -63,12 +71,13 @@ class Test(unittest.TestCase): # Example from Wikipedia Base64 article b64_str = "YW55IGNhcm5hbCBwbGVhcw" decoded_str = parsedmarc.utils.decode_base64(b64_str) - assert decoded_str == b"any carnal pleas" + self.assertEqual(decoded_str, b"any carnal pleas") def testPSLDownload(self): + """Test Public Suffix List domain lookups""" subdomain = "foo.example.com" result = parsedmarc.utils.get_base_domain(subdomain) - assert result == "example.com" + self.assertEqual(result, "example.com") # psl_overrides.txt intentionally folds CDN-customer PTRs so every # sender on the same network clusters under one display key. @@ -81,12 +90,10 @@ class Test(unittest.TestCase): def testExtractReportXMLComparator(self): """Test XML comparator function""" - xmlnice_file = open("samples/extract_report/nice-input.xml") - xmlnice = xmlnice_file.read() - xmlnice_file.close() - xmlchanged_file = open("samples/extract_report/changed-input.xml") - xmlchanged = minify_xml(xmlchanged_file.read()) - xmlchanged_file.close() + with open("samples/extract_report/nice-input.xml") as f: + xmlnice = f.read() + with open("samples/extract_report/changed-input.xml") as f: + xmlchanged = minify_xml(f.read()) self.assertTrue(compare_xml(xmlnice, xmlnice)) self.assertTrue(compare_xml(xmlchanged, xmlchanged)) self.assertFalse(compare_xml(xmlnice, xmlchanged)) @@ -101,21 +108,19 @@ class Test(unittest.TestCase): data = f.read() print("Testing {0}: ".format(file), end="") xmlout = parsedmarc.extract_report(data) - xmlin_file = open("samples/extract_report/nice-input.xml") - xmlin = xmlin_file.read() - xmlin_file.close() + with open("samples/extract_report/nice-input.xml") as f: + xmlin = f.read() self.assertTrue(compare_xml(xmlout, xmlin)) print("Passed!") def testExtractReportXML(self): """Test extract report function for XML input""" print() - report_path = "samples/extract_report/nice-input.xml" - print("Testing {0}: ".format(report_path), end="") - xmlout = parsedmarc.extract_report_from_file_path(report_path) - xmlin_file = open("samples/extract_report/nice-input.xml") - xmlin = xmlin_file.read() - xmlin_file.close() + file = "samples/extract_report/nice-input.xml" + print("Testing {0}: ".format(file), end="") + xmlout = parsedmarc.extract_report_from_file_path(file) + with open("samples/extract_report/nice-input.xml") as f: + xmlin = f.read() self.assertTrue(compare_xml(xmlout, xmlin)) print("Passed!") @@ -133,9 +138,8 @@ class Test(unittest.TestCase): file = "samples/extract_report/nice-input.xml.gz" print("Testing {0}: ".format(file), end="") xmlout = parsedmarc.extract_report_from_file_path(file) - xmlin_file = open("samples/extract_report/nice-input.xml") - xmlin = xmlin_file.read() - xmlin_file.close() + with open("samples/extract_report/nice-input.xml") as f: + xmlin = f.read() self.assertTrue(compare_xml(xmlout, xmlin)) print("Passed!") @@ -145,13 +149,11 @@ class Test(unittest.TestCase): file = "samples/extract_report/nice-input.xml.zip" print("Testing {0}: ".format(file), end="") xmlout = parsedmarc.extract_report_from_file_path(file) - xmlin_file = open("samples/extract_report/nice-input.xml") - xmlin = minify_xml(xmlin_file.read()) - xmlin_file.close() + with open("samples/extract_report/nice-input.xml") as f: + xmlin = minify_xml(f.read()) self.assertTrue(compare_xml(xmlout, xmlin)) - xmlin_file = open("samples/extract_report/changed-input.xml") - xmlin = xmlin_file.read() - xmlin_file.close() + with open("samples/extract_report/changed-input.xml") as f: + xmlin = f.read() self.assertFalse(compare_xml(xmlout, xmlin)) print("Passed!") @@ -164,7 +166,8 @@ class Test(unittest.TestCase): offline=True, ) assert result["report_type"] == "aggregate" - self.assertEqual(result["report"]["report_metadata"]["org_name"], "outlook.com") + report = cast(AggregateReport, result["report"]) + self.assertEqual(report["report_metadata"]["org_name"], "outlook.com") def testParseReportFileAcceptsPathForEmail(self): report_path = Path( @@ -175,7 +178,8 @@ class Test(unittest.TestCase): offline=True, ) assert result["report_type"] == "aggregate" - self.assertEqual(result["report"]["report_metadata"]["org_name"], "google.com") + report = cast(AggregateReport, result["report"]) + self.assertEqual(report["report_metadata"]["org_name"], "google.com") def testAggregateSamples(self): """Test sample aggregate/rua DMARC reports""" @@ -185,11 +189,14 @@ class Test(unittest.TestCase): if os.path.isdir(sample_path): continue print("Testing {0}: ".format(sample_path), end="") - result = parsedmarc.parse_report_file( - sample_path, always_use_local_files=True, offline=OFFLINE_MODE - ) - assert result["report_type"] == "aggregate" - parsedmarc.parsed_aggregate_reports_to_csv(result["report"]) + with self.subTest(sample=sample_path): + result = parsedmarc.parse_report_file( + sample_path, always_use_local_files=True, offline=OFFLINE_MODE + ) + assert result["report_type"] == "aggregate" + parsedmarc.parsed_aggregate_reports_to_csv( + cast(AggregateReport, result["report"]) + ) print("Passed!") def testEmptySample(self): @@ -197,23 +204,164 @@ class Test(unittest.TestCase): with self.assertRaises(parsedmarc.ParserError): parsedmarc.parse_report_file("samples/empty.xml", offline=OFFLINE_MODE) - def testForensicSamples(self): - """Test sample forensic/ruf/failure DMARC reports""" + def testFailureSamples(self): + """Test sample failure/ruf DMARC reports""" print() - sample_paths = glob("samples/forensic/*.eml") + sample_paths = glob("samples/failure/*.eml") for sample_path in sample_paths: print("Testing {0}: ".format(sample_path), end="") - with open(sample_path) as sample_file: - sample_content = sample_file.read() - email_result = parsedmarc.parse_report_email( - sample_content, offline=OFFLINE_MODE + with self.subTest(sample=sample_path): + with open(sample_path) as sample_file: + sample_content = sample_file.read() + email_result = parsedmarc.parse_report_email( + sample_content, offline=OFFLINE_MODE + ) + assert email_result["report_type"] == "failure" + result = parsedmarc.parse_report_file(sample_path, offline=OFFLINE_MODE) + assert result["report_type"] == "failure" + parsedmarc.parsed_failure_reports_to_csv( + cast(FailureReport, result["report"]) ) - assert email_result["report_type"] == "forensic" - result = parsedmarc.parse_report_file(sample_path, offline=OFFLINE_MODE) - assert result["report_type"] == "forensic" - parsedmarc.parsed_forensic_reports_to_csv(result["report"]) print("Passed!") + def testFailureReportBackwardCompat(self): + """Test that old forensic function aliases still work""" + self.assertIs( + parsedmarc.parse_forensic_report, + parsedmarc.parse_failure_report, + ) + self.assertIs( + parsedmarc.parsed_forensic_reports_to_csv, + parsedmarc.parsed_failure_reports_to_csv, + ) + self.assertIs( + parsedmarc.parsed_forensic_reports_to_csv_rows, + parsedmarc.parsed_failure_reports_to_csv_rows, + ) + self.assertIs( + parsedmarc.InvalidForensicReport, + parsedmarc.InvalidFailureReport, + ) + + def testDMARCbisDraftSample(self): + """Test parsing the sample report from the DMARCbis aggregate draft""" + print() + sample_path = "samples/aggregate/dmarcbis-draft-sample.xml" + print("Testing {0}: ".format(sample_path), end="") + result = parsedmarc.parse_report_file( + sample_path, always_use_local_files=True, offline=True + ) + report = cast(AggregateReport, result["report"]) + + # Verify report_type + self.assertEqual(result["report_type"], "aggregate") + + # Verify xml_schema + self.assertEqual(report["xml_schema"], "1.0") + + # Verify report_metadata + metadata = report["report_metadata"] + self.assertEqual(metadata["org_name"], "Sample Reporter") + self.assertEqual(metadata["org_email"], "report_sender@example-reporter.com") + self.assertEqual(metadata["org_extra_contact_info"], "...") + self.assertEqual(metadata["report_id"], "3v98abbp8ya9n3va8yr8oa3ya") + self.assertEqual( + metadata["generator"], + "Example DMARC Aggregate Reporter v1.2", + ) + + # Verify DMARCbis policy_published fields + pp = report["policy_published"] + self.assertEqual(pp["domain"], "example.com") + self.assertEqual(pp["p"], "quarantine") + self.assertEqual(pp["sp"], "none") + self.assertEqual(pp["np"], "none") + self.assertEqual(pp["testing"], "n") + self.assertEqual(pp["discovery_method"], "treewalk") + # adkim/aspf default when not in XML + self.assertEqual(pp["adkim"], "r") + self.assertEqual(pp["aspf"], "r") + # pct/fo are None on DMARCbis reports (not used) + self.assertIsNone(pp["pct"]) + self.assertIsNone(pp["fo"]) + + # Verify record + self.assertEqual(len(report["records"]), 1) + rec = report["records"][0] + self.assertEqual(rec["source"]["ip_address"], "192.0.2.123") + self.assertEqual(rec["count"], 123) + self.assertEqual(rec["policy_evaluated"]["disposition"], "pass") + self.assertEqual(rec["policy_evaluated"]["dkim"], "pass") + self.assertEqual(rec["policy_evaluated"]["spf"], "fail") + + # Verify DKIM auth result with human_result + self.assertEqual(len(rec["auth_results"]["dkim"]), 1) + dkim = rec["auth_results"]["dkim"][0] + self.assertEqual(dkim["domain"], "example.com") + self.assertEqual(dkim["selector"], "abc123") + self.assertEqual(dkim["result"], "pass") + self.assertIsNone(dkim["human_result"]) + + # Verify SPF auth result with human_result + self.assertEqual(len(rec["auth_results"]["spf"]), 1) + spf = rec["auth_results"]["spf"][0] + self.assertEqual(spf["domain"], "example.com") + self.assertEqual(spf["result"], "fail") + self.assertIsNone(spf["human_result"]) + + # Verify CSV output includes new fields + csv = parsedmarc.parsed_aggregate_reports_to_csv(report) + header = csv.split("\n")[0] + self.assertIn("np", header.split(",")) + self.assertIn("testing", header.split(",")) + self.assertIn("discovery_method", header.split(",")) + print("Passed!") + + def testDMARCbisFieldsWithRFC7489(self): + """Test that RFC 7489 reports have None for DMARCbis-only fields""" + print() + sample_path = ( + "samples/aggregate/example.net!example.com!1529366400!1529452799.xml" + ) + print("Testing {0}: ".format(sample_path), end="") + result = parsedmarc.parse_report_file( + sample_path, always_use_local_files=True, offline=True + ) + report = cast(AggregateReport, result["report"]) + pp = report["policy_published"] + + # RFC 7489 fields present + self.assertEqual(pp["pct"], "100") + self.assertEqual(pp["fo"], "0") + + # DMARCbis fields absent (None) + self.assertIsNone(pp["np"]) + self.assertIsNone(pp["testing"]) + self.assertIsNone(pp["discovery_method"]) + + # generator absent (None) + self.assertIsNone(report["report_metadata"]["generator"]) + print("Passed!") + + def testDMARCbisWithExplicitFields(self): + """Test DMARCbis report with explicit testing and discovery_method""" + print() + sample_path = ( + "samples/aggregate/" + "dmarcbis-example.net!example.com!1700000000!1700086399.xml" + ) + print("Testing {0}: ".format(sample_path), end="") + result = parsedmarc.parse_report_file( + sample_path, always_use_local_files=True, offline=True + ) + report = cast(AggregateReport, result["report"]) + pp = report["policy_published"] + + self.assertEqual(pp["np"], "reject") + self.assertEqual(pp["testing"], "y") + self.assertEqual(pp["discovery_method"], "treewalk") + print("Passed!") + def testSmtpTlsSamples(self): """Test sample SMTP TLS reports""" print() @@ -222,9 +370,12 @@ class Test(unittest.TestCase): if os.path.isdir(sample_path): continue print("Testing {0}: ".format(sample_path), end="") - result = parsedmarc.parse_report_file(sample_path, offline=OFFLINE_MODE) - assert result["report_type"] == "smtp_tls" - parsedmarc.parsed_smtp_tls_reports_to_csv(result["report"]) + with self.subTest(sample=sample_path): + result = parsedmarc.parse_report_file(sample_path, offline=OFFLINE_MODE) + assert result["report_type"] == "smtp_tls" + parsedmarc.parsed_smtp_tls_reports_to_csv( + cast(SMTPTLSReport, result["report"]) + ) print("Passed!") def testIpAddressInfoSurfacesASNFields(self): @@ -418,7 +569,7 @@ class Test(unittest.TestCase): mock_imap_connection.return_value = object() mock_get_reports.return_value = { "aggregate_reports": [], - "forensic_reports": [], + "failure_reports": [], "smtp_tls_reports": [], } @@ -468,7 +619,7 @@ aws_service = aoss mock_imap_connection.return_value = object() mock_get_reports.return_value = { "aggregate_reports": [{"policy_published": {"domain": "example.com"}}], - "forensic_reports": [], + "failure_reports": [], "smtp_tls_reports": [], } mock_save_aggregate.side_effect = parsedmarc.elastic.ElasticsearchError( @@ -518,7 +669,7 @@ hosts = localhost mock_imap_connection.return_value = object() mock_get_reports.return_value = { "aggregate_reports": [{"policy_published": {"domain": "example.com"}}], - "forensic_reports": [], + "failure_reports": [], "smtp_tls_reports": [], } mock_save_aggregate.side_effect = parsedmarc.elastic.ElasticsearchError( @@ -550,10 +701,10 @@ hosts = localhost mock_save_aggregate.assert_called_once() - @patch("parsedmarc.cli.opensearch.save_forensic_report_to_opensearch") + @patch("parsedmarc.cli.opensearch.save_failure_report_to_opensearch") @patch("parsedmarc.cli.opensearch.migrate_indexes") @patch("parsedmarc.cli.opensearch.set_hosts") - @patch("parsedmarc.cli.elastic.save_forensic_report_to_elasticsearch") + @patch("parsedmarc.cli.elastic.save_failure_report_to_elasticsearch") @patch("parsedmarc.cli.elastic.save_aggregate_report_to_elasticsearch") @patch("parsedmarc.cli.elastic.migrate_indexes") @patch("parsedmarc.cli.elastic.set_hosts") @@ -566,27 +717,27 @@ hosts = localhost _mock_es_set_hosts, _mock_es_migrate, mock_save_aggregate, - _mock_save_forensic_elastic, + _mock_save_failure_elastic, _mock_os_set_hosts, _mock_os_migrate, - mock_save_forensic_opensearch, + mock_save_failure_opensearch, ): mock_imap_connection.return_value = object() mock_get_reports.return_value = { "aggregate_reports": [{"policy_published": {"domain": "example.com"}}], - "forensic_reports": [{"reported_domain": "example.com"}], + "failure_reports": [{"reported_domain": "example.com"}], "smtp_tls_reports": [], } mock_save_aggregate.side_effect = parsedmarc.elastic.ElasticsearchError( "aggregate sink failed" ) - mock_save_forensic_opensearch.side_effect = ( - parsedmarc.cli.opensearch.OpenSearchError("forensic sink failed") + mock_save_failure_opensearch.side_effect = ( + parsedmarc.cli.opensearch.OpenSearchError("failure sink failed") ) config = """[general] save_aggregate = true -save_forensic = true +save_failure = true fail_on_output_error = true silent = true @@ -614,7 +765,7 @@ hosts = localhost self.assertEqual(ctx.exception.code, 1) mock_save_aggregate.assert_called_once() - mock_save_forensic_opensearch.assert_called_once() + mock_save_failure_opensearch.assert_called_once() class _FakeGraphResponse: @@ -1296,7 +1447,7 @@ class TestGmailAuthModes(unittest.TestCase): mock_gmail_connection.return_value = MagicMock() mock_get_mailbox_reports.return_value = { "aggregate_reports": [], - "forensic_reports": [], + "failure_reports": [], "smtp_tls_reports": [], } config = """[general] @@ -1330,7 +1481,7 @@ scopes = https://www.googleapis.com/auth/gmail.modify mock_gmail_connection.return_value = MagicMock() mock_get_reports.return_value = { "aggregate_reports": [], - "forensic_reports": [], + "failure_reports": [], "smtp_tls_reports": [], } config = """[general] @@ -1462,7 +1613,7 @@ class TestMailboxWatchSince(unittest.TestCase): mock_imap_connection.return_value = object() mock_get_mailbox_reports.return_value = { "aggregate_reports": [], - "forensic_reports": [], + "failure_reports": [], "smtp_tls_reports": [], } mock_watch_inbox.side_effect = FileExistsError("stop-watch-loop") @@ -1556,7 +1707,7 @@ class TestMailboxPerformance(unittest.TestCase): mock_graph_connection.return_value = object() mock_get_mailbox_reports.return_value = { "aggregate_reports": [], - "forensic_reports": [], + "failure_reports": [], "smtp_tls_reports": [], } @@ -1632,7 +1783,7 @@ mailbox = shared@example.com mock_graph_connection.return_value = object() mock_get_mailbox_reports.return_value = { "aggregate_reports": [], - "forensic_reports": [], + "failure_reports": [], "smtp_tls_reports": [], } @@ -1783,7 +1934,7 @@ class TestMSGraphCliValidation(unittest.TestCase): mock_graph_connection.return_value = object() mock_get_mailbox_reports.return_value = { "aggregate_reports": [], - "forensic_reports": [], + "failure_reports": [], "smtp_tls_reports": [], } @@ -1925,7 +2076,7 @@ tenant_id = tenant-id mock_graph_connection.return_value = object() mock_get_mailbox_reports.return_value = { "aggregate_reports": [], - "forensic_reports": [], + "failure_reports": [], "smtp_tls_reports": [], } @@ -2139,7 +2290,7 @@ watch = true mock_imap.return_value = object() mock_get_reports.return_value = { "aggregate_reports": [], - "forensic_reports": [], + "failure_reports": [], "smtp_tls_reports": [], } @@ -2212,7 +2363,7 @@ watch = true mock_imap.return_value = object() mock_get_reports.return_value = { "aggregate_reports": [], - "forensic_reports": [], + "failure_reports": [], "smtp_tls_reports": [], } @@ -2292,7 +2443,7 @@ watch = true mock_imap.return_value = object() mock_get_reports.return_value = { "aggregate_reports": [], - "forensic_reports": [], + "failure_reports": [], "smtp_tls_reports": [], } @@ -2367,7 +2518,7 @@ watch = true mock_imap.return_value = object() mock_get_reports.return_value = { "aggregate_reports": [], - "forensic_reports": [], + "failure_reports": [], "smtp_tls_reports": [], } mock_init_clients.return_value = {} @@ -2452,7 +2603,7 @@ watch = true mock_imap.return_value = object() mock_get_reports.return_value = { "aggregate_reports": [], - "forensic_reports": [], + "failure_reports": [], "smtp_tls_reports": [], } @@ -2537,7 +2688,7 @@ class TestIndexPrefixDomainMapTlsFiltering(unittest.TestCase): mock_imap_connection.return_value = object() mock_get_reports.return_value = { "aggregate_reports": [], - "forensic_reports": [], + "failure_reports": [], "smtp_tls_reports": [ { "organization_name": "Allowed Org", @@ -2929,16 +3080,15 @@ class TestMaildirUidHandling(unittest.TestCase): os.makedirs(os.path.join(d, subdir)) original_stat = os.stat + call_count = [0] def stat_that_fails_once(path, *args, **kwargs): """Fail on the first call (UID check), pass through after.""" - stat_that_fails_once.calls += 1 - if stat_that_fails_once.calls == 1: + call_count[0] += 1 + if call_count[0] == 1: raise OSError("no stat") return original_stat(path, *args, **kwargs) - stat_that_fails_once.calls = 0 - with patch( "parsedmarc.mail.maildir.os.stat", side_effect=stat_that_fails_once ): @@ -3196,6 +3346,1465 @@ class TestEnvVarConfig(unittest.TestCase): f"Expected falsy for {false_val!r}", ) + # ============================================================ # New tests for _bucket_interval_by_day + # ============================================================ + def testBucketIntervalBeginAfterEnd(self): + """begin > end should raise ValueError""" + begin = datetime(2024, 1, 2, tzinfo=timezone.utc) + end = datetime(2024, 1, 1, tzinfo=timezone.utc) + with self.assertRaises(ValueError): + parsedmarc._bucket_interval_by_day(begin, end, 100) + + def testBucketIntervalNaiveDatetime(self): + """Non-timezone-aware datetimes should raise ValueError""" + begin = datetime(2024, 1, 1) + end = datetime(2024, 1, 2) + with self.assertRaises(ValueError): + parsedmarc._bucket_interval_by_day(begin, end, 100) + + def testBucketIntervalDifferentTzinfo(self): + """Different tzinfo objects should raise ValueError""" + tz1 = timezone.utc + tz2 = timezone(timedelta(hours=5)) + begin = datetime(2024, 1, 1, tzinfo=tz1) + end = datetime(2024, 1, 2, tzinfo=tz2) + with self.assertRaises(ValueError): + parsedmarc._bucket_interval_by_day(begin, end, 100) + + def testBucketIntervalNegativeCount(self): + """Negative total_count should raise ValueError""" + begin = datetime(2024, 1, 1, tzinfo=timezone.utc) + end = datetime(2024, 1, 2, tzinfo=timezone.utc) + with self.assertRaises(ValueError): + parsedmarc._bucket_interval_by_day(begin, end, -1) + + def testBucketIntervalZeroCount(self): + """Zero total_count should return empty list""" + begin = datetime(2024, 1, 1, tzinfo=timezone.utc) + end = datetime(2024, 1, 2, tzinfo=timezone.utc) + result = parsedmarc._bucket_interval_by_day(begin, end, 0) + self.assertEqual(result, []) + + def testBucketIntervalSameBeginEnd(self): + """Same begin and end (zero interval) should return empty list""" + dt = datetime(2024, 1, 1, 12, 0, 0, tzinfo=timezone.utc) + result = parsedmarc._bucket_interval_by_day(dt, dt, 100) + self.assertEqual(result, []) + + def testBucketIntervalSingleDay(self): + """Single day interval should return one bucket with total count""" + begin = datetime(2024, 1, 1, 0, 0, 0, tzinfo=timezone.utc) + end = datetime(2024, 1, 1, 23, 59, 59, tzinfo=timezone.utc) + result = parsedmarc._bucket_interval_by_day(begin, end, 100) + self.assertEqual(len(result), 1) + self.assertEqual(result[0]["count"], 100) + self.assertEqual(result[0]["begin"], begin) + + def testBucketIntervalMultiDay(self): + """Multi-day interval should distribute counts proportionally""" + begin = datetime(2024, 1, 1, 0, 0, 0, tzinfo=timezone.utc) + end = datetime(2024, 1, 3, 0, 0, 0, tzinfo=timezone.utc) + result = parsedmarc._bucket_interval_by_day(begin, end, 100) + self.assertEqual(len(result), 2) + total = sum(b["count"] for b in result) + self.assertEqual(total, 100) + # Equal days => equal distribution + self.assertEqual(result[0]["count"], 50) + self.assertEqual(result[1]["count"], 50) + + def testBucketIntervalRemainderDistribution(self): + """Odd count across equal days distributes remainder correctly""" + begin = datetime(2024, 1, 1, 0, 0, 0, tzinfo=timezone.utc) + end = datetime(2024, 1, 4, 0, 0, 0, tzinfo=timezone.utc) + result = parsedmarc._bucket_interval_by_day(begin, end, 10) + total = sum(b["count"] for b in result) + self.assertEqual(total, 10) + self.assertEqual(len(result), 3) + + def testBucketIntervalPartialDays(self): + """Partial days: 12h on day1, 24h on day2 => 1/3 vs 2/3 split""" + begin = datetime(2024, 1, 1, 12, 0, 0, tzinfo=timezone.utc) + end = datetime(2024, 1, 3, 0, 0, 0, tzinfo=timezone.utc) + result = parsedmarc._bucket_interval_by_day(begin, end, 90) + total = sum(b["count"] for b in result) + self.assertEqual(total, 90) + # day1: 12h, day2: 24h => 1/3 vs 2/3 + self.assertEqual(result[0]["count"], 30) + self.assertEqual(result[1]["count"], 60) + + # ============================================================ # Tests for _append_parsed_record + # ============================================================ + def testAppendParsedRecordNoNormalize(self): + """No normalization: record appended as-is with interval fields""" + records = [] + rec = {"count": 10, "source": {"ip_address": "1.2.3.4"}} + begin = datetime(2024, 1, 1, 0, 0, 0, tzinfo=timezone.utc) + end = datetime(2024, 1, 2, 0, 0, 0, tzinfo=timezone.utc) + parsedmarc._append_parsed_record(rec, records, begin, end, False) + self.assertEqual(len(records), 1) + self.assertFalse(records[0]["normalized_timespan"]) # type: ignore[typeddict-item] + self.assertEqual(records[0]["interval_begin"], "2024-01-01 00:00:00") + self.assertEqual(records[0]["interval_end"], "2024-01-02 00:00:00") + + def testAppendParsedRecordNormalize(self): + """Normalization: record split into daily buckets""" + records = [] + rec = {"count": 100, "source": {"ip_address": "1.2.3.4"}} + begin = datetime(2024, 1, 1, 0, 0, 0, tzinfo=timezone.utc) + end = datetime(2024, 1, 3, 0, 0, 0, tzinfo=timezone.utc) + parsedmarc._append_parsed_record(rec, records, begin, end, True) + self.assertEqual(len(records), 2) + total = sum(r["count"] for r in records) + self.assertEqual(total, 100) + for r in records: + self.assertTrue(r["normalized_timespan"]) # type: ignore[typeddict-item] + + def testAppendParsedRecordNormalizeZeroCount(self): + """Normalization with zero count: nothing appended""" + records = [] + rec = {"count": 0, "source": {"ip_address": "1.2.3.4"}} + begin = datetime(2024, 1, 1, 0, 0, 0, tzinfo=timezone.utc) + end = datetime(2024, 1, 3, 0, 0, 0, tzinfo=timezone.utc) + parsedmarc._append_parsed_record(rec, records, begin, end, True) + self.assertEqual(len(records), 0) + + # ============================================================ # Tests for _parse_report_record + # ============================================================ + def testParseReportRecordNoneSourceIP(self): + """Record with None source_ip should raise ValueError""" + record = { + "row": { + "source_ip": None, + "count": "1", + "policy_evaluated": { + "disposition": "none", + "dkim": "pass", + "spf": "pass", + }, + }, + "identifiers": {"header_from": "example.com"}, + "auth_results": {"dkim": [], "spf": []}, + } + with self.assertRaises(ValueError): + parsedmarc._parse_report_record(record, offline=True) + + def testParseReportRecordMissingDkimSpf(self): + """Record with missing dkim/spf auth results defaults correctly""" + record = { + "row": { + "source_ip": "192.0.2.1", + "count": "5", + "policy_evaluated": { + "disposition": "none", + "dkim": "pass", + "spf": "fail", + }, + }, + "identifiers": {"header_from": "example.com"}, + "auth_results": {}, + } + result = parsedmarc._parse_report_record(record, offline=True) + self.assertEqual(result["auth_results"]["dkim"], []) + self.assertEqual(result["auth_results"]["spf"], []) + + def testParseReportRecordReasonHandling(self): + """Reasons in policy_evaluated get normalized with comment default""" + record = { + "row": { + "source_ip": "192.0.2.1", + "count": "1", + "policy_evaluated": { + "disposition": "none", + "dkim": "pass", + "spf": "pass", + "reason": {"type": "forwarded"}, + }, + }, + "identifiers": {"header_from": "example.com"}, + "auth_results": {"dkim": [], "spf": []}, + } + result = parsedmarc._parse_report_record(record, offline=True) + reasons = result["policy_evaluated"]["policy_override_reasons"] + self.assertEqual(len(reasons), 1) + self.assertEqual(reasons[0]["type"], "forwarded") + self.assertIsNone(reasons[0]["comment"]) + + def testParseReportRecordReasonList(self): + """Multiple reasons as a list are preserved""" + record = { + "row": { + "source_ip": "192.0.2.1", + "count": "1", + "policy_evaluated": { + "disposition": "none", + "dkim": "pass", + "spf": "pass", + "reason": [ + {"type": "forwarded", "comment": "relay"}, + {"type": "local_policy"}, + ], + }, + }, + "identifiers": {"header_from": "example.com"}, + "auth_results": {"dkim": [], "spf": []}, + } + result = parsedmarc._parse_report_record(record, offline=True) + reasons = result["policy_evaluated"]["policy_override_reasons"] + self.assertEqual(len(reasons), 2) + self.assertEqual(reasons[0]["comment"], "relay") + self.assertIsNone(reasons[1]["comment"]) + + def testParseReportRecordIdentities(self): + """'identities' key is mapped to 'identifiers'""" + record = { + "row": { + "source_ip": "192.0.2.1", + "count": "1", + "policy_evaluated": { + "disposition": "none", + "dkim": "pass", + "spf": "pass", + }, + }, + "identities": { + "header_from": "Example.COM", + "envelope_from": "example.com", + }, + "auth_results": {"dkim": [], "spf": []}, + } + result = parsedmarc._parse_report_record(record, offline=True) + self.assertIn("identifiers", result) + self.assertEqual(result["identifiers"]["header_from"], "example.com") + + def testParseReportRecordDkimDefaults(self): + """DKIM result defaults: selector='none', result='none' when missing""" + record = { + "row": { + "source_ip": "192.0.2.1", + "count": "1", + "policy_evaluated": { + "disposition": "none", + "dkim": "fail", + "spf": "fail", + }, + }, + "identifiers": {"header_from": "example.com"}, + "auth_results": { + "dkim": {"domain": "example.com"}, + "spf": [], + }, + } + result = parsedmarc._parse_report_record(record, offline=True) + dkim = result["auth_results"]["dkim"][0] + self.assertEqual(dkim["selector"], "none") + self.assertEqual(dkim["result"], "none") + self.assertIsNone(dkim["human_result"]) + + def testParseReportRecordSpfDefaults(self): + """SPF result defaults: scope='mfrom', result='none' when missing""" + record = { + "row": { + "source_ip": "192.0.2.1", + "count": "1", + "policy_evaluated": { + "disposition": "none", + "dkim": "fail", + "spf": "fail", + }, + }, + "identifiers": {"header_from": "example.com"}, + "auth_results": { + "dkim": [], + "spf": {"domain": "example.com"}, + }, + } + result = parsedmarc._parse_report_record(record, offline=True) + spf = result["auth_results"]["spf"][0] + self.assertEqual(spf["scope"], "mfrom") + self.assertEqual(spf["result"], "none") + self.assertIsNone(spf["human_result"]) + + def testParseReportRecordHumanResult(self): + """human_result field is included when present""" + record = { + "row": { + "source_ip": "192.0.2.1", + "count": "1", + "policy_evaluated": { + "disposition": "none", + "dkim": "pass", + "spf": "pass", + }, + }, + "identifiers": {"header_from": "example.com"}, + "auth_results": { + "dkim": [ + { + "domain": "example.com", + "selector": "s1", + "result": "pass", + "human_result": "good key", + } + ], + "spf": [ + { + "domain": "example.com", + "scope": "mfrom", + "result": "pass", + "human_result": "sender valid", + } + ], + }, + } + result = parsedmarc._parse_report_record(record, offline=True) + self.assertEqual(result["auth_results"]["dkim"][0]["human_result"], "good key") + self.assertEqual( + result["auth_results"]["spf"][0]["human_result"], "sender valid" + ) + + def testParseReportRecordEnvelopeFromFallback(self): + """envelope_from falls back to last SPF domain when missing""" + record = { + "row": { + "source_ip": "192.0.2.1", + "count": "1", + "policy_evaluated": { + "disposition": "none", + "dkim": "pass", + "spf": "pass", + }, + }, + "identifiers": {"header_from": "example.com"}, + "auth_results": { + "dkim": [], + "spf": [ + {"domain": "Bounce.Example.COM", "scope": "mfrom", "result": "pass"} + ], + }, + } + result = parsedmarc._parse_report_record(record, offline=True) + self.assertEqual(result["identifiers"]["envelope_from"], "bounce.example.com") + + def testParseReportRecordEnvelopeFromNullFallback(self): + """envelope_from None value falls back to SPF domain""" + record = { + "row": { + "source_ip": "192.0.2.1", + "count": "1", + "policy_evaluated": { + "disposition": "none", + "dkim": "pass", + "spf": "pass", + }, + }, + "identifiers": { + "header_from": "example.com", + "envelope_from": None, + }, + "auth_results": { + "dkim": [], + "spf": [ + {"domain": "SPF.Example.COM", "scope": "mfrom", "result": "pass"} + ], + }, + } + result = parsedmarc._parse_report_record(record, offline=True) + self.assertEqual(result["identifiers"]["envelope_from"], "spf.example.com") + + def testParseReportRecordEnvelopeTo(self): + """envelope_to is preserved and moved correctly""" + record = { + "row": { + "source_ip": "192.0.2.1", + "count": "1", + "policy_evaluated": { + "disposition": "none", + "dkim": "pass", + "spf": "pass", + }, + }, + "identifiers": { + "header_from": "example.com", + "envelope_from": "bounce@example.com", + "envelope_to": "recipient@example.com", + }, + "auth_results": {"dkim": [], "spf": []}, + } + result = parsedmarc._parse_report_record(record, offline=True) + self.assertEqual(result["identifiers"]["envelope_to"], "recipient@example.com") + + def testParseReportRecordAlignment(self): + """Alignment fields computed correctly from policy_evaluated""" + record = { + "row": { + "source_ip": "192.0.2.1", + "count": "1", + "policy_evaluated": { + "disposition": "none", + "dkim": "pass", + "spf": "fail", + }, + }, + "identifiers": {"header_from": "example.com"}, + "auth_results": {"dkim": [], "spf": []}, + } + result = parsedmarc._parse_report_record(record, offline=True) + self.assertTrue(result["alignment"]["dkim"]) + self.assertFalse(result["alignment"]["spf"]) + self.assertTrue(result["alignment"]["dmarc"]) + + # ============================================================ # Tests for _parse_smtp_tls_failure_details + # ============================================================ + def testParseSmtpTlsFailureDetailsMinimal(self): + """Minimal failure details with just required fields""" + details = { + "result-type": "certificate-expired", + "failed-session-count": 5, + } + result = parsedmarc._parse_smtp_tls_failure_details(details) + self.assertEqual(result["result_type"], "certificate-expired") + self.assertEqual(result["failed_session_count"], 5) + self.assertNotIn("sending_mta_ip", result) + + def testParseSmtpTlsFailureDetailsAllOptional(self): + """All optional fields included""" + details = { + "result-type": "starttls-not-supported", + "failed-session-count": 3, + "sending-mta-ip": "10.0.0.1", + "receiving-ip": "10.0.0.2", + "receiving-mx-hostname": "mx.example.com", + "receiving-mx-helo": "mx.example.com", + "additional-info-uri": "https://example.com/info", + "failure-reason-code": "TLS_ERROR", + } + result = parsedmarc._parse_smtp_tls_failure_details(details) + self.assertEqual(result["sending_mta_ip"], "10.0.0.1") + self.assertEqual(result["receiving_ip"], "10.0.0.2") + self.assertEqual(result["receiving_mx_hostname"], "mx.example.com") + self.assertEqual(result["receiving_mx_helo"], "mx.example.com") + self.assertEqual(result["additional_info_uri"], "https://example.com/info") + self.assertEqual(result["failure_reason_code"], "TLS_ERROR") + + def testParseSmtpTlsFailureDetailsMissingRequired(self): + """Missing required field raises InvalidSMTPTLSReport""" + with self.assertRaises(parsedmarc.InvalidSMTPTLSReport): + parsedmarc._parse_smtp_tls_failure_details({"result-type": "err"}) + + # ============================================================ # Tests for _parse_smtp_tls_report_policy + # ============================================================ + def testParseSmtpTlsReportPolicyValid(self): + """Valid STS policy parses correctly""" + policy = { + "policy": { + "policy-type": "sts", + "policy-domain": "example.com", + "policy-string": ["version: STSv1", "mode: enforce"], + "mx-host-pattern": ["*.example.com"], + }, + "summary": { + "total-successful-session-count": 100, + "total-failure-session-count": 2, + }, + } + result = parsedmarc._parse_smtp_tls_report_policy(policy) + self.assertEqual(result["policy_type"], "sts") + self.assertEqual(result["policy_domain"], "example.com") + self.assertEqual(result["policy_strings"], ["version: STSv1", "mode: enforce"]) + self.assertEqual(result["mx_host_patterns"], ["*.example.com"]) + self.assertEqual(result["successful_session_count"], 100) + self.assertEqual(result["failed_session_count"], 2) + + def testParseSmtpTlsReportPolicyInvalidType(self): + """Invalid policy type raises InvalidSMTPTLSReport""" + policy = { + "policy": { + "policy-type": "invalid", + "policy-domain": "example.com", + }, + "summary": { + "total-successful-session-count": 0, + "total-failure-session-count": 0, + }, + } + with self.assertRaises(parsedmarc.InvalidSMTPTLSReport): + parsedmarc._parse_smtp_tls_report_policy(policy) + + def testParseSmtpTlsReportPolicyEmptyPolicyString(self): + """Empty policy-string list is not included""" + policy = { + "policy": { + "policy-type": "sts", + "policy-domain": "example.com", + "policy-string": [], + "mx-host-pattern": [], + }, + "summary": { + "total-successful-session-count": 50, + "total-failure-session-count": 0, + }, + } + result = parsedmarc._parse_smtp_tls_report_policy(policy) + self.assertNotIn("policy_strings", result) + self.assertNotIn("mx_host_patterns", result) + + def testParseSmtpTlsReportPolicyWithFailureDetails(self): + """Policy with failure-details parses nested details""" + policy = { + "policy": { + "policy-type": "sts", + "policy-domain": "example.com", + }, + "summary": { + "total-successful-session-count": 10, + "total-failure-session-count": 1, + }, + "failure-details": [ + { + "result-type": "certificate-expired", + "failed-session-count": 1, + } + ], + } + result = parsedmarc._parse_smtp_tls_report_policy(policy) + self.assertEqual(len(result["failure_details"]), 1) + self.assertEqual( + result["failure_details"][0]["result_type"], "certificate-expired" + ) + + def testParseSmtpTlsReportPolicyMissingField(self): + """Missing required policy field raises InvalidSMTPTLSReport""" + policy = {"policy": {"policy-type": "sts"}, "summary": {}} + with self.assertRaises(parsedmarc.InvalidSMTPTLSReport): + parsedmarc._parse_smtp_tls_report_policy(policy) + + # ============================================================ # Tests for parse_smtp_tls_report_json + # ============================================================ + def testParseSmtpTlsReportJsonValid(self): + """Valid SMTP TLS JSON report parses correctly""" + report = json.dumps( + { + "organization-name": "Example Corp", + "date-range": { + "start-datetime": "2024-01-01T00:00:00Z", + "end-datetime": "2024-01-02T00:00:00Z", + }, + "contact-info": "admin@example.com", + "report-id": "report-123", + "policies": [ + { + "policy": { + "policy-type": "sts", + "policy-domain": "example.com", + }, + "summary": { + "total-successful-session-count": 50, + "total-failure-session-count": 0, + }, + } + ], + } + ) + result = parsedmarc.parse_smtp_tls_report_json(report) + self.assertEqual(result["organization_name"], "Example Corp") + self.assertEqual(result["report_id"], "report-123") + self.assertEqual(len(result["policies"]), 1) + + def testParseSmtpTlsReportJsonBytes(self): + """SMTP TLS report as bytes parses correctly""" + report = json.dumps( + { + "organization-name": "Org", + "date-range": { + "start-datetime": "2024-01-01", + "end-datetime": "2024-01-02", + }, + "contact-info": "a@b.com", + "report-id": "r1", + "policies": [ + { + "policy": {"policy-type": "tlsa", "policy-domain": "a.com"}, + "summary": { + "total-successful-session-count": 1, + "total-failure-session-count": 0, + }, + } + ], + } + ).encode("utf-8") + result = parsedmarc.parse_smtp_tls_report_json(report) + self.assertEqual(result["organization_name"], "Org") + + def testParseSmtpTlsReportJsonMissingField(self): + """Missing required field raises InvalidSMTPTLSReport""" + report = json.dumps({"organization-name": "Org"}) + with self.assertRaises(parsedmarc.InvalidSMTPTLSReport): + parsedmarc.parse_smtp_tls_report_json(report) + + def testParseSmtpTlsReportJsonPoliciesNotList(self): + """Non-list policies raises InvalidSMTPTLSReport""" + report = json.dumps( + { + "organization-name": "Org", + "date-range": { + "start-datetime": "2024-01-01", + "end-datetime": "2024-01-02", + }, + "contact-info": "a@b.com", + "report-id": "r1", + "policies": "not-a-list", + } + ) + with self.assertRaises(parsedmarc.InvalidSMTPTLSReport): + parsedmarc.parse_smtp_tls_report_json(report) + + # ============================================================ # Tests for aggregate report parsing (validation warnings, etc.) + # ============================================================ + def testAggregateReportInvalidNpWarning(self): + """Invalid np value is preserved but logs warning""" + xml = """ + + 1.0 + + Test Org + test@example.com + test-np-invalid + 17040672001704153599 + + + example.com +

none

+ banana + maybe + magic + + + + 192.0.2.1 + 1 + + none + pass + pass + + + example.com + + example.compass + + + """ + report = parsedmarc.parse_aggregate_report_xml(xml, offline=True) + # Invalid values are still stored + self.assertEqual(report["policy_published"]["np"], "banana") + self.assertEqual(report["policy_published"]["testing"], "maybe") + self.assertEqual(report["policy_published"]["discovery_method"], "magic") + + def testAggregateReportPassDisposition(self): + """'pass' as valid disposition is preserved""" + xml = """ + + + TestOrg + test@example.com + test-pass + 17040672001704153599 + + + example.com +

reject

+ + + + 192.0.2.1 + 1 + + pass + pass + pass + + + example.com + + example.compass + + + """ + report = parsedmarc.parse_aggregate_report_xml(xml, offline=True) + self.assertEqual( + report["records"][0]["policy_evaluated"]["disposition"], "pass" + ) + + def testAggregateReportMultipleRecords(self): + """Reports with multiple records are all parsed""" + xml = """ + + + TestOrg + test@example.com + test-multi + 17040672001704153599 + + + example.com +

none

+ + + + 192.0.2.1 + 10 + nonepasspass + + example.com + example.compass + + + + 192.0.2.2 + 5 + quarantinefailfail + + example.com + example.comfail + + """ + report = parsedmarc.parse_aggregate_report_xml(xml, offline=True) + self.assertEqual(len(report["records"]), 2) + self.assertEqual(report["records"][0]["count"], 10) + self.assertEqual(report["records"][1]["count"], 5) + + def testAggregateReportInvalidXmlRecovery(self): + """Badly formed XML is recovered via lxml""" + xml = 'Testt@e.comr117040672001704153599example.com

none

192.0.2.11nonepasspassexample.comexample.compass' + report = parsedmarc.parse_aggregate_report_xml(xml, offline=True) + self.assertEqual(report["report_metadata"]["report_id"], "r1") + + def testAggregateReportCsvRowsContainDMARCbisFields(self): + """CSV rows include np, testing, discovery_method columns""" + result = parsedmarc.parse_report_file( + "samples/aggregate/dmarcbis-draft-sample.xml", + always_use_local_files=True, + offline=True, + ) + report = cast(AggregateReport, result["report"]) + rows = parsedmarc.parsed_aggregate_reports_to_csv_rows(report) + self.assertTrue(len(rows) > 0) + row = rows[0] + self.assertIn("np", row) + self.assertIn("testing", row) + self.assertIn("discovery_method", row) + self.assertIn("source_ip_address", row) + self.assertIn("dkim_domains", row) + self.assertIn("spf_domains", row) + + def testAggregateReportSchemaVersion(self): + """DMARCbis report with returns correct xml_schema""" + xml = """ + + 1.0 + + TestOrg + test@example.com + test-version + 17040672001704153599 + + + example.com +

none

+ + + + 192.0.2.1 + 1 + nonepasspass + + example.com + example.compass + + """ + report = parsedmarc.parse_aggregate_report_xml(xml, offline=True) + self.assertEqual(report["xml_schema"], "1.0") + + def testAggregateReportDraftSchema(self): + """Report without defaults to 'draft' schema""" + xml = """ + + + TestOrg + test@example.com + test-draft + 17040672001704153599 + + + example.com +

none

+ + + + 192.0.2.1 + 1 + nonepasspass + + example.com + example.compass + + """ + report = parsedmarc.parse_aggregate_report_xml(xml, offline=True) + self.assertEqual(report["xml_schema"], "draft") + + def testAggregateReportGeneratorField(self): + """Generator field is correctly extracted""" + xml = """ + + + TestOrg + test@example.com + test-gen + My Reporter v1.0 + 17040672001704153599 + + + example.com +

none

+ + + + 192.0.2.1 + 1 + nonepasspass + + example.com + example.compass + + """ + report = parsedmarc.parse_aggregate_report_xml(xml, offline=True) + self.assertEqual(report["report_metadata"]["generator"], "My Reporter v1.0") + + def testAggregateReportReportErrors(self): + """Report errors in metadata are captured""" + xml = """ + + + TestOrg + test@example.com + test-err + Some error + 17040672001704153599 + + + example.com +

none

+ + + + 192.0.2.1 + 1 + nonepasspass + + example.com + example.compass + + """ + report = parsedmarc.parse_aggregate_report_xml(xml, offline=True) + self.assertIn("Some error", report["report_metadata"]["errors"]) + + def testAggregateReportPolicyDefaults(self): + """Policy defaults: adkim/aspf='r', sp=p, pct/fo=None""" + xml = """ + + + TestOrg + test@example.com + test-defaults + 17040672001704153599 + + + example.com +

reject

+ + + + 192.0.2.1 + 1 + nonepasspass + + example.com + example.compass + + """ + report = parsedmarc.parse_aggregate_report_xml(xml, offline=True) + pp = report["policy_published"] + self.assertEqual(pp["adkim"], "r") + self.assertEqual(pp["aspf"], "r") + self.assertEqual(pp["sp"], "reject") # defaults to p + self.assertIsNone(pp["pct"]) + self.assertIsNone(pp["fo"]) + self.assertIsNone(pp["np"]) + self.assertIsNone(pp["testing"]) + self.assertIsNone(pp["discovery_method"]) + + def testMagicXmlTagDetection(self): + """XML without declaration (starting with '<') is extracted""" + xml_no_decl = b"Ta@b.comr117040672001704153599example.com

none

192.0.2.11nonepasspassexample.comexample.compass" + self.assertTrue(xml_no_decl.startswith(parsedmarc.MAGIC_XML_TAG)) + # Ensure it extracts as XML + result = parsedmarc.extract_report(xml_no_decl) + self.assertIn("", result) + + # ============================================================ # Tests for parsedmarc/utils.py + # ============================================================ + def testTimestampToDatetime(self): + """timestamp_to_datetime converts UNIX timestamp to datetime""" + from datetime import datetime + + ts = 1704067200 + dt = parsedmarc.utils.timestamp_to_datetime(ts) + self.assertIsInstance(dt, datetime) + # Should match stdlib fromtimestamp (local time) + self.assertEqual(dt, datetime.fromtimestamp(ts)) + + def testTimestampToHuman(self): + """timestamp_to_human returns formatted string""" + result = parsedmarc.utils.timestamp_to_human(1704067200) + self.assertRegex(result, r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}") + + def testHumanTimestampToDatetime(self): + """human_timestamp_to_datetime parses timestamp string""" + dt = parsedmarc.utils.human_timestamp_to_datetime("2024-01-01 00:00:00") + self.assertIsInstance(dt, datetime) + self.assertEqual(dt.year, 2024) + self.assertEqual(dt.month, 1) + self.assertEqual(dt.day, 1) + + def testHumanTimestampToDatetimeUtc(self): + """human_timestamp_to_datetime with to_utc=True returns UTC""" + dt = parsedmarc.utils.human_timestamp_to_datetime( + "2024-01-01 12:00:00", to_utc=True + ) + self.assertEqual(dt.tzinfo, timezone.utc) + + def testHumanTimestampToDatetimeParenthesisStripping(self): + """Parenthesized content is stripped from timestamps""" + dt = parsedmarc.utils.human_timestamp_to_datetime( + "Mon, 01 Jan 2024 00:00:00 +0000 (UTC)" + ) + self.assertEqual(dt.year, 2024) + + def testHumanTimestampToDatetimeNegativeZero(self): + """-0000 timezone is handled""" + dt = parsedmarc.utils.human_timestamp_to_datetime("2024-01-01 00:00:00 -0000") + self.assertEqual(dt.year, 2024) + + def testHumanTimestampToUnixTimestamp(self): + """human_timestamp_to_unix_timestamp converts to int""" + ts = parsedmarc.utils.human_timestamp_to_unix_timestamp("2024-01-01 00:00:00") + self.assertIsInstance(ts, int) + + def testHumanTimestampToUnixTimestampWithT(self): + """T separator in timestamp is handled""" + ts = parsedmarc.utils.human_timestamp_to_unix_timestamp("2024-01-01T00:00:00") + self.assertIsInstance(ts, int) + + def testGetIpAddressCountry(self): + """get_ip_address_country returns country code using bundled DBIP""" + # 8.8.8.8 is a well-known Google DNS IP in US + country = parsedmarc.utils.get_ip_address_country("8.8.8.8") + self.assertEqual(country, "US") + + def testGetIpAddressCountryNotFound(self): + """get_ip_address_country returns None for reserved IP""" + country = parsedmarc.utils.get_ip_address_country("127.0.0.1") + self.assertIsNone(country) + + def testGetServiceFromReverseDnsBaseDomainOffline(self): + """get_service_from_reverse_dns_base_domain in offline mode""" + result = parsedmarc.utils.get_service_from_reverse_dns_base_domain( + "google.com", offline=True + ) + self.assertIn("Google", result["name"]) + self.assertIsNotNone(result["type"]) + + def testGetServiceFromReverseDnsBaseDomainUnknown(self): + """Unknown base domain returns domain as name and None as type""" + result = parsedmarc.utils.get_service_from_reverse_dns_base_domain( + "unknown-domain-xyz.example", offline=True + ) + self.assertEqual(result["name"], "unknown-domain-xyz.example") + self.assertIsNone(result["type"]) + + def testGetIpAddressInfoOffline(self): + """get_ip_address_info in offline mode returns country but no DNS""" + info = parsedmarc.utils.get_ip_address_info("8.8.8.8", offline=True) + self.assertEqual(info["ip_address"], "8.8.8.8") + self.assertEqual(info["country"], "US") + self.assertIsNone(info["reverse_dns"]) + + def testGetIpAddressInfoCache(self): + """get_ip_address_info uses cache on second call""" + from expiringdict import ExpiringDict + + cache = ExpiringDict(max_len=100, max_age_seconds=60) + with patch("parsedmarc.utils.get_reverse_dns", return_value="dns.google"): + info1 = parsedmarc.utils.get_ip_address_info( + "8.8.8.8", + offline=False, + cache=cache, + always_use_local_files=True, + ) + self.assertIn("8.8.8.8", cache) + info2 = parsedmarc.utils.get_ip_address_info( + "8.8.8.8", offline=False, cache=cache + ) + self.assertEqual(info1["ip_address"], info2["ip_address"]) + self.assertEqual(info2["reverse_dns"], "dns.google") + + def testParseEmailAddressWithDisplayName(self): + """parse_email_address with display name""" + result = parsedmarc.utils.parse_email_address(("John Doe", "john@example.com")) # type: ignore[arg-type] + self.assertEqual(result["display_name"], "John Doe") + self.assertEqual(result["address"], "john@example.com") + self.assertEqual(result["local"], "john") + self.assertEqual(result["domain"], "example.com") + + def testParseEmailAddressWithoutDisplayName(self): + """parse_email_address with empty display name""" + result = parsedmarc.utils.parse_email_address(("", "john@example.com")) # type: ignore[arg-type] + self.assertIsNone(result["display_name"]) + self.assertEqual(result["address"], "john@example.com") + + def testParseEmailAddressNoAt(self): + """parse_email_address with no @ returns None local/domain""" + result = parsedmarc.utils.parse_email_address(("", "localonly")) # type: ignore[arg-type] + self.assertIsNone(result["local"]) + self.assertIsNone(result["domain"]) + + def testGetFilenameSafeString(self): + """get_filename_safe_string removes invalid chars""" + result = parsedmarc.utils.get_filename_safe_string('file/name:with"bad*chars') + self.assertNotIn("/", result) + self.assertNotIn(":", result) + self.assertNotIn('"', result) + self.assertNotIn("*", result) + + def testGetFilenameSafeStringNone(self): + """get_filename_safe_string with None returns 'None'""" + result = parsedmarc.utils.get_filename_safe_string(None) # type: ignore[arg-type] + self.assertEqual(result, "None") + + def testGetFilenameSafeStringLong(self): + """get_filename_safe_string truncates to 100 chars""" + result = parsedmarc.utils.get_filename_safe_string("a" * 200) + self.assertEqual(len(result), 100) + + def testGetFilenameSafeStringTrailingDot(self): + """get_filename_safe_string strips trailing dots""" + result = parsedmarc.utils.get_filename_safe_string("filename...") + self.assertFalse(result.endswith(".")) + + def testIsMboxNonMbox(self): + """is_mbox returns False for non-mbox file""" + result = parsedmarc.utils.is_mbox("samples/empty.xml") + self.assertFalse(result) + + def testIsOutlookMsgNonMsg(self): + """is_outlook_msg returns False for non-MSG content""" + self.assertFalse(parsedmarc.utils.is_outlook_msg(b"not an outlook msg")) + self.assertFalse(parsedmarc.utils.is_outlook_msg("string content")) + + def testIsOutlookMsgMagic(self): + """is_outlook_msg returns True for correct magic bytes""" + magic = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 100 + self.assertTrue(parsedmarc.utils.is_outlook_msg(magic)) + + # ============================================================ # Tests for output modules (mocked) + # ============================================================ + def testWebhookClientInit(self): + """WebhookClient initializes with correct attributes""" + from parsedmarc.webhook import WebhookClient + + client = WebhookClient( + aggregate_url="http://agg.example.com", + failure_url="http://fail.example.com", + smtp_tls_url="http://tls.example.com", + ) + self.assertEqual(client.aggregate_url, "http://agg.example.com") + self.assertEqual(client.failure_url, "http://fail.example.com") + self.assertEqual(client.smtp_tls_url, "http://tls.example.com") + self.assertEqual(client.timeout, 60) + + def testWebhookClientSaveMethods(self): + """WebhookClient save methods call _send_to_webhook""" + from parsedmarc.webhook import WebhookClient + + client = WebhookClient("http://a", "http://f", "http://t") + client.session = MagicMock() + client.save_aggregate_report_to_webhook('{"test": 1}') + client.session.post.assert_called_with( + "http://a", data='{"test": 1}', timeout=60 + ) + client.save_failure_report_to_webhook('{"fail": 1}') + client.session.post.assert_called_with( + "http://f", data='{"fail": 1}', timeout=60 + ) + client.save_smtp_tls_report_to_webhook('{"tls": 1}') + client.session.post.assert_called_with( + "http://t", data='{"tls": 1}', timeout=60 + ) + + def testWebhookBackwardCompatAlias(self): + """WebhookClient forensic alias points to failure method""" + from parsedmarc.webhook import WebhookClient + + self.assertIs( + WebhookClient.save_forensic_report_to_webhook, # type: ignore[attr-defined] + WebhookClient.save_failure_report_to_webhook, + ) + + def testKafkaStripMetadata(self): + """KafkaClient.strip_metadata extracts metadata to root""" + from parsedmarc.kafkaclient import KafkaClient + + report = { + "report_metadata": { + "org_name": "TestOrg", + "org_email": "test@example.com", + "report_id": "r-123", + "begin_date": "2024-01-01", + "end_date": "2024-01-02", + }, + "records": [], + } + result = KafkaClient.strip_metadata(report) + self.assertEqual(result["org_name"], "TestOrg") + self.assertEqual(result["org_email"], "test@example.com") + self.assertEqual(result["report_id"], "r-123") + self.assertNotIn("report_metadata", result) + + def testKafkaGenerateDateRange(self): + """KafkaClient.generate_date_range generates date range list""" + from parsedmarc.kafkaclient import KafkaClient + + report = { + "report_metadata": { + "begin_date": "2024-01-01 00:00:00", + "end_date": "2024-01-02 00:00:00", + } + } + result = KafkaClient.generate_date_range(report) + self.assertEqual(len(result), 2) + self.assertIn("2024-01-01", result[0]) + self.assertIn("2024-01-02", result[1]) + + def testSplunkHECClientInit(self): + """HECClient initializes with correct URL and headers""" + from parsedmarc.splunk import HECClient + + client = HECClient( + url="https://splunk.example.com:8088", + access_token="my-token", + index="main", + ) + self.assertIn("/services/collector/event/1.0", client.url) + self.assertEqual(client.access_token, "my-token") + self.assertEqual(client.index, "main") + self.assertEqual(client.source, "parsedmarc") + self.assertIn("Splunk my-token", client.session.headers["Authorization"]) + + def testSplunkHECClientStripTokenPrefix(self): + """HECClient strips 'Splunk ' prefix from token""" + from parsedmarc.splunk import HECClient + + client = HECClient( + url="https://splunk.example.com", + access_token="Splunk my-token", + index="main", + ) + self.assertEqual(client.access_token, "my-token") + + def testSplunkBackwardCompatAlias(self): + """HECClient forensic alias points to failure method""" + from parsedmarc.splunk import HECClient + + self.assertIs( + HECClient.save_forensic_reports_to_splunk, # type: ignore[attr-defined] + HECClient.save_failure_reports_to_splunk, + ) + + def testSyslogClientUdpInit(self): + """SyslogClient creates UDP handler""" + from parsedmarc.syslog import SyslogClient + + client = SyslogClient("localhost", 514, protocol="udp") + self.assertEqual(client.server_name, "localhost") + self.assertEqual(client.server_port, 514) + self.assertEqual(client.protocol, "udp") + + def testSyslogClientInvalidProtocol(self): + """SyslogClient with invalid protocol raises ValueError""" + from parsedmarc.syslog import SyslogClient + + with self.assertRaises(ValueError): + SyslogClient("localhost", 514, protocol="invalid") + + def testSyslogBackwardCompatAlias(self): + """SyslogClient forensic alias points to failure method""" + from parsedmarc.syslog import SyslogClient + + self.assertIs( + SyslogClient.save_forensic_report_to_syslog, # type: ignore[attr-defined] + SyslogClient.save_failure_report_to_syslog, + ) + + def testLogAnalyticsConfig(self): + """LogAnalyticsConfig stores all fields""" + from parsedmarc.loganalytics import LogAnalyticsConfig + + config = LogAnalyticsConfig( + client_id="cid", + client_secret="csec", + tenant_id="tid", + dce="https://dce.example.com", + dcr_immutable_id="dcr-123", + dcr_aggregate_stream="agg-stream", + dcr_failure_stream="fail-stream", + dcr_smtp_tls_stream="tls-stream", + ) + self.assertEqual(config.client_id, "cid") + self.assertEqual(config.client_secret, "csec") + self.assertEqual(config.tenant_id, "tid") + self.assertEqual(config.dce, "https://dce.example.com") + self.assertEqual(config.dcr_immutable_id, "dcr-123") + self.assertEqual(config.dcr_aggregate_stream, "agg-stream") + self.assertEqual(config.dcr_failure_stream, "fail-stream") + self.assertEqual(config.dcr_smtp_tls_stream, "tls-stream") + + def testLogAnalyticsClientValidationError(self): + """LogAnalyticsClient raises on missing required config""" + from parsedmarc.loganalytics import LogAnalyticsClient, LogAnalyticsException + + with self.assertRaises(LogAnalyticsException): + LogAnalyticsClient( + client_id="", + client_secret="csec", + tenant_id="tid", + dce="https://dce.example.com", + dcr_immutable_id="dcr-123", + dcr_aggregate_stream="agg", + dcr_failure_stream="fail", + dcr_smtp_tls_stream="tls", + ) + + def testSmtpTlsCsvRows(self): + """parsed_smtp_tls_reports_to_csv_rows produces correct rows""" + report_json = json.dumps( + { + "organization-name": "Org", + "date-range": { + "start-datetime": "2024-01-01T00:00:00Z", + "end-datetime": "2024-01-02T00:00:00Z", + }, + "contact-info": "a@b.com", + "report-id": "r1", + "policies": [ + { + "policy": { + "policy-type": "sts", + "policy-domain": "example.com", + "policy-string": ["v: STSv1"], + "mx-host-pattern": ["*.example.com"], + }, + "summary": { + "total-successful-session-count": 10, + "total-failure-session-count": 1, + }, + "failure-details": [ + {"result-type": "cert-expired", "failed-session-count": 1} + ], + } + ], + } + ) + parsed = parsedmarc.parse_smtp_tls_report_json(report_json) + rows = parsedmarc.parsed_smtp_tls_reports_to_csv_rows(parsed) + self.assertTrue(len(rows) >= 2) + self.assertEqual(rows[0]["organization_name"], "Org") + self.assertEqual(rows[0]["policy_domain"], "example.com") + + def testParsedAggregateReportsToCsvRowsList(self): + """parsed_aggregate_reports_to_csv_rows handles list of reports""" + result = parsedmarc.parse_report_file( + "samples/aggregate/dmarcbis-draft-sample.xml", + always_use_local_files=True, + offline=True, + ) + report = cast(AggregateReport, result["report"]) + # Pass as a list + rows = parsedmarc.parsed_aggregate_reports_to_csv_rows([report]) + self.assertTrue(len(rows) > 0) + # Verify non-str/int/bool values are cleaned + for row in rows: + for v in row.values(): + self.assertIn(type(v), [str, int, bool]) + + def testExceptionHierarchy(self): + """Exception class hierarchy is correct""" + self.assertTrue(issubclass(parsedmarc.ParserError, RuntimeError)) + self.assertTrue( + issubclass(parsedmarc.InvalidDMARCReport, parsedmarc.ParserError) + ) + self.assertTrue( + issubclass(parsedmarc.InvalidAggregateReport, parsedmarc.InvalidDMARCReport) + ) + self.assertTrue( + issubclass(parsedmarc.InvalidFailureReport, parsedmarc.InvalidDMARCReport) + ) + self.assertTrue( + issubclass(parsedmarc.InvalidSMTPTLSReport, parsedmarc.ParserError) + ) + self.assertIs(parsedmarc.InvalidForensicReport, parsedmarc.InvalidFailureReport) + + def testAggregateReportNormalization(self): + """Reports spanning >24h get normalized per day""" + xml = """ + + + TestOrg + test@example.com + test-norm + 17040672001704326400 + + + example.com +

none

+ + + + 192.0.2.1 + 90 + nonepasspass + + example.com + example.compass + + """ + # Span is 259200 seconds (3 days), exceeds default 24h threshold + report = parsedmarc.parse_aggregate_report_xml(xml, offline=True) + self.assertTrue(report["report_metadata"]["timespan_requires_normalization"]) + # Records should be split across days + self.assertTrue(len(report["records"]) > 1) + total = sum(r["count"] for r in report["records"]) + self.assertEqual(total, 90) + for r in report["records"]: + self.assertTrue(r["normalized_timespan"]) # type: ignore[typeddict-item] + + # =================================================================== + # Additional backward compatibility alias tests + # =================================================================== + + def testGelfBackwardCompatAlias(self): + """GelfClient forensic alias points to failure method""" + from parsedmarc.gelf import GelfClient + + self.assertIs( + GelfClient.save_forensic_report_to_gelf, # type: ignore[attr-defined] + GelfClient.save_failure_report_to_gelf, + ) + + def testS3BackwardCompatAlias(self): + """S3Client forensic alias points to failure method""" + from parsedmarc.s3 import S3Client + + self.assertIs( + S3Client.save_forensic_report_to_s3, # type: ignore[attr-defined] + S3Client.save_failure_report_to_s3, + ) + + def testKafkaBackwardCompatAlias(self): + """KafkaClient forensic alias points to failure method""" + from parsedmarc.kafkaclient import KafkaClient + + self.assertIs( + KafkaClient.save_forensic_reports_to_kafka, # type: ignore[attr-defined] + KafkaClient.save_failure_reports_to_kafka, + ) + + # =================================================================== + # Additional extract/parse tests + # =================================================================== + + def testExtractReportFromFilePathNotFound(self): + """extract_report_from_file_path raises ParserError for missing file""" + with self.assertRaises(parsedmarc.ParserError): + parsedmarc.extract_report_from_file_path("nonexistent_file.xml") + + def testExtractReportInvalidArchive(self): + """extract_report raises ParserError for unrecognized binary content""" + with self.assertRaises(parsedmarc.ParserError): + parsedmarc.extract_report(b"\x00\x01\x02\x03\x04\x05\x06\x07") + + def testParseAggregateReportFile(self): + """parse_aggregate_report_file parses bytes input directly""" + print() + sample_path = "samples/aggregate/dmarcbis-draft-sample.xml" + print("Testing {0}: ".format(sample_path), end="") + with open(sample_path, "rb") as f: + data = f.read() + report = parsedmarc.parse_aggregate_report_file( + data, + offline=True, + always_use_local_files=True, + ) + self.assertEqual(report["report_metadata"]["org_name"], "Sample Reporter") + self.assertEqual(report["policy_published"]["domain"], "example.com") + print("Passed!") + + def testParseInvalidAggregateSample(self): + """Test invalid aggregate samples are handled""" + print() + sample_paths = glob("samples/aggregate_invalid/*") + for sample_path in sample_paths: + if os.path.isdir(sample_path): + continue + print("Testing {0}: ".format(sample_path), end="") + with self.subTest(sample=sample_path): + parsed_report = cast( + AggregateReport, + parsedmarc.parse_report_file( + sample_path, always_use_local_files=True, offline=OFFLINE_MODE + )["report"], + ) + parsedmarc.parsed_aggregate_reports_to_csv(parsed_report) + print("Passed!") + + def testParseReportFileWithBytes(self): + """parse_report_file handles bytes input""" + with open("samples/aggregate/dmarcbis-draft-sample.xml", "rb") as f: + data = f.read() + result = parsedmarc.parse_report_file( + data, always_use_local_files=True, offline=True + ) + self.assertEqual(result["report_type"], "aggregate") + + def testFailureReportCsvRoundtrip(self): + """Failure report CSV generation works on sample reports""" + print() + sample_paths = glob("samples/failure/*.eml") + for sample_path in sample_paths: + print("Testing CSV for {0}: ".format(sample_path), end="") + with self.subTest(sample=sample_path): + parsed_report = cast( + FailureReport, + parsedmarc.parse_report_file(sample_path, offline=OFFLINE_MODE)[ + "report" + ], + ) + csv_output = parsedmarc.parsed_failure_reports_to_csv(parsed_report) + self.assertIsNotNone(csv_output) + self.assertIn(",", csv_output) + rows = parsedmarc.parsed_failure_reports_to_csv_rows(parsed_report) + self.assertTrue(len(rows) > 0) + print("Passed!") + class TestLoadPSLOverrides(unittest.TestCase): """Covers `parsedmarc.utils.load_psl_overrides`.""" @@ -3344,6 +4953,803 @@ class TestGetBaseDomainWithOverrides(unittest.TestCase): self.assertEqual(result, "example.com") +class TestExtractReport(unittest.TestCase): + """Tests for parsedmarc.extract_report()""" + + def testExtractReportFromBytes(self): + """extract_report handles raw XML bytes""" + xml = b'' + result = parsedmarc.extract_report(xml) + self.assertIn("", result) + + def testExtractReportFromBase64Xml(self): + """extract_report handles base64-encoded XML string""" + import base64 + + xml = b'' + b64 = base64.b64encode(xml).decode() + result = parsedmarc.extract_report(b64) + self.assertIn("", result) + + def testExtractReportFromGzip(self): + """extract_report handles gzip compressed content""" + import gzip + + xml = b'' + compressed = gzip.compress(xml) + result = parsedmarc.extract_report(compressed) + self.assertIn("", result) + + def testExtractReportFromZip(self): + """extract_report handles zip compressed content""" + import zipfile + + xml = b'' + buf = BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + zf.writestr("report.xml", xml) + result = parsedmarc.extract_report(buf.getvalue()) + self.assertIn("", result) + + def testExtractReportFromBinaryIO(self): + """extract_report handles file-like BinaryIO objects""" + xml = b'' + bio = BytesIO(xml) + result = parsedmarc.extract_report(bio) + self.assertIn("", result) + + def testExtractReportFromNonSeekableStream(self): + """extract_report handles non-seekable streams""" + xml = b'' + + class NonSeekable: + def __init__(self, data): + self._data = data + self._pos = 0 + + def read(self, n=-1): + if n == -1: + result = self._data[self._pos :] + self._pos = len(self._data) + else: + result = self._data[self._pos : self._pos + n] + self._pos += n + return result + + def seekable(self): + return False + + def close(self): + pass + + result = parsedmarc.extract_report(cast(BinaryIO, NonSeekable(xml))) + self.assertIn("", result) + + def testExtractReportInvalidContent(self): + """extract_report raises ParserError for invalid content""" + with self.assertRaises(parsedmarc.ParserError): + parsedmarc.extract_report(b"this is not a valid archive") + + def testExtractReportTextModeRaises(self): + """extract_report raises ParserError for text-mode streams""" + + class TextStream: + def read(self, n=-1): + return "text data" + + def seekable(self): + return True + + def seek(self, pos): + pass + + def close(self): + pass + + with self.assertRaises(parsedmarc.ParserError): + parsedmarc.extract_report(cast(BinaryIO, TextStream())) + + +class TestMalformedXmlRecovery(unittest.TestCase): + """Tests for XML recovery in parse_aggregate_report_xml""" + + def testRecoversMalformedXml(self): + """Malformed XML triggers recovery path and still parses""" + # XML with a broken tag that xmltodict will reject but lxml can recover + malformed_xml = """ + + + example.com + dmarc@example.com + 12345 + 16800000001680086400 + + + example.com

none

+ + + 203.0.113.11 + nonepasspass + + example.com + example.compass + + """ + # lxml recovery may succeed or fail depending on how broken the XML is + # Either way, no unhandled exception should escape + try: + report = parsedmarc.parse_aggregate_report_xml(malformed_xml, offline=True) + self.assertIn("report_metadata", report) + except parsedmarc.InvalidAggregateReport: + pass # Also acceptable + + def testBytesXmlInput(self): + """XML bytes input is decoded""" + xml = b""" + + + example.com + dmarc@example.com + test-bytes-input + 16800000001680086400 + + + example.com

none

+ + + 203.0.113.11 + nonepasspass + + example.com + example.compass + +""" + report = parsedmarc.parse_aggregate_report_xml(xml.decode(), offline=True) + self.assertEqual(report["report_metadata"]["report_id"], "test-bytes-input") + + def testExpatErrorRaises(self): + """Completely invalid XML raises InvalidAggregateReport""" + with self.assertRaises(parsedmarc.InvalidAggregateReport): + parsedmarc.parse_aggregate_report_xml("not xml at all {}", offline=True) + + def testMissingOrgName(self): + """Missing org_name raises InvalidAggregateReport""" + xml = """ + + + dmarc@example.com + missing-org + 16800000001680086400 + + example.com

none

 + + 1.2.3.41 + nonepasspass + + example.com + example.compass + +""" + with self.assertRaises(parsedmarc.InvalidAggregateReport): + parsedmarc.parse_aggregate_report_xml(xml, offline=True) + + +class TestPolicyPublishedEdgeCases(unittest.TestCase): + """Tests for edge cases in policy_published parsing""" + + VALID_XML_TEMPLATE = """ + + + example.com + dmarc@example.com + test-{tag} + 16800000001680086400 + {extra_metadata} + + + example.com

reject

+ {policy_extra} + + + 203.0.113.11 + nonepasspass + + example.com + example.compass + +""" + + def _parse(self, tag="default", policy_extra="", extra_metadata=""): + xml = self.VALID_XML_TEMPLATE.format( + tag=tag, policy_extra=policy_extra, extra_metadata=extra_metadata + ) + return parsedmarc.parse_aggregate_report_xml(xml, offline=True) + + def testPolicyPublishedListHandled(self): + """policy_published as a list uses first element""" + # The code checks `if type(policy_published) is list` + # This is tested implicitly when xmltodict returns a list; + # we test via the np field presence + report = self._parse(tag="np", policy_extra="quarantine") + self.assertEqual(report["policy_published"]["np"], "quarantine") + + def testNpFieldValues(self): + """np field is parsed correctly""" + for val in ["none", "quarantine", "reject"]: + report = self._parse(tag=f"np-{val}", policy_extra=f"{val}") + self.assertEqual(report["policy_published"]["np"], val) + + def testTestingField(self): + """testing field is parsed correctly""" + for val in ["y", "n"]: + report = self._parse( + tag=f"testing-{val}", policy_extra=f"{val}" + ) + self.assertEqual(report["policy_published"]["testing"], val) + + def testDiscoveryMethodField(self): + """discovery_method field is parsed correctly""" + for val in ["psl", "treewalk"]: + report = self._parse( + tag=f"disc-{val}", + policy_extra=f"{val}", + ) + self.assertEqual(report["policy_published"]["discovery_method"], val) + + def testGeneratorField(self): + """generator field in report_metadata is parsed""" + report = self._parse( + tag="gen", extra_metadata="TestGen/1.0" + ) + self.assertEqual(report["report_metadata"]["generator"], "TestGen/1.0") + + def testPctFieldNone(self): + """pct defaults to None when absent (DMARCbis)""" + report = self._parse(tag="no-pct") + self.assertIsNone(report["policy_published"]["pct"]) + + def testFoFieldNone(self): + """fo defaults to None when absent (DMARCbis)""" + report = self._parse(tag="no-fo") + self.assertIsNone(report["policy_published"]["fo"]) + + def testReportMetadataErrors(self): + """Report metadata errors are captured""" + report = self._parse( + tag="errors", + extra_metadata="DNS timeout", + ) + self.assertIn("DNS timeout", report["report_metadata"]["errors"]) + + def testReportMetadataErrorsList(self): + """Report metadata errors as list are captured""" + report = self._parse( + tag="errors-list", + extra_metadata="error1error2", + ) + self.assertIn("error1", report["report_metadata"]["errors"]) + self.assertIn("error2", report["report_metadata"]["errors"]) + + def testRecordParseFailureSkipped(self): + """Bad records are skipped with a warning, not crashing""" + xml = """ + + + example.com + dmarc@example.com + bad-records + 16800000001680086400 + + example.com

none

 + + 203.0.113.11 + nonepasspass + + example.com + example.compass + + + bad-ipnot-a-number + nonepasspass + + example.com + example.compass + +""" + report = parsedmarc.parse_aggregate_report_xml(xml, offline=True) + # At least the valid record should be parsed + self.assertTrue(len(report["records"]) >= 1) + + +class TestParseReportFile(unittest.TestCase): + """Tests for parse_report_file with various input types""" + + def testParseReportFileFromBytes(self): + """parse_report_file works with bytes input""" + xml_path = "samples/aggregate/!example.com!1538204542!1538463818.xml" + with open(xml_path, "rb") as f: + content = f.read() + result = parsedmarc.parse_report_file(content, offline=True) + self.assertEqual(result["report_type"], "aggregate") + + def testParseReportFileFromBinaryIO(self): + """parse_report_file works with BinaryIO input""" + xml_path = "samples/aggregate/!example.com!1538204542!1538463818.xml" + with open(xml_path, "rb") as f: + result = parsedmarc.parse_report_file(f, offline=True) + self.assertEqual(result["report_type"], "aggregate") + + def testParseReportFileFromPathlib(self): + """parse_report_file works with pathlib.Path input""" + xml_path = Path("samples/aggregate/!example.com!1538204542!1538463818.xml") + result = parsedmarc.parse_report_file(xml_path, offline=True) + self.assertEqual(result["report_type"], "aggregate") + + def testParseReportFileSmtpTls(self): + """parse_report_file detects SMTP TLS reports""" + result = parsedmarc.parse_report_file( + "samples/smtp_tls/smtp_tls.json", offline=True + ) + self.assertEqual(result["report_type"], "smtp_tls") + + def testParseReportFileEmail(self): + """parse_report_file detects failure reports in email format""" + eml_path = "samples/failure/dmarc_ruf_report_linkedin.eml" + result = parsedmarc.parse_report_file(eml_path, offline=True) + self.assertEqual(result["report_type"], "failure") + + def testParseReportFileInvalid(self): + """parse_report_file raises ParserError for invalid content""" + with self.assertRaises(parsedmarc.ParserError): + parsedmarc.parse_report_file(b"this is not a report", offline=True) + + +class TestParseReportEmail(unittest.TestCase): + """Tests for parse_report_email edge cases""" + + def testSmtpTlsEmailReport(self): + """parse_report_email handles SMTP TLS reports in email format""" + eml_path = "samples/smtp_tls/google.com_smtp_tls_report.eml" + with open(eml_path, "rb") as f: + content = f.read() + result = parsedmarc.parse_report_email(content, offline=True) + self.assertEqual(result["report_type"], "smtp_tls") + + def testInvalidEmailRaisesError(self): + """parse_report_email raises error for non-DMARC email""" + email_str = """From: test@example.com +Subject: Hello World +Content-Type: text/plain + +This is not a DMARC report.""" + with self.assertRaises(parsedmarc.InvalidDMARCReport): + parsedmarc.parse_report_email(email_str, offline=True) + + +class TestFailureReportParsing(unittest.TestCase): + """Tests for failure report field defaults and edge cases""" + + def _make_feedback_report(self, **overrides): + """Create a minimal feedback report string""" + fields = { + "Feedback-Type": "auth-failure", + "User-Agent": "test/1.0", + "Version": "1", + "Original-Mail-From": "sender@example.com", + "Arrival-Date": "Thu, 1 Jan 2024 00:00:00 +0000", + "Source-IP": "203.0.113.1", + "Reported-Domain": "example.com", + "Auth-Failure": "dmarc", + } + fields.update(overrides) + return "\n".join(f"{k}: {v}" for k, v in fields.items()) + + def _make_sample(self): + return """From: sender@example.com +To: recipient@example.com +Subject: Test +Date: Thu, 1 Jan 2024 00:00:00 +0000 + +Test body""" + + def _default_msg_date(self): + return datetime(2024, 1, 1, 0, 0, 0, tzinfo=timezone.utc) + + def testMissingVersion(self): + """Missing version defaults to None""" + report_str = self._make_feedback_report() + lines = [ln for ln in report_str.split("\n") if not ln.startswith("Version:")] + report_str = "\n".join(lines) + report = parsedmarc.parse_failure_report( + report_str, self._make_sample(), self._default_msg_date(), offline=True + ) + self.assertIsNone(report["version"]) + + def testMissingUserAgent(self): + """Missing user_agent defaults to None""" + report_str = self._make_feedback_report() + lines = [ + ln for ln in report_str.split("\n") if not ln.startswith("User-Agent:") + ] + report_str = "\n".join(lines) + report = parsedmarc.parse_failure_report( + report_str, self._make_sample(), self._default_msg_date(), offline=True + ) + self.assertIsNone(report["user_agent"]) + + def testMissingDeliveryResult(self): + """Missing delivery_result maps to 'other' when field absent""" + report_str = self._make_feedback_report() + report = parsedmarc.parse_failure_report( + report_str, self._make_sample(), self._default_msg_date(), offline=True + ) + # When delivery_result is not in the parsed report, it's set to None, + # but then the validation check maps None (not in delivery_results list) to "other" + self.assertEqual(report["delivery_result"], "other") + + def testDeliveryResultMapped(self): + """Known delivery_result values are mapped correctly""" + for val in ["delivered", "spam", "policy", "reject"]: + report_str = self._make_feedback_report(**{"Delivery-Result": val}) + report = parsedmarc.parse_failure_report( + report_str, self._make_sample(), self._default_msg_date(), offline=True + ) + self.assertEqual(report["delivery_result"], val) + + def testDeliveryResultUnknownMapsToOther(self): + """Unknown delivery_result maps to 'other'""" + report_str = self._make_feedback_report(**{"Delivery-Result": "unknown-value"}) + report = parsedmarc.parse_failure_report( + report_str, self._make_sample(), self._default_msg_date(), offline=True + ) + self.assertEqual(report["delivery_result"], "other") + + def testIdentityAlignmentNone(self): + """identity_alignment='none' results in empty auth mechanisms""" + report_str = self._make_feedback_report(**{"Identity-Alignment": "none"}) + report = parsedmarc.parse_failure_report( + report_str, self._make_sample(), self._default_msg_date(), offline=True + ) + self.assertEqual(report["authentication_mechanisms"], []) + + def testIdentityAlignmentMultiple(self): + """identity_alignment with multiple values is split""" + report_str = self._make_feedback_report(**{"Identity-Alignment": "dkim,spf"}) + report = parsedmarc.parse_failure_report( + report_str, self._make_sample(), self._default_msg_date(), offline=True + ) + self.assertEqual(report["authentication_mechanisms"], ["dkim", "spf"]) + + def testMissingReportedDomainFallback(self): + """Missing reported_domain falls back to sample from domain""" + report_str = self._make_feedback_report() + lines = [ + ln for ln in report_str.split("\n") if not ln.startswith("Reported-Domain:") + ] + report_str = "\n".join(lines) + report = parsedmarc.parse_failure_report( + report_str, self._make_sample(), self._default_msg_date(), offline=True + ) + self.assertEqual(report["reported_domain"], "example.com") + + def testMissingArrivalDateWithMsgDate(self): + """Missing arrival_date uses msg_date fallback""" + report_str = self._make_feedback_report() + lines = [ + ln for ln in report_str.split("\n") if not ln.startswith("Arrival-Date:") + ] + report_str = "\n".join(lines) + msg_date = datetime(2024, 6, 15, 12, 0, 0, tzinfo=timezone.utc) + report = parsedmarc.parse_failure_report( + report_str, self._make_sample(), msg_date, offline=True + ) + self.assertIn("2024-06-15", report["arrival_date"]) + + def testMissingArrivalDateNoMsgDateRaises(self): + """Missing arrival_date with no msg_date raises""" + report_str = self._make_feedback_report() + lines = [ + ln for ln in report_str.split("\n") if not ln.startswith("Arrival-Date:") + ] + report_str = "\n".join(lines) + with self.assertRaises(parsedmarc.InvalidFailureReport): + parsedmarc.parse_failure_report( + report_str, + self._make_sample(), + cast(datetime, None), # intentionally None to test error path + offline=True, + ) + + +class TestWebhookClient(unittest.TestCase): + """Tests for webhook client initialization and close""" + + def testClose(self): + """WebhookClient.close() closes session""" + client = parsedmarc.webhook.WebhookClient( + aggregate_url="http://invalid.test/agg", + failure_url="http://invalid.test/fail", + smtp_tls_url="http://invalid.test/tls", + ) + mock_close = MagicMock() + client.session.close = mock_close + client.close() + mock_close.assert_called_once() + + +class TestUtilsDnsCaching(unittest.TestCase): + """Tests for DNS query caching and reverse DNS error handling""" + + def testQueryDnsUsesCacheHit(self): + """query_dns returns cached result without making DNS query""" + cache = ExpiringDict(max_len=100, max_age_seconds=60) + cache["example.com_A"] = ["1.2.3.4"] + result = parsedmarc.utils.query_dns("example.com", "A", cache=cache) + self.assertEqual(result, ["1.2.3.4"]) + + def testQueryDnsCachesResult(self): + """query_dns stores result in cache when cache is non-empty""" + cache = ExpiringDict(max_len=100, max_age_seconds=60) + # Pre-populate so ExpiringDict is truthy + cache["seed_key"] = ["seed"] + mock_record = MagicMock() + mock_record.to_text.return_value = '"1.2.3.4"' + mock_resolver = MagicMock() + mock_resolver.resolve.return_value = [mock_record] + with patch( + "parsedmarc.utils.dns.resolver.Resolver", return_value=mock_resolver + ): + result = parsedmarc.utils.query_dns( + "test-cache.example.com", "A", cache=cache + ) + self.assertEqual(result, ["1.2.3.4"]) + self.assertIn("test-cache.example.com_A", cache) + + def testReverseDnsReturnsNoneOnFailure(self): + """get_reverse_dns returns None on DNS exceptions""" + with patch( + "parsedmarc.utils.query_dns", + side_effect=dns.exception.DNSException("timeout"), + ): + result = parsedmarc.utils.get_reverse_dns("203.0.113.1") + self.assertIsNone(result) + + +class TestUtilsIpDbPaths(unittest.TestCase): + """Tests for IP database path validation""" + + def testCustomPathFallsBack(self): + """Non-existent custom db path falls back to default""" + result = parsedmarc.utils.get_ip_address_country( + "1.1.1.1", db_path="/nonexistent/path.mmdb" + ) + self.assertTrue(result is None or isinstance(result, str)) + + def testBundledDbWorks(self): + """Bundled IP database returns results""" + result = parsedmarc.utils.get_ip_address_country("8.8.8.8") + self.assertEqual(result, "US") + + +class TestUtilsParseEmail(unittest.TestCase): + """Tests for parse_email edge cases""" + + def testMinimalEmail(self): + """parse_email handles email with minimal headers""" + email_str = """From: test@example.com +Subject: Test + +Body text""" + result = parsedmarc.utils.parse_email(email_str) + self.assertEqual(result["subject"], "Test") + self.assertEqual(result["reply_to"], []) + + def testEmailWithNoSubject(self): + """parse_email defaults subject to None when missing""" + email_str = """From: test@example.com +To: other@example.com + +Body""" + result = parsedmarc.utils.parse_email(email_str) + self.assertIsNone(result["subject"]) + + def testEmailBytesInput(self): + """parse_email handles bytes input""" + email_bytes = b"""From: test@example.com +Subject: Bytes Test +To: other@example.com + +Body""" + result = parsedmarc.utils.parse_email(email_bytes) + self.assertEqual(result["subject"], "Bytes Test") + + def testEmailWithAttachments(self): + """parse_email with strip_attachment_payloads removes payloads""" + from email.mime.multipart import MIMEMultipart + from email.mime.text import MIMEText + from email.mime.base import MIMEBase + from email import encoders + + msg = MIMEMultipart() + msg["From"] = "test@example.com" + msg["To"] = "other@example.com" + msg["Subject"] = "Attachment Test" + msg.attach(MIMEText("Body text")) + + attachment = MIMEBase("application", "octet-stream") + attachment.set_payload(b"file content here") + encoders.encode_base64(attachment) + attachment.add_header("Content-Disposition", "attachment", filename="test.bin") + msg.attach(attachment) + + result = parsedmarc.utils.parse_email( + msg.as_string(), strip_attachment_payloads=True + ) + for att in result["attachments"]: + self.assertNotIn("payload", att) + + +class TestUtilsOutlookMsg(unittest.TestCase): + """Tests for Outlook MSG detection and conversion""" + + def testIsOutlookMsg(self): + """is_outlook_msg detects MSG magic bytes""" + msg_magic = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" + b"\x00" * 100 + self.assertTrue(parsedmarc.utils.is_outlook_msg(msg_magic)) + + def testIsNotOutlookMsg(self): + """is_outlook_msg rejects non-MSG content""" + self.assertFalse(parsedmarc.utils.is_outlook_msg(b"not an msg file")) + self.assertFalse(parsedmarc.utils.is_outlook_msg("string input")) + + def testConvertOutlookMsgInvalidInput(self): + """convert_outlook_msg raises ValueError for non-MSG bytes""" + with self.assertRaises(ValueError): + parsedmarc.utils.convert_outlook_msg(b"not an msg file") + + +class TestUtilsReverseDnsMap(unittest.TestCase): + """Tests for reverse DNS map loading""" + + def testLoadReverseDnsMapOffline(self): + """load_reverse_dns_map in offline mode loads bundled map""" + rdns_map = {} + parsedmarc.utils.load_reverse_dns_map(rdns_map, offline=True) + self.assertTrue(len(rdns_map) > 0) + + def testLoadReverseDnsMapLocalOverride(self): + """load_reverse_dns_map uses local_file_path when provided""" + with NamedTemporaryFile("w", suffix=".csv", delete=False) as f: + f.write("base_reverse_dns,name,type\n") + f.write("custom.example.com,Custom Service,hosting\n") + path = f.name + try: + rdns_map = {} + parsedmarc.utils.load_reverse_dns_map( + rdns_map, offline=True, local_file_path=path + ) + self.assertIn("custom.example.com", rdns_map) + self.assertEqual(rdns_map["custom.example.com"]["name"], "Custom Service") + finally: + os.remove(path) + + def testLoadReverseDnsMapNetworkFailureFallback(self): + """load_reverse_dns_map falls back to bundled on network error""" + rdns_map = {} + with patch( + "parsedmarc.utils.requests.get", + side_effect=requests.exceptions.ConnectionError("no network"), + ): + parsedmarc.utils.load_reverse_dns_map(rdns_map) + self.assertTrue(len(rdns_map) > 0) + + +class TestSmtpTlsReportErrors(unittest.TestCase): + """Tests for SMTP TLS report error handling""" + + def testMissingRequiredField(self): + """Missing required field raises InvalidSMTPTLSReport""" + json_str = json.dumps({"policies": []}) + with self.assertRaises(parsedmarc.InvalidSMTPTLSReport): + parsedmarc.parse_smtp_tls_report_json(json_str) + + def testInvalidJson(self): + """Invalid JSON raises InvalidSMTPTLSReport""" + with self.assertRaises(parsedmarc.InvalidSMTPTLSReport): + parsedmarc.parse_smtp_tls_report_json("not json {{{") + + +class TestBucketIntervalEdgeCases(unittest.TestCase): + """Tests for _bucket_interval_by_day edge cases""" + + def testDayCursorAdjustment(self): + """When begin is before midnight due to tz, day_cursor adjusts back""" + # Use a timezone where midnight calculation might cause day_cursor > begin + import pytz + + tz = pytz.FixedOffset(-600) # UTC-10 + begin = datetime(2024, 1, 1, 23, 30, 0, tzinfo=timezone.utc).astimezone(tz) + end = datetime(2024, 1, 3, 0, 0, 0, tzinfo=timezone.utc).astimezone(tz) + buckets = parsedmarc._bucket_interval_by_day(begin, end, 100) + total = sum(b["count"] for b in buckets) + self.assertEqual(total, 100) + + +class TestGetDmarcReportsFromMbox(unittest.TestCase): + """Tests for mbox parsing""" + + def testEmptyMbox(self): + """Empty mbox returns empty results""" + with NamedTemporaryFile(suffix=".mbox", delete=False) as f: + f.write(b"") + path = f.name + try: + results = parsedmarc.get_dmarc_reports_from_mbox(path, offline=True) + self.assertEqual(results["aggregate_reports"], []) + self.assertEqual(results["failure_reports"], []) + self.assertEqual(results["smtp_tls_reports"], []) + finally: + os.remove(path) + + def testMboxWithAggregateReport(self): + """Mbox with aggregate report email is parsed""" + from email.mime.multipart import MIMEMultipart + from email.mime.application import MIMEApplication + import gzip + + xml = b""" + + + example.com + dmarc@example.com + mbox-test-123 + 16800000001680086400 + + example.com

none

 + + 203.0.113.11 + nonepasspass + + example.com + example.compass + +""" + compressed = gzip.compress(xml) + + msg = MIMEMultipart() + msg["From"] = "dmarc@example.com" + msg["To"] = "postmaster@example.com" + msg["Subject"] = "DMARC Aggregate Report" + msg["Date"] = "Thu, 1 Jan 2024 00:00:00 +0000" + att = MIMEApplication(compressed, "gzip") + att.add_header("Content-Disposition", "attachment", filename="report.xml.gz") + msg.attach(att) + + with NamedTemporaryFile(suffix=".mbox", delete=False, mode="w") as f: + # mbox format requires "From " line + f.write("From dmarc@example.com Thu Jan 1 00:00:00 2024\n") + f.write(msg.as_string()) + f.write("\n") + path = f.name + try: + results = parsedmarc.get_dmarc_reports_from_mbox(path, offline=True) + self.assertTrue(len(results["aggregate_reports"]) >= 1) + finally: + os.remove(path) + + +class TestPslOverrides(unittest.TestCase): + """Tests for PSL override matching""" + + def testOverrideMatch(self): + """PSL overrides are applied when domain ends with override""" + # psl_overrides contains entries; test that get_base_domain + # handles them without error + result = parsedmarc.utils.get_base_domain("sub.example.com") + self.assertEqual(result, "example.com") + + class TestMapScriptsIPDetection(unittest.TestCase): """Full-IP detection and PSL folding in the map-maintenance scripts.""" @@ -3475,5 +5881,33 @@ class TestDetectPSLOverrides(unittest.TestCase): self.assertFalse(self.dpo.has_full_ip("example.com")) +class TestIsMbox(unittest.TestCase): + """Tests for is_mbox utility""" + + def testValidMbox(self): + """is_mbox returns True for valid mbox file""" + with NamedTemporaryFile(suffix=".mbox", delete=False, mode="w") as f: + f.write("From test@example.com Thu Jan 1 00:00:00 2024\n") + f.write("Subject: Test\n\nBody\n\n") + path = f.name + try: + self.assertTrue(parsedmarc.utils.is_mbox(path)) + finally: + os.remove(path) + + def testEmptyFileNotMbox(self): + """is_mbox returns False for empty file""" + with NamedTemporaryFile(suffix=".mbox", delete=False) as f: + path = f.name + try: + self.assertFalse(parsedmarc.utils.is_mbox(path)) + finally: + os.remove(path) + + def testNonExistentNotMbox(self): + """is_mbox returns False for non-existent file""" + self.assertFalse(parsedmarc.utils.is_mbox("/nonexistent/file.mbox")) + + if __name__ == "__main__": unittest.main(verbosity=2)