mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-07-05 16:25:09 +00:00
Prepare to test 3.0.0
This commit is contained in:
Sean Whalen
+3
-2
@@ -105,10 +105,10 @@ ENV/
|
||||
|
||||
# I/O files
|
||||
|
||||
output/
|
||||
*.xml
|
||||
*.zip
|
||||
*.gz
|
||||
*.json
|
||||
*.csv
|
||||
*.xls*
|
||||
*.eml
|
||||
@@ -117,6 +117,7 @@ ENV/
|
||||
# LibreOffice lock files
|
||||
.~*
|
||||
|
||||
# ignore data files
|
||||
# Data files
|
||||
*.dat
|
||||
*.mmdb
|
||||
|
||||
|
||||
+3
-2
@@ -1,9 +1,10 @@
|
||||
2.2.0
|
||||
3.0.0
|
||||
-----
|
||||
- Detect aggregate report email attachments by file content rather than
|
||||
file extension
|
||||
- In an aggregate report's `org_name` is a FQDN, the base is used
|
||||
- Add option to select the IMAP folder where reports are stored
|
||||
- Update CLI help
|
||||
- Add options to send data to Elasticsearch
|
||||
|
||||
2.1.2
|
||||
-----
|
||||
|
||||
@@ -0,0 +1,338 @@
|
||||
[
|
||||
{
|
||||
"_id": "269ba470-2871-11e8-b8b2-15742da3055c",
|
||||
"_type": "dashboard",
|
||||
"_source": {
|
||||
"title": "DMARC Summary",
|
||||
"hits": 0,
|
||||
"description": "",
|
||||
"panelsJSON": "[{\"panelIndex\":\"4\",\"gridData\":{\"x\":0,\"y\":3,\"w\":12,\"h\":3,\"i\":\"4\"},\"embeddableConfig\":{\"vis\":{\"legendOpen\":false}},\"id\":\"085eaa30-2870-11e8-b8b2-15742da3055c\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"7\",\"gridData\":{\"x\":0,\"y\":6,\"w\":4,\"h\":6,\"i\":\"7\"},\"embeddableConfig\":{\"spy\":null,\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"id\":\"620280a0-2886-11e8-b8b2-15742da3055c\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"8\",\"gridData\":{\"x\":4,\"y\":6,\"w\":4,\"h\":6,\"i\":\"8\"},\"id\":\"d787a580-2886-11e8-b8b2-15742da3055c\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"9\",\"gridData\":{\"x\":0,\"y\":0,\"w\":4,\"h\":3,\"i\":\"9\"},\"id\":\"356caa70-28d1-11e8-b8b2-15742da3055c\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"10\",\"gridData\":{\"x\":4,\"y\":0,\"w\":4,\"h\":3,\"i\":\"10\"},\"id\":\"7e26fb80-28d1-11e8-b8b2-15742da3055c\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"11\",\"gridData\":{\"x\":8,\"y\":0,\"w\":4,\"h\":3,\"i\":\"11\"},\"id\":\"93b823e0-28cf-11e8-b8b2-15742da3055c\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"12\",\"gridData\":{\"x\":1,\"y\":12,\"w\":10,\"h\":8,\"i\":\"12\"},\"embeddableConfig\":{\"mapZoom\":2,\"mapCenter\":[30.14512718337613,-0.703125]},\"id\":\"895f3a70-291d-11e8-b8b2-15742da3055c\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"13\",\"gridData\":{\"x\":8,\"y\":6,\"w\":4,\"h\":6,\"i\":\"13\"},\"version\":\"6.2.2\",\"type\":\"visualization\",\"id\":\"a69d0f40-2b02-11e8-8c8d-d3a0d2f2ba49\"}]",
|
||||
"optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}",
|
||||
"version": 1,
|
||||
"timeRestore": true,
|
||||
"timeTo": "now",
|
||||
"timeFrom": "now-7d",
|
||||
"refreshInterval": {
|
||||
"display": "30 seconds",
|
||||
"pause": true,
|
||||
"section": 1,
|
||||
"value": 30000
|
||||
},
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[],\"highlightAll\":true,\"version\":true}"
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "a41cfc70-2883-11e8-b8b2-15742da3055c",
|
||||
"_type": "dashboard",
|
||||
"_source": {
|
||||
"title": "DMARC Failures",
|
||||
"hits": 0,
|
||||
"description": "",
|
||||
"panelsJSON": "[{\"panelIndex\":\"2\",\"gridData\":{\"x\":5,\"y\":0,\"w\":7,\"h\":6,\"i\":\"2\"},\"id\":\"1fad3f60-2881-11e8-b8b2-15742da3055c\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"3\",\"gridData\":{\"x\":0,\"y\":6,\"w\":12,\"h\":6,\"i\":\"3\"},\"id\":\"40e7a5b0-2883-11e8-b8b2-15742da3055c\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"4\",\"gridData\":{\"x\":0,\"y\":0,\"w\":5,\"h\":6,\"i\":\"4\"},\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"}}}},\"id\":\"2ae719b0-2885-11e8-b8b2-15742da3055c\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"panelIndex\":\"5\",\"gridData\":{\"x\":1,\"y\":12,\"w\":10,\"h\":8,\"i\":\"5\"},\"embeddableConfig\":{\"mapCenter\":[30.14512718337613,-6.328125000000001],\"mapZoom\":2},\"id\":\"8b956350-2878-11e8-b8b2-15742da3055c\",\"type\":\"visualization\",\"version\":\"6.2.2\"}]",
|
||||
"optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}",
|
||||
"version": 1,
|
||||
"timeRestore": true,
|
||||
"timeTo": "now",
|
||||
"timeFrom": "now-7d",
|
||||
"refreshInterval": {
|
||||
"display": "Off",
|
||||
"pause": false,
|
||||
"section": 0,
|
||||
"value": 0
|
||||
},
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[],\"highlightAll\":true,\"version\":true}"
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "bbe4f890-295e-11e8-b8b2-15742da3055c",
|
||||
"_type": "dashboard",
|
||||
"_source": {
|
||||
"title": "DMARC Forensic Samples",
|
||||
"hits": 0,
|
||||
"description": "",
|
||||
"panelsJSON": "[{\"gridData\":{\"h\":8,\"i\":\"1\",\"w\":12,\"x\":0,\"y\":0},\"id\":\"def63400-295b-11e8-b8b2-15742da3055c\",\"panelIndex\":\"1\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"embeddableConfig\":{\"spy\":null,\"vis\":{\"params\":{\"sort\":{\"columnIndex\":4,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":6,\"i\":\"2\",\"w\":12,\"x\":0,\"y\":8},\"id\":\"316ef4e0-295e-11e8-b8b2-15742da3055c\",\"panelIndex\":\"2\",\"type\":\"visualization\",\"version\":\"6.2.2\"},{\"embeddableConfig\":{\"mapCenter\":[27.059125784374068,-0.703125],\"mapZoom\":2},\"gridData\":{\"h\":7,\"i\":\"3\",\"w\":10,\"x\":1,\"y\":14},\"id\":\"a386df70-295e-11e8-b8b2-15742da3055c\",\"panelIndex\":\"3\",\"type\":\"visualization\",\"version\":\"6.2.2\"}]",
|
||||
"optionsJSON": "{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}",
|
||||
"version": 1,
|
||||
"timeRestore": true,
|
||||
"timeTo": "now",
|
||||
"timeFrom": "now-7d",
|
||||
"refreshInterval": {
|
||||
"display": "Off",
|
||||
"pause": false,
|
||||
"section": 0,
|
||||
"value": 0
|
||||
},
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[],\"highlightAll\":true,\"version\":true}"
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "7e26fb80-28d1-11e8-b8b2-15742da3055c",
|
||||
"_type": "visualization",
|
||||
"_source": {
|
||||
"title": "DKIM Allignment",
|
||||
"visState": "{\"title\":\"DKIM Allignment\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"dkim_aligned\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"_term\",\"customLabel\":\"DKIM Alligned\"}}]}",
|
||||
"uiStateJSON": "{\"vis\":{\"legendOpen\":false}}",
|
||||
"description": "",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"index\":\"34fa53e0-28c1-11e8-b8b2-15742da3055c\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "356caa70-28d1-11e8-b8b2-15742da3055c",
|
||||
"_type": "visualization",
|
||||
"_source": {
|
||||
"title": "SPF Allignment",
|
||||
"visState": "{\"title\":\"SPF Allignment\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"spf_aligned\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"_term\",\"customLabel\":\"SPF Alligned\"}}]}",
|
||||
"uiStateJSON": "{\"vis\":{\"legendOpen\":false}}",
|
||||
"description": "",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"index\":\"34fa53e0-28c1-11e8-b8b2-15742da3055c\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "93b823e0-28cf-11e8-b8b2-15742da3055c",
|
||||
"_type": "visualization",
|
||||
"_source": {
|
||||
"title": "DMARC Passage",
|
||||
"visState": "{\"title\":\"DMARC Passage\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"passed_dmarc\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"_term\",\"customLabel\":\"Passed DMARC\"}}]}",
|
||||
"uiStateJSON": "{\"vis\":{\"legendOpen\":false}}",
|
||||
"description": "",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"index\":\"34fa53e0-28c1-11e8-b8b2-15742da3055c\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "8b956350-2878-11e8-b8b2-15742da3055c",
|
||||
"_type": "visualization",
|
||||
"_source": {
|
||||
"title": "Source Countries of Messages Failing DMARC",
|
||||
"visState": "{\"title\":\"Source Countries of Messages Failing DMARC\",\"type\":\"region_map\",\"params\":{\"legendPosition\":\"bottomright\",\"addTooltip\":true,\"colorSchema\":\"Yellow to Red\",\"selectedLayer\":{\"attribution\":\"<p><a href=\\\"http://www.naturalearthdata.com/about/terms-of-use\\\">Made with NaturalEarth</a> | <a href=\\\"https://www.elastic.co/elastic-maps-service\\\">Elastic Maps Service</a></p> \",\"name\":\"World Countries\",\"format\":{\"type\":\"geojson\"},\"url\":\"https://vector.maps.elastic.co/blob/5659313586569216?elastic_tile_service_tos=agree&my_app_version=6.2.2\",\"fields\":[{\"name\":\"iso2\",\"description\":\"Two letter abbreviation\"},{\"name\":\"name\",\"description\":\"Country name\"},{\"name\":\"iso3\",\"description\":\"Three letter abbreviation\"}],\"created_at\":\"2017-04-26T17:12:15.978370\",\"tags\":[],\"id\":5659313586569216,\"layerId\":\"elastic_maps_service.World Countries\"},\"selectedJoinField\":{\"name\":\"iso2\",\"description\":\"Two letter abbreviation\"},\"isDisplayWarning\":true,\"wms\":{\"enabled\":false,\"options\":{\"format\":\"image/png\",\"transparent\":true},\"baseLayersAreLoaded\":{\"_c\":[],\"_s\":1,\"_d\":true,\"_v\":true,\"_h\":0,\"_n\":false},\"tmsLayers\":[{\"id\":\"road_map\",\"url\":\"https://tiles.maps.elastic.co/v2/default/{z}/{x}/{y}.png?elastic_tile_service_tos=agree&my_app_name=kibana&my_app_version=6.2.2\",\"minZoom\":0,\"maxZoom\":10,\"attribution\":\"<p>© <a href=\\\"http://www.openstreetmap.org/copyright\\\">OpenStreetMap</a> contributors | <a href=\\\"https://www.elastic.co/elastic-maps-service\\\">Elastic Maps Service</a></p> \",\"subdomains\":[]}],\"selectedTmsLayer\":{\"id\":\"road_map\",\"url\":\"https://tiles.maps.elastic.co/v2/default/{z}/{x}/{y}.png?elastic_tile_service_tos=agree&my_app_name=kibana&my_app_version=6.2.2\",\"minZoom\":0,\"maxZoom\":10,\"attribution\":\"<p>© <a href=\\\"http://www.openstreetmap.org/copyright\\\">OpenStreetMap</a> contributors | <a href=\\\"https://www.elastic.co/elastic-maps-service\\\">Elastic Maps Service</a></p> \",\"subdomains\":[]}},\"mapZoom\":2,\"mapCenter\":[0,0],\"outlineWeight\":1,\"showAllShapes\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"source_country.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Country\"}}]}",
|
||||
"uiStateJSON": "{}",
|
||||
"description": "",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"index\":\"34fa53e0-28c1-11e8-b8b2-15742da3055c\",\"filter\":[{\"meta\":{\"index\":\"34fa53e0-28c1-11e8-b8b2-15742da3055c\",\"negate\":false,\"disabled\":false,\"alias\":null,\"type\":\"phrase\",\"key\":\"passed_dmarc\",\"value\":\"false\",\"params\":{\"query\":false,\"type\":\"phrase\"}},\"query\":{\"match\":{\"passed_dmarc\":{\"query\":false,\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "def63400-295b-11e8-b8b2-15742da3055c",
|
||||
"_type": "visualization",
|
||||
"_source": {
|
||||
"title": "Forensic Samples",
|
||||
"visState": "{\"title\":\"Forensic Samples\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"arrival_date\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"_term\",\"customLabel\":\"Arrival Date (UTC)\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"sample.headers.from.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"From\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"sample.headers.to.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"To\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"sample.reply_to.address.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"None\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Reply To\"}},{\"id\":\"6\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"sample.subject.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Subject\"}}]}",
|
||||
"uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
|
||||
"description": "",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"index\":\"3f4816b0-2958-11e8-b8b2-15742da3055c\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "895f3a70-291d-11e8-b8b2-15742da3055c",
|
||||
"_type": "visualization",
|
||||
"_source": {
|
||||
"title": "Message Source Countries",
|
||||
"visState": "{\"title\":\"Message Source Countries\",\"type\":\"region_map\",\"params\":{\"legendPosition\":\"bottomright\",\"addTooltip\":true,\"colorSchema\":\"Yellow to Red\",\"selectedLayer\":{\"attribution\":\"<p><a href=\\\"http://www.naturalearthdata.com/about/terms-of-use\\\">Made with NaturalEarth</a> | <a href=\\\"https://www.elastic.co/elastic-maps-service\\\">Elastic Maps Service</a></p> \",\"name\":\"World Countries\",\"format\":{\"type\":\"geojson\"},\"url\":\"https://vector.maps.elastic.co/blob/5659313586569216?elastic_tile_service_tos=agree&my_app_version=6.2.2\",\"fields\":[{\"name\":\"iso2\",\"description\":\"Two letter abbreviation\"},{\"name\":\"name\",\"description\":\"Country name\"},{\"name\":\"iso3\",\"description\":\"Three letter abbreviation\"}],\"created_at\":\"2017-04-26T17:12:15.978370\",\"tags\":[],\"id\":5659313586569216,\"layerId\":\"elastic_maps_service.World Countries\"},\"selectedJoinField\":{\"name\":\"iso2\",\"description\":\"Two letter abbreviation\"},\"isDisplayWarning\":true,\"wms\":{\"enabled\":false,\"options\":{\"format\":\"image/png\",\"transparent\":true},\"baseLayersAreLoaded\":{\"_c\":[],\"_s\":1,\"_d\":true,\"_v\":true,\"_h\":0,\"_n\":false},\"tmsLayers\":[{\"id\":\"road_map\",\"url\":\"https://tiles.maps.elastic.co/v2/default/{z}/{x}/{y}.png?elastic_tile_service_tos=agree&my_app_name=kibana&my_app_version=6.2.2\",\"minZoom\":0,\"maxZoom\":10,\"attribution\":\"<p>© <a href=\\\"http://www.openstreetmap.org/copyright\\\">OpenStreetMap</a> contributors | <a href=\\\"https://www.elastic.co/elastic-maps-service\\\">Elastic Maps Service</a></p> \",\"subdomains\":[]}],\"selectedTmsLayer\":{\"id\":\"road_map\",\"url\":\"https://tiles.maps.elastic.co/v2/default/{z}/{x}/{y}.png?elastic_tile_service_tos=agree&my_app_name=kibana&my_app_version=6.2.2\",\"minZoom\":0,\"maxZoom\":10,\"attribution\":\"<p>© <a href=\\\"http://www.openstreetmap.org/copyright\\\">OpenStreetMap</a> contributors | <a href=\\\"https://www.elastic.co/elastic-maps-service\\\">Elastic Maps Service</a></p> \",\"subdomains\":[]}},\"mapZoom\":2,\"mapCenter\":[0,0],\"outlineWeight\":1,\"showAllShapes\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"source_country.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":200,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Country\"}}]}",
|
||||
"uiStateJSON": "{\"mapZoom\":3,\"mapCenter\":[27.68352808378776,5.537109375000001]}",
|
||||
"description": "",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"index\":\"34fa53e0-28c1-11e8-b8b2-15742da3055c\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "a386df70-295e-11e8-b8b2-15742da3055c",
|
||||
"_type": "visualization",
|
||||
"_source": {
|
||||
"title": "Forensic Sample Source Countries",
|
||||
"visState": "{\"title\":\"Forensic Sample Source Countries\",\"type\":\"region_map\",\"params\":{\"legendPosition\":\"bottomright\",\"addTooltip\":true,\"colorSchema\":\"Yellow to Red\",\"selectedLayer\":{\"attribution\":\"<p><a href=\\\"http://www.naturalearthdata.com/about/terms-of-use\\\">Made with NaturalEarth</a> | <a href=\\\"https://www.elastic.co/elastic-maps-service\\\">Elastic Maps Service</a></p> \",\"name\":\"World Countries\",\"format\":{\"type\":\"geojson\"},\"url\":\"https://vector.maps.elastic.co/blob/5659313586569216?elastic_tile_service_tos=agree&my_app_version=6.2.2\",\"fields\":[{\"name\":\"iso2\",\"description\":\"Two letter abbreviation\"},{\"name\":\"name\",\"description\":\"Country name\"},{\"name\":\"iso3\",\"description\":\"Three letter abbreviation\"}],\"created_at\":\"2017-04-26T17:12:15.978370\",\"tags\":[],\"id\":5659313586569216,\"layerId\":\"elastic_maps_service.World Countries\"},\"selectedJoinField\":{\"name\":\"iso2\",\"description\":\"Two letter abbreviation\"},\"isDisplayWarning\":true,\"wms\":{\"enabled\":false,\"options\":{\"format\":\"image/png\",\"transparent\":true},\"baseLayersAreLoaded\":{\"_c\":[],\"_s\":1,\"_d\":true,\"_v\":true,\"_h\":0,\"_n\":false},\"tmsLayers\":[{\"id\":\"road_map\",\"url\":\"https://tiles.maps.elastic.co/v2/default/{z}/{x}/{y}.png?elastic_tile_service_tos=agree&my_app_name=kibana&my_app_version=6.2.2\",\"minZoom\":0,\"maxZoom\":10,\"attribution\":\"<p>© <a href=\\\"http://www.openstreetmap.org/copyright\\\">OpenStreetMap</a> contributors | <a href=\\\"https://www.elastic.co/elastic-maps-service\\\">Elastic Maps Service</a></p> \",\"subdomains\":[]}],\"selectedTmsLayer\":{\"id\":\"road_map\",\"url\":\"https://tiles.maps.elastic.co/v2/default/{z}/{x}/{y}.png?elastic_tile_service_tos=agree&my_app_name=kibana&my_app_version=6.2.2\",\"minZoom\":0,\"maxZoom\":10,\"attribution\":\"<p>© <a href=\\\"http://www.openstreetmap.org/copyright\\\">OpenStreetMap</a> contributors | <a href=\\\"https://www.elastic.co/elastic-maps-service\\\">Elastic Maps Service</a></p> \",\"subdomains\":[]}},\"mapZoom\":2,\"mapCenter\":[0,0],\"outlineWeight\":1,\"showAllShapes\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"source_country.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":200,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Country\"}}]}",
|
||||
"uiStateJSON": "{}",
|
||||
"description": "",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"index\":\"3f4816b0-2958-11e8-b8b2-15742da3055c\",\"filter\":[],\"query\":{\"language\":\"lucene\",\"query\":\"\"}}"
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "085eaa30-2870-11e8-b8b2-15742da3055c",
|
||||
"_type": "visualization",
|
||||
"_source": {
|
||||
"title": "DMARC Passage Over Time",
|
||||
"visState": "{\"title\":\"DMARC Passage Over Time\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Messages\"},\"drawLinesBetweenPoints\":true,\"mode\":\"normal\",\"show\":\"true\",\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"line\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Messages\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"date_range\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"passed_dmarc\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Passed DMARC\"}}]}",
|
||||
"uiStateJSON": "{}",
|
||||
"description": "",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"index\":\"34fa53e0-28c1-11e8-b8b2-15742da3055c\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "620280a0-2886-11e8-b8b2-15742da3055c",
|
||||
"_type": "visualization",
|
||||
"_source": {
|
||||
"title": "Reporting Organizations",
|
||||
"visState": "{\"title\":\"Reporting Organizations\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"org_name.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Name\"}}]}",
|
||||
"uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}}",
|
||||
"description": "",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"index\":\"34fa53e0-28c1-11e8-b8b2-15742da3055c\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "40e7a5b0-2883-11e8-b8b2-15742da3055c",
|
||||
"_type": "visualization",
|
||||
"_source": {
|
||||
"title": "DKIM Alignment Failures",
|
||||
"visState": "{\"title\":\"DKIM Alignment Failures\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":4,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"header_from.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Header From\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"dkim_results.domain.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Missing\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"DKIM Domain\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"dkim_results.result.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Missing\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"DKIM Result\"}},{\"id\":\"6\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_base_domain.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"No Reverse DNS\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Reverse DNS Base\"}}]}",
|
||||
"uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":4,\"direction\":\"desc\"}}}}",
|
||||
"description": "",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"index\":\"34fa53e0-28c1-11e8-b8b2-15742da3055c\",\"filter\":[{\"meta\":{\"index\":\"a9ba2300-286b-11e8-b8b2-15742da3055c\",\"negate\":false,\"disabled\":false,\"alias\":null,\"type\":\"phrase\",\"key\":\"dkim_aligned\",\"value\":false,\"params\":{\"query\":false,\"type\":\"phrase\"}},\"query\":{\"match\":{\"dkim_aligned\":{\"query\":false,\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "1fad3f60-2881-11e8-b8b2-15742da3055c",
|
||||
"_type": "visualization",
|
||||
"_source": {
|
||||
"title": "SPF Allignment Failures",
|
||||
"visState": "{\"title\":\"SPF Allignment Failures\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showMeticsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":3,\"direction\":\"desc\"},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"header_from.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Missing\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Header From\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"envelope_from.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Missing\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"_term\",\"customLabel\":\"Envelope From\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_base_domain.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"No Reverse DNS\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Reverse DNS Base\"}}]}",
|
||||
"uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":3,\"direction\":\"desc\"}}}}",
|
||||
"description": "",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"index\":\"34fa53e0-28c1-11e8-b8b2-15742da3055c\",\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"index\":\"a9ba2300-286b-11e8-b8b2-15742da3055c\",\"key\":\"spf_aligned\",\"negate\":false,\"params\":{\"query\":false,\"type\":\"phrase\"},\"type\":\"phrase\",\"value\":false},\"query\":{\"match\":{\"spf_aligned\":{\"query\":false,\"type\":\"phrase\"}}}}],\"query\":{\"language\":\"lucene\",\"query\":\"\"}}"
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "2ae719b0-2885-11e8-b8b2-15742da3055c",
|
||||
"_type": "visualization",
|
||||
"_source": {
|
||||
"title": "DMARC Falure Sources",
|
||||
"visState": "{\"title\":\"DMARC Falure Sources\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":3,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"header_from.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Missing\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Header From\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_base_domain.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"No Reverse DNS\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Reverse DNS Base\"}}]}",
|
||||
"uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":3,\"direction\":\"desc\"}}}}",
|
||||
"description": "",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"index\":\"34fa53e0-28c1-11e8-b8b2-15742da3055c\",\"filter\":[{\"meta\":{\"index\":\"34fa53e0-28c1-11e8-b8b2-15742da3055c\",\"negate\":false,\"disabled\":false,\"alias\":null,\"type\":\"phrase\",\"key\":\"passed_dmarc\",\"value\":\"false\",\"params\":{\"query\":false,\"type\":\"phrase\"}},\"query\":{\"match\":{\"passed_dmarc\":{\"query\":false,\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "316ef4e0-295e-11e8-b8b2-15742da3055c",
|
||||
"_type": "visualization",
|
||||
"_source": {
|
||||
"title": "Forensic Sample Sources",
|
||||
"visState": "{\"title\":\"Forensic Sample Sources\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_ip_address.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"IP Address\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_reverse_dns.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"None\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Reverse DNS\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_base_domain.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"None\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Reverse DNS Base\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_country.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Country\"}}]}",
|
||||
"uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
|
||||
"description": "",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"index\":\"3f4816b0-2958-11e8-b8b2-15742da3055c\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "d787a580-2886-11e8-b8b2-15742da3055c",
|
||||
"_type": "visualization",
|
||||
"_source": {
|
||||
"title": "Message Sources by Reverse DNS",
|
||||
"visState": "{\"title\":\"Message Sources by Reverse DNS\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_base_domain.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"No reverse DNS\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Reverse DNS Base\"}}]}",
|
||||
"uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}}",
|
||||
"description": "",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"index\":\"34fa53e0-28c1-11e8-b8b2-15742da3055c\",\"filter\":[],\"query\":{\"language\":\"lucene\",\"query\":\"\"}}"
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"_id": "a69d0f40-2b02-11e8-8c8d-d3a0d2f2ba49",
|
||||
"_type": "visualization",
|
||||
"_source": {
|
||||
"title": "Message Volume by Header From",
|
||||
"visState": "{\"title\":\"Message Volume by Header From\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"domain.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Header From\"}}]}",
|
||||
"uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
|
||||
"description": "",
|
||||
"version": 1,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": "{\"index\":\"34fa53e0-28c1-11e8-b8b2-15742da3055c\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}"
|
||||
}
|
||||
},
|
||||
"_meta": {
|
||||
"savedObjectVersion": 2
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -1,7 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""A Python module and CLI for parsing DMARC reports"""
|
||||
"""A Python module for parsing DMARC reports"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
@@ -18,13 +17,10 @@ import re
|
||||
from base64 import b64decode
|
||||
import binascii
|
||||
import shutil
|
||||
from argparse import ArgumentParser
|
||||
from glob import glob
|
||||
import email
|
||||
import tempfile
|
||||
import subprocess
|
||||
import socket
|
||||
from time import sleep
|
||||
from email.mime.application import MIMEApplication
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.text import MIMEText
|
||||
@@ -45,7 +41,7 @@ import imapclient.exceptions
|
||||
import dateparser
|
||||
import mailparser
|
||||
|
||||
__version__ = "2.2.0"
|
||||
__version__ = "3.0.0"
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.INFO)
|
||||
@@ -130,6 +126,7 @@ def _query_dns(domain, record_type, nameservers=None, timeout=6.0):
|
||||
domain (str): The domain or subdomain to query about
|
||||
record_type (str): The record type to query for
|
||||
nameservers (list): A list of one or more nameservers to use
|
||||
(8.8.8.8 and 4.4.4.4 by default)
|
||||
timeout (float): Sets the DNS timeout in seconds
|
||||
|
||||
Returns:
|
||||
@@ -137,8 +134,9 @@ def _query_dns(domain, record_type, nameservers=None, timeout=6.0):
|
||||
"""
|
||||
resolver = dns.resolver.Resolver()
|
||||
timeout = float(timeout)
|
||||
if nameservers:
|
||||
resolver.nameservers = nameservers
|
||||
if nameservers is None:
|
||||
nameservers = ["8.8.8.8", "4.4.4.4"]
|
||||
resolver.nameservers = nameservers
|
||||
resolver.timeout = timeout
|
||||
resolver.lifetime = timeout
|
||||
return list(map(
|
||||
@@ -152,7 +150,8 @@ def _get_reverse_dns(ip_address, nameservers=None, timeout=6.0):
|
||||
|
||||
Args:
|
||||
ip_address (str): The IP address to resolve
|
||||
nameservers (list): A list of nameservers to query
|
||||
nameservers (list): A list of one or more nameservers to use
|
||||
(8.8.8.8 and 4.4.4.4 by default)
|
||||
timeout (float): Sets the DNS query timeout in seconds
|
||||
|
||||
Returns:
|
||||
@@ -277,6 +276,7 @@ def _get_ip_address_info(ip_address, nameservers=None, timeout=6.0):
|
||||
Args:
|
||||
ip_address (str): The IP address to check
|
||||
nameservers (list): A list of one or more nameservers to use
|
||||
(8.8.8.8 and 4.4.4.4 by default)
|
||||
timeout (float): Sets the DNS timeout in seconds
|
||||
|
||||
Returns:
|
||||
@@ -308,11 +308,14 @@ def _parse_report_record(record, nameservers=None, timeout=6.0):
|
||||
Args:
|
||||
record (OrderedDict): The record to convert
|
||||
nameservers (list): A list of one or more nameservers to use
|
||||
(8.8.8.8 and 4.4.4.4 by default)
|
||||
timeout (float): Sets the DNS timeout in seconds
|
||||
|
||||
Returns:
|
||||
OrderedDict: The converted record
|
||||
"""
|
||||
if nameservers is None:
|
||||
nameservers = ["8.8.8.8", "4.4.4.4"]
|
||||
record = record.copy()
|
||||
new_record = OrderedDict()
|
||||
new_record["source"] = _get_ip_address_info(record["row"]["source_ip"],
|
||||
@@ -399,6 +402,7 @@ def parse_aggregate_report_xml(xml, nameservers=None, timeout=6.0):
|
||||
Args:
|
||||
xml (str): A string of DMARC aggregate report XML
|
||||
nameservers (list): A list of one or more nameservers to use
|
||||
(8.8.8.8 and 4.4.4.4 by default)
|
||||
timeout (float): Sets the DNS timeout in seconds
|
||||
|
||||
Returns:
|
||||
@@ -412,7 +416,8 @@ def parse_aggregate_report_xml(xml, nameservers=None, timeout=6.0):
|
||||
schema = report["version"]
|
||||
new_report = OrderedDict([("xml_schema", schema)])
|
||||
new_report_metadata = OrderedDict()
|
||||
new_report_metadata["org_name"] = report_metadata["org_name"]
|
||||
org_name = _get_base_domain(report_metadata["org_name"])
|
||||
new_report_metadata["org_name"] = org_name
|
||||
new_report_metadata["org_email"] = report_metadata["email"]
|
||||
extra = None
|
||||
if "extra_contact_info" in report_metadata:
|
||||
@@ -530,6 +535,7 @@ def parse_aggregate_report_file(_input, nameservers=None, timeout=6.0):
|
||||
Args:
|
||||
_input: A path to a file, a file like object, or bytes
|
||||
nameservers (list): A list of one or more nameservers to use
|
||||
(8.8.8.8 and 4.4.4.4 by default)
|
||||
timeout (float): Sets the DNS timeout in seconds
|
||||
|
||||
Returns:
|
||||
@@ -656,6 +662,7 @@ def parse_forensic_report(feedback_report, sample, sample_headers_only,
|
||||
sample (str): The RFC 822 headers or RFC 822 message sample
|
||||
sample_headers_only (bool): Set true if the sample is only headers
|
||||
nameservers (list): A list of one or more nameservers to use
|
||||
(8.8.8.8 and 4.4.4.4 by default)
|
||||
timeout (float): Sets the DNS timeout in seconds
|
||||
|
||||
Returns:
|
||||
@@ -977,6 +984,7 @@ def parse_report_file(input_, nameservers=None, timeout=6.0):
|
||||
Args:
|
||||
input_: A path to a file, a file like object, or bytes
|
||||
nameservers (list): A list of one or more nameservers to use
|
||||
(8.8.8.8 and 4.4.4.4 by default)
|
||||
timeout (float): Sets the DNS timeout in seconds
|
||||
|
||||
Returns:
|
||||
@@ -1317,7 +1325,8 @@ def watch_inbox(host, username, password, callback, reports_folder="INBOX",
|
||||
delete (bool): Delete messages after processing them
|
||||
test (bool): Do not move or delete messages after processing them
|
||||
wait (int): Number of seconds to wait for a IMAP IDLE response
|
||||
nameservers (list): A list of DNS nameservers to query
|
||||
nameservers (list): A list of one or more nameservers to use
|
||||
(8.8.8.8 and 4.4.4.4 by default)
|
||||
dns_timeout (float): Set the DNS query timeout
|
||||
"""
|
||||
rf = reports_folder
|
||||
@@ -1374,152 +1383,3 @@ def watch_inbox(host, username, password, callback, reports_folder="INBOX",
|
||||
server.logout()
|
||||
|
||||
|
||||
def _main():
|
||||
"""Called when the module in executed"""
|
||||
def print_results(results_):
|
||||
"""
|
||||
Print results in human readable format
|
||||
|
||||
Args:
|
||||
results_ (OrderedDict): Parsing results
|
||||
"""
|
||||
print(json.dumps(results_, ensure_ascii=False, indent=2), "\n")
|
||||
|
||||
arg_parser = ArgumentParser(description="Parses DMARC reports")
|
||||
arg_parser.add_argument("file_path", nargs="*",
|
||||
help="one or more paths to aggregate or forensic "
|
||||
"report files or emails")
|
||||
arg_parser.add_argument("-o", "--output",
|
||||
help="Write output files to the given directory")
|
||||
arg_parser.add_argument("-n", "--nameservers", nargs="+",
|
||||
help="nameservers to query")
|
||||
arg_parser.add_argument("-t", "--timeout",
|
||||
help="number of seconds to wait for an answer "
|
||||
"from DNS (default 6.0)",
|
||||
type=float,
|
||||
default=6.0)
|
||||
arg_parser.add_argument("-H", "--host", help="IMAP hostname or IP address")
|
||||
arg_parser.add_argument("-u", "--user", help="IMAP user")
|
||||
arg_parser.add_argument("-p", "--password", help="IMAP password")
|
||||
arg_parser.add_argument("-r", "--reports-folder", default="INBOX",
|
||||
help="The IMAP folder containing the reports\n"
|
||||
"Default: INBOX")
|
||||
arg_parser.add_argument("-a", "--archive-folder",
|
||||
help="Specifies the IMAP folder to move "
|
||||
"messages to after processing them\n"
|
||||
"Default: Archive",
|
||||
default="Archive")
|
||||
arg_parser.add_argument("-d", "--delete",
|
||||
help="Delete the reports after processing them",
|
||||
action="store_true", default=False)
|
||||
arg_parser.add_argument("-O", "--outgoing-host",
|
||||
help="Email the results using this host")
|
||||
arg_parser.add_argument("-U", "--outgoing-user",
|
||||
help="Email the results using this user")
|
||||
arg_parser.add_argument("-P", "--outgoing-password",
|
||||
help="Email the results using this password")
|
||||
arg_parser.add_argument("-F", "--outgoing-from",
|
||||
help="Email the results using this from address")
|
||||
arg_parser.add_argument("-T", "--outgoing-to", nargs="+",
|
||||
help="Email the results to these addresses")
|
||||
arg_parser.add_argument("-S", "--outgoing-subject",
|
||||
help="Email the results using this subject")
|
||||
arg_parser.add_argument("-A", "--outgoing-attachment",
|
||||
help="Email the results using this filename")
|
||||
arg_parser.add_argument("-M", "--outgoing-message",
|
||||
help="Email the results using this message")
|
||||
|
||||
arg_parser.add_argument("-i", "--idle", action="store_true",
|
||||
help="Use an IMAP IDLE connection to process "
|
||||
"reports as they arrive in the inbox")
|
||||
arg_parser.add_argument("--test",
|
||||
help="Do not move or delete IMAP messages",
|
||||
action="store_true", default=False)
|
||||
arg_parser.add_argument("-v", "--version", action="version",
|
||||
version=__version__)
|
||||
|
||||
aggregate_reports = []
|
||||
forensic_reports = []
|
||||
|
||||
args = arg_parser.parse_args()
|
||||
file_paths = []
|
||||
for file_path in args.file_path:
|
||||
file_paths += glob(file_path)
|
||||
file_paths = list(set(file_paths))
|
||||
|
||||
for file_path in file_paths:
|
||||
try:
|
||||
file_results = parse_report_file(file_path,
|
||||
nameservers=args.nameservers,
|
||||
timeout=args.timeout)
|
||||
if file_results["report_type"] == "aggregate":
|
||||
aggregate_reports.append(file_results["report"])
|
||||
elif file_results["report_type"] == "forensic":
|
||||
forensic_reports.append(file_results["report"])
|
||||
|
||||
except ParserError as error:
|
||||
logger.error("Failed to parse {0} - {1}".format(file_path,
|
||||
error))
|
||||
|
||||
if args.host:
|
||||
try:
|
||||
if args.user is None or args.password is None:
|
||||
logger.error("user and password must be specified if"
|
||||
"host is specified")
|
||||
|
||||
rf = args.reports_folder
|
||||
af = args.archive_folder
|
||||
reports = get_dmarc_reports_from_inbox(args.host,
|
||||
args.user,
|
||||
args.password,
|
||||
reports_folder=rf,
|
||||
archive_folder=af,
|
||||
delete=args.delete,
|
||||
test=args.test)
|
||||
|
||||
aggregate_reports += reports["aggregate_reports"]
|
||||
forensic_reports += reports["forensic_reports"]
|
||||
|
||||
except IMAPError as error:
|
||||
logger.error("IMAP Error: {0}".format(error.__str__()))
|
||||
exit(1)
|
||||
|
||||
results = OrderedDict([("aggregate_reports", aggregate_reports),
|
||||
("forensic_reports", forensic_reports)])
|
||||
|
||||
if args.output:
|
||||
save_output(results, output_directory=args.output)
|
||||
|
||||
print_results(results)
|
||||
|
||||
if args.outgoing_host:
|
||||
if args.outgoing_from is None or args.outgoing_to is None:
|
||||
logger.error("--outgoing-from and --outgoing-to must "
|
||||
"be provided if --outgoing-host is used")
|
||||
exit(1)
|
||||
|
||||
try:
|
||||
email_results(results, args.outgoing_host, args.outgoing_from,
|
||||
args.outgoing_to, user=args.outgoing_user,
|
||||
password=args.outgoing_password,
|
||||
subject=args.outgoing_subject)
|
||||
except SMTPError as error:
|
||||
logger.error("SMTP Error: {0}".format(error.__str__()))
|
||||
exit(1)
|
||||
|
||||
if args.host and args.idle:
|
||||
sleep(2)
|
||||
logger.warning("The IMAP Connection is now in IDLE mode. "
|
||||
"Send yourself an email, or quit with ^c")
|
||||
try:
|
||||
watch_inbox(args.host, args.username, args.password, print_results,
|
||||
archive_folder=args.archive_folder, delete=args.delete,
|
||||
test=args.test, nameservers=args.nameservers,
|
||||
dns_timeout=args.timeout)
|
||||
except IMAPError as error:
|
||||
logger.error("IMAP Error: {0}".format(error.__str__()))
|
||||
exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
_main()
|
||||
@@ -0,0 +1,189 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""A CLI for parsing DMARC reports"""
|
||||
|
||||
from argparse import ArgumentParser
|
||||
from glob import glob
|
||||
from time import sleep
|
||||
|
||||
from parsedmarc import *
|
||||
from parsedmarc import elastic
|
||||
|
||||
__version__ = "3.0.0"
|
||||
|
||||
|
||||
def _main():
|
||||
"""Called when the module in executed"""
|
||||
def callback(results_, save_aggregate=False, save_forensic=False):
|
||||
"""
|
||||
Prints results and optionally saves them to Elasticsearch
|
||||
|
||||
Args:
|
||||
results_ (OrderedDict): Parsing results
|
||||
"""
|
||||
print(json.dumps(results_, ensure_ascii=False, indent=2), "\n")
|
||||
|
||||
arg_parser = ArgumentParser(description="Parses DMARC reports")
|
||||
arg_parser.add_argument("file_path", nargs="*",
|
||||
help="one or more paths to aggregate or forensic "
|
||||
"report files or emails")
|
||||
arg_parser.add_argument("-o", "--output",
|
||||
help="Write output files to the given directory")
|
||||
arg_parser.add_argument("-n", "--nameservers", nargs="+",
|
||||
help="nameservers to query "
|
||||
"(Default 8.8.8.8 4.4.4.4)")
|
||||
arg_parser.add_argument("-t", "--timeout",
|
||||
help="number of seconds to wait for an answer "
|
||||
"from DNS (default 6.0)",
|
||||
type=float,
|
||||
default=6.0)
|
||||
arg_parser.add_argument("-H", "--host", help="IMAP hostname or IP address")
|
||||
arg_parser.add_argument("-u", "--user", help="IMAP user")
|
||||
arg_parser.add_argument("-p", "--password", help="IMAP password")
|
||||
arg_parser.add_argument("-r", "--reports-folder", default="INBOX",
|
||||
help="The IMAP folder containing the reports\n"
|
||||
"Default: INBOX")
|
||||
arg_parser.add_argument("-a", "--archive-folder",
|
||||
help="Specifies the IMAP folder to move "
|
||||
"messages to after processing them\n"
|
||||
"Default: Archive",
|
||||
default="Archive")
|
||||
arg_parser.add_argument("-d", "--delete",
|
||||
help="Delete the reports after processing them",
|
||||
action="store_true", default=False)
|
||||
|
||||
arg_parser.add_argument("-E", "--elasticsearch-hosts", nargs="*",
|
||||
help="A list of one or more Elasticsearch "
|
||||
"hostnames or URLs to use (Default "
|
||||
"localhost:9200)",
|
||||
default=["localhost:9200"])
|
||||
arg_parser.add_argument("-A", "--save-aggregate", action="store_true",
|
||||
default=False,
|
||||
help="Save aggregate reports to Elasticsearch")
|
||||
arg_parser.add_argument("--save-forensic", action="store_true",
|
||||
default=False,
|
||||
help="Save forensic reports to Elasticsearch")
|
||||
arg_parser.add_argument("-O", "--outgoing-host",
|
||||
help="Email the results using this host")
|
||||
arg_parser.add_argument("-U", "--outgoing-user",
|
||||
help="Email the results using this user")
|
||||
arg_parser.add_argument("-P", "--outgoing-password",
|
||||
help="Email the results using this password")
|
||||
arg_parser.add_argument("-F", "--outgoing-from",
|
||||
help="Email the results using this from address")
|
||||
arg_parser.add_argument("-T", "--outgoing-to", nargs="+",
|
||||
help="Email the results to these addresses")
|
||||
arg_parser.add_argument("-S", "--outgoing-subject",
|
||||
help="Email the results using this subject")
|
||||
arg_parser.add_argument("-A", "--outgoing-attachment",
|
||||
help="Email the results using this filename")
|
||||
arg_parser.add_argument("-M", "--outgoing-message",
|
||||
help="Email the results using this message")
|
||||
|
||||
arg_parser.add_argument("-i", "--idle", action="store_true",
|
||||
help="Use an IMAP IDLE connection to process "
|
||||
"reports as they arrive in the inbox")
|
||||
arg_parser.add_argument("--test",
|
||||
help="Do not move or delete IMAP messages",
|
||||
action="store_true", default=False)
|
||||
arg_parser.add_argument("-v", "--version", action="version",
|
||||
version=__version__)
|
||||
|
||||
aggregate_reports = []
|
||||
forensic_reports = []
|
||||
|
||||
args = arg_parser.parse_args()
|
||||
|
||||
if args.host is None and len(args.file_path) == 0:
|
||||
arg_parser.print_help()
|
||||
exit(1)
|
||||
|
||||
if args.save_aggregate or args.save_forensic:
|
||||
elastic.set_hosts(args.elasticsearch_hosts)
|
||||
elastic.create_indexes()
|
||||
|
||||
file_paths = []
|
||||
for file_path in args.file_path:
|
||||
file_paths += glob(file_path)
|
||||
file_paths = list(set(file_paths))
|
||||
|
||||
for file_path in file_paths:
|
||||
try:
|
||||
file_results = parse_report_file(file_path,
|
||||
nameservers=args.nameservers,
|
||||
timeout=args.timeout)
|
||||
if file_results["report_type"] == "aggregate":
|
||||
aggregate_reports.append(file_results["report"])
|
||||
elif file_results["report_type"] == "forensic":
|
||||
forensic_reports.append(file_results["report"])
|
||||
|
||||
except ParserError as error:
|
||||
logger.error("Failed to parse {0} - {1}".format(file_path,
|
||||
error))
|
||||
|
||||
if args.host:
|
||||
try:
|
||||
if args.user is None or args.password is None:
|
||||
logger.error("user and password must be specified if"
|
||||
"host is specified")
|
||||
|
||||
rf = args.reports_folder
|
||||
af = args.archive_folder
|
||||
reports = get_dmarc_reports_from_inbox(args.host,
|
||||
args.user,
|
||||
args.password,
|
||||
reports_folder=rf,
|
||||
archive_folder=af,
|
||||
delete=args.delete,
|
||||
test=args.test)
|
||||
|
||||
aggregate_reports += reports["aggregate_reports"]
|
||||
forensic_reports += reports["forensic_reports"]
|
||||
|
||||
except IMAPError as error:
|
||||
logger.error("IMAP Error: {0}".format(error.__str__()))
|
||||
exit(1)
|
||||
|
||||
results = OrderedDict([("aggregate_reports", aggregate_reports),
|
||||
("forensic_reports", forensic_reports)])
|
||||
|
||||
if args.output:
|
||||
save_output(results, output_directory=args.output)
|
||||
|
||||
callback(results,
|
||||
save_aggregate=args.save_aggregate,
|
||||
save_forensic=args.save_forensic)
|
||||
|
||||
if args.outgoing_host:
|
||||
if args.outgoing_from is None or args.outgoing_to is None:
|
||||
logger.error("--outgoing-from and --outgoing-to must "
|
||||
"be provided if --outgoing-host is used")
|
||||
exit(1)
|
||||
|
||||
try:
|
||||
email_results(results, args.outgoing_host, args.outgoing_from,
|
||||
args.outgoing_to, user=args.outgoing_user,
|
||||
password=args.outgoing_password,
|
||||
subject=args.outgoing_subject)
|
||||
except SMTPError as error:
|
||||
logger.error("SMTP Error: {0}".format(error.__str__()))
|
||||
exit(1)
|
||||
|
||||
if args.host and args.idle:
|
||||
sleep(2)
|
||||
logger.warning("The IMAP Connection is now in IDLE mode. "
|
||||
"Quit with ^c")
|
||||
try:
|
||||
watch_inbox(args.host, args.username, args.password, callback,
|
||||
reports_folder=args.reports_folder,
|
||||
archive_folder=args.archive_folder, delete=args.delete,
|
||||
test=args.test, nameservers=args.nameservers,
|
||||
dns_timeout=args.timeout)
|
||||
except IMAPError as error:
|
||||
logger.error("IMAP Error: {0}".format(error.__str__()))
|
||||
exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
_main()
|
||||
@@ -0,0 +1,341 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from collections import OrderedDict
|
||||
|
||||
import parsedmarc
|
||||
from elasticsearch_dsl.search import Q
|
||||
from elasticsearch_dsl import connections, Object, DocType, Index, Nested, \
|
||||
InnerDoc, Integer, Text, Boolean, DateRange, Ip, Date
|
||||
|
||||
aggregate_index = Index("dmarc_aggregate")
|
||||
forensic_index = Index("dmarc_forensic")
|
||||
|
||||
|
||||
class PolicyOverride(InnerDoc):
|
||||
type = Text()
|
||||
comment = Text()
|
||||
|
||||
|
||||
class PublishedPolicy(InnerDoc):
|
||||
adkim = Text()
|
||||
aspf = Text()
|
||||
p = Text()
|
||||
sp = Text()
|
||||
pct = Integer()
|
||||
fo = Integer()
|
||||
|
||||
|
||||
class DKIMResult(InnerDoc):
|
||||
domain = Text()
|
||||
selector = Text()
|
||||
result = Text()
|
||||
|
||||
|
||||
class SPFResult(InnerDoc):
|
||||
domain = Text()
|
||||
scope = Text()
|
||||
results = Text()
|
||||
|
||||
|
||||
class AggregateReportDoc(DocType):
|
||||
class Meta:
|
||||
index = "dmarc_aggregate"
|
||||
|
||||
xml_schema = Text()
|
||||
org_name = Text()
|
||||
org_extra_contact_info = Text()
|
||||
report_id = Text()
|
||||
date_range = DateRange()
|
||||
errors = Text()
|
||||
domain = Text()
|
||||
published_policy = Object(PublishedPolicy)
|
||||
source_ip_address = Ip()
|
||||
source_country = Text()
|
||||
source_reverse_dns = Text()
|
||||
source_Base_domain = Text()
|
||||
message_count = Integer
|
||||
disposition = Text()
|
||||
dkim_aligned = Boolean()
|
||||
spf_aligned = Boolean()
|
||||
passed_dmarc = Boolean()
|
||||
policy_overrides = Nested(PolicyOverride)
|
||||
header_from = Text()
|
||||
envelope_from = Text()
|
||||
envelope_to = Text()
|
||||
dkim_results = Nested(DKIMResult)
|
||||
spf_results = Nested(SPFResult)
|
||||
|
||||
def add_policy_override(self, type_, comment):
|
||||
self.policy_overrides.append(PolicyOverride(type=type_,
|
||||
comment=comment))
|
||||
|
||||
def add_dkim_result(self, domain, selector, result):
|
||||
self.dkim_results.append(DKIMResult(domain=domain,
|
||||
selector=selector,
|
||||
result=result))
|
||||
|
||||
def add_spf_result(self, domain, scope, result):
|
||||
self.spf_results.append(SPFResult(domain=domain,
|
||||
scope=scope,
|
||||
result=result))
|
||||
|
||||
def save(self, ** kwargs):
|
||||
self.passed_dmarc = False
|
||||
self.passed_dmarc = self.spf_aligned or self.dkim_aligned
|
||||
return super().save(** kwargs)
|
||||
|
||||
|
||||
class EmailAddressDoc(InnerDoc):
|
||||
display_name = Text()
|
||||
address = Text()
|
||||
|
||||
|
||||
class EmailAttachmentDoc(DocType):
|
||||
filename = Text()
|
||||
content_type = Text()
|
||||
|
||||
|
||||
class ForensicSampleDoc(InnerDoc):
|
||||
raw = Text()
|
||||
headers = Object()
|
||||
headers_only = Boolean()
|
||||
to = Nested(EmailAddressDoc)
|
||||
subject = Text()
|
||||
filename_safe_subject = Text()
|
||||
_from = Object(EmailAddressDoc)
|
||||
date = Date()
|
||||
reply_to = Nested(EmailAddressDoc)
|
||||
cc = Nested(EmailAddressDoc)
|
||||
bcc = Nested(EmailAddressDoc)
|
||||
body = Text()
|
||||
attachments = Nested(EmailAttachmentDoc)
|
||||
|
||||
def add_to(self, display_name, address):
|
||||
self.to.append(EmailAddressDoc(display_name=display_name,
|
||||
address=address))
|
||||
|
||||
def add_reply_to(self, display_name, address):
|
||||
self.reply_to.append(EmailAddressDoc(display_name=display_name,
|
||||
address=address))
|
||||
|
||||
def add_cc(self, display_name, address):
|
||||
self.cc.append(EmailAddressDoc(display_name=display_name,
|
||||
address=address))
|
||||
|
||||
def add_bcc(self, display_name, address):
|
||||
self.bcc.append(EmailAddressDoc(display_name=display_name,
|
||||
address=address))
|
||||
|
||||
def add_attachment(self, filename, content_type):
|
||||
self.attachments.append(filename=filename,
|
||||
content_type=content_type)
|
||||
|
||||
|
||||
class ForensicReportDoc(DocType):
|
||||
class Meta:
|
||||
index = "dmarc_forensic"
|
||||
|
||||
feedback_type = Text()
|
||||
user_agent = Text()
|
||||
version = Text()
|
||||
original_mail_from = Text()
|
||||
arrival_date = Date()
|
||||
domain = Text()
|
||||
original_envelope_id = Text()
|
||||
authentication_results = Text()
|
||||
delivery_results = Text()
|
||||
source_ip_address = Ip()
|
||||
source_country = Text()
|
||||
source_reverse_dns = Text()
|
||||
source_authentication_mechanisms = Text()
|
||||
source_auth_failures = Text()
|
||||
dkim_domain = Text()
|
||||
original_rcpt_to = Text()
|
||||
sample = Object(ForensicSampleDoc)
|
||||
|
||||
|
||||
class ExistingReport(RuntimeError):
|
||||
"""Raised when a report to be saved matches an existing report"""
|
||||
|
||||
|
||||
def set_hosts(hosts):
|
||||
"""
|
||||
Sets the Elasticsearch hosts to use
|
||||
|
||||
Args:
|
||||
hosts: A single hostname or URL, or list of hostnames or URLs
|
||||
"""
|
||||
if type(hosts) != list:
|
||||
hosts = [hosts]
|
||||
connections.create_connection(hosts=hosts, timeout=20)
|
||||
|
||||
|
||||
def create_indexes():
|
||||
"""Creates the required indexes"""
|
||||
if not aggregate_index.exists():
|
||||
aggregate_index.create()
|
||||
if not forensic_index.exists():
|
||||
forensic_index.create()
|
||||
|
||||
|
||||
def save_aggregate_report_to_elasticsearch(aggregate_report):
|
||||
"""
|
||||
Saves a parsed DMARC aggregate report to ElasticSearch
|
||||
|
||||
Args:
|
||||
aggregate_report (OrderedDict): A parsed forensic report
|
||||
|
||||
Raises:
|
||||
ExistingReport
|
||||
|
||||
"""
|
||||
metadata = aggregate_report["report_metadata"]
|
||||
org_name = metadata["org_name"]
|
||||
report_id = metadata["report_id"]
|
||||
domain = aggregate_report["policy_published"]["domain"]
|
||||
|
||||
org_name_query = Q(dict(match=dict(org_name=org_name)))
|
||||
report_id_query = Q(dict(match=dict(report_id=report_id)))
|
||||
domain_query = Q(dict(match=dict(domain=domain)))
|
||||
|
||||
search = aggregate_index.search()
|
||||
search.query = org_name_query & report_id_query & domain_query
|
||||
existing = search.execute()
|
||||
if len(existing) > 0:
|
||||
raise ExistingReport("A matching aggregate report already exists")
|
||||
|
||||
aggregate_report["begin_date"] = parsedmarc.human_timestamp_to_datetime(
|
||||
metadata["begin_date"])
|
||||
aggregate_report["end_date"] = parsedmarc.human_timestamp_to_datetime(
|
||||
metadata["end_date"])
|
||||
date_range = (aggregate_report["begin_date"],
|
||||
aggregate_report["end_date"])
|
||||
published_policy = PublishedPolicy(
|
||||
adkim=aggregate_report["policy_published"]["adkim"],
|
||||
aspf=aggregate_report["policy_published"]["aspf"],
|
||||
p=aggregate_report["policy_published"]["p"],
|
||||
sp=aggregate_report["policy_published"]["sp"],
|
||||
pct=aggregate_report["policy_published"]["pct"],
|
||||
fo=aggregate_report["policy_published"]["fo"]
|
||||
)
|
||||
|
||||
for record in aggregate_report["records"]:
|
||||
agg_doc = AggregateReportDoc(
|
||||
xml_schemea=aggregate_report["xml_schema"],
|
||||
org_name=metadata["org_name"],
|
||||
org_email=metadata["org_email"],
|
||||
org_extra_contact_info=metadata["org_extra_contact_info"],
|
||||
report_id=metadata["report_id"],
|
||||
date_range=date_range,
|
||||
errors=metadata["errors"],
|
||||
domain=aggregate_report["policy_published"]["domain"],
|
||||
published_policy=published_policy,
|
||||
source_ip_address=record["source"]["ip_address"],
|
||||
source_country=record["source"]["country"],
|
||||
source_reverse_dns=record["source"]["reverse_dns"],
|
||||
source_base_domain=record["source"]["base_domain"],
|
||||
message_count=record["count"],
|
||||
disposition=record["policy_evaluated"]["disposition"],
|
||||
dkim_aligned=record["policy_evaluated"]["dkim"] == "pass",
|
||||
spf_aligned=record["policy_evaluated"]["spf"] == "pass",
|
||||
header_from=record["identifiers"]["header_from"],
|
||||
envelope_from=record["identifiers"]["envelope_from"],
|
||||
envelope_to=record["identifiers"]["envelope_to"]
|
||||
)
|
||||
|
||||
for override in record["policy_evaluated"]["policy_override_reasons"]:
|
||||
agg_doc.add_policy_override(type_=override["type"],
|
||||
comment=override["comment"])
|
||||
|
||||
for dkim_result in record["auth_results"]["dkim"]:
|
||||
agg_doc.add_dkim_result(domain=dkim_result["domain"],
|
||||
selector=dkim_result["selector"],
|
||||
result=dkim_result["result"])
|
||||
|
||||
for spf_result in record["auth_results"]["spf"]:
|
||||
agg_doc.add_spf_result(domain=spf_result["domain"],
|
||||
scope=spf_result["scope"],
|
||||
result=spf_result["result"])
|
||||
agg_doc.save()
|
||||
|
||||
|
||||
def save_forensic_report_to_elasticsearch(forensic_report):
|
||||
"""
|
||||
Saves a parsed DMARC forensic report to ElasticSearch
|
||||
|
||||
Args:
|
||||
forensic_report (OrderedDict): A parsed forensic report
|
||||
|
||||
Raises:
|
||||
ExistingReport
|
||||
|
||||
"""
|
||||
sample_date = forensic_report["parsed_sample"]["date"]
|
||||
sample_date = parsedmarc.human_timestamp_to_datetime(sample_date)
|
||||
original_headers = forensic_report["parsed_sample"]["headers"]
|
||||
headers = OrderedDict()
|
||||
for original_header in original_headers:
|
||||
headers[original_header.lower()] = original_headers[original_header]
|
||||
|
||||
arrival_date = forensic_report["arrival_date_utc"]
|
||||
arrival_date = parsedmarc.human_timestamp_to_datetime(arrival_date)
|
||||
|
||||
search = forensic_index.search()
|
||||
to_query = {"match": {"sample.headers.to": headers["to"]}}
|
||||
from_query = {"match": {"sample.headers.from": headers["from"]}}
|
||||
subject_query = {"match": {"sample.headers.subject": headers["subject"]}}
|
||||
search.query = Q(to_query) & Q(from_query) & Q(subject_query)
|
||||
existing = search.execute()
|
||||
|
||||
if len(existing) > 0:
|
||||
raise ExistingReport(" A matching forensic report already exists")
|
||||
|
||||
parsed_sample = forensic_report["parsed_sample"]
|
||||
sample = ForensicSampleDoc(
|
||||
raw=forensic_report["sample"],
|
||||
headers=headers,
|
||||
headers_only=forensic_report["sample_headers_only"],
|
||||
date=sample_date,
|
||||
subject=forensic_report["parsed_sample"]["subject"],
|
||||
filename_safe_subject=parsed_sample["filename_safe_subject"],
|
||||
body=forensic_report["parsed_sample"]["body"]
|
||||
)
|
||||
|
||||
for address in forensic_report["parsed_sample"]["to"]:
|
||||
sample.add_to(display_name=address["display_name"],
|
||||
address=address["address"])
|
||||
for address in forensic_report["parsed_sample"]["reply_to"]:
|
||||
sample.add_reply_to(display_name=address["display_name"],
|
||||
address=address["address"])
|
||||
for address in forensic_report["parsed_sample"]["cc"]:
|
||||
sample.add_cc(display_name=address["display_name"],
|
||||
address=address["address"])
|
||||
for address in forensic_report["parsed_sample"]["bcc"]:
|
||||
sample.add_bcc(display_name=address["display_name"],
|
||||
address=address["address"])
|
||||
for attachment in forensic_report["parsed_sample"]["attachments"]:
|
||||
sample.add_attachment(filename=attachment["filename"],
|
||||
content_type=attachment["mail_content_type"])
|
||||
|
||||
forensic_doc = ForensicReportDoc(
|
||||
feedback_type=forensic_report["feedback_type"],
|
||||
user_agent=forensic_report["user_agent"],
|
||||
version=forensic_report["version"],
|
||||
original_mail_from=forensic_report["original_mail_from"],
|
||||
arrival_date=arrival_date,
|
||||
domain=forensic_report["reported_domain"],
|
||||
original_envelope_id=forensic_report["original_envelope_id"],
|
||||
authentication_results=forensic_report["authentication_results"],
|
||||
delivery_results=forensic_report["delivery_result"],
|
||||
source_ip_address=forensic_report["source"]["ip_address"],
|
||||
source_country=forensic_report["source"]["country"],
|
||||
source_reverse_dns=forensic_report["source"]["reverse_dns"],
|
||||
source_base_domain=forensic_report["source"]["base_domain"],
|
||||
authentication_mechanisms=forensic_report["authentication_mechanisms"],
|
||||
auth_failure=forensic_report["auth_failure"],
|
||||
dkim_domain=forensic_report["dkim_domain"],
|
||||
original_rcpt_to=forensic_report["original_rcpt_to"],
|
||||
sample=sample
|
||||
)
|
||||
|
||||
forensic_doc.save()
|
||||
@@ -7,6 +7,7 @@ dnspython
|
||||
imapclient
|
||||
mail-parser
|
||||
dateparser
|
||||
elasticsearch-dsl
|
||||
flake8
|
||||
sphinx
|
||||
sphinx_rtd_theme
|
||||
|
||||
@@ -80,21 +80,23 @@ setup(
|
||||
|
||||
# You can just specify the packages manually here if your project is
|
||||
# simple. Or you can use find_packages().
|
||||
# packages=find_packages(exclude=['contrib', 'docs', 'samples']),
|
||||
packages=["parsedmarc"],
|
||||
|
||||
|
||||
# Alternatively, if you want to distribute just a my_module.py, uncomment
|
||||
# this:
|
||||
py_modules=["parsedmarc"],
|
||||
# py_modules=["parsedmarc"],
|
||||
|
||||
# List run-time dependencies here. These will be installed by pip when
|
||||
# your project is installed. For an analysis of "install_requires" vs pip's
|
||||
# requirements files see:
|
||||
# https://packaging.python.org/en/latest/requirements.html
|
||||
install_requires=['dnspython', 'publicsuffix', 'xmltodict', 'geoip2',
|
||||
'dnspython', 'imapclient', 'mail-parser', 'dateparser'],
|
||||
'dnspython', 'imapclient', 'mail-parser', 'dateparser',
|
||||
'elasticsearch-dsl'
|
||||
],
|
||||
|
||||
entry_points={
|
||||
'console_scripts': ['parsedmarc=parsedmarc:_main'],
|
||||
'console_scripts': ['parsedmarc=parsedmarc.cli:_main'],
|
||||
}
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user