Made with NaturalEarth | Elastic Maps Service
\",\"name\":\"World Countries\",\"format\":{\"type\":\"geojson\"},\"url\":\"https://vector.maps.elastic.co/blob/5659313586569216?elastic_tile_service_tos=agree&my_app_version=6.2.2\",\"fields\":[{\"name\":\"iso2\",\"description\":\"Two letter abbreviation\"},{\"name\":\"name\",\"description\":\"Country name\"},{\"name\":\"iso3\",\"description\":\"Three letter abbreviation\"}],\"created_at\":\"2017-04-26T17:12:15.978370\",\"tags\":[],\"id\":5659313586569216,\"layerId\":\"elastic_maps_service.World Countries\"},\"selectedJoinField\":{\"name\":\"iso2\",\"description\":\"Two letter abbreviation\"},\"isDisplayWarning\":true,\"wms\":{\"enabled\":false,\"options\":{\"format\":\"image/png\",\"transparent\":true},\"baseLayersAreLoaded\":{\"_c\":[],\"_s\":1,\"_d\":true,\"_v\":true,\"_h\":0,\"_n\":false},\"tmsLayers\":[{\"id\":\"road_map\",\"url\":\"https://tiles.maps.elastic.co/v2/default/{z}/{x}/{y}.png?elastic_tile_service_tos=agree&my_app_name=kibana&my_app_version=6.2.2\",\"minZoom\":0,\"maxZoom\":10,\"attribution\":\"© OpenStreetMap contributors | Elastic Maps Service
\",\"subdomains\":[]}],\"selectedTmsLayer\":{\"id\":\"road_map\",\"url\":\"https://tiles.maps.elastic.co/v2/default/{z}/{x}/{y}.png?elastic_tile_service_tos=agree&my_app_name=kibana&my_app_version=6.2.2\",\"minZoom\":0,\"maxZoom\":10,\"attribution\":\"© OpenStreetMap contributors | Elastic Maps Service
\",\"subdomains\":[]}},\"mapZoom\":2,\"mapCenter\":[0,0],\"outlineWeight\":1,\"showAllShapes\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"source_country.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Country\"}}]}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"34fa53e0-28c1-11e8-b8b2-15742da3055c\",\"filter\":[{\"meta\":{\"index\":\"34fa53e0-28c1-11e8-b8b2-15742da3055c\",\"negate\":false,\"disabled\":false,\"alias\":null,\"type\":\"phrase\",\"key\":\"passed_dmarc\",\"value\":\"false\",\"params\":{\"query\":false,\"type\":\"phrase\"}},\"query\":{\"match\":{\"passed_dmarc\":{\"query\":false,\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}" + } + }, + "_meta": { + "savedObjectVersion": 2 + } + }, + { + "_id": "def63400-295b-11e8-b8b2-15742da3055c", + "_type": "visualization", + "_source": { + "title": "Forensic Samples", + "visState": "{\"title\":\"Forensic Samples\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"arrival_date\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"_term\",\"customLabel\":\"Arrival Date (UTC)\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"sample.headers.from.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"From\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"sample.headers.to.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"To\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"sample.reply_to.address.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"None\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Reply To\"}},{\"id\":\"6\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"sample.subject.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Subject\"}}]}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"3f4816b0-2958-11e8-b8b2-15742da3055c\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}" + } + }, + "_meta": { + "savedObjectVersion": 2 + } + }, + { + "_id": "895f3a70-291d-11e8-b8b2-15742da3055c", + "_type": "visualization", + "_source": { + "title": "Message Source Countries", + "visState": "{\"title\":\"Message Source Countries\",\"type\":\"region_map\",\"params\":{\"legendPosition\":\"bottomright\",\"addTooltip\":true,\"colorSchema\":\"Yellow to Red\",\"selectedLayer\":{\"attribution\":\"Made with NaturalEarth | Elastic Maps Service
\",\"name\":\"World Countries\",\"format\":{\"type\":\"geojson\"},\"url\":\"https://vector.maps.elastic.co/blob/5659313586569216?elastic_tile_service_tos=agree&my_app_version=6.2.2\",\"fields\":[{\"name\":\"iso2\",\"description\":\"Two letter abbreviation\"},{\"name\":\"name\",\"description\":\"Country name\"},{\"name\":\"iso3\",\"description\":\"Three letter abbreviation\"}],\"created_at\":\"2017-04-26T17:12:15.978370\",\"tags\":[],\"id\":5659313586569216,\"layerId\":\"elastic_maps_service.World Countries\"},\"selectedJoinField\":{\"name\":\"iso2\",\"description\":\"Two letter abbreviation\"},\"isDisplayWarning\":true,\"wms\":{\"enabled\":false,\"options\":{\"format\":\"image/png\",\"transparent\":true},\"baseLayersAreLoaded\":{\"_c\":[],\"_s\":1,\"_d\":true,\"_v\":true,\"_h\":0,\"_n\":false},\"tmsLayers\":[{\"id\":\"road_map\",\"url\":\"https://tiles.maps.elastic.co/v2/default/{z}/{x}/{y}.png?elastic_tile_service_tos=agree&my_app_name=kibana&my_app_version=6.2.2\",\"minZoom\":0,\"maxZoom\":10,\"attribution\":\"© OpenStreetMap contributors | Elastic Maps Service
\",\"subdomains\":[]}],\"selectedTmsLayer\":{\"id\":\"road_map\",\"url\":\"https://tiles.maps.elastic.co/v2/default/{z}/{x}/{y}.png?elastic_tile_service_tos=agree&my_app_name=kibana&my_app_version=6.2.2\",\"minZoom\":0,\"maxZoom\":10,\"attribution\":\"© OpenStreetMap contributors | Elastic Maps Service
\",\"subdomains\":[]}},\"mapZoom\":2,\"mapCenter\":[0,0],\"outlineWeight\":1,\"showAllShapes\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"source_country.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":200,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Country\"}}]}", + "uiStateJSON": "{\"mapZoom\":3,\"mapCenter\":[27.68352808378776,5.537109375000001]}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"34fa53e0-28c1-11e8-b8b2-15742da3055c\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}" + } + }, + "_meta": { + "savedObjectVersion": 2 + } + }, + { + "_id": "a386df70-295e-11e8-b8b2-15742da3055c", + "_type": "visualization", + "_source": { + "title": "Forensic Sample Source Countries", + "visState": "{\"title\":\"Forensic Sample Source Countries\",\"type\":\"region_map\",\"params\":{\"legendPosition\":\"bottomright\",\"addTooltip\":true,\"colorSchema\":\"Yellow to Red\",\"selectedLayer\":{\"attribution\":\"Made with NaturalEarth | Elastic Maps Service
\",\"name\":\"World Countries\",\"format\":{\"type\":\"geojson\"},\"url\":\"https://vector.maps.elastic.co/blob/5659313586569216?elastic_tile_service_tos=agree&my_app_version=6.2.2\",\"fields\":[{\"name\":\"iso2\",\"description\":\"Two letter abbreviation\"},{\"name\":\"name\",\"description\":\"Country name\"},{\"name\":\"iso3\",\"description\":\"Three letter abbreviation\"}],\"created_at\":\"2017-04-26T17:12:15.978370\",\"tags\":[],\"id\":5659313586569216,\"layerId\":\"elastic_maps_service.World Countries\"},\"selectedJoinField\":{\"name\":\"iso2\",\"description\":\"Two letter abbreviation\"},\"isDisplayWarning\":true,\"wms\":{\"enabled\":false,\"options\":{\"format\":\"image/png\",\"transparent\":true},\"baseLayersAreLoaded\":{\"_c\":[],\"_s\":1,\"_d\":true,\"_v\":true,\"_h\":0,\"_n\":false},\"tmsLayers\":[{\"id\":\"road_map\",\"url\":\"https://tiles.maps.elastic.co/v2/default/{z}/{x}/{y}.png?elastic_tile_service_tos=agree&my_app_name=kibana&my_app_version=6.2.2\",\"minZoom\":0,\"maxZoom\":10,\"attribution\":\"© OpenStreetMap contributors | Elastic Maps Service
\",\"subdomains\":[]}],\"selectedTmsLayer\":{\"id\":\"road_map\",\"url\":\"https://tiles.maps.elastic.co/v2/default/{z}/{x}/{y}.png?elastic_tile_service_tos=agree&my_app_name=kibana&my_app_version=6.2.2\",\"minZoom\":0,\"maxZoom\":10,\"attribution\":\"© OpenStreetMap contributors | Elastic Maps Service
\",\"subdomains\":[]}},\"mapZoom\":2,\"mapCenter\":[0,0],\"outlineWeight\":1,\"showAllShapes\":true},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"source_country.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":200,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Country\"}}]}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"3f4816b0-2958-11e8-b8b2-15742da3055c\",\"filter\":[],\"query\":{\"language\":\"lucene\",\"query\":\"\"}}" + } + }, + "_meta": { + "savedObjectVersion": 2 + } + }, + { + "_id": "085eaa30-2870-11e8-b8b2-15742da3055c", + "_type": "visualization", + "_source": { + "title": "DMARC Passage Over Time", + "visState": "{\"title\":\"DMARC Passage Over Time\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Messages\"},\"drawLinesBetweenPoints\":true,\"mode\":\"normal\",\"show\":\"true\",\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"line\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Messages\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"date_range\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"passed_dmarc\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Passed DMARC\"}}]}", + "uiStateJSON": "{}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"34fa53e0-28c1-11e8-b8b2-15742da3055c\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}" + } + }, + "_meta": { + "savedObjectVersion": 2 + } + }, + { + "_id": "620280a0-2886-11e8-b8b2-15742da3055c", + "_type": "visualization", + "_source": { + "title": "Reporting Organizations", + "visState": "{\"title\":\"Reporting Organizations\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"org_name.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Name\"}}]}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"34fa53e0-28c1-11e8-b8b2-15742da3055c\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}" + } + }, + "_meta": { + "savedObjectVersion": 2 + } + }, + { + "_id": "40e7a5b0-2883-11e8-b8b2-15742da3055c", + "_type": "visualization", + "_source": { + "title": "DKIM Alignment Failures", + "visState": "{\"title\":\"DKIM Alignment Failures\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":4,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"header_from.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Header From\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"dkim_results.domain.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Missing\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"DKIM Domain\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"dkim_results.result.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Missing\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"DKIM Result\"}},{\"id\":\"6\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_base_domain.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"No Reverse DNS\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Reverse DNS Base\"}}]}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":4,\"direction\":\"desc\"}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"34fa53e0-28c1-11e8-b8b2-15742da3055c\",\"filter\":[{\"meta\":{\"index\":\"a9ba2300-286b-11e8-b8b2-15742da3055c\",\"negate\":false,\"disabled\":false,\"alias\":null,\"type\":\"phrase\",\"key\":\"dkim_aligned\",\"value\":false,\"params\":{\"query\":false,\"type\":\"phrase\"}},\"query\":{\"match\":{\"dkim_aligned\":{\"query\":false,\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}" + } + }, + "_meta": { + "savedObjectVersion": 2 + } + }, + { + "_id": "1fad3f60-2881-11e8-b8b2-15742da3055c", + "_type": "visualization", + "_source": { + "title": "SPF Allignment Failures", + "visState": "{\"title\":\"SPF Allignment Failures\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showMeticsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":3,\"direction\":\"desc\"},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"header_from.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Missing\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Header From\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"envelope_from.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Missing\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"_term\",\"customLabel\":\"Envelope From\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_base_domain.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"No Reverse DNS\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Reverse DNS Base\"}}]}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":3,\"direction\":\"desc\"}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"34fa53e0-28c1-11e8-b8b2-15742da3055c\",\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"index\":\"a9ba2300-286b-11e8-b8b2-15742da3055c\",\"key\":\"spf_aligned\",\"negate\":false,\"params\":{\"query\":false,\"type\":\"phrase\"},\"type\":\"phrase\",\"value\":false},\"query\":{\"match\":{\"spf_aligned\":{\"query\":false,\"type\":\"phrase\"}}}}],\"query\":{\"language\":\"lucene\",\"query\":\"\"}}" + } + }, + "_meta": { + "savedObjectVersion": 2 + } + }, + { + "_id": "2ae719b0-2885-11e8-b8b2-15742da3055c", + "_type": "visualization", + "_source": { + "title": "DMARC Falure Sources", + "visState": "{\"title\":\"DMARC Falure Sources\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":3,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"header_from.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"Missing\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Header From\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_base_domain.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"No Reverse DNS\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Reverse DNS Base\"}}]}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":3,\"direction\":\"desc\"}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"34fa53e0-28c1-11e8-b8b2-15742da3055c\",\"filter\":[{\"meta\":{\"index\":\"34fa53e0-28c1-11e8-b8b2-15742da3055c\",\"negate\":false,\"disabled\":false,\"alias\":null,\"type\":\"phrase\",\"key\":\"passed_dmarc\",\"value\":\"false\",\"params\":{\"query\":false,\"type\":\"phrase\"}},\"query\":{\"match\":{\"passed_dmarc\":{\"query\":false,\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}" + } + }, + "_meta": { + "savedObjectVersion": 2 + } + }, + { + "_id": "316ef4e0-295e-11e8-b8b2-15742da3055c", + "_type": "visualization", + "_source": { + "title": "Forensic Sample Sources", + "visState": "{\"title\":\"Forensic Sample Sources\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_ip_address.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"IP Address\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_reverse_dns.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"None\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Reverse DNS\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_base_domain.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"None\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Reverse DNS Base\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_country.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Country\"}}]}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"3f4816b0-2958-11e8-b8b2-15742da3055c\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}" + } + }, + "_meta": { + "savedObjectVersion": 2 + } + }, + { + "_id": "d787a580-2886-11e8-b8b2-15742da3055c", + "_type": "visualization", + "_source": { + "title": "Message Sources by Reverse DNS", + "visState": "{\"title\":\"Message Sources by Reverse DNS\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source_base_domain.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":true,\"missingBucketLabel\":\"No reverse DNS\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Reverse DNS Base\"}}]}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"34fa53e0-28c1-11e8-b8b2-15742da3055c\",\"filter\":[],\"query\":{\"language\":\"lucene\",\"query\":\"\"}}" + } + }, + "_meta": { + "savedObjectVersion": 2 + } + }, + { + "_id": "a69d0f40-2b02-11e8-8c8d-d3a0d2f2ba49", + "_type": "visualization", + "_source": { + "title": "Message Volume by Header From", + "visState": "{\"title\":\"Message Volume by Header From\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"message_count\",\"customLabel\":\"Messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"domain.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Header From\"}}]}", + "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", + "description": "", + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"index\":\"34fa53e0-28c1-11e8-b8b2-15742da3055c\",\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"}}" + } + }, + "_meta": { + "savedObjectVersion": 2 + } + } +] \ No newline at end of file diff --git a/parsedmarc.py b/parsedmarc/__init__.py similarity index 87% rename from parsedmarc.py rename to parsedmarc/__init__.py index c908a03..13df735 100644 --- a/parsedmarc.py +++ b/parsedmarc/__init__.py @@ -1,7 +1,6 @@ -#!/usr/bin/env python3 # -*- coding: utf-8 -*- -"""A Python module and CLI for parsing DMARC reports""" +"""A Python module for parsing DMARC reports""" import logging import os @@ -18,13 +17,10 @@ import re from base64 import b64decode import binascii import shutil -from argparse import ArgumentParser -from glob import glob import email import tempfile import subprocess import socket -from time import sleep from email.mime.application import MIMEApplication from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText @@ -45,7 +41,7 @@ import imapclient.exceptions import dateparser import mailparser -__version__ = "2.2.0" +__version__ = "3.0.0" logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -130,6 +126,7 @@ def _query_dns(domain, record_type, nameservers=None, timeout=6.0): domain (str): The domain or subdomain to query about record_type (str): The record type to query for nameservers (list): A list of one or more nameservers to use + (8.8.8.8 and 4.4.4.4 by default) timeout (float): Sets the DNS timeout in seconds Returns: @@ -137,8 +134,9 @@ def _query_dns(domain, record_type, nameservers=None, timeout=6.0): """ resolver = dns.resolver.Resolver() timeout = float(timeout) - if nameservers: - resolver.nameservers = nameservers + if nameservers is None: + nameservers = ["8.8.8.8", "4.4.4.4"] + resolver.nameservers = nameservers resolver.timeout = timeout resolver.lifetime = timeout return list(map( @@ -152,7 +150,8 @@ def _get_reverse_dns(ip_address, nameservers=None, timeout=6.0): Args: ip_address (str): The IP address to resolve - nameservers (list): A list of nameservers to query + nameservers (list): A list of one or more nameservers to use + (8.8.8.8 and 4.4.4.4 by default) timeout (float): Sets the DNS query timeout in seconds Returns: @@ -277,6 +276,7 @@ def _get_ip_address_info(ip_address, nameservers=None, timeout=6.0): Args: ip_address (str): The IP address to check nameservers (list): A list of one or more nameservers to use + (8.8.8.8 and 4.4.4.4 by default) timeout (float): Sets the DNS timeout in seconds Returns: @@ -308,11 +308,14 @@ def _parse_report_record(record, nameservers=None, timeout=6.0): Args: record (OrderedDict): The record to convert nameservers (list): A list of one or more nameservers to use + (8.8.8.8 and 4.4.4.4 by default) timeout (float): Sets the DNS timeout in seconds Returns: OrderedDict: The converted record """ + if nameservers is None: + nameservers = ["8.8.8.8", "4.4.4.4"] record = record.copy() new_record = OrderedDict() new_record["source"] = _get_ip_address_info(record["row"]["source_ip"], @@ -399,6 +402,7 @@ def parse_aggregate_report_xml(xml, nameservers=None, timeout=6.0): Args: xml (str): A string of DMARC aggregate report XML nameservers (list): A list of one or more nameservers to use + (8.8.8.8 and 4.4.4.4 by default) timeout (float): Sets the DNS timeout in seconds Returns: @@ -412,7 +416,8 @@ def parse_aggregate_report_xml(xml, nameservers=None, timeout=6.0): schema = report["version"] new_report = OrderedDict([("xml_schema", schema)]) new_report_metadata = OrderedDict() - new_report_metadata["org_name"] = report_metadata["org_name"] + org_name = _get_base_domain(report_metadata["org_name"]) + new_report_metadata["org_name"] = org_name new_report_metadata["org_email"] = report_metadata["email"] extra = None if "extra_contact_info" in report_metadata: @@ -530,6 +535,7 @@ def parse_aggregate_report_file(_input, nameservers=None, timeout=6.0): Args: _input: A path to a file, a file like object, or bytes nameservers (list): A list of one or more nameservers to use + (8.8.8.8 and 4.4.4.4 by default) timeout (float): Sets the DNS timeout in seconds Returns: @@ -656,6 +662,7 @@ def parse_forensic_report(feedback_report, sample, sample_headers_only, sample (str): The RFC 822 headers or RFC 822 message sample sample_headers_only (bool): Set true if the sample is only headers nameservers (list): A list of one or more nameservers to use + (8.8.8.8 and 4.4.4.4 by default) timeout (float): Sets the DNS timeout in seconds Returns: @@ -977,6 +984,7 @@ def parse_report_file(input_, nameservers=None, timeout=6.0): Args: input_: A path to a file, a file like object, or bytes nameservers (list): A list of one or more nameservers to use + (8.8.8.8 and 4.4.4.4 by default) timeout (float): Sets the DNS timeout in seconds Returns: @@ -1317,7 +1325,8 @@ def watch_inbox(host, username, password, callback, reports_folder="INBOX", delete (bool): Delete messages after processing them test (bool): Do not move or delete messages after processing them wait (int): Number of seconds to wait for a IMAP IDLE response - nameservers (list): A list of DNS nameservers to query + nameservers (list): A list of one or more nameservers to use + (8.8.8.8 and 4.4.4.4 by default) dns_timeout (float): Set the DNS query timeout """ rf = reports_folder @@ -1374,152 +1383,3 @@ def watch_inbox(host, username, password, callback, reports_folder="INBOX", server.logout() -def _main(): - """Called when the module in executed""" - def print_results(results_): - """ - Print results in human readable format - - Args: - results_ (OrderedDict): Parsing results - """ - print(json.dumps(results_, ensure_ascii=False, indent=2), "\n") - - arg_parser = ArgumentParser(description="Parses DMARC reports") - arg_parser.add_argument("file_path", nargs="*", - help="one or more paths to aggregate or forensic " - "report files or emails") - arg_parser.add_argument("-o", "--output", - help="Write output files to the given directory") - arg_parser.add_argument("-n", "--nameservers", nargs="+", - help="nameservers to query") - arg_parser.add_argument("-t", "--timeout", - help="number of seconds to wait for an answer " - "from DNS (default 6.0)", - type=float, - default=6.0) - arg_parser.add_argument("-H", "--host", help="IMAP hostname or IP address") - arg_parser.add_argument("-u", "--user", help="IMAP user") - arg_parser.add_argument("-p", "--password", help="IMAP password") - arg_parser.add_argument("-r", "--reports-folder", default="INBOX", - help="The IMAP folder containing the reports\n" - "Default: INBOX") - arg_parser.add_argument("-a", "--archive-folder", - help="Specifies the IMAP folder to move " - "messages to after processing them\n" - "Default: Archive", - default="Archive") - arg_parser.add_argument("-d", "--delete", - help="Delete the reports after processing them", - action="store_true", default=False) - arg_parser.add_argument("-O", "--outgoing-host", - help="Email the results using this host") - arg_parser.add_argument("-U", "--outgoing-user", - help="Email the results using this user") - arg_parser.add_argument("-P", "--outgoing-password", - help="Email the results using this password") - arg_parser.add_argument("-F", "--outgoing-from", - help="Email the results using this from address") - arg_parser.add_argument("-T", "--outgoing-to", nargs="+", - help="Email the results to these addresses") - arg_parser.add_argument("-S", "--outgoing-subject", - help="Email the results using this subject") - arg_parser.add_argument("-A", "--outgoing-attachment", - help="Email the results using this filename") - arg_parser.add_argument("-M", "--outgoing-message", - help="Email the results using this message") - - arg_parser.add_argument("-i", "--idle", action="store_true", - help="Use an IMAP IDLE connection to process " - "reports as they arrive in the inbox") - arg_parser.add_argument("--test", - help="Do not move or delete IMAP messages", - action="store_true", default=False) - arg_parser.add_argument("-v", "--version", action="version", - version=__version__) - - aggregate_reports = [] - forensic_reports = [] - - args = arg_parser.parse_args() - file_paths = [] - for file_path in args.file_path: - file_paths += glob(file_path) - file_paths = list(set(file_paths)) - - for file_path in file_paths: - try: - file_results = parse_report_file(file_path, - nameservers=args.nameservers, - timeout=args.timeout) - if file_results["report_type"] == "aggregate": - aggregate_reports.append(file_results["report"]) - elif file_results["report_type"] == "forensic": - forensic_reports.append(file_results["report"]) - - except ParserError as error: - logger.error("Failed to parse {0} - {1}".format(file_path, - error)) - - if args.host: - try: - if args.user is None or args.password is None: - logger.error("user and password must be specified if" - "host is specified") - - rf = args.reports_folder - af = args.archive_folder - reports = get_dmarc_reports_from_inbox(args.host, - args.user, - args.password, - reports_folder=rf, - archive_folder=af, - delete=args.delete, - test=args.test) - - aggregate_reports += reports["aggregate_reports"] - forensic_reports += reports["forensic_reports"] - - except IMAPError as error: - logger.error("IMAP Error: {0}".format(error.__str__())) - exit(1) - - results = OrderedDict([("aggregate_reports", aggregate_reports), - ("forensic_reports", forensic_reports)]) - - if args.output: - save_output(results, output_directory=args.output) - - print_results(results) - - if args.outgoing_host: - if args.outgoing_from is None or args.outgoing_to is None: - logger.error("--outgoing-from and --outgoing-to must " - "be provided if --outgoing-host is used") - exit(1) - - try: - email_results(results, args.outgoing_host, args.outgoing_from, - args.outgoing_to, user=args.outgoing_user, - password=args.outgoing_password, - subject=args.outgoing_subject) - except SMTPError as error: - logger.error("SMTP Error: {0}".format(error.__str__())) - exit(1) - - if args.host and args.idle: - sleep(2) - logger.warning("The IMAP Connection is now in IDLE mode. " - "Send yourself an email, or quit with ^c") - try: - watch_inbox(args.host, args.username, args.password, print_results, - archive_folder=args.archive_folder, delete=args.delete, - test=args.test, nameservers=args.nameservers, - dns_timeout=args.timeout) - except IMAPError as error: - logger.error("IMAP Error: {0}".format(error.__str__())) - exit(1) - - -if __name__ == "__main__": - _main() diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py new file mode 100644 index 0000000..139e18b --- /dev/null +++ b/parsedmarc/cli.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +"""A CLI for parsing DMARC reports""" + +from argparse import ArgumentParser +from glob import glob +from time import sleep + +from parsedmarc import * +from parsedmarc import elastic + +__version__ = "3.0.0" + + +def _main(): + """Called when the module in executed""" + def callback(results_, save_aggregate=False, save_forensic=False): + """ + Prints results and optionally saves them to Elasticsearch + + Args: + results_ (OrderedDict): Parsing results + """ + print(json.dumps(results_, ensure_ascii=False, indent=2), "\n") + + arg_parser = ArgumentParser(description="Parses DMARC reports") + arg_parser.add_argument("file_path", nargs="*", + help="one or more paths to aggregate or forensic " + "report files or emails") + arg_parser.add_argument("-o", "--output", + help="Write output files to the given directory") + arg_parser.add_argument("-n", "--nameservers", nargs="+", + help="nameservers to query " + "(Default 8.8.8.8 4.4.4.4)") + arg_parser.add_argument("-t", "--timeout", + help="number of seconds to wait for an answer " + "from DNS (default 6.0)", + type=float, + default=6.0) + arg_parser.add_argument("-H", "--host", help="IMAP hostname or IP address") + arg_parser.add_argument("-u", "--user", help="IMAP user") + arg_parser.add_argument("-p", "--password", help="IMAP password") + arg_parser.add_argument("-r", "--reports-folder", default="INBOX", + help="The IMAP folder containing the reports\n" + "Default: INBOX") + arg_parser.add_argument("-a", "--archive-folder", + help="Specifies the IMAP folder to move " + "messages to after processing them\n" + "Default: Archive", + default="Archive") + arg_parser.add_argument("-d", "--delete", + help="Delete the reports after processing them", + action="store_true", default=False) + + arg_parser.add_argument("-E", "--elasticsearch-hosts", nargs="*", + help="A list of one or more Elasticsearch " + "hostnames or URLs to use (Default " + "localhost:9200)", + default=["localhost:9200"]) + arg_parser.add_argument("-A", "--save-aggregate", action="store_true", + default=False, + help="Save aggregate reports to Elasticsearch") + arg_parser.add_argument("--save-forensic", action="store_true", + default=False, + help="Save forensic reports to Elasticsearch") + arg_parser.add_argument("-O", "--outgoing-host", + help="Email the results using this host") + arg_parser.add_argument("-U", "--outgoing-user", + help="Email the results using this user") + arg_parser.add_argument("-P", "--outgoing-password", + help="Email the results using this password") + arg_parser.add_argument("-F", "--outgoing-from", + help="Email the results using this from address") + arg_parser.add_argument("-T", "--outgoing-to", nargs="+", + help="Email the results to these addresses") + arg_parser.add_argument("-S", "--outgoing-subject", + help="Email the results using this subject") + arg_parser.add_argument("-A", "--outgoing-attachment", + help="Email the results using this filename") + arg_parser.add_argument("-M", "--outgoing-message", + help="Email the results using this message") + + arg_parser.add_argument("-i", "--idle", action="store_true", + help="Use an IMAP IDLE connection to process " + "reports as they arrive in the inbox") + arg_parser.add_argument("--test", + help="Do not move or delete IMAP messages", + action="store_true", default=False) + arg_parser.add_argument("-v", "--version", action="version", + version=__version__) + + aggregate_reports = [] + forensic_reports = [] + + args = arg_parser.parse_args() + + if args.host is None and len(args.file_path) == 0: + arg_parser.print_help() + exit(1) + + if args.save_aggregate or args.save_forensic: + elastic.set_hosts(args.elasticsearch_hosts) + elastic.create_indexes() + + file_paths = [] + for file_path in args.file_path: + file_paths += glob(file_path) + file_paths = list(set(file_paths)) + + for file_path in file_paths: + try: + file_results = parse_report_file(file_path, + nameservers=args.nameservers, + timeout=args.timeout) + if file_results["report_type"] == "aggregate": + aggregate_reports.append(file_results["report"]) + elif file_results["report_type"] == "forensic": + forensic_reports.append(file_results["report"]) + + except ParserError as error: + logger.error("Failed to parse {0} - {1}".format(file_path, + error)) + + if args.host: + try: + if args.user is None or args.password is None: + logger.error("user and password must be specified if" + "host is specified") + + rf = args.reports_folder + af = args.archive_folder + reports = get_dmarc_reports_from_inbox(args.host, + args.user, + args.password, + reports_folder=rf, + archive_folder=af, + delete=args.delete, + test=args.test) + + aggregate_reports += reports["aggregate_reports"] + forensic_reports += reports["forensic_reports"] + + except IMAPError as error: + logger.error("IMAP Error: {0}".format(error.__str__())) + exit(1) + + results = OrderedDict([("aggregate_reports", aggregate_reports), + ("forensic_reports", forensic_reports)]) + + if args.output: + save_output(results, output_directory=args.output) + + callback(results, + save_aggregate=args.save_aggregate, + save_forensic=args.save_forensic) + + if args.outgoing_host: + if args.outgoing_from is None or args.outgoing_to is None: + logger.error("--outgoing-from and --outgoing-to must " + "be provided if --outgoing-host is used") + exit(1) + + try: + email_results(results, args.outgoing_host, args.outgoing_from, + args.outgoing_to, user=args.outgoing_user, + password=args.outgoing_password, + subject=args.outgoing_subject) + except SMTPError as error: + logger.error("SMTP Error: {0}".format(error.__str__())) + exit(1) + + if args.host and args.idle: + sleep(2) + logger.warning("The IMAP Connection is now in IDLE mode. " + "Quit with ^c") + try: + watch_inbox(args.host, args.username, args.password, callback, + reports_folder=args.reports_folder, + archive_folder=args.archive_folder, delete=args.delete, + test=args.test, nameservers=args.nameservers, + dns_timeout=args.timeout) + except IMAPError as error: + logger.error("IMAP Error: {0}".format(error.__str__())) + exit(1) + + +if __name__ == "__main__": + _main() diff --git a/parsedmarc/elastic.py b/parsedmarc/elastic.py new file mode 100644 index 0000000..2ec913d --- /dev/null +++ b/parsedmarc/elastic.py @@ -0,0 +1,341 @@ +# -*- coding: utf-8 -*- + +from collections import OrderedDict + +import parsedmarc +from elasticsearch_dsl.search import Q +from elasticsearch_dsl import connections, Object, DocType, Index, Nested, \ + InnerDoc, Integer, Text, Boolean, DateRange, Ip, Date + +aggregate_index = Index("dmarc_aggregate") +forensic_index = Index("dmarc_forensic") + + +class PolicyOverride(InnerDoc): + type = Text() + comment = Text() + + +class PublishedPolicy(InnerDoc): + adkim = Text() + aspf = Text() + p = Text() + sp = Text() + pct = Integer() + fo = Integer() + + +class DKIMResult(InnerDoc): + domain = Text() + selector = Text() + result = Text() + + +class SPFResult(InnerDoc): + domain = Text() + scope = Text() + results = Text() + + +class AggregateReportDoc(DocType): + class Meta: + index = "dmarc_aggregate" + + xml_schema = Text() + org_name = Text() + org_extra_contact_info = Text() + report_id = Text() + date_range = DateRange() + errors = Text() + domain = Text() + published_policy = Object(PublishedPolicy) + source_ip_address = Ip() + source_country = Text() + source_reverse_dns = Text() + source_Base_domain = Text() + message_count = Integer + disposition = Text() + dkim_aligned = Boolean() + spf_aligned = Boolean() + passed_dmarc = Boolean() + policy_overrides = Nested(PolicyOverride) + header_from = Text() + envelope_from = Text() + envelope_to = Text() + dkim_results = Nested(DKIMResult) + spf_results = Nested(SPFResult) + + def add_policy_override(self, type_, comment): + self.policy_overrides.append(PolicyOverride(type=type_, + comment=comment)) + + def add_dkim_result(self, domain, selector, result): + self.dkim_results.append(DKIMResult(domain=domain, + selector=selector, + result=result)) + + def add_spf_result(self, domain, scope, result): + self.spf_results.append(SPFResult(domain=domain, + scope=scope, + result=result)) + + def save(self, ** kwargs): + self.passed_dmarc = False + self.passed_dmarc = self.spf_aligned or self.dkim_aligned + return super().save(** kwargs) + + +class EmailAddressDoc(InnerDoc): + display_name = Text() + address = Text() + + +class EmailAttachmentDoc(DocType): + filename = Text() + content_type = Text() + + +class ForensicSampleDoc(InnerDoc): + raw = Text() + headers = Object() + headers_only = Boolean() + to = Nested(EmailAddressDoc) + subject = Text() + filename_safe_subject = Text() + _from = Object(EmailAddressDoc) + date = Date() + reply_to = Nested(EmailAddressDoc) + cc = Nested(EmailAddressDoc) + bcc = Nested(EmailAddressDoc) + body = Text() + attachments = Nested(EmailAttachmentDoc) + + def add_to(self, display_name, address): + self.to.append(EmailAddressDoc(display_name=display_name, + address=address)) + + def add_reply_to(self, display_name, address): + self.reply_to.append(EmailAddressDoc(display_name=display_name, + address=address)) + + def add_cc(self, display_name, address): + self.cc.append(EmailAddressDoc(display_name=display_name, + address=address)) + + def add_bcc(self, display_name, address): + self.bcc.append(EmailAddressDoc(display_name=display_name, + address=address)) + + def add_attachment(self, filename, content_type): + self.attachments.append(filename=filename, + content_type=content_type) + + +class ForensicReportDoc(DocType): + class Meta: + index = "dmarc_forensic" + + feedback_type = Text() + user_agent = Text() + version = Text() + original_mail_from = Text() + arrival_date = Date() + domain = Text() + original_envelope_id = Text() + authentication_results = Text() + delivery_results = Text() + source_ip_address = Ip() + source_country = Text() + source_reverse_dns = Text() + source_authentication_mechanisms = Text() + source_auth_failures = Text() + dkim_domain = Text() + original_rcpt_to = Text() + sample = Object(ForensicSampleDoc) + + +class ExistingReport(RuntimeError): + """Raised when a report to be saved matches an existing report""" + + +def set_hosts(hosts): + """ + Sets the Elasticsearch hosts to use + + Args: + hosts: A single hostname or URL, or list of hostnames or URLs + """ + if type(hosts) != list: + hosts = [hosts] + connections.create_connection(hosts=hosts, timeout=20) + + +def create_indexes(): + """Creates the required indexes""" + if not aggregate_index.exists(): + aggregate_index.create() + if not forensic_index.exists(): + forensic_index.create() + + +def save_aggregate_report_to_elasticsearch(aggregate_report): + """ + Saves a parsed DMARC aggregate report to ElasticSearch + + Args: + aggregate_report (OrderedDict): A parsed forensic report + + Raises: + ExistingReport + + """ + metadata = aggregate_report["report_metadata"] + org_name = metadata["org_name"] + report_id = metadata["report_id"] + domain = aggregate_report["policy_published"]["domain"] + + org_name_query = Q(dict(match=dict(org_name=org_name))) + report_id_query = Q(dict(match=dict(report_id=report_id))) + domain_query = Q(dict(match=dict(domain=domain))) + + search = aggregate_index.search() + search.query = org_name_query & report_id_query & domain_query + existing = search.execute() + if len(existing) > 0: + raise ExistingReport("A matching aggregate report already exists") + + aggregate_report["begin_date"] = parsedmarc.human_timestamp_to_datetime( + metadata["begin_date"]) + aggregate_report["end_date"] = parsedmarc.human_timestamp_to_datetime( + metadata["end_date"]) + date_range = (aggregate_report["begin_date"], + aggregate_report["end_date"]) + published_policy = PublishedPolicy( + adkim=aggregate_report["policy_published"]["adkim"], + aspf=aggregate_report["policy_published"]["aspf"], + p=aggregate_report["policy_published"]["p"], + sp=aggregate_report["policy_published"]["sp"], + pct=aggregate_report["policy_published"]["pct"], + fo=aggregate_report["policy_published"]["fo"] + ) + + for record in aggregate_report["records"]: + agg_doc = AggregateReportDoc( + xml_schemea=aggregate_report["xml_schema"], + org_name=metadata["org_name"], + org_email=metadata["org_email"], + org_extra_contact_info=metadata["org_extra_contact_info"], + report_id=metadata["report_id"], + date_range=date_range, + errors=metadata["errors"], + domain=aggregate_report["policy_published"]["domain"], + published_policy=published_policy, + source_ip_address=record["source"]["ip_address"], + source_country=record["source"]["country"], + source_reverse_dns=record["source"]["reverse_dns"], + source_base_domain=record["source"]["base_domain"], + message_count=record["count"], + disposition=record["policy_evaluated"]["disposition"], + dkim_aligned=record["policy_evaluated"]["dkim"] == "pass", + spf_aligned=record["policy_evaluated"]["spf"] == "pass", + header_from=record["identifiers"]["header_from"], + envelope_from=record["identifiers"]["envelope_from"], + envelope_to=record["identifiers"]["envelope_to"] + ) + + for override in record["policy_evaluated"]["policy_override_reasons"]: + agg_doc.add_policy_override(type_=override["type"], + comment=override["comment"]) + + for dkim_result in record["auth_results"]["dkim"]: + agg_doc.add_dkim_result(domain=dkim_result["domain"], + selector=dkim_result["selector"], + result=dkim_result["result"]) + + for spf_result in record["auth_results"]["spf"]: + agg_doc.add_spf_result(domain=spf_result["domain"], + scope=spf_result["scope"], + result=spf_result["result"]) + agg_doc.save() + + +def save_forensic_report_to_elasticsearch(forensic_report): + """ + Saves a parsed DMARC forensic report to ElasticSearch + + Args: + forensic_report (OrderedDict): A parsed forensic report + + Raises: + ExistingReport + + """ + sample_date = forensic_report["parsed_sample"]["date"] + sample_date = parsedmarc.human_timestamp_to_datetime(sample_date) + original_headers = forensic_report["parsed_sample"]["headers"] + headers = OrderedDict() + for original_header in original_headers: + headers[original_header.lower()] = original_headers[original_header] + + arrival_date = forensic_report["arrival_date_utc"] + arrival_date = parsedmarc.human_timestamp_to_datetime(arrival_date) + + search = forensic_index.search() + to_query = {"match": {"sample.headers.to": headers["to"]}} + from_query = {"match": {"sample.headers.from": headers["from"]}} + subject_query = {"match": {"sample.headers.subject": headers["subject"]}} + search.query = Q(to_query) & Q(from_query) & Q(subject_query) + existing = search.execute() + + if len(existing) > 0: + raise ExistingReport(" A matching forensic report already exists") + + parsed_sample = forensic_report["parsed_sample"] + sample = ForensicSampleDoc( + raw=forensic_report["sample"], + headers=headers, + headers_only=forensic_report["sample_headers_only"], + date=sample_date, + subject=forensic_report["parsed_sample"]["subject"], + filename_safe_subject=parsed_sample["filename_safe_subject"], + body=forensic_report["parsed_sample"]["body"] + ) + + for address in forensic_report["parsed_sample"]["to"]: + sample.add_to(display_name=address["display_name"], + address=address["address"]) + for address in forensic_report["parsed_sample"]["reply_to"]: + sample.add_reply_to(display_name=address["display_name"], + address=address["address"]) + for address in forensic_report["parsed_sample"]["cc"]: + sample.add_cc(display_name=address["display_name"], + address=address["address"]) + for address in forensic_report["parsed_sample"]["bcc"]: + sample.add_bcc(display_name=address["display_name"], + address=address["address"]) + for attachment in forensic_report["parsed_sample"]["attachments"]: + sample.add_attachment(filename=attachment["filename"], + content_type=attachment["mail_content_type"]) + + forensic_doc = ForensicReportDoc( + feedback_type=forensic_report["feedback_type"], + user_agent=forensic_report["user_agent"], + version=forensic_report["version"], + original_mail_from=forensic_report["original_mail_from"], + arrival_date=arrival_date, + domain=forensic_report["reported_domain"], + original_envelope_id=forensic_report["original_envelope_id"], + authentication_results=forensic_report["authentication_results"], + delivery_results=forensic_report["delivery_result"], + source_ip_address=forensic_report["source"]["ip_address"], + source_country=forensic_report["source"]["country"], + source_reverse_dns=forensic_report["source"]["reverse_dns"], + source_base_domain=forensic_report["source"]["base_domain"], + authentication_mechanisms=forensic_report["authentication_mechanisms"], + auth_failure=forensic_report["auth_failure"], + dkim_domain=forensic_report["dkim_domain"], + original_rcpt_to=forensic_report["original_rcpt_to"], + sample=sample + ) + + forensic_doc.save() diff --git a/requirements.txt b/requirements.txt index a53bde5..2fe92e3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,7 @@ dnspython imapclient mail-parser dateparser +elasticsearch-dsl flake8 sphinx sphinx_rtd_theme diff --git a/setup.py b/setup.py index c983da6..772afc4 100644 --- a/setup.py +++ b/setup.py @@ -80,21 +80,23 @@ setup( # You can just specify the packages manually here if your project is # simple. Or you can use find_packages(). - # packages=find_packages(exclude=['contrib', 'docs', 'samples']), + packages=["parsedmarc"], # Alternatively, if you want to distribute just a my_module.py, uncomment # this: - py_modules=["parsedmarc"], + # py_modules=["parsedmarc"], # List run-time dependencies here. These will be installed by pip when # your project is installed. For an analysis of "install_requires" vs pip's # requirements files see: # https://packaging.python.org/en/latest/requirements.html install_requires=['dnspython', 'publicsuffix', 'xmltodict', 'geoip2', - 'dnspython', 'imapclient', 'mail-parser', 'dateparser'], + 'dnspython', 'imapclient', 'mail-parser', 'dateparser', + 'elasticsearch-dsl' + ], entry_points={ - 'console_scripts': ['parsedmarc=parsedmarc:_main'], + 'console_scripts': ['parsedmarc=parsedmarc.cli:_main'], } )