diff --git a/CHANGELOG.md b/CHANGELOG.md index fb7e4bd8..1916fc9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ as `text` instead of `long` (Closes issue #31) - Bugfix: IDLE email processing in Gmail/G-Suite accounts (closes issue #33) - Bugfix: Fix inaccurate DNS timeout in CLI documentation (closes issue #34) +- Bugfix: Forensic report processing via CLI +- Bugfix: Duplicate aggregate report Elasticsearch query broken - Bugfix: Crash when `Arrival-Date` header is missing in a forensic/fialure/ruf report - IMAP reliability improvements diff --git a/README.rst b/README.rst index a9282c0d..a593c507 100644 --- a/README.rst +++ b/README.rst @@ -25,7 +25,8 @@ Features * Consistent data structures * Simple JSON and/or CSV output * Optionally email the results -* Optionally send the results to Elasticsearch and/or Splunk, for use with premade dashboards +* Optionally send the results to Elasticsearch and/or Splunk, for use with + premade dashboards * Optionally send reports to Apache Kafka Resources @@ -46,8 +47,9 @@ check out the sister project, Lookalike domains ----------------- -DMARC protects against domain spoofing, not lookalike domains. for open source -lookalike domain monitoring, check out `DomainAware `_. +DMARC protects against domain spoofing, not lookalike domains. For open source +lookalike domain monitoring, check out +`DomainAware `_. CLI help @@ -55,26 +57,26 @@ CLI help :: - usage: cli.py [-h] [--strip-attachment-payloads] [-o OUTPUT] - [-n NAMESERVERS [NAMESERVERS ...]] [-t TIMEOUT] [-H HOST] - [-u USER] [-p PASSWORD] [--imap-port IMAP_PORT] - [--imap-skip-certificate-verification] [--imap-no-ssl] - [-r REPORTS_FOLDER] [-a ARCHIVE_FOLDER] [-d] - [-E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]]] - [--elasticsearch-index-prefix ELASTICSEARCH_INDEX_PREFIX] - [--elasticsearch-index-suffix ELASTICSEARCH_INDEX_SUFFIX] - [--hec HEC] [--hec-token HEC_TOKEN] [--hec-index HEC_INDEX] - [--hec-skip-certificate-verification] - [-K [KAFKA_HOSTS [KAFKA_HOSTS ...]]] - [--kafka-aggregate-topic KAFKA_AGGREGATE_TOPIC] - [--kafka-forensic_topic KAFKA_FORENSIC_TOPIC] [--save-aggregate] - [--save-forensic] [-O OUTGOING_HOST] [-U OUTGOING_USER] - [-P OUTGOING_PASSWORD] [--outgoing-port OUTGOING_PORT] - [--outgoing-ssl OUTGOING_SSL] [-F OUTGOING_FROM] - [-T OUTGOING_TO [OUTGOING_TO ...]] [-S OUTGOING_SUBJECT] - [-A OUTGOING_ATTACHMENT] [-M OUTGOING_MESSAGE] [-w] [--test] - [-s] [--debug] [-v] - [file_path [file_path ...]] + usage: parsedmarc [-h] [--strip-attachment-payloads] [-o OUTPUT] + [-n NAMESERVERS [NAMESERVERS ...]] [-t TIMEOUT] [-H HOST] + [-u USER] [-p PASSWORD] [--imap-port IMAP_PORT] + [--imap-skip-certificate-verification] [--imap-no-ssl] + [-r REPORTS_FOLDER] [-a ARCHIVE_FOLDER] [-d] + [-E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]]] + [--elasticsearch-index-suffix ELASTICSEARCH_INDEX_SUFFIX] + [--hec HEC] [--hec-token HEC_TOKEN] [--hec-index HEC_INDEX] + [--hec-skip-certificate-verification] + [-K [KAFKA_HOSTS [KAFKA_HOSTS ...]]] + [--kafka-aggregate-topic KAFKA_AGGREGATE_TOPIC] + [--kafka-forensic_topic KAFKA_FORENSIC_TOPIC] + [--save-aggregate] [--save-forensic] [-O OUTGOING_HOST] + [-U OUTGOING_USER] [-P OUTGOING_PASSWORD] + [--outgoing-port OUTGOING_PORT] + [--outgoing-ssl OUTGOING_SSL] [-F OUTGOING_FROM] + [-T OUTGOING_TO [OUTGOING_TO ...]] [-S OUTGOING_SUBJECT] + [-A OUTGOING_ATTACHMENT] [-M OUTGOING_MESSAGE] [-w] [--test] + [-s] [--debug] [-v] + [file_path [file_path ...]] Parses DMARC reports @@ -113,9 +115,6 @@ CLI help -E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]], --elasticsearch-host [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]] One or more Elasticsearch hostnames or URLs to use (e.g. localhost:9200) - --elasticsearch-index-prefix ELASTICSEARCH_INDEX_PREFIX - Prefix to add in front of the dmarc_aggregate and - dmarc_forensic Elasticsearch index names, joined by _ --elasticsearch-index-suffix ELASTICSEARCH_INDEX_SUFFIX Append this suffix to the dmarc_aggregate and dmarc_forensic Elasticsearch index names, joined by _ diff --git a/docs/index.rst b/docs/index.rst index f0e8891d..23ab5377 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -67,7 +67,6 @@ CLI help [--imap-skip-certificate-verification] [--imap-no-ssl] [-r REPORTS_FOLDER] [-a ARCHIVE_FOLDER] [-d] [-E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]]] - [--elasticsearch-index-prefix ELASTICSEARCH_INDEX_PREFIX] [--elasticsearch-index-suffix ELASTICSEARCH_INDEX_SUFFIX] [--hec HEC] [--hec-token HEC_TOKEN] [--hec-index HEC_INDEX] [--hec-skip-certificate-verification] @@ -120,9 +119,6 @@ CLI help -E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]], --elasticsearch-host [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]] One or more Elasticsearch hostnames or URLs to use (e.g. localhost:9200) - --elasticsearch-index-prefix ELASTICSEARCH_INDEX_PREFIX - Prefix to add in front of the dmarc_aggregate and - dmarc_forensic Elasticsearch index names, joined by _ --elasticsearch-index-suffix ELASTICSEARCH_INDEX_SUFFIX Append this suffix to the dmarc_aggregate and dmarc_forensic Elasticsearch index names, joined by _ @@ -608,7 +604,8 @@ Configure Davmail by creating a ``davmail.properties`` file # Enable IDLE support, set polling delay in minutes davmail.imapIdleDelay=1 - # Always reply to IMAP RFC822.SIZE requests with Exchange approximate message size for performance reasons + # Always reply to IMAP RFC822.SIZE requests with Exchange approximate + # message size for performance reasons davmail.imapAlwaysApproxMsgSize=true ############################################################# @@ -849,12 +846,33 @@ the commercial X-Pack_. :align: center :target: _static/screenshots/confirm-overwrite.png +Upgrading Kibana index patterns +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``parsedmarc`` 5.0.0 makes some changes to the way data is indexed in +Elasticsearch. if you are upgrading prom a previous release of +``parsedmarc``, you need to complete the following steps to replace the +Kibana index patterns with versions that match the upgraded indexes: + +1. Login in to Kibana, and click on Management +2. Under Kibana, click on Saved Objects +3. Check the checkboxes for the ``dmarc_aggregate`` and ``dmarc_forensic`` +index patterns +4. Click Delete +5. Click Delete on the conformation message +6. Download (right click the link and click save as) +the latest version of kibana_saved_objects.json_ +7. Import ``kibana_saved_objects.json`` by clicking Import from the Kibana +Saved Objects page + Records retention ~~~~~~~~~~~~~~~~~ -To prevent your indexes from growing too large, or to comply with records -retention regulations such as GDPR, you need to use `time-based indexes +Starting in version 5.0.0, ``parsedmarc`` stores data in a separate +index for each day to make it easy to comply with records +retention regulations such as GDPR. For fore information, +check out the Elastic guide to `managing time-based indexes efficiently `_. Splunk