From 95f58018f23b379746404b2f08fb4afcf2bf8e4f Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Thu, 29 Mar 2018 17:02:58 -0400 Subject: [PATCH] Prepare for the 2.4.0 release --- CHANGELOG.md | 7 + ...e-index.png => define-dmarc-aggregate.png} | Bin ...rt-saved-objects.png => saved-objects.png} | Bin docs/index.rst | 279 +++++++++++++++++- parsedmarc/__init__.py | 4 +- parsedmarc/cli.py | 6 +- 6 files changed, 289 insertions(+), 7 deletions(-) rename docs/_static/screenshots/{define-dmarc-aggregate-index.png => define-dmarc-aggregate.png} (100%) rename docs/_static/screenshots/{import-saved-objects.png => saved-objects.png} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5899e2a..6c14c5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +3.4.0 +----- +- Maintain IMAP IDLE state when watching the inbox +- The `-i`/`--idle` CLI option is now `-w`/`--watch` +- Improved Exception handling and documentation + + 3.3.0 ----- - Fix errors when saving to Elasticsearch diff --git a/docs/_static/screenshots/define-dmarc-aggregate-index.png b/docs/_static/screenshots/define-dmarc-aggregate.png similarity index 100% rename from docs/_static/screenshots/define-dmarc-aggregate-index.png rename to docs/_static/screenshots/define-dmarc-aggregate.png diff --git a/docs/_static/screenshots/import-saved-objects.png b/docs/_static/screenshots/saved-objects.png similarity index 100% rename from docs/_static/screenshots/import-saved-objects.png rename to docs/_static/screenshots/saved-objects.png diff --git a/docs/index.rst b/docs/index.rst index 0ad0cbd..f7b1211 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -9,7 +9,7 @@ Welcome to parsedmarc's documentation! |Build Status| -.. image:: /_static/screenshots/dmarc-summary-charts.png +.. image:: _static/screenshots/dmarc-summary-charts.png :alt: A screenshot of DMARC summary charts in Kibana :scale: 50 % :align: center @@ -247,6 +247,277 @@ On Debian or Ubuntu systems, run: $ sudo apt-get install libemail-outlook-message-perl +Elasticsearch and Kibana +------------------------ + +To set up visual dashboards of DMARC data, install Elasticsearch and Kibana. + +.. code-block:: bash + + sudo apt-get install -y openjdk-8-jre apt-transport-https + wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - + echo "deb https://artifacts.elastic.co/packages/6.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-6.x.list + sudo apt-get update + sudo apt-get install -y elasticsearch kibana + sudo systemctl daemon-reload + sudo systemctl enable elasticsearch.service + sudo systemctl enable kibana.service + sudo service start elasticsearch + sudo service start kibana + +Without the commercial X-Pack_, Kibana does not have any authentication +mechanism of its own. You can use nginx as a reverse proxy that provides basic +authentication. + +.. code-block:: bash + + sudo apt-get install -y nginx apache2-utils + +Create a directory to store the certificates and keys: + +.. code-block:: bash + + mkdir ~/ssl + cd ~/ssl + +To create a self-signed certificate, run: + +.. code-block:: bash + + openssl req -x509 -nodes -days 365 -newkey rsa:4096 -keyout kibana.key -out kibana.crt + +Or, to create a Certificate Signing Request (CSR) for a CA, run: + +.. code-block:: bash + + openssl req -newkey rsa:4096-nodes -keyout kibana.key -out kibana.csr + +Fill in the prompts. Watch out for Common Name (e.g. server FQDN or YOUR +domain name), which is the IP address or domain name that you will be hosting +Kibana on. it is the most important field. + +If you generated a CSR, remove the CSR after you have your certs + +.. code-block:: bash + + rm -f kibana.csr + + +Move the keys into place and secure them: + +.. code-block:: bash + + cd + sudo mv ssl /etc/nginx + sudo chown -R root:www-data /etc/nginx/ssl + sudo chmod -R u=rX,g=rX,o= /etc/nginx/ssl + +Disable the default nginx configuration: + +.. code-block:: bash + + sudo rm /etc/nginx/sites-enabled/default + +Create the Cuckoo web server configuration + +.. code-block:: bash + + sudo nano /etc/nginx/sites-available/kibana + +.. code-block:: nginx + + server { + listen 443 ssl http2; + ssl_certificate /etc/nginx/ssl/kibana.crt; + ssl_certificate_key /etc/nginx/ssl/kibana.key; + ssl_dhparam /etc/nginx/ssl/dhparam.pem; + ssl_session_timeout 1d; + ssl_session_cache shared:SSL:50m; + ssl_session_tickets off; + + ssl_protocols TLSv1.2; + ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHAC ssl_prefer_server_ciphers on; + + # Uncomment this next line if you are using a signed, trusted cert + #add_header Strict-Transport-Security "max-age=63072000; includeSubdomains; preload"; + add_header X-Frame-Options SAMEORIGIN; + add_header X-Content-Type-Options nosniff; + auth_basic "Login required"; + auth_basic_user_file /etc/nginx/htpasswd; + + location / { + proxy_pass http://127.0.0.1:5601; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + } + + server { + listen 80; + return 301 https://$server_name$request_uri; + } + + +Enable the nginx configuration for Kibana: + +.. code-block:: bash + + sudo ln -s /etc/nginx/sites-available/kibana /etc/nginx/sites-enabled/kibana + +Add a user to basic authentication: + +.. code-block:: bash + + sudo htpasswd -c /etc/nginx/htpasswd exampleuser + +Where ``exampleuser`` is the name of the user you want to add. + +Secure the permissions of the httpasswd file: + +.. code-block:: bash + + sudo chown root:www-data /etc/nginx/htpasswd + sudo chmod u=rw,g=r,o= /etc/nginx/htpasswd + +Restart nginx: + +.. code-block:: bash + + sudo service nginx restart + + +Now that Elasticsearch is up and running, use ``parsedmarc`` to send data to +it. + +Om the same system as Elasticsearch, pass ``--save-aggregate`` and/or +``--save-forensic`` to ``parsedmarc`` save the results in Elasticsearch. + +.. warning:: + + ``--save-aggregate`` and ``--save-forensic`` are separate options because + you may not want to save forensic reports to your Elasticsearch instance, + particularly if you are in a highly-regulated industry that handles + sensitive data, such as healthcare or finance. If your legitimate outgoing + email fails DMARC, it is possible that email may appear later in a + forensic report. + + Forensic reports contain the original headers of an email that failed a + DMARC check, and sometimes may also include the full message body, + depending on the policy of the reporting organisation. + + Most reporting organisations do not send forensic reports of any kind for + privacy reasons. While aggregate DMARC reports are sent at least daily, + it is normal to receive very few forensic reports. + + +When you first visit Kibana, it will prompt you to create an index pattern. +Start by creating the index pattern ``dmarc_aggregate`` (without an ``*``), +and select ``date_range`` as the time field. Once the ``dmarc_aggregate`` +index pattern ``dmarc_aggregate`` has been saved, create a ``dmarc_forensic`` +index pattern, with ``arrival_date`` as the time field. + +.. image:: _static/screenshots/define-dmarc-aggregate.png + :alt: A screenshot of defining the dmarc_aggregate index pattern + :align: center + :target: _static/screenshots/define-dmarc-aggregate.png + +.. image:: _static/screenshots/dmarc-aggregate-time-field.png + :alt: A screenshot of setting the time field for the dmarc_aggregate index pattern + :align: center + :target: _static/screenshots/dmarc-aggregate-time-field.png + +.. image:: _static/screenshots/define-dmarc-forensic.png + :alt: A screenshot of defining the dmarc_forensic index pattern + :align: center + :target: _static/screenshots/define-dmarc-forensic.png + +.. image:: _static/screenshots/dmarc-forensic-time-field.png + :alt: A screenshot of setting the time field for the dmarc_forensic index pattern + :align: center + :target: _static/screenshots/dmarc-forensic-time-field.png + +Once the index patterns have been created, you can import the dashboards. + +Download (right click the link and click save as) kibana_saved_objects.json_. + +Import ``kibana_saved_objects.json`` the Saved Objects tab of the management +page of Kibana. + +It will give you the option to overwrite existing saved dashboards or +visualizations, which could be used to restore them if you or someone else +breaks them, as there are no permissions/access controls in Kibana without +the commercial X-Pack_. + +.. image:: _static/screenshots/saved-objects.png + :alt: A screenshot of setting the Saved Objects management UI in Kibana + :align: center + :target: _static/screenshots/saved-objects.png + +.. image:: _static/screenshots/confirm-overwrite.png + :alt: A screenshot of the overwrite conformation prompt + :align: center + :target: _static/screenshots/confirm-overwrite.png + +Kibana will then ask you to match the newly imported objects to your index +patterns. Select ``dmarc_forensic`` for the set of forensic objects, and +select ``dmarc_aggregate`` for the other saved objects, as shown below. + + +.. image:: _static/screenshots/index-pattern-conflicts.png + :alt: A screenshot showing how to resolve index pattern conflicts after importing saved objects + :align: center + :target: _static/screenshots/index-pattern-conflicts.png + +Running parsedmarc as a systemd service +--------------------------------------- + +Use systemd to run ``parsedmarc`` as a service and process reports as they +arrive. + +Create the service configuration file + +.. code-block:: bash + + sudo nano /etc/systemd/system/parsedmarc.service + +Edit the command line options of ``parsedmarc`` in the service's ``ExecStart`` +setting to suit your needs. + +.. note:: + + Always pass the ``--watch`` option to ``parsedmarc`` when running it as a + service. + +.. code-block:: ini + + [Unit] + Description=parsedmarc DMARC mailbox watcher + Documentation=https://domainaware.github.io/parsedmarc/ + + [Service] + ExecStart=/usr/local/bin/parsedmarc --watch --save-aggregate -H "outlook.office365.com" -u "dmarc@example.com" -p "FooBar!" --save-aggregate + User=nobody + Group=nobody + Restart=always + RestartSec=5m + + [Install] + WantedBy=multi-user.target + +Then, enable the service + +.. code-block:: bash + + sudo systemctl daemon-reload + sudo systemctl enable parsedmarc.service + sudo service parsedmarc restart + +.. note:: + + You must also run the above commands anytime you edit + ``parsedmarc.service``. + API === @@ -272,4 +543,8 @@ Indices and tables .. |Build Status| image:: https://travis-ci.org/domainaware/parsedmarc.svg?branch=master - :target: https://travis-ci.org/domainaware/parsedmarc \ No newline at end of file + :target: https://travis-ci.org/domainaware/parsedmarc + +.. _X-Pack: https://www.elastic.co/products/x-pack + +.. _kibana_saved_objects.json: https://raw.githubusercontent.com/domainaware/parsedmarc/master/kibana/kibana_saved_objects.json \ No newline at end of file diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 0154b6e..9d0fcd0 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -"""A Python module for parsing DMARC reports""" +"""A Python package for parsing DMARC reports""" import logging import os @@ -156,7 +156,7 @@ def _get_reverse_dns(ip_address, nameservers=None, timeout=6.0): timeout (float): Sets the DNS query timeout in seconds Returns: - + str: The reverse DNS hostname (if any) """ hostname = None try: diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index 1919f36..a3c63fd 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -96,7 +96,7 @@ def _main(): help="Email the results using this filename") arg_parser.add_argument("-M", "--outgoing-message", help="Email the results using this message") - arg_parser.add_argument("-i", "--idle", action="store_true", + arg_parser.add_argument("-w", "--watch", action="store_true", help="Use an IMAP IDLE connection to process " "reports as they arrive in the inbox") arg_parser.add_argument("--test", @@ -187,8 +187,8 @@ def _main(): logger.error("SMTP Error: {0}".format(error.__str__())) exit(1) - if args.host and args.idle: - logger.warning("The IMAP Connection is now in IDLE mode. " + if args.host and args.watch: + logger.warning("Watching for email\n" "Quit with ^c") try: watch_inbox(args.host, args.user, args.password, process_reports,