From 9bed81ad07a52429923a844e9316cbe1c9457d9a Mon Sep 17 00:00:00 2001 From: arnydo Date: Fri, 15 Feb 2019 15:10:45 -0500 Subject: [PATCH 01/19] Correct typo in docs --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index b7f09ff..0b4bdb0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -135,7 +135,7 @@ The full set of configuration options are: - ``general`` - ``save_aggregate`` - bool: Save aggregate report data to the Elasticsearch and/or Splunk - ``save_forensic`` - bool: Save forensic report data to the Elasticsearch and/or Splunk - - ``strip_attachments_payloads`` - bool: Remove attachment payloads from results + - ``strip_attachment_payloads`` - bool: Remove attachment payloads from results - ``output`` - str: Directory to place JSON and CSV files in - ``nameservers`` - str: A comma separated list of DNS resolvers (Default: `Cloudflare's public resolvers`_) - ``dns_timeout`` - float: DNS timeout period From b3d2efe0b013f93528d1c88c40390a4d6289efbf Mon Sep 17 00:00:00 2001 From: arnydo Date: Fri, 15 Feb 2019 16:19:12 -0500 Subject: [PATCH 02/19] Add doc on running davmail as systemd service --- docs/index.rst | 85 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 20f40a7..462839a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1023,6 +1023,89 @@ To check the status of the service, run: .. code-block:: bash journalctl -u parsedmarc.service -r + +Running DavMail as a systemd service +--------------------------------------- + +Use systemd to run ``davmail`` as a service and process reports as they +arrive. + + +Create a system user + +.. code-block:: bash + + sudo useradd davmail -r -s /bin/false + +Protect the ``davmail`` configuration file from prying eyes + +.. code-block:: bash + + sudo chown root:parsedmarc /opt/davmail/davmail.properties + sudo chmod u=rw,g=r,o= /opt/davmail/davmail.properties + +Create the service configuration file + +.. code-block:: bash + + sudo nano /etc/systemd/system/davmail.service + +.. code-block:: ini + + [Unit] + Description=DavMail gateway service + Documentation=https://sourceforge.net/projects/davmail/ + Wants=network-online.target + After=syslog.target network.target + + [Service] + ExecStart=/opt/davmail/davmail /opt/davmail/davmail.properties + User=davmail + Group=davmail + Restart=always + RestartSec=5m + + [Install] + WantedBy=multi-user.target + +Then, enable the service + +.. code-block:: bash + + sudo systemctl daemon-reload + sudo systemctl enable parsedmarc.service + sudo service davmail restart + +.. note:: + + You must also run the above commands whenever you edit + ``davmail.service``. + +.. warning:: + + Always restart the service every time you upgrade to a new version of + ``davmail``: + + .. code-block:: bash + + sudo service davmail restart + +To check the status of the service, run: + +.. code-block:: bash + + service davmail status + +.. note:: + + In the event of a crash, systemd will restart the service after 10 minutes, + but the `service davmail status` command will only show the logs for the + current process. To vew the logs for previous runs as well as the + current process (newest to oldest), run: + + .. code-block:: bash + + journalctl -u davmail.service -r Using the Kibana dashboards @@ -1282,4 +1365,4 @@ Indices and tables .. _XML files: https://github.com/domainaware/parsedmarc/tree/master/splunk -.. _LISTSERV 16.0-2017a: https://www.lsoft.com/news/dmarc-issue1-2018.asp \ No newline at end of file +.. _LISTSERV 16.0-2017a: https://www.lsoft.com/news/dmarc-issue1-2018.asp From c2b5ec9fbdbdc446b0a30a1cdcb780d13d029ab2 Mon Sep 17 00:00:00 2001 From: arnydo Date: Fri, 15 Feb 2019 16:20:04 -0500 Subject: [PATCH 03/19] Update index.rst --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 462839a..42098ee 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1041,7 +1041,7 @@ Protect the ``davmail`` configuration file from prying eyes .. code-block:: bash - sudo chown root:parsedmarc /opt/davmail/davmail.properties + sudo chown root:davmail /opt/davmail/davmail.properties sudo chmod u=rw,g=r,o= /opt/davmail/davmail.properties Create the service configuration file From 2e2e47b202eb9559025571a8832d9258e02c87f6 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Fri, 15 Feb 2019 19:14:08 -0500 Subject: [PATCH 04/19] 6.1.2 - Use local Public Suffix List file instead of downloading it --- CHANGELOG.md | 5 +++++ parsedmarc/__init__.py | 2 +- parsedmarc/utils.py | 42 ++++++++++++++++++++---------------------- setup.py | 2 +- 4 files changed, 27 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d8e4c6..3fcf667 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +6.1.2 +----- + +- Use local Public Suffix List file instead of downloading it + 6.1.1 ----- diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index b6d0731..92b97ba 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -38,7 +38,7 @@ from parsedmarc.utils import is_outlook_msg, convert_outlook_msg from parsedmarc.utils import timestamp_to_human, human_timestamp_to_datetime from parsedmarc.utils import parse_email -__version__ = "6.1.1" +__version__ = "6.1.2" logging.basicConfig( format='%(levelname)8s:%(filename)s:%(lineno)d:' diff --git a/parsedmarc/utils.py b/parsedmarc/utils.py index adcfd51..1ef8ec1 100644 --- a/parsedmarc/utils.py +++ b/parsedmarc/utils.py @@ -26,12 +26,9 @@ import geoip2.errors import requests import publicsuffix2 -__version__ = "6.1.1" - -USER_AGENT = "Mozilla/5.0 ((0 {1})) parsedmarc/{2}".format( +USER_AGENT = "Mozilla/5.0 ((0 {1})) parsedmarc".format( platform.system(), platform.release(), - __version__ ) @@ -70,7 +67,7 @@ def decode_base64(data): return base64.b64decode(data) -def get_base_domain(domain): +def get_base_domain(domain, use_fresh_psl=False): """ Gets the base domain name for the given domain @@ -78,11 +75,9 @@ def get_base_domain(domain): Results are based on a list of public domain suffixes at https://publicsuffix.org/list/public_suffix_list.dat. - This file is saved to the current working directory, - where it is used as a cache file for 24 hours. - Args: domain (str): A domain or subdomain + use_fresh_psl (bool): Download a fresh Public Suffix List Returns: str: The base domain of the given domain @@ -98,21 +93,24 @@ def get_base_domain(domain): with open(psl_path, "w", encoding="utf-8") as fresh_psl_file: fresh_psl_file.write(fresh_psl) - if not os.path.exists(psl_path): - download_psl() - else: - psl_age = datetime.now() - datetime.fromtimestamp( - os.stat(psl_path).st_mtime) - if psl_age > timedelta(hours=24): - try: - download_psl() - except Exception as error: - logger.warning( - "Failed to download an updated PSL {0}".format(error)) - with open(psl_path, encoding="utf-8") as psl_file: - psl = publicsuffix2.PublicSuffixList(psl_file) + if use_fresh_psl: + if not os.path.exists(psl_path): + download_psl() + else: + psl_age = datetime.now() - datetime.fromtimestamp( + os.stat(psl_path).st_mtime) + if psl_age > timedelta(hours=24): + try: + download_psl() + except Exception as error: + logger.warning( + "Failed to download an updated PSL {0}".format(error)) + with open(psl_path, encoding="utf-8") as psl_file: + psl = publicsuffix2.PublicSuffixList(psl_file) - return psl.get_public_suffix(domain) + return psl.get_public_suffix(domain) + else: + return publicsuffix2.get_public_suffix(domain) def query_dns(domain, record_type, cache=None, nameservers=None, timeout=2.0): diff --git a/setup.py b/setup.py index 696a0b2..8db06be 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ from setuptools import setup from codecs import open from os import path -__version__ = "6.1.1" +__version__ = "6.1.2" description = "A Python package and CLI for parsing aggregate and " \ "forensic DMARC reports" From 3a5d97e8f729577d1708a51ef14f94db3e85a8a0 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Fri, 15 Feb 2019 19:35:52 -0500 Subject: [PATCH 05/19] Fix argument name for `send_email()` (closes issue #60) --- CHANGELOG.md | 1 + parsedmarc/__init__.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fcf667..c39ca0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ----- - Use local Public Suffix List file instead of downloading it +- Fix argument name for `send_email()` (closes issue #60) 6.1.1 ----- diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 92b97ba..cfbc8af 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -1380,7 +1380,7 @@ def get_report_zip(results): def email_results(results, host, mail_from, mail_to, port=0, - use_ssl=False, user=None, password=None, subject=None, + ssl=False, user=None, password=None, subject=None, attachment_filename=None, message=None, ssl_context=None): """ Emails parsing results as a zip file @@ -1391,7 +1391,7 @@ def email_results(results, host, mail_from, mail_to, port=0, mail_from: The value of the message from header mail_to : A list of addresses to mail to port (int): Port to use - use_ssl (bool): Require a SSL connection from the start + ssl (bool): Require a SSL connection from the start user: An optional username password: An optional password subject: Overrides the default message subject @@ -1428,7 +1428,7 @@ def email_results(results, host, mail_from, mail_to, port=0, try: if ssl_context is None: ssl_context = create_default_context() - if use_ssl: + if ssl: server = smtplib.SMTP_SSL(host, port=port, context=ssl_context) server.connect(host, port) server.ehlo_or_helo_if_needed() From 264ed68b1400fb0511ea113a32c4910f9dafed9b Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Fri, 15 Feb 2019 20:09:50 -0500 Subject: [PATCH 06/19] Release 6.1.2 --- docs/index.rst | 172 ++++++++++++++++++++++++------------------------- 1 file changed, 85 insertions(+), 87 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 42098ee..2531f79 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -660,14 +660,95 @@ Configure Davmail by creating a ``davmail.properties`` file ############################################################# -Run Davmail + +Running DavMail as a systemd service +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use systemd to run ``davmail`` as a service and process reports as they +arrive. + + +Create a system user .. code-block:: bash - ./davmail.sh + sudo useradd davmail -r -s /bin/false + +Protect the ``davmail`` configuration file from prying eyes + +.. code-block:: bash + + sudo chown root:davmail /opt/davmail/davmail.properties + sudo chmod u=rw,g=r,o= /opt/davmail/davmail.properties + +Create the service configuration file + +.. code-block:: bash + + sudo nano /etc/systemd/system/davmail.service + +.. code-block:: ini + + [Unit] + Description=DavMail gateway service + Documentation=https://sourceforge.net/projects/davmail/ + Wants=network-online.target + After=syslog.target network.target + + [Service] + ExecStart=/opt/davmail/davmail /opt/davmail/davmail.properties + User=davmail + Group=davmail + Restart=always + RestartSec=5m + + [Install] + WantedBy=multi-user.target + +Then, enable the service + +.. code-block:: bash + + sudo systemctl daemon-reload + sudo systemctl enable parsedmarc.service + sudo service davmail restart + +.. note:: + + You must also run the above commands whenever you edit + ``davmail.service``. + +.. warning:: + + Always restart the service every time you upgrade to a new version of + ``davmail``: + + .. code-block:: bash + + sudo service davmail restart + +To check the status of the service, run: + +.. code-block:: bash + + service davmail status + +.. note:: + + In the event of a crash, systemd will restart the service after 5 minutes, + but the `service davmail status` command will only show the logs for the + current process. To vew the logs for previous runs as well as the + current process (newest to oldest), run: + + .. code-block:: bash + + journalctl -u davmail.service -r -Because you are interacting with Davmail server over the loopback +Configuring parsedmarc for DavMail +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Because you are interacting with DavMail server over the loopback (i.e. ``127.0.0.1``), add the following options to ``parsedmarc.ini`` config file: @@ -677,7 +758,7 @@ config file: host=127.0.0.1 port=1143 ssl=False - watch = True + watch=True Elasticsearch and Kibana ------------------------ @@ -1023,89 +1104,6 @@ To check the status of the service, run: .. code-block:: bash journalctl -u parsedmarc.service -r - -Running DavMail as a systemd service ---------------------------------------- - -Use systemd to run ``davmail`` as a service and process reports as they -arrive. - - -Create a system user - -.. code-block:: bash - - sudo useradd davmail -r -s /bin/false - -Protect the ``davmail`` configuration file from prying eyes - -.. code-block:: bash - - sudo chown root:davmail /opt/davmail/davmail.properties - sudo chmod u=rw,g=r,o= /opt/davmail/davmail.properties - -Create the service configuration file - -.. code-block:: bash - - sudo nano /etc/systemd/system/davmail.service - -.. code-block:: ini - - [Unit] - Description=DavMail gateway service - Documentation=https://sourceforge.net/projects/davmail/ - Wants=network-online.target - After=syslog.target network.target - - [Service] - ExecStart=/opt/davmail/davmail /opt/davmail/davmail.properties - User=davmail - Group=davmail - Restart=always - RestartSec=5m - - [Install] - WantedBy=multi-user.target - -Then, enable the service - -.. code-block:: bash - - sudo systemctl daemon-reload - sudo systemctl enable parsedmarc.service - sudo service davmail restart - -.. note:: - - You must also run the above commands whenever you edit - ``davmail.service``. - -.. warning:: - - Always restart the service every time you upgrade to a new version of - ``davmail``: - - .. code-block:: bash - - sudo service davmail restart - -To check the status of the service, run: - -.. code-block:: bash - - service davmail status - -.. note:: - - In the event of a crash, systemd will restart the service after 10 minutes, - but the `service davmail status` command will only show the logs for the - current process. To vew the logs for previous runs as well as the - current process (newest to oldest), run: - - .. code-block:: bash - - journalctl -u davmail.service -r Using the Kibana dashboards From 6ac5305db5f090149fd2f88c5ac992b17fab68ba Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Fri, 15 Feb 2019 20:10:52 -0500 Subject: [PATCH 07/19] 6.1.2 Release --- docs/index.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 2531f79..97c286c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -664,8 +664,7 @@ Configure Davmail by creating a ``davmail.properties`` file Running DavMail as a systemd service ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Use systemd to run ``davmail`` as a service and process reports as they -arrive. +Use systemd to run ``davmail`` as a service. Create a system user From 559b5dff07daf7db8aa3d5eeae49d31d32ead138 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Sat, 16 Feb 2019 10:51:40 -0500 Subject: [PATCH 08/19] Update pypy3 download --- docs/index.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 97c286c..97b6bf9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -509,11 +509,11 @@ symlink: .. code-block:: bash - wget https://bitbucket.org/squeaky/portable-pypy/downloads/pypy3.5-6.0.0-linux_x86_64-portable.tar.bz2 - tar -jxf pypy3.5-6.0.0-linux_x86_64-portable.tar.bz2 + wget https://bitbucket.org/squeaky/portable-pypy/downloads/pypy3.5-7.0.0-linux_x86_64-portable.tar.bz2 + tar -jxf pypy3.5-7.0.0-linux_x86_64-portable.tar.bz2 rm pypy3.5-6.0.0-linux_x86_64-portable.tar.bz2 - sudo chown -R root:root pypy3.5-6.0.0-linux_x86_64-portable - sudo mv pypy3.5-6.0.0-linux_x86_64-portable /opt/pypy3 + sudo chown -R root:root pypy3.5-7.0.0-linux_x86_64-portable + sudo mv pypy3.5-7.0.0-linux_x86_64-portable /opt/pypy3 sudo ln -s /opt/pypy3/bin/pypy3 /usr/local/bin/pypy3 Install ``virtualenv`` on your system: From ecdff4d339940d64f2b7f59e554412b289839603 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Sat, 16 Feb 2019 11:06:16 -0500 Subject: [PATCH 09/19] 6.1.3 Fix package requirements --- CHANGELOG.md | 5 +++++ setup.py | 13 +++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c39ca0a..3bf27ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +6.1.3 +----- + +- Fix package requirements + 6.1.2 ----- diff --git a/setup.py b/setup.py index 8db06be..4be3af7 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ from setuptools import setup from codecs import open from os import path -__version__ = "6.1.2" +__version__ = "6.1.3" description = "A Python package and CLI for parsing aggregate and " \ "forensic DMARC reports" @@ -92,11 +92,12 @@ setup( # your project is installed. For an analysis of "install_requires" vs pip's # requirements files see: # https://packaging.python.org/en/latest/requirements.html - install_requires=['dnspython', 'expiringdict', 'publicsuffix', - 'xmltodict', 'geoip2', 'urllib3>=1.21.1', - 'requests', 'imapclient', 'mail-parser', 'dateparser', - 'elasticsearch', - 'elasticsearch-dsl', 'kafka-python' + install_requires=['dnspython>=1.16.0', 'expiringdict>=1.1.4', + 'publicsuffix2', 'xmltodict>=0.12.0', 'geoip2>2.9.0', + 'urllib3>=1.21.1', 'requests>=2.2.16.0', + 'imapclient>=2.1.0', 'mail-parser>=3.9.2', + 'dateparser>=0.7.1', 'elasticsearch>=6.3.1', + 'elasticsearch-dsl>=0.0.12', 'kafka-python>=1.4.4' ], entry_points={ From 0e5247d79f952b6a62880952e39d6ed07cb7e627 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Sat, 16 Feb 2019 11:12:17 -0500 Subject: [PATCH 10/19] 6.1.4 - Actually package requirements --- CHANGELOG.md | 5 +++++ parsedmarc/__init__.py | 2 +- setup.py | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bf27ee..e10beca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +6.1.4 +----- + +- Actually package requirements + 6.1.3 ----- diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index cfbc8af..1da952e 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -38,7 +38,7 @@ from parsedmarc.utils import is_outlook_msg, convert_outlook_msg from parsedmarc.utils import timestamp_to_human, human_timestamp_to_datetime from parsedmarc.utils import parse_email -__version__ = "6.1.2" +__version__ = "6.1.4" logging.basicConfig( format='%(levelname)8s:%(filename)s:%(lineno)d:' diff --git a/setup.py b/setup.py index 4be3af7..0b93b08 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ from setuptools import setup from codecs import open from os import path -__version__ = "6.1.3" +__version__ = "6.1.4" description = "A Python package and CLI for parsing aggregate and " \ "forensic DMARC reports" @@ -93,7 +93,7 @@ setup( # requirements files see: # https://packaging.python.org/en/latest/requirements.html install_requires=['dnspython>=1.16.0', 'expiringdict>=1.1.4', - 'publicsuffix2', 'xmltodict>=0.12.0', 'geoip2>2.9.0', + 'publicsuffix2', 'xmltodict>=0.12.0', 'geoip2>=2.9.0', 'urllib3>=1.21.1', 'requests>=2.2.16.0', 'imapclient>=2.1.0', 'mail-parser>=3.9.2', 'dateparser>=0.7.1', 'elasticsearch>=6.3.1', From a16b5c5627f93c8c6079b4f2175d8b9787aebe69 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Sat, 16 Feb 2019 13:29:07 -0500 Subject: [PATCH 11/19] 6.1.5 --- .travis.yml | 2 +- CHANGELOG.md | 7 +++++++ docs/index.rst | 13 ++++++++++--- parsedmarc/__init__.py | 6 ++++-- parsedmarc/utils.py | 7 ++++--- setup.py | 2 +- 6 files changed, 27 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7a867f8..c99facb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ python: # commands to install dependencies before_install: - "sudo apt-get update" - - "sudo apt-get install -y libemail-outlook-message-perl" + - "sudo apt-get install -y libemail-outlook-message-perl geoipupdate" - "curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.5.1.deb && sudo dpkg -i --force-confnew elasticsearch-6.5.1.deb && sudo service elasticsearch restart" install: diff --git a/CHANGELOG.md b/CHANGELOG.md index e10beca..424ec67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +6.1.5 +----- + +- Always use Cloudflare's nameservers by default instead of Google's +- Avoid re-downloading the Geolite2 database (and tripping their DDoS protection) +- Add `geoipupdate` to install instructions + 6.1.4 ----- diff --git a/docs/index.rst b/docs/index.rst index 97b6bf9..d961e45 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -465,19 +465,24 @@ On Debian or Ubuntu systems, run: .. code-block:: bash - sudo apt-get install python3-pip - + sudo apt-get install -y python3-pip geoipupdate On CentOS systems, run: .. code-block:: bash - sudo yum install -y python34-setuptools + sudo yum install -y python34-setuptools GeoIP-Update sudo easy_install-3.4 pip + sudo geoipupdate Python 3 installers for Windows and macOS can be found at https://www.python.org/downloads/ +.. note:: + + Windows users should also download a copy of Maxmind's free + `GeoLite2-Country.mmdb`_ to ``C:\GeoIP\GeoLite2-Country.mmdb``. + To install or upgrade to the latest stable release of ``parsedmarc`` on macOS or Linux, run @@ -1346,6 +1351,8 @@ Indices and tables .. _Modern Auth/multi-factor authentication: http://davmail.sourceforge.net/faq.html +.. _GeoLite2-Country.mmdb:https://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz + .. _download the latest portable Linux version of pypy3: https://github.com/squeaky-pl/portable-pypy#portable-pypy-distribution-for-linux .. _Elasticsearch: https://www.elastic.co/guide/en/elasticsearch/reference/current/rpm.html diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 1da952e..619e39d 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -38,7 +38,7 @@ from parsedmarc.utils import is_outlook_msg, convert_outlook_msg from parsedmarc.utils import timestamp_to_human, human_timestamp_to_datetime from parsedmarc.utils import parse_email -__version__ = "6.1.4" +__version__ = "6.1.5" logging.basicConfig( format='%(levelname)8s:%(filename)s:%(lineno)d:' @@ -98,7 +98,9 @@ def _parse_report_record(record, nameservers=None, dns_timeout=2.0): OrderedDict: The converted record """ if nameservers is None: - nameservers = ["8.8.8.8", "4.4.4.4"] + nameservers = ["1.1.1.1", "1.0.0.1", + "2606:4700:4700::1111", "2606:4700:4700::1001", + ] record = record.copy() new_record = OrderedDict() new_record_source = get_ip_address_info(record["row"]["source_ip"], diff --git a/parsedmarc/utils.py b/parsedmarc/utils.py index 1ef8ec1..40fd836 100644 --- a/parsedmarc/utils.py +++ b/parsedmarc/utils.py @@ -261,7 +261,7 @@ def get_ip_address_country(ip_address): Returns: str: And ISO country code associated with the given IP address """ - def download_country_database(location=".GeoLite2-Country.mmdb"): + def download_country_database(location="GeoLite2-Country.mmdb"): """Downloads the MaxMind Geolite2 Country database Args: @@ -281,7 +281,8 @@ def get_ip_address_country(ip_address): shutil.rmtree(tar_dir) system_paths = ["/usr/local/share/GeoIP/GeoLite2-Country.mmdb", - "/usr/share/GeoIP/GeoLite2-Country.mmdb"] + "/usr/share/GeoIP/GeoLite2-Country.mmdb" + "C:\\GeoIP\\GeoLite2-Country.mmdb"] db_path = None for system_path in system_paths: @@ -296,7 +297,7 @@ def get_ip_address_country(ip_address): else: db_age = datetime.now() - datetime.fromtimestamp( os.stat(db_path).st_mtime) - if db_age > timedelta(days=60): + if db_age > timedelta(days=7): download_country_database() db_path = db_path diff --git a/setup.py b/setup.py index 0b93b08..5bd11f3 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ from setuptools import setup from codecs import open from os import path -__version__ = "6.1.4" +__version__ = "6.1.5" description = "A Python package and CLI for parsing aggregate and " \ "forensic DMARC reports" From 89ce95e2cdbc5f767135036039576485afbc589f Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Sat, 16 Feb 2019 13:33:33 -0500 Subject: [PATCH 12/19] Update index.rst --- docs/index.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index d961e45..360a3e6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -483,6 +483,7 @@ https://www.python.org/downloads/ Windows users should also download a copy of Maxmind's free `GeoLite2-Country.mmdb`_ to ``C:\GeoIP\GeoLite2-Country.mmdb``. + To install or upgrade to the latest stable release of ``parsedmarc`` on macOS or Linux, run @@ -1351,7 +1352,7 @@ Indices and tables .. _Modern Auth/multi-factor authentication: http://davmail.sourceforge.net/faq.html -.. _GeoLite2-Country.mmdb:https://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz +.. _GeoLite2-Country.mmdb: https://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz .. _download the latest portable Linux version of pypy3: https://github.com/squeaky-pl/portable-pypy#portable-pypy-distribution-for-linux From 97b581f404b324ec4fbf4d076ba2e95e0a3818e4 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Sat, 16 Feb 2019 13:35:11 -0500 Subject: [PATCH 13/19] Update index.rst --- docs/index.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 360a3e6..a9f1a85 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -483,7 +483,6 @@ https://www.python.org/downloads/ Windows users should also download a copy of Maxmind's free `GeoLite2-Country.mmdb`_ to ``C:\GeoIP\GeoLite2-Country.mmdb``. - To install or upgrade to the latest stable release of ``parsedmarc`` on macOS or Linux, run From 39d71968f1ea0b9d482e7a484700f6f34d294fb4 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Sat, 16 Feb 2019 13:36:56 -0500 Subject: [PATCH 14/19] Update index.rst --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index a9f1a85..4628ff0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -481,7 +481,7 @@ https://www.python.org/downloads/ .. note:: Windows users should also download a copy of Maxmind's free - `GeoLite2-Country.mmdb`_ to ``C:\GeoIP\GeoLite2-Country.mmdb``. + `GeoLite2-Country.mmdb`_ to ``C:\GeoIP\GeoLite2-Country.mmdb``. To install or upgrade to the latest stable release of ``parsedmarc`` on macOS or Linux, run From de8b4f936ca2ef4852ed659db59b16f252d37ae7 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Sat, 16 Feb 2019 13:50:39 -0500 Subject: [PATCH 15/19] 6.1.6 - Better GeoIP error handling --- CHANGELOG.md | 5 +++++ parsedmarc/__init__.py | 2 +- parsedmarc/utils.py | 22 +++++++++++++++------- setup.py | 2 +- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 424ec67..6384d37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +6.1.6 +----- + +- Better GeoIP error handling + 6.1.5 ----- diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 619e39d..21018fb 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -38,7 +38,7 @@ from parsedmarc.utils import is_outlook_msg, convert_outlook_msg from parsedmarc.utils import timestamp_to_human, human_timestamp_to_datetime from parsedmarc.utils import parse_email -__version__ = "6.1.5" +__version__ = "6.1.6" logging.basicConfig( format='%(levelname)8s:%(filename)s:%(lineno)d:' diff --git a/parsedmarc/utils.py b/parsedmarc/utils.py index 40fd836..0648cf1 100644 --- a/parsedmarc/utils.py +++ b/parsedmarc/utils.py @@ -272,13 +272,19 @@ def get_ip_address_country(ip_address): # Use a browser-like user agent string to bypass some proxy blocks headers = {"User-Agent": USER_AGENT} original_filename = "GeoLite2-Country.mmdb" - tar_bytes = requests.get(url, headers=headers).content - tar_file = tarfile.open(fileobj=BytesIO(tar_bytes), mode="r:gz") - tar_dir = tar_file.getnames()[0] - tar_path = "{0}/{1}".format(tar_dir, original_filename) - tar_file.extract(tar_path) - shutil.move(tar_path, location) - shutil.rmtree(tar_dir) + try: + response = requests.get(url, headers=headers) + response.raise_for_status() + tar_bytes = response.content + tar_file = tarfile.open(fileobj=BytesIO(tar_bytes), mode="r:gz") + tar_dir = tar_file.getnames()[0] + tar_path = "{0}/{1}".format(tar_dir, original_filename) + tar_file.extract(tar_path) + shutil.move(tar_path, location) + shutil.rmtree(tar_dir) + except Exception as e: + logging.debug("Error downloading {0}: {1}".format(url, + e.__str__())) system_paths = ["/usr/local/share/GeoIP/GeoLite2-Country.mmdb", "/usr/share/GeoIP/GeoLite2-Country.mmdb" @@ -294,6 +300,8 @@ def get_ip_address_country(ip_address): db_path = os.path.join(tempdir, "GeoLite2-Country.mmdb") if not os.path.exists(db_path): download_country_database(db_path) + if not os.path.exists(db_path): + return None else: db_age = datetime.now() - datetime.fromtimestamp( os.stat(db_path).st_mtime) diff --git a/setup.py b/setup.py index 5bd11f3..bd4aeaa 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ from setuptools import setup from codecs import open from os import path -__version__ = "6.1.5" +__version__ = "6.1.6" description = "A Python package and CLI for parsing aggregate and " \ "forensic DMARC reports" From d29ae43dd7fb5f822fa0fccc77cb30f8cfaeedd4 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Sat, 16 Feb 2019 14:08:58 -0500 Subject: [PATCH 16/19] Fix CI builds --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c99facb..7a867f8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ python: # commands to install dependencies before_install: - "sudo apt-get update" - - "sudo apt-get install -y libemail-outlook-message-perl geoipupdate" + - "sudo apt-get install -y libemail-outlook-message-perl" - "curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.5.1.deb && sudo dpkg -i --force-confnew elasticsearch-6.5.1.deb && sudo service elasticsearch restart" install: From 7681f302958daa4c21f30bc8017a5475ac9f15d7 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Sat, 16 Feb 2019 15:31:27 -0500 Subject: [PATCH 17/19] 6.1.7 - Fix GeoIP lookups --- CHANGELOG.md | 5 +++++ parsedmarc/__init__.py | 2 +- parsedmarc/utils.py | 3 ++- setup.py | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6384d37..92fc4ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +6.1.7 +----- + +- Fix GeoIP lookups + 6.1.6 ----- diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index 21018fb..c495b2d 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -38,7 +38,7 @@ from parsedmarc.utils import is_outlook_msg, convert_outlook_msg from parsedmarc.utils import timestamp_to_human, human_timestamp_to_datetime from parsedmarc.utils import parse_email -__version__ = "6.1.6" +__version__ = "6.1.7" logging.basicConfig( format='%(levelname)8s:%(filename)s:%(lineno)d:' diff --git a/parsedmarc/utils.py b/parsedmarc/utils.py index 0648cf1..78cb7ea 100644 --- a/parsedmarc/utils.py +++ b/parsedmarc/utils.py @@ -287,7 +287,8 @@ def get_ip_address_country(ip_address): e.__str__())) system_paths = ["/usr/local/share/GeoIP/GeoLite2-Country.mmdb", - "/usr/share/GeoIP/GeoLite2-Country.mmdb" + "/usr/share/GeoIP/GeoLite2-Country.mmdb", + "/var/lib/GeoIP/GeoLite2-Country.mmdb" "C:\\GeoIP\\GeoLite2-Country.mmdb"] db_path = None diff --git a/setup.py b/setup.py index bd4aeaa..75fee9c 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ from setuptools import setup from codecs import open from os import path -__version__ = "6.1.6" +__version__ = "6.1.7" description = "A Python package and CLI for parsing aggregate and " \ "forensic DMARC reports" From 47598d9de9f667fdc4116ecb27a56687a461ed3f Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Sat, 16 Feb 2019 15:37:14 -0500 Subject: [PATCH 18/19] 6.1.8 - Actually fix GeoIP lookups --- CHANGELOG.md | 5 +++++ parsedmarc/__init__.py | 2 +- parsedmarc/utils.py | 2 +- setup.py | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92fc4ea..f0ad0b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +6.1.8 +----- + +- Actually fix GeoIP lookups + 6.1.7 ----- diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index c495b2d..1514a8e 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -38,7 +38,7 @@ from parsedmarc.utils import is_outlook_msg, convert_outlook_msg from parsedmarc.utils import timestamp_to_human, human_timestamp_to_datetime from parsedmarc.utils import parse_email -__version__ = "6.1.7" +__version__ = "6.1.8" logging.basicConfig( format='%(levelname)8s:%(filename)s:%(lineno)d:' diff --git a/parsedmarc/utils.py b/parsedmarc/utils.py index 78cb7ea..9d9dfbe 100644 --- a/parsedmarc/utils.py +++ b/parsedmarc/utils.py @@ -288,7 +288,7 @@ def get_ip_address_country(ip_address): system_paths = ["/usr/local/share/GeoIP/GeoLite2-Country.mmdb", "/usr/share/GeoIP/GeoLite2-Country.mmdb", - "/var/lib/GeoIP/GeoLite2-Country.mmdb" + "/var/lib/GeoIP/GeoLite2-Country.mmdb", "C:\\GeoIP\\GeoLite2-Country.mmdb"] db_path = None diff --git a/setup.py b/setup.py index 75fee9c..2959e13 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ from setuptools import setup from codecs import open from os import path -__version__ = "6.1.7" +__version__ = "6.1.8" description = "A Python package and CLI for parsing aggregate and " \ "forensic DMARC reports" From dc5dd1dc5413a450f055411c0ad8200134145bf0 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Sat, 16 Feb 2019 22:04:05 -0500 Subject: [PATCH 19/19] Logging improvements --- parsedmarc/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/parsedmarc/utils.py b/parsedmarc/utils.py index 9d9dfbe..85c126f 100644 --- a/parsedmarc/utils.py +++ b/parsedmarc/utils.py @@ -283,12 +283,13 @@ def get_ip_address_country(ip_address): shutil.move(tar_path, location) shutil.rmtree(tar_dir) except Exception as e: - logging.debug("Error downloading {0}: {1}".format(url, - e.__str__())) + logger.warning("Error downloading {0}: {1}".format(url, + e.__str__())) system_paths = ["/usr/local/share/GeoIP/GeoLite2-Country.mmdb", "/usr/share/GeoIP/GeoLite2-Country.mmdb", "/var/lib/GeoIP/GeoLite2-Country.mmdb", + "/var/local/lib/GeoIP/GeoLite2-Country.mmdb", "C:\\GeoIP\\GeoLite2-Country.mmdb"] db_path = None