diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d8e4c6..f0ad0b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,41 @@ +6.1.8 +----- + +- Actually fix GeoIP lookups + +6.1.7 +----- + +- Fix GeoIP lookups + +6.1.6 +----- + +- Better GeoIP error handling + +6.1.5 +----- + +- Always use Cloudflare's nameservers by default instead of Google's +- Avoid re-downloading the Geolite2 database (and tripping their DDoS protection) +- Add `geoipupdate` to install instructions + +6.1.4 +----- + +- Actually package requirements + +6.1.3 +----- + +- Fix package requirements + +6.1.2 +----- + +- Use local Public Suffix List file instead of downloading it +- Fix argument name for `send_email()` (closes issue #60) + 6.1.1 ----- diff --git a/docs/index.rst b/docs/index.rst index 4a8f81f..4628ff0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -135,7 +135,7 @@ The full set of configuration options are: - ``general`` - ``save_aggregate`` - bool: Save aggregate report data to the Elasticsearch and/or Splunk - ``save_forensic`` - bool: Save forensic report data to the Elasticsearch and/or Splunk - - ``strip_attachments_payloads`` - bool: Remove attachment payloads from results + - ``strip_attachment_payloads`` - bool: Remove attachment payloads from results - ``output`` - str: Directory to place JSON and CSV files in - ``nameservers`` - str: A comma separated list of DNS resolvers (Default: `Cloudflare's public resolvers`_) - ``dns_timeout`` - float: DNS timeout period @@ -465,19 +465,24 @@ On Debian or Ubuntu systems, run: .. code-block:: bash - sudo apt-get install python3-pip - + sudo apt-get install -y python3-pip geoipupdate On CentOS systems, run: .. code-block:: bash - sudo yum install -y python34-setuptools + sudo yum install -y python34-setuptools GeoIP-Update sudo easy_install-3.4 pip + sudo geoipupdate Python 3 installers for Windows and macOS can be found at https://www.python.org/downloads/ +.. note:: + + Windows users should also download a copy of Maxmind's free + `GeoLite2-Country.mmdb`_ to ``C:\GeoIP\GeoLite2-Country.mmdb``. + To install or upgrade to the latest stable release of ``parsedmarc`` on macOS or Linux, run @@ -509,11 +514,11 @@ symlink: .. code-block:: bash - wget https://bitbucket.org/squeaky/portable-pypy/downloads/pypy3.5-6.0.0-linux_x86_64-portable.tar.bz2 - tar -jxf pypy3.5-6.0.0-linux_x86_64-portable.tar.bz2 + wget https://bitbucket.org/squeaky/portable-pypy/downloads/pypy3.5-7.0.0-linux_x86_64-portable.tar.bz2 + tar -jxf pypy3.5-7.0.0-linux_x86_64-portable.tar.bz2 rm pypy3.5-6.0.0-linux_x86_64-portable.tar.bz2 - sudo chown -R root:root pypy3.5-6.0.0-linux_x86_64-portable - sudo mv pypy3.5-6.0.0-linux_x86_64-portable /opt/pypy3 + sudo chown -R root:root pypy3.5-7.0.0-linux_x86_64-portable + sudo mv pypy3.5-7.0.0-linux_x86_64-portable /opt/pypy3 sudo ln -s /opt/pypy3/bin/pypy3 /usr/local/bin/pypy3 Install ``virtualenv`` on your system: @@ -660,14 +665,94 @@ Configure Davmail by creating a ``davmail.properties`` file ############################################################# -Run Davmail + +Running DavMail as a systemd service +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use systemd to run ``davmail`` as a service. + + +Create a system user .. code-block:: bash - ./davmail.sh + sudo useradd davmail -r -s /bin/false + +Protect the ``davmail`` configuration file from prying eyes + +.. code-block:: bash + + sudo chown root:davmail /opt/davmail/davmail.properties + sudo chmod u=rw,g=r,o= /opt/davmail/davmail.properties + +Create the service configuration file + +.. code-block:: bash + + sudo nano /etc/systemd/system/davmail.service + +.. code-block:: ini + + [Unit] + Description=DavMail gateway service + Documentation=https://sourceforge.net/projects/davmail/ + Wants=network-online.target + After=syslog.target network.target + + [Service] + ExecStart=/opt/davmail/davmail /opt/davmail/davmail.properties + User=davmail + Group=davmail + Restart=always + RestartSec=5m + + [Install] + WantedBy=multi-user.target + +Then, enable the service + +.. code-block:: bash + + sudo systemctl daemon-reload + sudo systemctl enable parsedmarc.service + sudo service davmail restart + +.. note:: + + You must also run the above commands whenever you edit + ``davmail.service``. + +.. warning:: + + Always restart the service every time you upgrade to a new version of + ``davmail``: + + .. code-block:: bash + + sudo service davmail restart + +To check the status of the service, run: + +.. code-block:: bash + + service davmail status + +.. note:: + + In the event of a crash, systemd will restart the service after 5 minutes, + but the `service davmail status` command will only show the logs for the + current process. To vew the logs for previous runs as well as the + current process (newest to oldest), run: + + .. code-block:: bash + + journalctl -u davmail.service -r -Because you are interacting with Davmail server over the loopback +Configuring parsedmarc for DavMail +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Because you are interacting with DavMail server over the loopback (i.e. ``127.0.0.1``), add the following options to ``parsedmarc.ini`` config file: @@ -677,7 +762,7 @@ config file: host=127.0.0.1 port=1143 ssl=False - watch = True + watch=True Elasticsearch and Kibana ------------------------ @@ -1266,6 +1351,8 @@ Indices and tables .. _Modern Auth/multi-factor authentication: http://davmail.sourceforge.net/faq.html +.. _GeoLite2-Country.mmdb: https://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz + .. _download the latest portable Linux version of pypy3: https://github.com/squeaky-pl/portable-pypy#portable-pypy-distribution-for-linux .. _Elasticsearch: https://www.elastic.co/guide/en/elasticsearch/reference/current/rpm.html @@ -1282,4 +1369,4 @@ Indices and tables .. _XML files: https://github.com/domainaware/parsedmarc/tree/master/splunk -.. _LISTSERV 16.0-2017a: https://www.lsoft.com/news/dmarc-issue1-2018.asp \ No newline at end of file +.. _LISTSERV 16.0-2017a: https://www.lsoft.com/news/dmarc-issue1-2018.asp diff --git a/parsedmarc/__init__.py b/parsedmarc/__init__.py index c73dde9..2b3c93f 100644 --- a/parsedmarc/__init__.py +++ b/parsedmarc/__init__.py @@ -38,7 +38,7 @@ from parsedmarc.utils import is_outlook_msg, convert_outlook_msg from parsedmarc.utils import timestamp_to_human, human_timestamp_to_datetime from parsedmarc.utils import parse_email -__version__ = "6.1.1" +__version__ = "6.1.8" logging.basicConfig( format='%(levelname)8s:%(filename)s:%(lineno)d:' @@ -98,7 +98,9 @@ def _parse_report_record(record, nameservers=None, dns_timeout=2.0, parallel=Fal OrderedDict: The converted record """ if nameservers is None: - nameservers = ["8.8.8.8", "4.4.4.4"] + nameservers = ["1.1.1.1", "1.0.0.1", + "2606:4700:4700::1111", "2606:4700:4700::1001", + ] record = record.copy() new_record = OrderedDict() new_record_source = get_ip_address_info(record["row"]["source_ip"], @@ -1392,7 +1394,7 @@ def get_report_zip(results): def email_results(results, host, mail_from, mail_to, port=0, - use_ssl=False, user=None, password=None, subject=None, + ssl=False, user=None, password=None, subject=None, attachment_filename=None, message=None, ssl_context=None): """ Emails parsing results as a zip file @@ -1403,7 +1405,7 @@ def email_results(results, host, mail_from, mail_to, port=0, mail_from: The value of the message from header mail_to : A list of addresses to mail to port (int): Port to use - use_ssl (bool): Require a SSL connection from the start + ssl (bool): Require a SSL connection from the start user: An optional username password: An optional password subject: Overrides the default message subject @@ -1440,7 +1442,7 @@ def email_results(results, host, mail_from, mail_to, port=0, try: if ssl_context is None: ssl_context = create_default_context() - if use_ssl: + if ssl: server = smtplib.SMTP_SSL(host, port=port, context=ssl_context) server.connect(host, port) server.ehlo_or_helo_if_needed() diff --git a/parsedmarc/utils.py b/parsedmarc/utils.py index 409593f..11f1e9c 100644 --- a/parsedmarc/utils.py +++ b/parsedmarc/utils.py @@ -26,12 +26,9 @@ import geoip2.errors import requests import publicsuffix2 -__version__ = "6.1.1" - -USER_AGENT = "Mozilla/5.0 ((0 {1})) parsedmarc/{2}".format( +USER_AGENT = "Mozilla/5.0 ((0 {1})) parsedmarc".format( platform.system(), platform.release(), - __version__ ) @@ -70,7 +67,7 @@ def decode_base64(data): return base64.b64decode(data) -def get_base_domain(domain): +def get_base_domain(domain, use_fresh_psl=False): """ Gets the base domain name for the given domain @@ -78,11 +75,9 @@ def get_base_domain(domain): Results are based on a list of public domain suffixes at https://publicsuffix.org/list/public_suffix_list.dat. - This file is saved to the current working directory, - where it is used as a cache file for 24 hours. - Args: domain (str): A domain or subdomain + use_fresh_psl (bool): Download a fresh Public Suffix List Returns: str: The base domain of the given domain @@ -98,21 +93,24 @@ def get_base_domain(domain): with open(psl_path, "w", encoding="utf-8") as fresh_psl_file: fresh_psl_file.write(fresh_psl) - if not os.path.exists(psl_path): - download_psl() - else: - psl_age = datetime.now() - datetime.fromtimestamp( - os.stat(psl_path).st_mtime) - if psl_age > timedelta(hours=24): - try: - download_psl() - except Exception as error: - logger.warning( - "Failed to download an updated PSL {0}".format(error)) - with open(psl_path, encoding="utf-8") as psl_file: - psl = publicsuffix2.PublicSuffixList(psl_file) + if use_fresh_psl: + if not os.path.exists(psl_path): + download_psl() + else: + psl_age = datetime.now() - datetime.fromtimestamp( + os.stat(psl_path).st_mtime) + if psl_age > timedelta(hours=24): + try: + download_psl() + except Exception as error: + logger.warning( + "Failed to download an updated PSL {0}".format(error)) + with open(psl_path, encoding="utf-8") as psl_file: + psl = publicsuffix2.PublicSuffixList(psl_file) - return psl.get_public_suffix(domain) + return psl.get_public_suffix(domain) + else: + return publicsuffix2.get_public_suffix(domain) def query_dns(domain, record_type, cache=None, nameservers=None, timeout=2.0): @@ -263,7 +261,7 @@ def get_ip_address_country(ip_address): Returns: str: And ISO country code associated with the given IP address """ - def download_country_database(location=".GeoLite2-Country.mmdb"): + def download_country_database(location="GeoLite2-Country.mmdb"): """Downloads the MaxMind Geolite2 Country database Args: @@ -275,16 +273,25 @@ def get_ip_address_country(ip_address): # Use a browser-like user agent string to bypass some proxy blocks headers = {"User-Agent": USER_AGENT} original_filename = "GeoLite2-Country.mmdb" - tar_bytes = requests.get(url, headers=headers).content - tar_file = tarfile.open(fileobj=BytesIO(tar_bytes), mode="r:gz") - tar_dir = tar_file.getnames()[0] - tar_path = "{0}/{1}".format(tar_dir, original_filename) - tar_file.extract(tar_path) - shutil.move(tar_path, location) - shutil.rmtree(tar_dir) + try: + response = requests.get(url, headers=headers) + response.raise_for_status() + tar_bytes = response.content + tar_file = tarfile.open(fileobj=BytesIO(tar_bytes), mode="r:gz") + tar_dir = tar_file.getnames()[0] + tar_path = "{0}/{1}".format(tar_dir, original_filename) + tar_file.extract(tar_path) + shutil.move(tar_path, location) + shutil.rmtree(tar_dir) + except Exception as e: + logger.warning("Error downloading {0}: {1}".format(url, + e.__str__())) system_paths = ["/usr/local/share/GeoIP/GeoLite2-Country.mmdb", - "/usr/share/GeoIP/GeoLite2-Country.mmdb"] + "/usr/share/GeoIP/GeoLite2-Country.mmdb", + "/var/lib/GeoIP/GeoLite2-Country.mmdb", + "/var/local/lib/GeoIP/GeoLite2-Country.mmdb", + "C:\\GeoIP\\GeoLite2-Country.mmdb"] db_path = None for system_path in system_paths: @@ -296,10 +303,12 @@ def get_ip_address_country(ip_address): db_path = os.path.join(tempdir, "GeoLite2-Country.mmdb") if not os.path.exists(db_path): download_country_database(db_path) + if not os.path.exists(db_path): + return None else: db_age = datetime.now() - datetime.fromtimestamp( os.stat(db_path).st_mtime) - if db_age > timedelta(days=60): + if db_age > timedelta(days=7): download_country_database() db_path = db_path diff --git a/setup.py b/setup.py index 696a0b2..2959e13 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ from setuptools import setup from codecs import open from os import path -__version__ = "6.1.1" +__version__ = "6.1.8" description = "A Python package and CLI for parsing aggregate and " \ "forensic DMARC reports" @@ -92,11 +92,12 @@ setup( # your project is installed. For an analysis of "install_requires" vs pip's # requirements files see: # https://packaging.python.org/en/latest/requirements.html - install_requires=['dnspython', 'expiringdict', 'publicsuffix', - 'xmltodict', 'geoip2', 'urllib3>=1.21.1', - 'requests', 'imapclient', 'mail-parser', 'dateparser', - 'elasticsearch', - 'elasticsearch-dsl', 'kafka-python' + install_requires=['dnspython>=1.16.0', 'expiringdict>=1.1.4', + 'publicsuffix2', 'xmltodict>=0.12.0', 'geoip2>=2.9.0', + 'urllib3>=1.21.1', 'requests>=2.2.16.0', + 'imapclient>=2.1.0', 'mail-parser>=3.9.2', + 'dateparser>=0.7.1', 'elasticsearch>=6.3.1', + 'elasticsearch-dsl>=0.0.12', 'kafka-python>=1.4.4' ], entry_points={