diff --git a/.gitignore b/.gitignore index 89f65eb..7dfcf7e 100644 --- a/.gitignore +++ b/.gitignore @@ -107,12 +107,8 @@ ENV/ # I/O files output/ -*.zip -*.gz *.csv *.xls* -*.eml -*.msg # LibreOffice lock files .~* diff --git a/.travis.yml b/.travis.yml index cb9f03c..bce20d0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: python -sudo: false +sudo: true python: - '3.4' @@ -8,8 +8,11 @@ python: - '3.6' # commands to install dependencies +before_install: + - "sudo apt-get update" + - "sudo apt-get install -y libemail-outlook-message-perl" + install: - - "pip install flake8 pytest-cov pytest coveralls" - "pip install -r requirements.txt" # commands to run samples diff --git a/README.rst b/README.rst index af02632..d77810f 100644 --- a/README.rst +++ b/README.rst @@ -31,33 +31,50 @@ Features Resources ========= +DMARC guides +------------ + * `Demystifying DMARC`_ - A complete guide to SPF, DKIM, and DMARC +SPF and DMARC record validation +------------------------------- + +If you are looking for SPF and DMARC record validation and parsing, +check out the sister project, +`checkdmarc `_. + +Lookalike domains +----------------- + +DMARC protects against domain spoofing, not lookalike domains. for open source +lookalike domain monitoring, check out `DomainAware `_. + + CLI help ======== :: -usage: parsedmarc [-h] [-o OUTPUT] [-n NAMESERVERS [NAMESERVERS ...]] - [-t TIMEOUT] [-H HOST] [-u USER] [-p PASSWORD] - [--imap-port IMAP_PORT] [--imap-no-ssl] [-r REPORTS_FOLDER] - [-a ARCHIVE_FOLDER] [-d] - [-E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]]] - [--elasticsearch-index-prefix ELASTICSEARCH_INDEX_PREFIX] - [--elasticsearch-index-suffix ELASTICSEARCH_INDEX_SUFFIX] - [--hec HEC] [--hec-token HEC_TOKEN] [--hec-index HEC_INDEX] - [--hec-skip-certificate-verification] - [-K [KAFKA_HOSTS [KAFKA_HOSTS ...]]] - [--kafka-aggregate-topic KAFKA_AGGREGATE_TOPIC] - [--kafka-forensic_topic KAFKA_FORENSIC_TOPIC] - [--save-aggregate] [--save-forensic] [-O OUTGOING_HOST] - [-U OUTGOING_USER] [-P OUTGOING_PASSWORD] - [--outgoing-port OUTGOING_PORT] - [--outgoing-ssl OUTGOING_SSL] [-F OUTGOING_FROM] - [-T OUTGOING_TO [OUTGOING_TO ...]] [-S OUTGOING_SUBJECT] - [-A OUTGOING_ATTACHMENT] [-M OUTGOING_MESSAGE] [-w] [--test] - [-s] [--debug] [-v] - [file_path [file_path ...]] + usage: parsedmarc [-h] [-o OUTPUT] [-n NAMESERVERS [NAMESERVERS ...]] + [-t TIMEOUT] [-H HOST] [-u USER] [-p PASSWORD] + [--imap-port IMAP_PORT] [--imap-no-ssl] [-r REPORTS_FOLDER] + [-a ARCHIVE_FOLDER] [-d] + [-E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]]] + [--elasticsearch-index-prefix ELASTICSEARCH_INDEX_PREFIX] + [--elasticsearch-index-suffix ELASTICSEARCH_INDEX_SUFFIX] + [--hec HEC] [--hec-token HEC_TOKEN] [--hec-index HEC_INDEX] + [--hec-skip-certificate-verification] + [-K [KAFKA_HOSTS [KAFKA_HOSTS ...]]] + [--kafka-aggregate-topic KAFKA_AGGREGATE_TOPIC] + [--kafka-forensic_topic KAFKA_FORENSIC_TOPIC] + [--save-aggregate] [--save-forensic] [-O OUTGOING_HOST] + [-U OUTGOING_USER] [-P OUTGOING_PASSWORD] + [--outgoing-port OUTGOING_PORT] + [--outgoing-ssl OUTGOING_SSL] [-F OUTGOING_FROM] + [-T OUTGOING_TO [OUTGOING_TO ...]] [-S OUTGOING_SUBJECT] + [-A OUTGOING_ATTACHMENT] [-M OUTGOING_MESSAGE] [-w] [--test] + [-s] [--debug] [-v] + [file_path [file_path ...]] Parses DMARC reports @@ -145,18 +162,6 @@ usage: parsedmarc [-h] [-o OUTPUT] [-n NAMESERVERS [NAMESERVERS ...]] --debug Print debugging information -v, --version show program's version number and exit -SPF and DMARC record validation -=============================== - -If you are looking for SPF and DMARC record validation and parsing, -check out the sister project, -`checkdmarc `_. - -Lookalike domains -================= - -DMARC protects against domain spoofing, not lookalike domains. for open source -lookalike domain monitoring, check out `DomainAware `_. Sample aggregate report output ============================== @@ -255,125 +260,6 @@ Sample forensic report output I don't have a sample I can share for privacy reasons. If you have a sample forensic report that you can share publicly, please contact me! -Installation -============ - -``parsedmarc`` works with Python 2 or 3, but Python 3 is preferred. - -On Debian or Ubuntu systems, run: - -.. code-block:: bash - - $ sudo apt-get install python3-pip - - -Python 3 installers for Windows and macOS can be found at -https://www.python.org/downloads/ - -To install or upgrade to the latest stable release of ``parsedmarc`` on -macOS or Linux, run - -.. code-block:: bash - - $ sudo -H pip3 install -U parsedmarc - -Or, install the latest development release directly from GitHub: - -.. code-block:: bash - - $ sudo -H pip3 install -U git+https://github.com/domainaware/parsedmarc.git - -.. note:: - - On Windows, ``pip3`` is ``pip``, even with Python 3. So on Windows, simply - substitute ``pip`` as an administrator in place of ``sudo pip3``, in the - above commands. - -Installation using pypy3 ------------------------- - -For the best possible processing speed, consider using `parsedmarc` inside a ``pypy3`` -virtualenv. First, `download the latest version of pypy3`_. Extract it to -``/opt/pypy3`` (``sudo mkdir /opt`` if ``/opt`` does not exist), then create a -symlink: - -.. code-block:: bash - - $ sudo ln -s /opt/pypy3/bin/pypy3 /usr/local/bin/pypy3 - -Install ``virtualenv`` on your system: - -.. code-block:: bash - - $ sudo apt-get install python3-pip - $ sudo -H pip3 install -U virtualenv - -Uninstall any instance of ``parsedmarc`` that you may have installed globally - -.. code-block:: bash - - $ sudo -H pip3 uninstall -y parsedmarc - -Next, create a ``pypy3`` virtualenv for parsedmarc - - -.. code-block:: bash - - $ sudo mkdir /opt/venvs - $ cd /opt/venvs - $ sudo -H pip3 install -U virtualenv - $ sudo virtualenv --download -p /usr/local/bin/pypy3 parsedmarc - $ sudo -H /opt/venvs/parsedmarc/bin/pip3 install -U parsedmarc - $ sudo ln -s /opt/venvs/parsedmarc/bin/parsedmarc /usr/local/bin/parsedmarc - -To upgrade ``parsedmarc`` inside the virtualenv, run: - - -.. code-block:: bash - - $ sudo -H /opt/venvs/parsedmarc/bin/pip3 install -U parsedmarc - -Or, install the latest development release directly from GitHub: - -.. code-block:: bash - - $ sudo -H /opt/venvs/parsedmarc/bin/pip3 install -U git+https://github.com/domainaware/parsedmarc.git - -Optional dependencies ---------------------- - -If you would like to be able to parse emails saved from Microsoft Outlook -(i.e. OLE .msg files), install ``msgconvert``: - -On Debian or Ubuntu systems, run: - -.. code-block:: bash - - $ sudo apt-get install libemail-outlook-message-perl - -DNS performance ---------------- - -You can often improve performance by providing one or more local nameservers -to the CLI or function calls, as long as those nameservers return the same -records as the public DNS. - - -.. note:: - - If you do not specify any nameservers, Cloudflare's public nameservers are - used by default, **not the system's default nameservers**. - - This is done to avoid a situation where records in a local nameserver do - not match records in the public DNS. - -Testing multiple report analyzers ---------------------------------- - -If you would like to test parsedmarc and another report processing solution -at the same time, you can have up to two mailto URIs each in the rua and ruf -tags tgs in your DMARC record, separated by commas. - Documentation ============= diff --git a/docs/index.rst b/docs/index.rst index 47e1b88..4330888 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -37,8 +37,24 @@ Features Resources ========= +DMARC guides +------------ + * `Demystifying DMARC`_ - A complete guide to SPF, DKIM, and DMARC +SPF and DMARC record validation +------------------------------- + +If you are looking for SPF and DMARC record validation and parsing, +check out the sister project, +`checkdmarc `_. + +Lookalike domains +----------------- + +DMARC protects against domain spoofing, not lookalike domains. for open source +lookalike domain monitoring, check out `DomainAware `_. + CLI help ======== @@ -152,20 +168,6 @@ CLI help --debug Print debugging information -v, --version show program's version number and exit -SPF and DMARC record validation -=============================== - -If you are looking for SPF and DMARC record validation and parsing, -check out the sister project, -`checkdmarc `_. - -SPF and DMARC record validation -=============================== - -If you are looking for SPF and DMARC record validation and parsing, -check out the sister project, -`checkdmarc `_. - Sample aggregate report output ============================== diff --git a/requirements.txt b/requirements.txt index 6d978e4..3087d84 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,8 @@ elasticsearch>=6.3.0,<7.0.0 elasticsearch-dsl>=6.2.1,<7.0.0 kafka-python flake8 +pytest +pytest-cov sphinx==1.7.9 sphinx_rtd_theme collective.checkdocs diff --git a/samples/!example.com!1538204542!1538463818.xml.sample b/samples/aggregate/!example.com!1538204542!1538463818.xml similarity index 100% rename from samples/!example.com!1538204542!1538463818.xml.sample rename to samples/aggregate/!example.com!1538204542!1538463818.xml diff --git a/samples/addisonfoods.com!example.com!1536105600!1536191999.xml.sample b/samples/aggregate/addisonfoods.com!example.com!1536105600!1536191999.xml similarity index 100% rename from samples/addisonfoods.com!example.com!1536105600!1536191999.xml.sample rename to samples/aggregate/addisonfoods.com!example.com!1536105600!1536191999.xml diff --git a/samples/aggregate/estadocuenta1.infonacot.gob.mx!example.com!1536853302!1536939702!2940.xml.zip b/samples/aggregate/estadocuenta1.infonacot.gob.mx!example.com!1536853302!1536939702!2940.xml.zip new file mode 100644 index 0000000..beca144 Binary files /dev/null and b/samples/aggregate/estadocuenta1.infonacot.gob.mx!example.com!1536853302!1536939702!2940.xml.zip differ diff --git a/samples/example.net!example.com!1529366400!1529452799.xml.sample b/samples/aggregate/example.net!example.com!1529366400!1529452799.xml similarity index 100% rename from samples/example.net!example.com!1529366400!1529452799.xml.sample rename to samples/aggregate/example.net!example.com!1529366400!1529452799.xml diff --git a/samples/fastmail.com!example.com!1516060800!1516147199!102675056.xml.gz.sample b/samples/aggregate/fastmail.com!example.com!1516060800!1516147199!102675056.xml.gz similarity index 100% rename from samples/fastmail.com!example.com!1516060800!1516147199!102675056.xml.gz.sample rename to samples/aggregate/fastmail.com!example.com!1516060800!1516147199!102675056.xml.gz diff --git a/samples/ikea.com!example.de!1538690400!1538776800.xml.sample b/samples/aggregate/ikea.com!example.de!1538690400!1538776800.xml similarity index 100% rename from samples/ikea.com!example.de!1538690400!1538776800.xml.sample rename to samples/aggregate/ikea.com!example.de!1538690400!1538776800.xml diff --git a/samples/old_draft_from_wiki.xml.sample b/samples/aggregate/old_draft_from_wiki.xml similarity index 100% rename from samples/old_draft_from_wiki.xml.sample rename to samples/aggregate/old_draft_from_wiki.xml diff --git a/samples/usssa.com!example.com!1538784000!1538870399.xml.sample b/samples/aggregate/usssa.com!example.com!1538784000!1538870399.xml similarity index 100% rename from samples/usssa.com!example.com!1538784000!1538870399.xml.sample rename to samples/aggregate/usssa.com!example.com!1538784000!1538870399.xml diff --git a/samples/veeam.com!example.com!1530133200!1530219600.xml.sample b/samples/aggregate/veeam.com!example.com!1530133200!1530219600.xml similarity index 100% rename from samples/veeam.com!example.com!1530133200!1530219600.xml.sample rename to samples/aggregate/veeam.com!example.com!1530133200!1530219600.xml diff --git a/samples/estadocuenta1.infonacot.gob.mx!example.com!1536853302!1536939702!2940.xml.sample b/samples/estadocuenta1.infonacot.gob.mx!example.com!1536853302!1536939702!2940.xml.sample deleted file mode 100644 index 795e9d9..0000000 --- a/samples/estadocuenta1.infonacot.gob.mx!example.com!1536853302!1536939702!2940.xml.sample +++ /dev/null @@ -1,34 +0,0 @@ - - - - 2940 - XYZ Corporation - admin@estadocuenta1.infonacot.gob.mx - http://estadocuenta1.infonacot.gob.mx - - 1536853302 - 1536939702 - - - - example.com -

none

- - - - 148.243.137.254 - 1 - - none - fail - fail - - - - estadocuenta1.infonacot.gob.mx - example.com - - - - - diff --git a/samples/forensic/subject.eml b/samples/forensic/subject.eml new file mode 100644 index 0000000..04a342b --- /dev/null +++ b/samples/forensic/subject.eml @@ -0,0 +1,95 @@ +MIME-Version: 1.0 +Received: from mailrelay.de ([234.234.234.234]) + by Servername.domain.local (IBM Domino Release 9.0.1FP10 HF197) + with ESMTP id 2018100111202767-369529 ; + Mon, 1 Oct 2018 11:20:27 +0200 +Return-Path: +Received: from [127.0.0.1] ([local]) + by mailrelay.de (envelope-from ) + (ecelerity 4.2.39.63080 r(Core:4.2.39.2)) with UNKNOWN + id 48/E7-30937-BD6E1BB5; Mon, 01 Oct 2018 11:20:27 +0200 +Subject: DMARC Failure Report for domain.de (mail-from=sharepoint@domain.de, ip=10.10.10.10) +To: dmarc-report@domain.de +From: dmarc-report@domain.de +X-MIMETrack: Itemize by SMTP Server on Servername/DOMAIN(Release 9.0.1FP10 HF197|April + 16, 2018) at 01.10.2018 11:20:27, + Serialize by Notes Client on Peter Pan/DOMAIN(Release 9.0.1|October + 14, 2013) at 05.10.2018 23:38:45 +X-Notes-Item: Memo; + name=Form +Date: Mon, 1 Oct 2018 11:20:27 +0200 +X-Notes-Item: CN=Servername/O=DOMAIN; + type=501; flags=44; name=$UpdatedBy +Message-ID: +X-Notes-Item: CN=Servername/O=DOMAIN, + CN=DE9899SL4/O=DOMAIN; + type=501; flags=0; name=RouteServers +X-Notes-Item: =?UTF-8?B?MDEtT2N0LTIwMTggMTE6MjA6MjcgQ0VEVC8wMS1PY3QtMjAxOCAxMToyMDo=?= + =?UTF-8?B?MjcgQ0VEVCwgMDEtT2N0LTIwMTggMTE6MjA6MjcgQ0VEVC8wMS1PY3Qt?= + =?UTF-8?B?MjAxOCAxMToyMDoyNyBDRURU?=; + type=401; flags=0; name=RouteTimes +X-Notes-Item: 587285BA:CB01D107-C1258319:00334FCF; + type=4; name=$Orig +X-Notes-Item: ; + type=501; name=Categories +X-Notes-Item: ; + type=401; name=$Revisions +X-Notes-Item: Mon, 1 Oct 2018 11:20:27 +0200; + type=400; name=DeliveredDate +X-Notes-Item: =?UTF-8?B?VGhpcyBpcyBhbiBlbWFpbCBhYnVzZSByZXBvcnQgZm9yIGFuIGVtYWls?= + =?UTF-8?B?IG1lc3NhZ2UgcmVjZWl2ZWQgZnJvbSBJUCAxMC4yLjMwLjEwMiBvbiBNb24=?= + =?UTF-8?B?LCAwMSBPY3QgMjAxOA==?=; + flags=6; name=$Abstract +X-Notes-Item: 587285BA:CB01D107-C1258319:00334FCF; + type=4; name=$TUA +X-Notes-Item: 1; + name=$NoteHasNativeMIME +Content-Type: multipart/report; report-type=feedback-report; + boundary="_----jqB1YyrX3TKBNru++5PX3w===_48/E7-30937-BD6E1BB5" + +--_----jqB1YyrX3TKBNru++5PX3w===_48/E7-30937-BD6E1BB5 +Content-Transfer-Encoding: 7bit +Content-Type: text/plain; charset="US-ASCII" + +This is an email abuse report for an email message received from IP 10.10.10.10 on Mon, 01 Oct 2018 11:20:27 +0200. +The message below did not meet the sending domain's DMARC policy. +For more information about this format please see http://tools.ietf.org/html/rfc6591 . + +--_----jqB1YyrX3TKBNru++5PX3w===_48/E7-30937-BD6E1BB5 +Content-Type: message/feedback-report; name=report + +Feedback-Type: auth-failure +User-Agent: Lua/1.0 +Version: 1.0 +Original-Mail-From: sharepoint@domain.de +Original-Rcpt-To: peter.pan@domain.de +Arrival-Date: Mon, 01 Oct 2018 11:20:27 +0200 +Message-ID: <38.E7.30937.BD6E1BB5@ mailrelay.de> +Authentication-Results: dmarc=fail (p=none, dis=none) header.from=domain.de +Source-IP: 10.10.10.10 +Delivery-Result: smg-policy-action +Auth-Failure: dmarc +Reported-Domain: domain.de + +--_----jqB1YyrX3TKBNru++5PX3w===_48/E7-30937-BD6E1BB5 +Content-Type: message/rfc822 +Content-Disposition: inline + +Received: from Servernameone.domain.local (Servernameone.domain.local [10.10.10.10]) + by mailrelay.de (mail.DOMAIN.de) with SMTP id 38.E7.30937.BD6E1BB5; Mon, 1 Oct 2018 11:20:27 +0200 (CEST) +Date: 01 Oct 2018 11:20:27 +0200 +Message-ID: <38.E7.30937.BD6E1BB5@ mailrelay.de> +To: +from: "=?utf-8?B?SW50ZXJha3RpdmUgV2V0dGJld2VyYmVyLcOcYmVyc2ljaHQ=?=" +Subject: Subject +MIME-Version: 1.0 +X-Mailer: Microsoft SharePoint Foundation 2010 +Content-Type: text/html; charset=utf-8 +Content-Transfer-Encoding: quoted-printable + + + +--_----jqB1YyrX3TKBNru++5PX3w===_48/E7-30937-BD6E1BB5-- \ No newline at end of file diff --git a/tests.py b/tests.py index 0a727fa..ddcb915 100644 --- a/tests.py +++ b/tests.py @@ -8,15 +8,28 @@ import parsedmarc class Test(unittest.TestCase): - def testSamples(self): - """Test sample aggregate DMARC reports""" - sample_paths = glob("samples/*.sample") - for sample_path in sample_paths: - print("Testing {0}...\n".format(sample_path)) - parsed_report = parsedmarc.parse_aggregate_report_file(sample_path) - print(json.dumps(parsed_report, ensure_ascii=False, indent=2)) - print("\n") - print(parsedmarc.parsed_aggregate_reports_to_csv(parsed_report)) + class Test(unittest.TestCase): + def testAggregateSamples(self): + """Test sample aggregate.rua DMARC reports""" + sample_paths = glob("samples/aggregate/*") + for sample_path in sample_paths: + print("Testing {0}...\n".format(sample_path)) + parsed_report = parsedmarc.parse_aggregate_report_file( + sample_path) + print(json.dumps(parsed_report, ensure_ascii=False, indent=2)) + print("\n") + print( + parsedmarc.parsed_aggregate_reports_to_csv(parsed_report)) + + def testForensicSamples(self): + """Test sample forensic/ruf/failure DMARC reports""" + sample_paths = glob("samples/forensic/*.eml") + for sample_path in sample_paths: + print("Testing {0}...\n".format(sample_path)) + parsed_report = parsedmarc.parse_report_email(sample_path) + print(json.dumps(parsed_report, ensure_ascii=False, indent=2)) + print("\n") + print(parsedmarc.parsed_forensic_reports_to_csv(parsed_report)) if __name__ == "__main__":