mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-10-31 06:35:42 +00:00
Update test suitw
This commit is contained in:
Sean Whalen committed
1 parent
babdc661ac
commit
f45ab94e06
17 files changed
+177
-214
No files matched your search
@@ -107,12 +107,8 @@ ENV/
|
||||
|
||||
# I/O files
|
||||
output/
|
||||
*.zip
|
||||
*.gz
|
||||
*.csv
|
||||
*.xls*
|
||||
*.eml
|
||||
*.msg
|
||||
|
||||
# LibreOffice lock files
|
||||
.~*
|
||||
|
||||
+5
-2
@@ -1,6 +1,6 @@
|
||||
language: python
|
||||
|
||||
sudo: false
|
||||
sudo: true
|
||||
|
||||
python:
|
||||
- '3.4'
|
||||
@@ -8,8 +8,11 @@ python:
|
||||
- '3.6'
|
||||
|
||||
# commands to install dependencies
|
||||
before_install:
|
||||
- "sudo apt-get update"
|
||||
- "sudo apt-get install -y libemail-outlook-message-perl"
|
||||
|
||||
install:
|
||||
- "pip install flake8 pytest-cov pytest coveralls"
|
||||
- "pip install -r requirements.txt"
|
||||
|
||||
# commands to run samples
|
||||
|
||||
+37
-151
@@ -31,33 +31,50 @@ Features
|
||||
Resources
|
||||
=========
|
||||
|
||||
DMARC guides
|
||||
------------
|
||||
|
||||
* `Demystifying DMARC`_ - A complete guide to SPF, DKIM, and DMARC
|
||||
|
||||
SPF and DMARC record validation
|
||||
-------------------------------
|
||||
|
||||
If you are looking for SPF and DMARC record validation and parsing,
|
||||
check out the sister project,
|
||||
`checkdmarc <https://domainaware.github.io/checkdmarc/>`_.
|
||||
|
||||
Lookalike domains
|
||||
-----------------
|
||||
|
||||
DMARC protects against domain spoofing, not lookalike domains. for open source
|
||||
lookalike domain monitoring, check out `DomainAware <https://github.com/seanthegeek/domainaware>`_.
|
||||
|
||||
|
||||
CLI help
|
||||
========
|
||||
|
||||
::
|
||||
|
||||
usage: parsedmarc [-h] [-o OUTPUT] [-n NAMESERVERS [NAMESERVERS ...]]
|
||||
[-t TIMEOUT] [-H HOST] [-u USER] [-p PASSWORD]
|
||||
[--imap-port IMAP_PORT] [--imap-no-ssl] [-r REPORTS_FOLDER]
|
||||
[-a ARCHIVE_FOLDER] [-d]
|
||||
[-E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]]]
|
||||
[--elasticsearch-index-prefix ELASTICSEARCH_INDEX_PREFIX]
|
||||
[--elasticsearch-index-suffix ELASTICSEARCH_INDEX_SUFFIX]
|
||||
[--hec HEC] [--hec-token HEC_TOKEN] [--hec-index HEC_INDEX]
|
||||
[--hec-skip-certificate-verification]
|
||||
[-K [KAFKA_HOSTS [KAFKA_HOSTS ...]]]
|
||||
[--kafka-aggregate-topic KAFKA_AGGREGATE_TOPIC]
|
||||
[--kafka-forensic_topic KAFKA_FORENSIC_TOPIC]
|
||||
[--save-aggregate] [--save-forensic] [-O OUTGOING_HOST]
|
||||
[-U OUTGOING_USER] [-P OUTGOING_PASSWORD]
|
||||
[--outgoing-port OUTGOING_PORT]
|
||||
[--outgoing-ssl OUTGOING_SSL] [-F OUTGOING_FROM]
|
||||
[-T OUTGOING_TO [OUTGOING_TO ...]] [-S OUTGOING_SUBJECT]
|
||||
[-A OUTGOING_ATTACHMENT] [-M OUTGOING_MESSAGE] [-w] [--test]
|
||||
[-s] [--debug] [-v]
|
||||
[file_path [file_path ...]]
|
||||
usage: parsedmarc [-h] [-o OUTPUT] [-n NAMESERVERS [NAMESERVERS ...]]
|
||||
[-t TIMEOUT] [-H HOST] [-u USER] [-p PASSWORD]
|
||||
[--imap-port IMAP_PORT] [--imap-no-ssl] [-r REPORTS_FOLDER]
|
||||
[-a ARCHIVE_FOLDER] [-d]
|
||||
[-E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]]]
|
||||
[--elasticsearch-index-prefix ELASTICSEARCH_INDEX_PREFIX]
|
||||
[--elasticsearch-index-suffix ELASTICSEARCH_INDEX_SUFFIX]
|
||||
[--hec HEC] [--hec-token HEC_TOKEN] [--hec-index HEC_INDEX]
|
||||
[--hec-skip-certificate-verification]
|
||||
[-K [KAFKA_HOSTS [KAFKA_HOSTS ...]]]
|
||||
[--kafka-aggregate-topic KAFKA_AGGREGATE_TOPIC]
|
||||
[--kafka-forensic_topic KAFKA_FORENSIC_TOPIC]
|
||||
[--save-aggregate] [--save-forensic] [-O OUTGOING_HOST]
|
||||
[-U OUTGOING_USER] [-P OUTGOING_PASSWORD]
|
||||
[--outgoing-port OUTGOING_PORT]
|
||||
[--outgoing-ssl OUTGOING_SSL] [-F OUTGOING_FROM]
|
||||
[-T OUTGOING_TO [OUTGOING_TO ...]] [-S OUTGOING_SUBJECT]
|
||||
[-A OUTGOING_ATTACHMENT] [-M OUTGOING_MESSAGE] [-w] [--test]
|
||||
[-s] [--debug] [-v]
|
||||
[file_path [file_path ...]]
|
||||
|
||||
Parses DMARC reports
|
||||
|
||||
@@ -145,18 +162,6 @@ usage: parsedmarc [-h] [-o OUTPUT] [-n NAMESERVERS [NAMESERVERS ...]]
|
||||
--debug Print debugging information
|
||||
-v, --version show program's version number and exit
|
||||
|
||||
SPF and DMARC record validation
|
||||
===============================
|
||||
|
||||
If you are looking for SPF and DMARC record validation and parsing,
|
||||
check out the sister project,
|
||||
`checkdmarc <https://domainaware.github.io/checkdmarc/>`_.
|
||||
|
||||
Lookalike domains
|
||||
=================
|
||||
|
||||
DMARC protects against domain spoofing, not lookalike domains. for open source
|
||||
lookalike domain monitoring, check out `DomainAware <https://github.com/seanthegeek/domainaware>`_.
|
||||
|
||||
Sample aggregate report output
|
||||
==============================
|
||||
@@ -255,125 +260,6 @@ Sample forensic report output
|
||||
I don't have a sample I can share for privacy reasons. If you have a sample
|
||||
forensic report that you can share publicly, please contact me!
|
||||
|
||||
Installation
|
||||
============
|
||||
|
||||
``parsedmarc`` works with Python 2 or 3, but Python 3 is preferred.
|
||||
|
||||
On Debian or Ubuntu systems, run:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ sudo apt-get install python3-pip
|
||||
|
||||
|
||||
Python 3 installers for Windows and macOS can be found at
|
||||
https://www.python.org/downloads/
|
||||
|
||||
To install or upgrade to the latest stable release of ``parsedmarc`` on
|
||||
macOS or Linux, run
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ sudo -H pip3 install -U parsedmarc
|
||||
|
||||
Or, install the latest development release directly from GitHub:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ sudo -H pip3 install -U git+https://github.com/domainaware/parsedmarc.git
|
||||
|
||||
.. note::
|
||||
|
||||
On Windows, ``pip3`` is ``pip``, even with Python 3. So on Windows, simply
|
||||
substitute ``pip`` as an administrator in place of ``sudo pip3``, in the
|
||||
above commands.
|
||||
|
||||
Installation using pypy3
|
||||
------------------------
|
||||
|
||||
For the best possible processing speed, consider using `parsedmarc` inside a ``pypy3``
|
||||
virtualenv. First, `download the latest version of pypy3`_. Extract it to
|
||||
``/opt/pypy3`` (``sudo mkdir /opt`` if ``/opt`` does not exist), then create a
|
||||
symlink:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ sudo ln -s /opt/pypy3/bin/pypy3 /usr/local/bin/pypy3
|
||||
|
||||
Install ``virtualenv`` on your system:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ sudo apt-get install python3-pip
|
||||
$ sudo -H pip3 install -U virtualenv
|
||||
|
||||
Uninstall any instance of ``parsedmarc`` that you may have installed globally
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ sudo -H pip3 uninstall -y parsedmarc
|
||||
|
||||
Next, create a ``pypy3`` virtualenv for parsedmarc
|
||||
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ sudo mkdir /opt/venvs
|
||||
$ cd /opt/venvs
|
||||
$ sudo -H pip3 install -U virtualenv
|
||||
$ sudo virtualenv --download -p /usr/local/bin/pypy3 parsedmarc
|
||||
$ sudo -H /opt/venvs/parsedmarc/bin/pip3 install -U parsedmarc
|
||||
$ sudo ln -s /opt/venvs/parsedmarc/bin/parsedmarc /usr/local/bin/parsedmarc
|
||||
|
||||
To upgrade ``parsedmarc`` inside the virtualenv, run:
|
||||
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ sudo -H /opt/venvs/parsedmarc/bin/pip3 install -U parsedmarc
|
||||
|
||||
Or, install the latest development release directly from GitHub:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ sudo -H /opt/venvs/parsedmarc/bin/pip3 install -U git+https://github.com/domainaware/parsedmarc.git
|
||||
|
||||
Optional dependencies
|
||||
---------------------
|
||||
|
||||
If you would like to be able to parse emails saved from Microsoft Outlook
|
||||
(i.e. OLE .msg files), install ``msgconvert``:
|
||||
|
||||
On Debian or Ubuntu systems, run:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ sudo apt-get install libemail-outlook-message-perl
|
||||
|
||||
DNS performance
|
||||
---------------
|
||||
|
||||
You can often improve performance by providing one or more local nameservers
|
||||
to the CLI or function calls, as long as those nameservers return the same
|
||||
records as the public DNS.
|
||||
|
||||
|
||||
.. note::
|
||||
|
||||
If you do not specify any nameservers, Cloudflare's public nameservers are
|
||||
used by default, **not the system's default nameservers**.
|
||||
|
||||
This is done to avoid a situation where records in a local nameserver do
|
||||
not match records in the public DNS.
|
||||
|
||||
Testing multiple report analyzers
|
||||
---------------------------------
|
||||
|
||||
If you would like to test parsedmarc and another report processing solution
|
||||
at the same time, you can have up to two mailto URIs each in the rua and ruf
|
||||
tags tgs in your DMARC record, separated by commas.
|
||||
|
||||
Documentation
|
||||
=============
|
||||
|
||||
|
||||
+16
-14
@@ -37,8 +37,24 @@ Features
|
||||
Resources
|
||||
=========
|
||||
|
||||
DMARC guides
|
||||
------------
|
||||
|
||||
* `Demystifying DMARC`_ - A complete guide to SPF, DKIM, and DMARC
|
||||
|
||||
SPF and DMARC record validation
|
||||
-------------------------------
|
||||
|
||||
If you are looking for SPF and DMARC record validation and parsing,
|
||||
check out the sister project,
|
||||
`checkdmarc <https://domainaware.github.io/checkdmarc/>`_.
|
||||
|
||||
Lookalike domains
|
||||
-----------------
|
||||
|
||||
DMARC protects against domain spoofing, not lookalike domains. for open source
|
||||
lookalike domain monitoring, check out `DomainAware <https://github.com/seanthegeek/domainaware>`_.
|
||||
|
||||
|
||||
CLI help
|
||||
========
|
||||
@@ -152,20 +168,6 @@ CLI help
|
||||
--debug Print debugging information
|
||||
-v, --version show program's version number and exit
|
||||
|
||||
SPF and DMARC record validation
|
||||
===============================
|
||||
|
||||
If you are looking for SPF and DMARC record validation and parsing,
|
||||
check out the sister project,
|
||||
`checkdmarc <https://domainaware.github.io/checkdmarc/>`_.
|
||||
|
||||
SPF and DMARC record validation
|
||||
===============================
|
||||
|
||||
If you are looking for SPF and DMARC record validation and parsing,
|
||||
check out the sister project,
|
||||
`checkdmarc <https://domainaware.github.io/checkdmarc/>`_.
|
||||
|
||||
Sample aggregate report output
|
||||
==============================
|
||||
|
||||
|
||||
@@ -12,6 +12,8 @@ elasticsearch>=6.3.0,<7.0.0
|
||||
elasticsearch-dsl>=6.2.1,<7.0.0
|
||||
kafka-python
|
||||
flake8
|
||||
pytest
|
||||
pytest-cov
|
||||
sphinx==1.7.9
|
||||
sphinx_rtd_theme
|
||||
collective.checkdocs
|
||||
|
||||
File renamed without changes.
File renamed without changes.
BIN
Binary file not shown.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
-34
@@ -1,34 +0,0 @@
|
||||
<?xml version="1.0"?>
|
||||
<feedback>
|
||||
<report_metadata>
|
||||
<report_id>2940</report_id>
|
||||
<org_name>XYZ Corporation</org_name>
|
||||
<email>admin@estadocuenta1.infonacot.gob.mx</email>
|
||||
<extra_contact_info>http://estadocuenta1.infonacot.gob.mx</extra_contact_info>
|
||||
<date_range>
|
||||
<begin>1536853302</begin>
|
||||
<end>1536939702</end>
|
||||
</date_range>
|
||||
</report_metadata>
|
||||
<policy_published>
|
||||
<domain>example.com</domain>
|
||||
<p>none</p>
|
||||
</policy_published>
|
||||
<record>
|
||||
<row>
|
||||
<source_ip>148.243.137.254</source_ip>
|
||||
<count>1</count>
|
||||
<policy_evaluated>
|
||||
<disposition>none</disposition>
|
||||
<dkim>fail</dkim>
|
||||
<spf>fail</spf>
|
||||
</policy_evaluated>
|
||||
</row>
|
||||
<identifiers>
|
||||
<envelope_to>estadocuenta1.infonacot.gob.mx</envelope_to>
|
||||
<header_from>example.com</header_from>
|
||||
</identifiers>
|
||||
<auth_results>
|
||||
</auth_results>
|
||||
</record>
|
||||
</feedback>
|
||||
@@ -0,0 +1,95 @@
|
||||
MIME-Version: 1.0
|
||||
Received: from mailrelay.de ([234.234.234.234])
|
||||
by Servername.domain.local (IBM Domino Release 9.0.1FP10 HF197)
|
||||
with ESMTP id 2018100111202767-369529 ;
|
||||
Mon, 1 Oct 2018 11:20:27 +0200
|
||||
Return-Path: <dmarc-report@domain.de>
|
||||
Received: from [127.0.0.1] ([local])
|
||||
by mailrelay.de (envelope-from <dmarc-report@domain.de>)
|
||||
(ecelerity 4.2.39.63080 r(Core:4.2.39.2)) with UNKNOWN
|
||||
id 48/E7-30937-BD6E1BB5; Mon, 01 Oct 2018 11:20:27 +0200
|
||||
Subject: DMARC Failure Report for domain.de (mail-from=sharepoint@domain.de, ip=10.10.10.10)
|
||||
To: dmarc-report@domain.de
|
||||
From: dmarc-report@domain.de
|
||||
X-MIMETrack: Itemize by SMTP Server on Servername/DOMAIN(Release 9.0.1FP10 HF197|April
|
||||
16, 2018) at 01.10.2018 11:20:27,
|
||||
Serialize by Notes Client on Peter Pan/DOMAIN(Release 9.0.1|October
|
||||
14, 2013) at 05.10.2018 23:38:45
|
||||
X-Notes-Item: Memo;
|
||||
name=Form
|
||||
Date: Mon, 1 Oct 2018 11:20:27 +0200
|
||||
X-Notes-Item: CN=Servername/O=DOMAIN;
|
||||
type=501; flags=44; name=$UpdatedBy
|
||||
Message-ID: <OF587285BA.CB01D107-ONC1258319.00334FCF@LocalDomain>
|
||||
X-Notes-Item: CN=Servername/O=DOMAIN,
|
||||
CN=DE9899SL4/O=DOMAIN;
|
||||
type=501; flags=0; name=RouteServers
|
||||
X-Notes-Item: =?UTF-8?B?MDEtT2N0LTIwMTggMTE6MjA6MjcgQ0VEVC8wMS1PY3QtMjAxOCAxMToyMDo=?=
|
||||
=?UTF-8?B?MjcgQ0VEVCwgMDEtT2N0LTIwMTggMTE6MjA6MjcgQ0VEVC8wMS1PY3Qt?=
|
||||
=?UTF-8?B?MjAxOCAxMToyMDoyNyBDRURU?=;
|
||||
type=401; flags=0; name=RouteTimes
|
||||
X-Notes-Item: 587285BA:CB01D107-C1258319:00334FCF;
|
||||
type=4; name=$Orig
|
||||
X-Notes-Item: ;
|
||||
type=501; name=Categories
|
||||
X-Notes-Item: ;
|
||||
type=401; name=$Revisions
|
||||
X-Notes-Item: Mon, 1 Oct 2018 11:20:27 +0200;
|
||||
type=400; name=DeliveredDate
|
||||
X-Notes-Item: =?UTF-8?B?VGhpcyBpcyBhbiBlbWFpbCBhYnVzZSByZXBvcnQgZm9yIGFuIGVtYWls?=
|
||||
=?UTF-8?B?IG1lc3NhZ2UgcmVjZWl2ZWQgZnJvbSBJUCAxMC4yLjMwLjEwMiBvbiBNb24=?=
|
||||
=?UTF-8?B?LCAwMSBPY3QgMjAxOA==?=;
|
||||
flags=6; name=$Abstract
|
||||
X-Notes-Item: 587285BA:CB01D107-C1258319:00334FCF;
|
||||
type=4; name=$TUA
|
||||
X-Notes-Item: 1;
|
||||
name=$NoteHasNativeMIME
|
||||
Content-Type: multipart/report; report-type=feedback-report;
|
||||
boundary="_----jqB1YyrX3TKBNru++5PX3w===_48/E7-30937-BD6E1BB5"
|
||||
|
||||
--_----jqB1YyrX3TKBNru++5PX3w===_48/E7-30937-BD6E1BB5
|
||||
Content-Transfer-Encoding: 7bit
|
||||
Content-Type: text/plain; charset="US-ASCII"
|
||||
|
||||
This is an email abuse report for an email message received from IP 10.10.10.10 on Mon, 01 Oct 2018 11:20:27 +0200.
|
||||
The message below did not meet the sending domain's DMARC policy.
|
||||
For more information about this format please see http://tools.ietf.org/html/rfc6591 .
|
||||
|
||||
--_----jqB1YyrX3TKBNru++5PX3w===_48/E7-30937-BD6E1BB5
|
||||
Content-Type: message/feedback-report; name=report
|
||||
|
||||
Feedback-Type: auth-failure
|
||||
User-Agent: Lua/1.0
|
||||
Version: 1.0
|
||||
Original-Mail-From: sharepoint@domain.de
|
||||
Original-Rcpt-To: peter.pan@domain.de
|
||||
Arrival-Date: Mon, 01 Oct 2018 11:20:27 +0200
|
||||
Message-ID: <38.E7.30937.BD6E1BB5@ mailrelay.de>
|
||||
Authentication-Results: dmarc=fail (p=none, dis=none) header.from=domain.de
|
||||
Source-IP: 10.10.10.10
|
||||
Delivery-Result: smg-policy-action
|
||||
Auth-Failure: dmarc
|
||||
Reported-Domain: domain.de
|
||||
|
||||
--_----jqB1YyrX3TKBNru++5PX3w===_48/E7-30937-BD6E1BB5
|
||||
Content-Type: message/rfc822
|
||||
Content-Disposition: inline
|
||||
|
||||
Received: from Servernameone.domain.local (Servernameone.domain.local [10.10.10.10])
|
||||
by mailrelay.de (mail.DOMAIN.de) with SMTP id 38.E7.30937.BD6E1BB5; Mon, 1 Oct 2018 11:20:27 +0200 (CEST)
|
||||
Date: 01 Oct 2018 11:20:27 +0200
|
||||
Message-ID: <38.E7.30937.BD6E1BB5@ mailrelay.de>
|
||||
To: <peter.pan@domain.de>
|
||||
from: "=?utf-8?B?SW50ZXJha3RpdmUgV2V0dGJld2VyYmVyLcOcYmVyc2ljaHQ=?=" <sharepoint@domain.de>
|
||||
Subject: Subject
|
||||
MIME-Version: 1.0
|
||||
X-Mailer: Microsoft SharePoint Foundation 2010
|
||||
Content-Type: text/html; charset=utf-8
|
||||
Content-Transfer-Encoding: quoted-printable
|
||||
|
||||
<html><head><base href=3D'
|
||||
wettbewerb' /></head><body><!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2//EN"=
|
||||
><HTML><HEAD><META NAME=3D"Generator" CONTENT=3D"MS Exchange Server version=
|
||||
08.01.0240.003"></html>
|
||||
|
||||
--_----jqB1YyrX3TKBNru++5PX3w===_48/E7-30937-BD6E1BB5--
|
||||
@@ -8,15 +8,28 @@ import parsedmarc
|
||||
|
||||
|
||||
class Test(unittest.TestCase):
|
||||
def testSamples(self):
|
||||
"""Test sample aggregate DMARC reports"""
|
||||
sample_paths = glob("samples/*.sample")
|
||||
for sample_path in sample_paths:
|
||||
print("Testing {0}...\n".format(sample_path))
|
||||
parsed_report = parsedmarc.parse_aggregate_report_file(sample_path)
|
||||
print(json.dumps(parsed_report, ensure_ascii=False, indent=2))
|
||||
print("\n")
|
||||
print(parsedmarc.parsed_aggregate_reports_to_csv(parsed_report))
|
||||
class Test(unittest.TestCase):
|
||||
def testAggregateSamples(self):
|
||||
"""Test sample aggregate.rua DMARC reports"""
|
||||
sample_paths = glob("samples/aggregate/*")
|
||||
for sample_path in sample_paths:
|
||||
print("Testing {0}...\n".format(sample_path))
|
||||
parsed_report = parsedmarc.parse_aggregate_report_file(
|
||||
sample_path)
|
||||
print(json.dumps(parsed_report, ensure_ascii=False, indent=2))
|
||||
print("\n")
|
||||
print(
|
||||
parsedmarc.parsed_aggregate_reports_to_csv(parsed_report))
|
||||
|
||||
def testForensicSamples(self):
|
||||
"""Test sample forensic/ruf/failure DMARC reports"""
|
||||
sample_paths = glob("samples/forensic/*.eml")
|
||||
for sample_path in sample_paths:
|
||||
print("Testing {0}...\n".format(sample_path))
|
||||
parsed_report = parsedmarc.parse_report_email(sample_path)
|
||||
print(json.dumps(parsed_report, ensure_ascii=False, indent=2))
|
||||
print("\n")
|
||||
print(parsedmarc.parsed_forensic_reports_to_csv(parsed_report))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in new issue
Block a user