From 906b9c7d70df0019167ec0e93a067f117ee98488 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Fri, 12 Oct 2018 12:37:10 -0400 Subject: [PATCH] 4.3.0 --- _modules/index.html | 8 +- _modules/parsedmarc.html | 541 ++++--------------------- _modules/parsedmarc/elastic.html | 21 +- _modules/parsedmarc/splunk.html | 355 +++++++++++++++++ _modules/parsedmarc/utils.html | 662 +++++++++++++++++++++++++++++++ _sources/index.rst.txt | 186 +++++++-- _static/documentation_options.js | 2 +- genindex.html | 66 ++- index.html | 558 ++++++++++++++++++++++---- objects.inv | Bin 561 -> 723 bytes py-modindex.html | 16 +- search.html | 6 +- searchindex.js | 2 +- 13 files changed, 1840 insertions(+), 583 deletions(-) create mode 100644 _modules/parsedmarc/splunk.html create mode 100644 _modules/parsedmarc/utils.html diff --git a/_modules/index.html b/_modules/index.html index 9cdae99..4efc963 100644 --- a/_modules/index.html +++ b/_modules/index.html @@ -8,7 +8,7 @@ - Overview: module code — parsedmarc 4.2.0k documentation + Overview: module code — parsedmarc 4.3.0 documentation @@ -56,7 +56,7 @@
- 4.2.0k + 4.3.0
@@ -143,6 +143,8 @@

All modules for which code is available

@@ -179,7 +181,7 @@ + + + + + + +
+ + + + +
+ + + + + +
+ +
+ + + + + + + + + + + + + + + + + +
+ + + + +
+
+
+
+ +

Source code for parsedmarc.splunk

+import logging
+from urllib.parse import urlparse
+import socket
+import json
+import urllib3
+
+import requests
+
+from parsedmarc.__version__ import __version__
+from parsedmarc.utils import human_timestamp_to_timestamp
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+logger = logging.getLogger("parsedmarc")
+
+
+
[docs]class SplunkError(RuntimeError):
+    """Raised when a Splunk API error occurs"""

+
+
+
[docs]class HECClient(object):
+    """A client for a Splunk HTTP Events Collector (HEC)"""
+
+    # http://docs.splunk.com/Documentation/Splunk/latest/Data/AboutHEC
+    # http://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTinput#services.2Fcollector
+
+    def __init__(self, url, access_token, index,
+                 source="parsedmarc", verify=True):
+        """
+        Initializes the HECClient
+        Args:
+            url (str): The URL of the HEC
+            access_token (str): The HEC access token
+            index (str): The name of the index
+            source (str): The source name
+            verify (bool): Verify SSL certificates
+        """
+        url = urlparse(url)
+        self.url = "{0}://{1}/services/collector/event/1.0".format(url.scheme,
+                                                                   url.netloc)
+        self.access_token = access_token.lstrip("Splunk ")
+        self.index = index
+        self.host = socket.getfqdn()
+        self.source = source
+        self.session = requests.Session()
+        self.session.verify = verify
+        self._common_data = dict(host=self.host, source=self.source,
+                                 index=self.index)
+
+        self.session.headers = {
+            "User-Agent": "parsedmarc/{0}".format(__version__),
+            "Authorization": "Splunk {0}".format(self.access_token)
+        }
+
+
[docs]    def save_aggregate_reports_to_splunk(self, aggregate_reports):
+        """
+        Saves aggregate DMARC reports to Splunk
+
+        Args:
+            aggregate_reports: A list of aggregate report dictionaries
+            to save in Splunk
+
+        """
+        logger.debug("Saving aggregate reports to Splunk")
+        if type(aggregate_reports) == dict:
+            aggregate_reports = [aggregate_reports]
+
+        if len(aggregate_reports) < 1:
+            return
+
+        data = self._common_data.copy()
+        json_str = ""
+        for report in aggregate_reports:
+            for record in report["records"]:
+                new_report = dict()
+                for metadata in report["report_metadata"]:
+                    new_report[metadata] = report["report_metadata"][metadata]
+                new_report["published_policy"] = report["policy_published"]
+                new_report["source_ip_address"] = record["source"][
+                    "ip_address"]
+                new_report["source_country"] = record["source"]["country"]
+                new_report["source_reverse_dns"] = record["source"][
+                    "reverse_dns"]
+                new_report["source_base_domain"] = record["source"][
+                    "base_domain"]
+                new_report["message_count"] = record["count"]
+                new_report["disposition"] = record["policy_evaluated"][
+                    "disposition"
+                ]
+                new_report["spf_aligned"] = record["alignment"]["spf"]
+                new_report["dkim_aligned"] = record["alignment"]["dkim"]
+                new_report["passed_dmarc"] = record["alignment"]["dmarc"]
+                new_report["header_from"] = record["identifiers"][
+                    "header_from"]
+                new_report["envelope_from"] = record["identifiers"][
+                    "envelope_from"]
+                if "dkim" in record["auth_results"]:
+                    new_report["dkim_results"] = record["auth_results"][
+                        "dkim"]
+                if "spf" in record["auth_results"]:
+                    new_report["spf_results"] = record["auth_results"][
+                        "spf"]
+
+                data["sourcetype"] = "dmarc:aggregate"
+                timestamp = human_timestamp_to_timestamp(
+                    new_report["begin_date"])
+                data["time"] = timestamp
+                data["event"] = new_report.copy()
+                json_str += "{0}\n".format(json.dumps(data))
+        try:
+            response = self.session.post(self.url, data=json_str).json()
+        except Exception as e:
+            raise SplunkError(e.__str__())
+        if response["code"] != 0:
+            raise SplunkError(response["text"])

+
+
[docs]    def save_forensic_reports_to_splunk(self, forensic_reports):
+        """
+        Saves forensic DMARC reports to Splunk
+
+        Args:
+            forensic_reports (list):  A list of forensic report dictionaries
+            to save in Splunk
+
+        """
+        logger.debug("Saving forensic reports to Splunk")
+        if type(forensic_reports) == dict:
+            forensic_reports = [forensic_reports]
+
+        if len(forensic_reports) < 1:
+            return
+
+        json_str = ""
+        for report in forensic_reports:
+            data = self._common_data.copy()
+            data["sourcetype"] = "dmarc:forensic"
+            timestamp = human_timestamp_to_timestamp(
+                report["arrival_date_utc"])
+            data["time"] = timestamp
+            data["event"] = report.copy()
+            json_str += "{0}\n".format(json.dumps(data))
+        try:
+            response = self.session.post(self.url, data=json_str).json()
+        except Exception as e:
+            raise SplunkError(e.__str__())
+        if response["code"] != 0:
+            raise SplunkError(response["text"])

+
+ +
+ +
+ + +
+
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/_modules/parsedmarc/utils.html b/_modules/parsedmarc/utils.html new file mode 100644 index 0000000..aaac1b8 --- /dev/null +++ b/_modules/parsedmarc/utils.html @@ -0,0 +1,662 @@ + + + + + + + + + + + parsedmarc.utils — parsedmarc 4.3.0 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + +
+ + + + + +
+ +
+ + + + + + + + + + + + + + + + + +
+ + + + +
+
+
+
+ +

Source code for parsedmarc.utils

+"""Utility functions that might be useful for other projects"""
+
+import logging
+import os
+from datetime import datetime
+from datetime import timedelta
+from collections import OrderedDict
+from io import BytesIO
+import tarfile
+import tempfile
+import subprocess
+import shutil
+import mailparser
+import json
+
+import dateparser
+import dns.reversename
+import dns.resolver
+import dns.exception
+import geoip2.database
+import geoip2.errors
+import requests
+import publicsuffix
+
+from parsedmarc.__version__ import USER_AGENT
+
+
+logger = logging.getLogger("parsedmarc")
+
+
+
[docs]class EmailParserError(RuntimeError):
+    """Raised when an error parsing the email occurs"""

+
+
+
[docs]def get_base_domain(domain):
+    """
+    Gets the base domain name for the given domain
+
+    .. note::
+        Results are based on a list of public domain suffixes at
+        https://publicsuffix.org/list/public_suffix_list.dat.
+
+        This file is saved to the current working directory,
+        where it is used as a cache file for 24 hours.
+
+    Args:
+        domain (str): A domain or subdomain
+
+    Returns:
+        str: The base domain of the given domain
+
+    """
+    psl_path = ".public_suffix_list.dat"
+
+    def download_psl():
+        url = "https://publicsuffix.org/list/public_suffix_list.dat"
+        # Use a browser-like user agent string to bypass some proxy blocks
+        headers = {"User-Agent": USER_AGENT}
+        fresh_psl = requests.get(url, headers=headers).text
+        with open(psl_path, "w", encoding="utf-8") as fresh_psl_file:
+            fresh_psl_file.write(fresh_psl)
+
+    if not os.path.exists(psl_path):
+        download_psl()
+    else:
+        psl_age = datetime.now() - datetime.fromtimestamp(
+            os.stat(psl_path).st_mtime)
+        if psl_age > timedelta(hours=24):
+            try:
+                download_psl()
+            except Exception as error:
+                logger.warning(
+                    "Failed to download an updated PSL {0}".format(error))
+    with open(psl_path, encoding="utf-8") as psl_file:
+        psl = publicsuffix.PublicSuffixList(psl_file)
+
+    return psl.get_public_suffix(domain)

+
+
+
[docs]def query_dns(domain, record_type, nameservers=None, timeout=2.0):
+    """
+    Queries DNS
+
+    Args:
+        domain (str): The domain or subdomain to query about
+        record_type (str): The record type to query for
+        nameservers (list): A list of one or more nameservers to use
+        (Cloudflare's public DNS resolvers by default)
+        timeout (float): Sets the DNS timeout in seconds
+
+    Returns:
+        list: A list of answers
+    """
+    resolver = dns.resolver.Resolver()
+    timeout = float(timeout)
+    if nameservers is None:
+        nameservers = ["1.1.1.1", "1.0.0.1",
+                       "2606:4700:4700::1111", "2606:4700:4700::1001",
+                       ]
+    resolver.nameservers = nameservers
+    resolver.timeout = timeout
+    resolver.lifetime = timeout
+    if record_type == "TXT":
+        resource_records = list(map(
+            lambda r: r.strings,
+            resolver.query(domain, record_type, tcp=True)))
+        _resource_record = [
+            resource_record[0][:0].join(resource_record)
+            for resource_record in resource_records if resource_record]
+        return [r.decode() for r in _resource_record]
+    else:
+        return list(map(
+            lambda r: r.to_text().replace('"', '').rstrip("."),
+            resolver.query(domain, record_type, tcp=True)))

+
+
+
[docs]def get_reverse_dns(ip_address, nameservers=None, timeout=2.0):
+    """
+    Resolves an IP address to a hostname using a reverse DNS query
+
+    Args:
+        ip_address (str): The IP address to resolve
+        nameservers (list): A list of one or more nameservers to use
+        (Cloudflare's public DNS resolvers by default)
+        timeout (float): Sets the DNS query timeout in seconds
+
+    Returns:
+        str: The reverse DNS hostname (if any)
+    """
+    hostname = None
+    try:
+        address = dns.reversename.from_address(ip_address)
+        hostname = query_dns(address, "PTR",
+                             nameservers=nameservers,
+                             timeout=timeout)[0]
+
+    except dns.exception.DNSException:
+        pass
+
+    return hostname

+
+
+
[docs]def timestamp_to_datetime(timestamp):
+    """
+    Converts a UNIX/DMARC timestamp to a Python ``DateTime`` object
+
+    Args:
+        timestamp (int): The timestamp
+
+    Returns:
+        DateTime: The converted timestamp as a Python ``DateTime`` object
+    """
+    return datetime.fromtimestamp(int(timestamp))

+
+
+
[docs]def timestamp_to_human(timestamp):
+    """
+    Converts a UNIX/DMARC timestamp to a human-readable string
+
+    Args:
+        timestamp: The timestamp
+
+    Returns:
+        str: The converted timestamp in ``YYYY-MM-DD HH:MM:SS`` format
+    """
+    return timestamp_to_datetime(timestamp).strftime("%Y-%m-%d %H:%M:%S")

+
+
+
[docs]def human_timestamp_to_datetime(human_timestamp, to_utc=False):
+    """
+    Converts a human-readable timestamp into a Python ``DateTime`` object
+
+    Args:
+        human_timestamp (str): A timestamp string
+        to_utc (bool): Convert the timestamp to UTC
+
+    Returns:
+        DateTime: The converted timestamp
+    """
+
+    settings = {}
+
+    if to_utc:
+        settings = {"TO_TIMEZONE": "UTC"}
+
+    return dateparser.parse(human_timestamp, settings=settings)

+
+
+
[docs]def human_timestamp_to_timestamp(human_timestamp):
+    """
+    Converts a human-readable timestamp into a into a UNIX timestamp
+
+    Args:
+        human_timestamp (str): A timestamp in `YYYY-MM-DD HH:MM:SS`` format
+
+    Returns:
+        float: The converted timestamp
+    """
+    human_timestamp = human_timestamp.replace("T", " ")
+    return human_timestamp_to_datetime(human_timestamp).timestamp()

+
+
+
[docs]def get_ip_address_country(ip_address):
+    """
+    Uses the MaxMind Geolite2 Country database to return the ISO code for the
+    country associated with the given IPv4 or IPv6 address
+
+    Args:
+        ip_address (str): The IP address to query for
+
+    Returns:
+        str: And ISO country code associated with the given IP address
+    """
+    db_filename = ".GeoLite2-Country.mmdb"
+
+    def download_country_database(location=".GeoLite2-Country.mmdb"):
+        """Downloads the MaxMind Geolite2 Country database
+
+        Args:
+            location (str): Local location for the database file
+        """
+        url = "https://geolite.maxmind.com/download/geoip/database/" \
+              "GeoLite2-Country.tar.gz"
+        # Use a browser-like user agent string to bypass some proxy blocks
+        headers = {"User-Agent": USER_AGENT}
+        original_filename = "GeoLite2-Country.mmdb"
+        tar_bytes = requests.get(url, headers=headers).content
+        tar_file = tarfile.open(fileobj=BytesIO(tar_bytes), mode="r:gz")
+        tar_dir = tar_file.getnames()[0]
+        tar_path = "{0}/{1}".format(tar_dir, original_filename)
+        tar_file.extract(tar_path)
+        shutil.move(tar_path, location)
+        shutil.rmtree(tar_dir)
+
+    system_paths = ["/usr/local/share/GeoIP/GeoLite2-Country.mmdb",
+                    "/usr/share/GeoIP/GeoLite2-Country.mmdb"]
+    db_path = ""
+
+    for system_path in system_paths:
+        if os.path.exists(system_path):
+            db_path = system_path
+            break
+
+    if db_path == "":
+        if not os.path.exists(db_filename):
+            download_country_database(db_filename)
+        else:
+            db_age = datetime.now() - datetime.fromtimestamp(
+                os.stat(db_filename).st_mtime)
+            if db_age > timedelta(days=60):
+                download_country_database()
+        db_path = db_filename
+
+    db_reader = geoip2.database.Reader(db_path)
+
+    country = None
+
+    try:
+        country = db_reader.country(ip_address).country.iso_code
+    except geoip2.errors.AddressNotFoundError:
+        pass
+
+    return country

+
+
+
[docs]def get_ip_address_info(ip_address, nameservers=None, timeout=2.0):
+    """
+    Returns reverse DNS and country information for the given IP address
+
+    Args:
+        ip_address (str): The IP address to check
+        nameservers (list): A list of one or more nameservers to use
+        (Cloudflare's public DNS resolvers by default)
+        timeout (float): Sets the DNS timeout in seconds
+
+    Returns:
+        OrderedDict: ``ip_address``, ``reverse_dns``
+
+    """
+    ip_address = ip_address.lower()
+    info = OrderedDict()
+    info["ip_address"] = ip_address
+    reverse_dns = get_reverse_dns(ip_address,
+                                  nameservers=nameservers,
+                                  timeout=timeout)
+    country = get_ip_address_country(ip_address)
+    info["country"] = country
+    info["reverse_dns"] = reverse_dns
+    info["base_domain"] = None
+    if reverse_dns is not None:
+        base_domain = get_base_domain(reverse_dns)
+        info["base_domain"] = base_domain
+
+    return info

+
+
+def parse_email_address(original_address):
+    if original_address[0] == "":
+        display_name = None
+    else:
+        display_name = original_address[0]
+    address = original_address[1]
+    address_parts = address.split("@")
+    local = None
+    domain = None
+    if len(address_parts) > 1:
+        local = address_parts[0].lower()
+        domain = address_parts[-1].lower()
+
+    return OrderedDict([("display_name", display_name),
+                        ("address", address),
+                        ("local", local),
+                        ("domain", domain)])
+
+
+
[docs]def get_filename_safe_string(string):
+    """
+    Converts a string to a string that is safe for a filename
+    Args:
+        string (str): A string to make safe for a filename
+
+    Returns:
+        str: A string safe for a filename
+    """
+    invalid_filename_chars = ['\\', '/', ':', '"', '*', '?', '|', '\n',
+                              '\r']
+    if string is None:
+        string = "None"
+    for char in invalid_filename_chars:
+        string = string.replace(char, "")
+    string = string.rstrip(".")
+
+    return string

+
+
+
[docs]def is_outlook_msg(content):
+    """
+    Checks if the given content is a Outlook msg OLE file
+
+    Args:
+        content: Content to check
+
+    Returns:
+        bool: A flag the indicates if a file is a Outlook MSG file
+    """
+    return type(content) == bytes and content.startswith(
+        b"\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1")

+
+
+
[docs]def convert_outlook_msg(msg_bytes):
+    """
+    Uses the ``msgconvert`` Perl utility to convert an Outlook MS file to
+    standard RFC 822 format
+
+    Args:
+        msg_bytes (bytes): the content of the .msg file
+
+    Returns:
+        A RFC 822 string
+    """
+    if not is_outlook_msg(msg_bytes):
+        raise ValueError("The supplied bytes are not an Outlook MSG file")
+    orig_dir = os.getcwd()
+    tmp_dir = tempfile.mkdtemp()
+    os.chdir(tmp_dir)
+    with open("sample.msg", "wb") as msg_file:
+        msg_file.write(msg_bytes)
+    try:
+        subprocess.check_call(["msgconvert", "sample.msg"])
+        eml_path = "sample.eml"
+        with open(eml_path, "rb") as eml_file:
+            rfc822 = eml_file.read()
+    except FileNotFoundError:
+        raise EmailParserError(
+            "Failed to convert Outlook MSG: msgconvert utility not found")
+    finally:
+        os.chdir(orig_dir)
+        shutil.rmtree(tmp_dir)
+
+    return rfc822

+
+
+
[docs]def parse_email(data):
+    """
+    A simplified email parser
+
+    Args:
+        data: The RFC 822 message string, or MSG binary
+
+    Returns (dict): Parsed email data
+    """
+
+    if type(data) == bytes:
+        if is_outlook_msg(data):
+            data = convert_outlook_msg(data)
+        data = data.decode("utf-8", errors="replace")
+    parsed_email = mailparser.parse_from_string(data)
+    headers = json.loads(parsed_email.headers_json).copy()
+    parsed_email = json.loads(parsed_email.mail_json).copy()
+    parsed_email["headers"] = headers
+    if "received" in parsed_email:
+        for received in parsed_email["received"]:
+            if "date_utc" in received:
+                received["date_utc"] = received["date_utc"].replace("T",
+                                                                    " ")
+    parsed_email["from"] = parse_email_address(parsed_email["from"][0])
+
+    if "date" in parsed_email:
+        parsed_email["date"] = parsed_email["date"].replace("T", " ")
+    else:
+        parsed_email["date"] = None
+    if "reply_to" in parsed_email:
+        parsed_email["reply_to"] = list(map(lambda x: parse_email_address(x),
+                                            parsed_email["reply_to"]))
+    else:
+        parsed_email["reply_to"] = []
+
+    if "to" in parsed_email:
+        parsed_email["to"] = list(map(lambda x: parse_email_address(x),
+                                      parsed_email["to"]))
+    else:
+        parsed_email["to"] = []
+
+    if "cc" in parsed_email:
+        parsed_email["cc"] = list(map(lambda x: parse_email_address(x),
+                                      parsed_email["cc"]))
+    else:
+        parsed_email["cc"] = []
+
+    if "bcc" in parsed_email:
+        parsed_email["bcc"] = list(map(lambda x: parse_email_address(x),
+                                       parsed_email["bcc"]))
+    else:
+        parsed_email["bcc"] = []
+
+    if "delivered_to" in parsed_email:
+        parsed_email["delivered_to"] = list(
+            map(lambda x: parse_email_address(x),
+                parsed_email["delivered_to"])
+        )
+
+    if "attachments" not in parsed_email:
+        parsed_email["attachments"] = []
+
+    if "subject" not in parsed_email:
+        parsed_email["subject"] = None
+
+    parsed_email["filename_safe_subject"] = get_filename_safe_string(
+        parsed_email["subject"])
+
+    if "body" not in parsed_email:
+        parsed_email["body"] = None
+
+    return parsed_email

+
+ +
+ +
+ + +
+
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/_sources/index.rst.txt b/_sources/index.rst.txt index 18b24a0..e2277f6 100644 --- a/_sources/index.rst.txt +++ b/_sources/index.rst.txt @@ -37,8 +37,24 @@ Features Resources ========= +DMARC guides +------------ + * `Demystifying DMARC`_ - A complete guide to SPF, DKIM, and DMARC +SPF and DMARC record validation +------------------------------- + +If you are looking for SPF and DMARC record validation and parsing, +check out the sister project, +`checkdmarc `_. + +Lookalike domains +----------------- + +DMARC protects against domain spoofing, not lookalike domains. for open source +lookalike domain monitoring, check out `DomainAware `_. + CLI help ======== @@ -53,9 +69,13 @@ CLI help [--elasticsearch-index-prefix ELASTICSEARCH_INDEX_PREFIX] [--elasticsearch-index-suffix ELASTICSEARCH_INDEX_SUFFIX] [--hec HEC] [--hec-token HEC_TOKEN] [--hec-index HEC_INDEX] - [--hec-skip-certificate-verification] [--save-aggregate] - [--save-forensic] [-O OUTGOING_HOST] [-U OUTGOING_USER] - [-P OUTGOING_PASSWORD] [--outgoing-port OUTGOING_PORT] + [--hec-skip-certificate-verification] + [-K [KAFKA_HOSTS [KAFKA_HOSTS ...]]] + [--kafka-aggregate-topic KAFKA_AGGREGATE_TOPIC] + [--kafka-forensic_topic KAFKA_FORENSIC_TOPIC] + [--save-aggregate] [--save-forensic] [-O OUTGOING_HOST] + [-U OUTGOING_USER] [-P OUTGOING_PASSWORD] + [--outgoing-port OUTGOING_PORT] [--outgoing-ssl OUTGOING_SSL] [-F OUTGOING_FROM] [-T OUTGOING_TO [OUTGOING_TO ...]] [-S OUTGOING_SUBJECT] [-A OUTGOING_ATTACHMENT] [-M OUTGOING_MESSAGE] [-w] [--test] @@ -73,10 +93,11 @@ CLI help -o OUTPUT, --output OUTPUT Write output files to the given directory -n NAMESERVERS [NAMESERVERS ...], --nameservers NAMESERVERS [NAMESERVERS ...] - nameservers to query (Default is Cloudflare's) + nameservers to query (Default is Cloudflare's + nameservers) -t TIMEOUT, --timeout TIMEOUT number of seconds to wait for an answer from DNS - (default 2.0) + (Default: 2.0) -H HOST, --host HOST IMAP hostname or IP address -u USER, --user USER IMAP user -p PASSWORD, --password PASSWORD @@ -85,14 +106,15 @@ CLI help IMAP port --imap-no-ssl Do not use SSL/TLS when connecting to IMAP -r REPORTS_FOLDER, --reports-folder REPORTS_FOLDER - The IMAP folder containing the reports Default: INBOX + The IMAP folder containing the reports (Default: + INBOX) -a ARCHIVE_FOLDER, --archive-folder ARCHIVE_FOLDER Specifies the IMAP folder to move messages to after - processing them Default: Archive + processing them (Default: Archive) -d, --delete Delete the reports after processing them -E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]], --elasticsearch-host [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]] - A list of one or more Elasticsearch hostnames or URLs - to use (e.g. localhost:9200) + One or more Elasticsearch hostnames or URLs to use + (e.g. localhost:9200) --elasticsearch-index-prefix ELASTICSEARCH_INDEX_PREFIX Prefix to add in front of the dmarc_aggregate and dmarc_forensic Elasticsearch index names, joined by _ @@ -108,6 +130,14 @@ CLI help HTTP Event Collector (HEC) --hec-skip-certificate-verification Skip certificate verification for Splunk HEC + -K [KAFKA_HOSTS [KAFKA_HOSTS ...]], --kafka-hosts [KAFKA_HOSTS [KAFKA_HOSTS ...]] + A list of one or more Kafka hostnames or URLs + --kafka-aggregate-topic KAFKA_AGGREGATE_TOPIC + The Kafka topic to publish aggregate reports to + (Default: dmarc_aggregate) + --kafka-forensic_topic KAFKA_FORENSIC_TOPIC + The Kafka topic to publish forensic reports to + (Default: dmarc_forensic) --save-aggregate Save aggregate reports to search indexes --save-forensic Save forensic reports to search indexes -O OUTGOING_HOST, --outgoing-host OUTGOING_HOST @@ -138,20 +168,6 @@ CLI help --debug Print debugging information -v, --version show program's version number and exit -SPF and DMARC record validation -=============================== - -If you are looking for SPF and DMARC record validation and parsing, -check out the sister project, -`checkdmarc `_. - -SPF and DMARC record validation -=============================== - -If you are looking for SPF and DMARC record validation and parsing, -check out the sister project, -`checkdmarc `_. - Sample aggregate report output ============================== @@ -242,12 +258,114 @@ CSV xml_schema,org_name,org_email,org_extra_contact_info,report_id,begin_date,end_date,errors,domain,adkim,aspf,p,sp,pct,fo,source_ip_address,source_country,source_reverse_dns,source_base_domain,count,disposition,dkim_alignment,spf_alignment,policy_override_reasons,policy_override_comments,envelope_from,header_from,envelope_to,dkim_domains,dkim_selectors,dkim_results,spf_domains,spf_scopes,spf_results draft,acme.com,noreply-dmarc-support@acme.com,http://acme.com/dmarc/support,9391651994964116463,2012-04-27 20:00:00,2012-04-28 19:59:59,,example.com,r,r,none,none,100,0,72.150.241.94,US,adsl-72-150-241-94.shv.bellsouth.net,bellsouth.net,2,none,fail,pass,,,example.com,example.com,,example.com,none,fail,example.com,mfrom,pass - Sample forensic report output ============================= -I don't have a sample I can share for privacy reasons. If you have a sample -forensic report that you can share publicly, please contact me! +Thanks to Github user `xennn `_ for the anonymized +`forensic report email sample +`_. + +JSON +---- + + +.. code-block:: json + + { + "feedback_type": "auth-failure", + "user_agent": "Lua/1.0", + "version": "1.0", + "original_mail_from": "sharepoint@domain.de", + "original_rcpt_to": "peter.pan@domain.de", + "arrival_date": "Mon, 01 Oct 2018 11:20:27 +0200", + "message_id": "<38.E7.30937.BD6E1BB5@ mailrelay.de>", + "authentication_results": "dmarc=fail (p=none, dis=none) header.from=domain.de", + "delivery_result": "smg-policy-action", + "auth_failure": [ + "dmarc" + ], + "reported_domain": "domain.de", + "arrival_date_utc": "2018-10-01 09:20:27", + "source": { + "ip_address": "10.10.10.10", + "country": null, + "reverse_dns": null, + "base_domain": null + }, + "authentication_mechanisms": [], + "original_envelope_id": null, + "dkim_domain": null, + "sample_headers_only": false, + "sample": "Content-Type: message/rfc822\nContent-Disposition: inline\nReceived: from Servernameone.domain.local (Servernameone.domain.local [10.10.10.10])\n by mailrelay.de (mail.DOMAIN.de) with SMTP id 38.E7.30937.BD6E1BB5; Mon, 1 Oct 2018 11:20:27 +0200 (CEST)\nDate: 01 Oct 2018 11:20:27 +0200\nMessage-ID: <38.E7.30937.BD6E1BB5@ mailrelay.de>\nTo: \nfrom: \"=?utf-8?B?SW50ZXJha3RpdmUgV2V0dGJld2VyYmVyLcOcYmVyc2ljaHQ=?=\" \nSubject: Subject\nMIME-Version: 1.0\nX-Mailer: Microsoft SharePoint Foundation 2010\nContent-Type: text/html; charset=utf-8\nContent-Transfer-Encoding: quoted-printable\n\n\n\n ", + "parsed_sample": { + "content-transfer-encoding": "quoted-printable", + "x-mailer": "Microsoft SharePoint Foundation 2010", + "message-id": "<38.E7.30937.BD6E1BB5@ mailrelay.de>", + "body": "", + "to": [ + { + "display_name": null, + "address": "peter.pan@domain.de", + "local": "peter.pan", + "domain": "domain.de" + } + ], + "date": "2018-10-01 09:20:27", + "to_domains": [ + "domain.de" + ], + "received": [ + { + "from": "Servernameone.domain.local Servernameone.domain.local 10.10.10.10", + "by": "mailrelay.de mail.DOMAIN.de", + "with": "SMTP id 38.E7.30937.BD6E1BB5", + "date": "Mon, 1 Oct 2018 11:20:27 +0200 CEST", + "hop": 1, + "date_utc": "2018-10-01 09:20:27", + "delay": 0 + } + ], + "content-disposition": "inline", + "mime-version": "1.0", + "subject": "Subject", + "timezone": "+2", + "from": { + "display_name": "Interaktive Wettbewerber-Übersicht", + "address": "sharepoint@domain.de", + "local": "sharepoint", + "domain": "domain.de" + }, + "content-type": "message/rfc822", + "has_defects": false, + "headers": { + "Content-Type": "text/html; charset=utf-8", + "Content-Disposition": "inline", + "Received": "from Servernameone.domain.local (Servernameone.domain.local [10.10.10.10])\n by mailrelay.de (mail.DOMAIN.de) with SMTP id 38.E7.30937.BD6E1BB5; Mon, 1 Oct 2018 11:20:27 +0200 (CEST)", + "Date": "01 Oct 2018 11:20:27 +0200", + "Message-ID": "<38.E7.30937.BD6E1BB5@ mailrelay.de>", + "To": "", + "from": "\"Interaktive Wettbewerber-Übersicht\" ", + "Subject": "Subject", + "MIME-Version": "1.0", + "X-Mailer": "Microsoft SharePoint Foundation 2010", + "Content-Transfer-Encoding": "quoted-printable" + }, + "reply_to": [], + "cc": [], + "bcc": [], + "attachments": [], + "filename_safe_subject": "Subject" + } + } + + +CSV +--- + +:: + + feedback_type,user_agent,version,original_envelope_id,original_mail_from,original_rcpt_to,arrival_date,arrival_date_utc,subject,message_id,authentication_results,dkim_domain,source_ip_address,source_country,source_reverse_dns,source_base_domain,delivery_result,auth_failure,reported_domain,authentication_mechanisms,sample_headers_only + auth-failure,Lua/1.0,1.0,,sharepoint@domain.de,peter.pan@domain.de,"Mon, 01 Oct 2018 11:20:27 +0200",2018-10-01 09:20:27,Subject,<38.E7.30937.BD6E1BB5@ mailrelay.de>,"dmarc=fail (p=none, dis=none) header.from=domain.de",,10.10.10.10,,,,smg-policy-action,dmarc,domain.de,,False Bug reports =========== @@ -382,7 +500,7 @@ Elasticsearch and Kibana .. note:: - Splunk is also supported starting with ``parsedmarc`` 4.1.3 + Splunk is also supported starting with ``parsedmarc`` 4.3.0 To set up visual dashboards of DMARC data, install Elasticsearch and Kibana. @@ -616,11 +734,17 @@ select ``dmarc_aggregate`` for the other saved objects, as shown below. :align: center :target: _static/screenshots/index-pattern-conflicts.png +Records retention +~~~~~~~~~~~~~~~~~ + +To prevent your indexes from growing too large, or to comply with records +retention regulations such as GDPR, you need to use `time-based indexes +`_. Splunk ------ -Starting in version 4.1.3 ``parsedmarc`` supports sending aggregate and/or +Starting in version 4.3.0 ``parsedmarc`` supports sending aggregate and/or forensic DMARC data to a Splunk `HTTP Event collector (HEC)`_. Simply use the following command line options, along with ``--save-aggregate`` and/or ``--save-forensic``: @@ -902,6 +1026,12 @@ parsedmarc.elastic .. automodule:: parsedmarc.elastic :members: +.. automodule:: parsedmarc.splunk + :members: + +.. automodule:: parsedmarc.utils + :members: + .. toctree:: :maxdepth: 2 :caption: Contents: diff --git a/_static/documentation_options.js b/_static/documentation_options.js index 005c8d2..7060e4a 100644 --- a/_static/documentation_options.js +++ b/_static/documentation_options.js @@ -1,6 +1,6 @@ var DOCUMENTATION_OPTIONS = { URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'), - VERSION: '4.2.0k', + VERSION: '4.3.0', LANGUAGE: 'None', COLLAPSE_INDEX: false, FILE_SUFFIX: '.html', diff --git a/genindex.html b/genindex.html index bf0d7c4..a44614c 100644 --- a/genindex.html +++ b/genindex.html @@ -9,7 +9,7 @@ - Index — parsedmarc 4.2.0k documentation + Index — parsedmarc 4.3.0 documentation @@ -57,7 +57,7 @@
- 4.2.0k + 4.3.0
@@ -154,7 +154,9 @@ | H | I | P + | Q | S + | T | W @@ -168,6 +170,10 @@

C

+ @@ -189,13 +197,23 @@

G

@@ -203,11 +221,13 @@

H

@@ -224,6 +244,8 @@
  • InvalidDMARCReport
  • InvalidForensicReport +
    • +
  • is_outlook_msg() (in module parsedmarc.utils)
    • @@ -234,6 +256,8 @@
  • parse_aggregate_report_file() (in module parsedmarc)
  • parse_aggregate_report_xml() (in module parsedmarc) +
    • +
  • parse_email() (in module parsedmarc.utils)
  • parse_forensic_report() (in module parsedmarc)
    • @@ -250,18 +274,34 @@
  • parsedmarc (module)
  • parsedmarc.elastic (module) +
    • +
  • parsedmarc.splunk (module) +
    • +
  • parsedmarc.utils (module)
  • ParserError
    • +

    Q

    + + +
    +

    S

    +
    + +

    T

    + + +
    @@ -318,7 +372,7 @@