From 0e1e0142b1dd6523405d8ab650f5316145109364 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Tue, 27 Mar 2018 11:04:57 -0400 Subject: [PATCH] 3.3.0 --- _modules/elasticsearch_dsl/field.html | 579 +++++++++++++++++++++++++ _modules/index.html | 14 +- _modules/parsedmarc.html | 99 ++--- _modules/parsedmarc/elastic.html | 584 ++++++++++++++++++++++++++ _sources/index.rst.txt | 6 + _static/basic.css | 19 +- _static/documentation_options.js | 9 + _static/websupport.js | 2 +- genindex.html | 53 ++- index.html | 130 ++++-- objects.inv | 5 +- py-modindex.html | 20 +- search.html | 8 +- searchindex.js | 2 +- 14 files changed, 1396 insertions(+), 134 deletions(-) create mode 100644 _modules/elasticsearch_dsl/field.html create mode 100644 _modules/parsedmarc/elastic.html create mode 100644 _static/documentation_options.js diff --git a/_modules/elasticsearch_dsl/field.html b/_modules/elasticsearch_dsl/field.html new file mode 100644 index 00000000..975cbb07 --- /dev/null +++ b/_modules/elasticsearch_dsl/field.html @@ -0,0 +1,579 @@ + + + + + + + + + + + elasticsearch_dsl.field — parsedmarc 3.3.0 documentation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + +
+ + + + + + +
+
+ + + + + + + + + + + + + + + + +
+ +
    + +
  • Docs »
    • + +
  • Module code »
    • + +
  • elasticsearch_dsl.field
    • + + +
  • + + + +
    • + +
+ + +
+
+
+
+ +

Source code for elasticsearch_dsl.field

+import base64
+import ipaddress
+
+import collections
+
+from datetime import date, datetime
+
+from dateutil import parser, tz
+from six import itervalues, string_types, iteritems
+from six.moves import map
+
+from .utils import DslBase, ObjectBase, AttrDict, AttrList
+from .exceptions import ValidationException
+
+unicode = type(u'')
+
+def construct_field(name_or_field, **params):
+    # {"type": "text", "analyzer": "snowball"}
+    if isinstance(name_or_field, collections.Mapping):
+        if params:
+            raise ValueError('construct_field() cannot accept parameters when passing in a dict.')
+        params = name_or_field.copy()
+        if 'type' not in params:
+            # inner object can be implicitly defined
+            if 'properties' in params:
+                name = 'object'
+            else:
+                raise ValueError('construct_field() needs to have a "type" key.')
+        else:
+            name = params.pop('type')
+        return Field.get_dsl_class(name)(**params)
+
+    # Text()
+    if isinstance(name_or_field, Field):
+        if params:
+            raise ValueError('construct_field() cannot accept parameters when passing in a construct_field object.')
+        return name_or_field
+
+    # "text", analyzer="snowball"
+    return Field.get_dsl_class(name_or_field)(**params)
+
+class Field(DslBase):
+    _type_name = 'field'
+    _type_shortcut = staticmethod(construct_field)
+    # all fields can be multifields
+    _param_defs = {'fields': {'type': 'field', 'hash': True}}
+    name = None
+    _coerce = False
+
+    def __init__(self, *args, **kwargs):
+        self._multi = kwargs.pop('multi', False)
+        self._required = kwargs.pop('required', False)
+        super(Field, self).__init__(*args, **kwargs)
+
+    def __getitem__(self, subfield):
+        return self._params.get('fields', {})[subfield]
+
+    def _serialize(self, data):
+        return data
+
+    def _deserialize(self, data):
+        return data
+
+    def _empty(self):
+        return None
+
+    def empty(self):
+        if self._multi:
+            return AttrList([])
+        return self._empty()
+
+    def serialize(self, data):
+        if isinstance(data, (list, AttrList)):
+            return list(map(self._serialize, data))
+        return self._serialize(data)
+
+    def deserialize(self, data):
+        if isinstance(data, (list, AttrList)):
+            data[:] = [
+                None if d is None else self._deserialize(d)
+                for d in data
+            ]
+            return data
+        if data is None:
+            return None
+        return self._deserialize(data)
+
+    def clean(self, data):
+        if data is not None:
+            data = self.deserialize(data)
+        if data in (None, [], {}) and self._required:
+            raise ValidationException("Value required for this field.")
+        return data
+
+    def to_dict(self):
+        d = super(Field, self).to_dict()
+        name, value = d.popitem()
+        value['type'] = name
+        return value
+
+class CustomField(Field):
+    name = 'custom'
+    _coerce = True
+
+    def to_dict(self):
+        if isinstance(self.builtin_type, Field):
+            return self.builtin_type.to_dict()
+
+        d = super(CustomField, self).to_dict()
+        d['type'] = self.builtin_type
+        return d
+
+class Object(Field):
+    name = 'object'
+    _coerce = True
+
+    def __init__(self, doc_class=None, **kwargs):
+        self._doc_class = doc_class
+        if doc_class is None:
+            # FIXME import
+            from .document import InnerDoc
+            # no InnerDoc subclass, creating one instead...
+            self._doc_class = type('InnerDoc', (InnerDoc, ), {})
+            for name, field in iteritems(kwargs.pop('properties', {})):
+                self._doc_class._doc_type.mapping.field(name, field)
+            if 'dynamic' in kwargs:
+                self._doc_class._doc_type.mapping.meta('dynamic', kwargs.pop('dynamic'))
+
+        self._mapping = self._doc_class._doc_type.mapping
+        super(Object, self).__init__(**kwargs)
+
+    def __getitem__(self, name):
+        return self._mapping[name]
+
+    def __contains__(self, name):
+        return name in self._mapping
+
+    def _empty(self):
+        return self._wrap({})
+
+    def _wrap(self, data):
+        return self._doc_class.from_es(data)
+
+    def empty(self):
+        if self._multi:
+            return AttrList([], self._wrap)
+        return self._empty()
+
+    def to_dict(self):
+        d = self._mapping.to_dict()
+        _, d = d.popitem()
+        d["type"] = self.name
+        return d
+
+    def _collect_fields(self):
+        return self._mapping.properties._collect_fields()
+
+    def _deserialize(self, data):
+        # don't wrap already wrapped data
+        if isinstance(data, self._doc_class):
+            return data
+
+        if isinstance(data, AttrDict):
+            data = data._d_
+
+        return self._wrap(data)
+
+    def _serialize(self, data):
+        if data is None:
+            return None
+
+        # somebody assigned raw dict to the field, we should tolerate that
+        if isinstance(data, collections.Mapping):
+            return data
+
+        return data.to_dict()
+
+    def clean(self, data):
+        data = super(Object, self).clean(data)
+        if data is None:
+            return None
+        if isinstance(data, (list, AttrList)):
+            for d in data:
+                d.full_clean()
+        else:
+            data.full_clean()
+        return data
+
+    def update(self, other):
+        if not isinstance(other, Object):
+            # not an inner/nested object, no merge possible
+            return
+
+        self._mapping.update(other._mapping)
+
+class Nested(Object):
+    name = 'nested'
+
+    def __init__(self, *args, **kwargs):
+        kwargs.setdefault('multi', True)
+        super(Nested, self).__init__(*args, **kwargs)
+
+class Date(Field):
+    name = 'date'
+    _coerce = True
+
+    def __init__(self, *args, **kwargs):
+        self._default_timezone = kwargs.pop('default_timezone', None)
+        if isinstance(self._default_timezone, string_types):
+            self._default_timezone = tz.gettz(self._default_timezone)
+        super(Date, self).__init__(*args, **kwargs)
+
+    def _deserialize(self, data):
+        if isinstance(data, string_types):
+            try:
+                data = parser.parse(data)
+            except Exception as e:
+                raise ValidationException('Could not parse date from the value (%r)' % data, e)
+
+        if isinstance(data, datetime):
+            if self._default_timezone and data.tzinfo is None:
+                data = data.replace(tzinfo=self._default_timezone)
+            return data
+        if isinstance(data, date):
+            return data
+        if isinstance(data, int):
+            # Divide by a float to preserve milliseconds on the datetime.
+            return datetime.utcfromtimestamp(data / 1000.0)
+
+        raise ValidationException('Could not parse date from the value (%r)' % data)
+
+class Text(Field):
+    _param_defs = {
+        'fields': {'type': 'field', 'hash': True},
+        'analyzer': {'type': 'analyzer'},
+        'search_analyzer': {'type': 'analyzer'},
+        'search_quote_analyzer': {'type': 'analyzer'},
+    }
+    name = 'text'
+
+class Keyword(Field):
+    _param_defs = {
+        'fields': {'type': 'field', 'hash': True},
+        'search_analyzer': {'type': 'analyzer'},
+        'normalizer': {'type': 'normalizer'}
+    }
+    name = 'keyword'
+
+class Boolean(Field):
+    name = 'boolean'
+    _coerce = True
+
+    def _deserialize(self, data):
+        if data == "false":
+            return False
+        return bool(data)
+
+    def clean(self, data):
+        if data is not None:
+            data = self.deserialize(data)
+        if data is None and self._required:
+            raise ValidationException("Value required for this field.")
+        return data
+
+class Float(Field):
+    name = 'float'
+    _coerce = True
+
+    def _deserialize(self, data):
+        return float(data)
+
+class HalfFloat(Float):
+    name = 'half_float'
+
+class ScaledFloat(Float):
+    name = 'scaled_float'
+
+    def __init__(self, scaling_factor, *args, **kwargs):
+        super(ScaledFloat, self).__init__(scaling_factor=scaling_factor, *args, **kwargs)
+
+class Double(Float):
+    name = 'double'
+
+class Integer(Field):
+    name = 'integer'
+    _coerce = True
+
+    def _deserialize(self, data):
+        return int(data)
+
+class Byte(Integer):
+    name = 'byte'
+
+class Short(Integer):
+    name = 'short'
+
+class Long(Integer):
+    name = 'long'
+
+class Ip(Field):
+    name = 'ip'
+    _coerce = True
+
+    def _deserialize(self, data):
+        # the ipaddress library for pypy, python2.5 and 2.6 only accepts unicode.
+        return ipaddress.ip_address(unicode(data))
+
+    def _serialize(self, data):
+        if data is None:
+            return None
+        return str(data)
+
+class Binary(Field):
+    name = 'binary'
+    _coerce = True
+
+    def _deserialize(self, data):
+        return base64.b64decode(data)
+
+    def _serialize(self, data):
+        if data is None:
+            return None
+        return base64.b64encode(data)
+
+class GeoPoint(Field):
+    name = 'geo_point'
+
+class GeoShape(Field):
+    name = 'geo_shape'
+
+class Completion(Field):
+    name = 'completion'
+
+class Percolator(Field):
+    name = 'percolator'
+
+class IntegerRange(Field):
+    name = 'integer_range'
+
+class FloatRange(Field):
+    name = 'float_range'
+
+class LongRange(Field):
+    name = 'long_range'
+
+class DoubleRange(Field):
+    name = 'double_ranged'
+
+class DateRange(Field):
+    name = 'date_range'
+
+class Join(Field):
+    name = 'join'
+
+class TokenCount(Field):
+    name = 'token_count'
+
+class Murmur3(Field):
+    name = 'murmur3'
+
+ +
+
+ +
+
+ + +
+
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/_modules/index.html b/_modules/index.html index abc9f19c..1619cdd3 100644 --- a/_modules/index.html +++ b/_modules/index.html @@ -8,7 +8,7 @@ - Overview: module code — parsedmarc 3.2.0 documentation + Overview: module code — parsedmarc 3.3.0 documentation @@ -35,7 +35,7 @@ - + @@ -64,7 +64,7 @@
- 3.2.0 + 3.3.0
@@ -150,8 +150,10 @@

All modules for which code is available

- +
@@ -187,7 +189,7 @@ + + + + + + +
+ + + + +
+ + + + + + +
+
+ + + + + + + + + + + + + + + + +
+ + + + +
+
+
+
+ +

Source code for parsedmarc.elastic

+# -*- coding: utf-8 -*-
+
+from collections import OrderedDict
+
+import parsedmarc
+from elasticsearch_dsl.search import Q
+from elasticsearch_dsl import connections, Object, DocType, Index, Nested, \
+    InnerDoc, Integer, Text, Boolean, DateRange, Ip, Date
+
+aggregate_index = Index("dmarc_aggregate")
+forensic_index = Index("dmarc_forensic")
+
+
+class _PolicyOverride(InnerDoc):
+    type = Text()
+    comment = Text()
+
+
+class _PublishedPolicy(InnerDoc):
+    adkim = Text()
+    aspf = Text()
+    p = Text()
+    sp = Text()
+    pct = Integer()
+    fo = Integer()
+
+
+class _DKIMResult(InnerDoc):
+    domain = Text()
+    selector = Text()
+    result = Text()
+
+
+class _SPFResult(InnerDoc):
+    domain = Text()
+    scope = Text()
+    results = Text()
+
+
+class _AggregateReportDoc(DocType):
+    class Meta:
+        index = "dmarc_aggregate"
+
+    xml_schema = Text()
+    org_name = Text()
+    org_email = Text()
+    org_extra_contact_info = Text()
+    report_id = Text()
+    date_range = DateRange()
+    errors = Text()
+    domain = Text()
+    published_policy = Object(_PublishedPolicy)
+    source_ip_address = Ip()
+    source_country = Text()
+    source_reverse_dns = Text()
+    source_Base_domain = Text()
+    message_count = Integer
+    disposition = Text()
+    dkim_aligned = Boolean()
+    spf_aligned = Boolean()
+    passed_dmarc = Boolean()
+    policy_overrides = Nested(_PolicyOverride)
+    header_from = Text()
+    envelope_from = Text()
+    envelope_to = Text()
+    dkim_results = Nested(_DKIMResult)
+    spf_results = Nested(_SPFResult)
+
+    def add_policy_override(self, type_, comment):
+        self.policy_overrides.append(_PolicyOverride(type=type_,
+                                                     comment=comment))
+
+    def add_dkim_result(self, domain, selector, result):
+        self.dkim_results.append(_DKIMResult(domain=domain,
+                                             selector=selector,
+                                             result=result))
+
+    def add_spf_result(self, domain, scope, result):
+        self.spf_results.append(_SPFResult(domain=domain,
+                                           scope=scope,
+                                           result=result))
+
+    def save(self, ** kwargs):
+        self.passed_dmarc = False
+        self.passed_dmarc = self.spf_aligned or self.dkim_aligned
+
+        return super().save(** kwargs)
+
+
+class _EmailAddressDoc(InnerDoc):
+    display_name = Text()
+    address = Text()
+
+
+class _EmailAttachmentDoc(DocType):
+    filename = Text()
+    content_type = Text()
+
+
+class _ForensicSampleDoc(InnerDoc):
+    raw = Text()
+    headers = Object()
+    headers_only = Boolean()
+    to = Nested(_EmailAddressDoc)
+    subject = Text()
+    filename_safe_subject = Text()
+    _from = Object(_EmailAddressDoc)
+    date = Date()
+    reply_to = Nested(_EmailAddressDoc)
+    cc = Nested(_EmailAddressDoc)
+    bcc = Nested(_EmailAddressDoc)
+    body = Text()
+    attachments = Nested(_EmailAttachmentDoc)
+
+    def add_to(self, display_name, address):
+        self.to.append(_EmailAddressDoc(display_name=display_name,
+                                        address=address))
+
+    def add_reply_to(self, display_name, address):
+        self.reply_to.append(_EmailAddressDoc(display_name=display_name,
+                                              address=address))
+
+    def add_cc(self, display_name, address):
+        self.cc.append(_EmailAddressDoc(display_name=display_name,
+                                        address=address))
+
+    def add_bcc(self, display_name, address):
+        self.bcc.append(_EmailAddressDoc(display_name=display_name,
+                                         address=address))
+
+    def add_attachment(self, filename, content_type):
+        self.attachments.append(filename=filename,
+                                content_type=content_type)
+
+
+class _ForensicReportDoc(DocType):
+    class Meta:
+        index = "dmarc_forensic"
+
+    feedback_type = Text()
+    user_agent = Text()
+    version = Text()
+    original_mail_from = Text()
+    arrival_date = Date()
+    domain = Text()
+    original_envelope_id = Text()
+    authentication_results = Text()
+    delivery_results = Text()
+    source_ip_address = Ip()
+    source_country = Text()
+    source_reverse_dns = Text()
+    source_authentication_mechanisms = Text()
+    source_auth_failures = Text()
+    dkim_domain = Text()
+    original_rcpt_to = Text()
+    sample = Object(_ForensicSampleDoc)
+
+
+
[docs]class AlreadySaved(ValueError):
+    """Raised when a report to be saved matches an existing report"""

+
+
+
[docs]def set_hosts(hosts):
+    """
+    Sets the Elasticsearch hosts to use
+
+    Args:
+        hosts: A single hostname or URL, or list of hostnames or URLs
+    """
+    if type(hosts) != list:
+        hosts = [hosts]
+    connections.create_connection(hosts=hosts, timeout=20)

+
+
+
[docs]def create_indexes():
+    """Creates the required indexes"""
+    if not aggregate_index.exists():
+        aggregate_index.create()
+    if not forensic_index.exists():
+        forensic_index.create()

+
+
+
[docs]def save_aggregate_report_to_elasticsearch(aggregate_report):
+    """
+    Saves a parsed DMARC aggregate report to ElasticSearch
+
+    Args:
+        aggregate_report (OrderedDict): A parsed forensic report
+
+    Raises:
+            AlreadySaved
+    """
+    aggregate_report = aggregate_report.copy()
+    metadata = aggregate_report["report_metadata"]
+    org_name = metadata["org_name"]
+    domain = aggregate_report["policy_published"]["domain"]
+    begin_date = parsedmarc.human_timestamp_to_datetime(metadata["begin_date"])
+    end_date = parsedmarc.human_timestamp_to_datetime(metadata["end_date"])
+    begin_date_human = begin_date.strftime("%Y-%m-%d %H:%M:%S")
+    end_date_human = end_date.strftime("%Y-%m-%d %H:%M:%S")
+    aggregate_report["begin_date"] = begin_date
+    aggregate_report["end_date"] = end_date
+    date_range = (aggregate_report["begin_date"],
+                  aggregate_report["end_date"])
+
+    org_name_query = Q(dict(match=dict(org_name=org_name)))
+    domain_query = Q(dict(match=dict(domain=domain)))
+    begin_date_query = Q(dict(match=dict(date_range=begin_date)))
+    end_date_query = Q(dict(match=dict(date_range=end_date)))
+
+    search = aggregate_index.search()
+    search.query = org_name_query & domain_query & begin_date_query & \
+        end_date_query
+
+    existing = search.execute()
+    if len(existing) > 0:
+        raise AlreadySaved("An aggregate report from {0} about {1} with a "
+                           "date range of {2} UTC to {3} UTC already exists "
+                           "in Elasticsearch".format(org_name,
+                                                     domain,
+                                                     begin_date_human,
+                                                     end_date_human))
+    published_policy = _PublishedPolicy(
+        adkim=aggregate_report["policy_published"]["adkim"],
+        aspf=aggregate_report["policy_published"]["aspf"],
+        p=aggregate_report["policy_published"]["p"],
+        sp=aggregate_report["policy_published"]["sp"],
+        pct=aggregate_report["policy_published"]["pct"],
+        fo=aggregate_report["policy_published"]["fo"]
+    )
+
+    for record in aggregate_report["records"]:
+        agg_doc = _AggregateReportDoc(
+            xml_schemea=aggregate_report["xml_schema"],
+            org_name=metadata["org_name"],
+            org_email=metadata["org_email"],
+            org_extra_contact_info=metadata["org_extra_contact_info"],
+            report_id=metadata["report_id"],
+            date_range=date_range,
+            errors=metadata["errors"],
+            domain=aggregate_report["policy_published"]["domain"],
+            published_policy=published_policy,
+            source_ip_address=record["source"]["ip_address"],
+            source_country=record["source"]["country"],
+            source_reverse_dns=record["source"]["reverse_dns"],
+            source_base_domain=record["source"]["base_domain"],
+            message_count=record["count"],
+            disposition=record["policy_evaluated"]["disposition"],
+            dkim_aligned=record["policy_evaluated"]["dkim"] == "pass",
+            spf_aligned=record["policy_evaluated"]["spf"] == "pass",
+            header_from=record["identifiers"]["header_from"],
+            envelope_from=record["identifiers"]["envelope_from"],
+            envelope_to=record["identifiers"]["envelope_to"]
+        )
+
+        for override in record["policy_evaluated"]["policy_override_reasons"]:
+            agg_doc.add_policy_override(type_=override["type"],
+                                        comment=override["comment"])
+
+        for dkim_result in record["auth_results"]["dkim"]:
+            agg_doc.add_dkim_result(domain=dkim_result["domain"],
+                                    selector=dkim_result["selector"],
+                                    result=dkim_result["result"])
+
+        for spf_result in record["auth_results"]["spf"]:
+            agg_doc.add_spf_result(domain=spf_result["domain"],
+                                   scope=spf_result["scope"],
+                                   result=spf_result["result"])
+        agg_doc.save()

+
+
+
[docs]def save_forensic_report_to_elasticsearch(forensic_report):
+    """
+        Saves a parsed DMARC forensic report to ElasticSearch
+
+        Args:
+            forensic_report (OrderedDict): A parsed forensic report
+
+        Raises:
+            AlreadySaved
+
+        """
+    forensic_report = forensic_report.copy()
+    sample_date = forensic_report["parsed_sample"]["date"]
+    sample_date = parsedmarc.human_timestamp_to_datetime(sample_date)
+    original_headers = forensic_report["parsed_sample"]["headers"]
+    headers = OrderedDict()
+    for original_header in original_headers:
+        headers[original_header.lower()] = original_headers[original_header]
+
+    arrival_date_human = forensic_report["arrival_date_utc"]
+    arrival_date = parsedmarc.human_timestamp_to_datetime(arrival_date_human)
+
+    search = forensic_index.search()
+    to_query = {"match": {"sample.headers.to": headers["to"]}}
+    from_query = {"match": {"sample.headers.from": headers["from"]}}
+    subject_query = {"match": {"sample.headers.subject": headers["subject"]}}
+    arrival_date_query = {"match": {"sample.headers.arrival_date": arrival_date
+                                    }}
+    q = Q(to_query) & Q(from_query) & Q(subject_query) & Q(arrival_date_query)
+    search.query = q
+    existing = search.execute()
+
+    if len(existing) > 0:
+        raise AlreadySaved("A forensic sample to {0} from {1} "
+                           "with a subject of {2} and arrival date of {3} "
+                           "already exists in "
+                           "Elasticsearch".format(headers["to"],
+                                                  headers["from"],
+                                                  headers["subject"],
+                                                  arrival_date_human
+                                                  ))
+
+    parsed_sample = forensic_report["parsed_sample"]
+    sample = _ForensicSampleDoc(
+        raw=forensic_report["sample"],
+        headers=headers,
+        headers_only=forensic_report["sample_headers_only"],
+        date=sample_date,
+        subject=forensic_report["parsed_sample"]["subject"],
+        filename_safe_subject=parsed_sample["filename_safe_subject"],
+        body=forensic_report["parsed_sample"]["body"]
+    )
+
+    for address in forensic_report["parsed_sample"]["to"]:
+        sample.add_to(display_name=address["display_name"],
+                      address=address["address"])
+    for address in forensic_report["parsed_sample"]["reply_to"]:
+        sample.add_reply_to(display_name=address["display_name"],
+                            address=address["address"])
+    for address in forensic_report["parsed_sample"]["cc"]:
+        sample.add_cc(display_name=address["display_name"],
+                      address=address["address"])
+    for address in forensic_report["parsed_sample"]["bcc"]:
+        sample.add_bcc(display_name=address["display_name"],
+                       address=address["address"])
+    for attachment in forensic_report["parsed_sample"]["attachments"]:
+        sample.add_attachment(filename=attachment["filename"],
+                              content_type=attachment["mail_content_type"])
+
+    forensic_doc = _ForensicReportDoc(
+        feedback_type=forensic_report["feedback_type"],
+        user_agent=forensic_report["user_agent"],
+        version=forensic_report["version"],
+        original_mail_from=forensic_report["original_mail_from"],
+        arrival_date=arrival_date,
+        domain=forensic_report["reported_domain"],
+        original_envelope_id=forensic_report["original_envelope_id"],
+        authentication_results=forensic_report["authentication_results"],
+        delivery_results=forensic_report["delivery_result"],
+        source_ip_address=forensic_report["source"]["ip_address"],
+        source_country=forensic_report["source"]["country"],
+        source_reverse_dns=forensic_report["source"]["reverse_dns"],
+        source_base_domain=forensic_report["source"]["base_domain"],
+        authentication_mechanisms=forensic_report["authentication_mechanisms"],
+        auth_failure=forensic_report["auth_failure"],
+        dkim_domain=forensic_report["dkim_domain"],
+        original_rcpt_to=forensic_report["original_rcpt_to"],
+        sample=sample
+    )
+
+    forensic_doc.save()

+
+ +
+
+ +
+
+ + +
+
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/_sources/index.rst.txt b/_sources/index.rst.txt index 3d9684f8..0ad0cbd0 100644 --- a/_sources/index.rst.txt +++ b/_sources/index.rst.txt @@ -253,6 +253,12 @@ API .. automodule:: parsedmarc :members: +parsedmarc.elastic +------------------ + +.. automodule:: parsedmarc.elastic + :members: + .. toctree:: :maxdepth: 2 :caption: Contents: diff --git a/_static/basic.css b/_static/basic.css index 607b5f55..19ced105 100644 --- a/_static/basic.css +++ b/_static/basic.css @@ -82,9 +82,21 @@ div.sphinxsidebar input { } div.sphinxsidebar #searchbox input[type="text"] { - width: 170px; + float: left; + width: 80%; + padding: 0.25em; + box-sizing: border-box; } +div.sphinxsidebar #searchbox input[type="submit"] { + float: left; + width: 20%; + border-left: none; + padding: 0.25em; + box-sizing: border-box; +} + + img { border: 0; max-width: 100%; @@ -199,6 +211,11 @@ table.modindextable td { /* -- general body styles --------------------------------------------------- */ +div.body { + min-width: 450px; + max-width: 800px; +} + div.body p, div.body dd, div.body li, div.body blockquote { -moz-hyphens: auto; -ms-hyphens: auto; diff --git a/_static/documentation_options.js b/_static/documentation_options.js new file mode 100644 index 00000000..a85cb519 --- /dev/null +++ b/_static/documentation_options.js @@ -0,0 +1,9 @@ +var DOCUMENTATION_OPTIONS = { + URL_ROOT: '', + VERSION: '3.3.0', + LANGUAGE: 'None', + COLLAPSE_INDEX: false, + FILE_SUFFIX: '.html', + HAS_SOURCE: true, + SOURCELINK_SUFFIX: '.txt' +}; \ No newline at end of file diff --git a/_static/websupport.js b/_static/websupport.js index 79b18e38..78e14bb4 100644 --- a/_static/websupport.js +++ b/_static/websupport.js @@ -301,7 +301,7 @@ li.hide(); // Determine where in the parents children list to insert this comment. - for(i=0; i < siblings.length; i++) { + for(var i=0; i < siblings.length; i++) { if (comp(comment, siblings[i]) <= 0) { $('#cd' + siblings[i].id) .parent() diff --git a/genindex.html b/genindex.html index 4988fb3d..7e133b1d 100644 --- a/genindex.html +++ b/genindex.html @@ -9,7 +9,7 @@ - Index — parsedmarc 3.2.0 documentation + Index — parsedmarc 3.3.0 documentation @@ -36,7 +36,7 @@ - + @@ -65,7 +65,7 @@
- 3.2.0 + 3.3.0
@@ -154,16 +154,33 @@

Index

- E + A + | C + | E | G | H | I | P | S - | T | W
+

A

+ + +
+ +

C

+ + +
+

E

- + -
-
  • API
    • +
  • API +
  • Indices and tables
    • @@ -176,7 +179,7 @@

    Welcome to parsedmarc’s documentation!

    Build Status

    A screenshot of DMARC summary charts in Kibana -

    pasedmarc is a Python module and CLI utility for parsing DMARC reports.

    +

    pasedmarc is a Python module and CLI utility for parsing DMARC reports.

    Features

      @@ -192,7 +195,7 @@

    CLI help

    -
    usage: parsedmarc [-h] [-o OUTPUT] [-n NAMESERVERS [NAMESERVERS ...]]
+
    usage: parsedmarc [-h] [-o OUTPUT] [-n NAMESERVERS [NAMESERVERS ...]]
               [-t TIMEOUT] [-H HOST] [-u USER] [-p PASSWORD]
               [-r REPORTS_FOLDER] [-a ARCHIVE_FOLDER] [-d]
               [-E [ELASTICSEARCH_HOST [ELASTICSEARCH_HOST ...]]]
@@ -268,11 +271,11 @@ report from the dmarc.org wiki. It’s actually an older draft of the the 1.0
 report schema standardized in
 RFC 7480 Appendix C.
 This draft schema is still in wide use.
    
-
    parsedmarc produces consistent, normalized output, regardless of the report
+
    parsedmarc produces consistent, normalized output, regardless of the report
 schema.
    
 
    
 
    JSON
    
-
    {
+
    {
   "xml_schema": "draft",
   "report_metadata": {
     "org_name": "acme.com",
@@ -336,7 +339,7 @@ schema.
    
 
    
 
    
 
    CSV
    
-
    xml_schema,org_name,org_email,org_extra_contact_info,report_id,begin_date,end_date,errors,domain,adkim,aspf,p,sp,pct,fo,source_ip_address,source_country,source_reverse_dns,source_base_domain,count,disposition,dkim_alignment,spf_alignment,policy_override_reasons,policy_override_comments,envelope_from,header_from,envelope_to,dkim_domains,dkim_selectors,dkim_results,spf_domains,spf_scopes,spf_results
+
    xml_schema,org_name,org_email,org_extra_contact_info,report_id,begin_date,end_date,errors,domain,adkim,aspf,p,sp,pct,fo,source_ip_address,source_country,source_reverse_dns,source_base_domain,count,disposition,dkim_alignment,spf_alignment,policy_override_reasons,policy_override_comments,envelope_from,header_from,envelope_to,dkim_domains,dkim_selectors,dkim_results,spf_domains,spf_scopes,spf_results
 draft,acme.com,noreply-dmarc-support@acme.com,http://acme.com/dmarc/support,9391651994964116463,2012-04-27 20:00:00,2012-04-28 19:59:59,,example.com,r,r,none,none,100,0,72.150.241.94,US,adsl-72-150-241-94.shv.bellsouth.net,bellsouth.net,2,none,fail,pass,,,example.com,example.com,,example.com,none,fail,example.com,mfrom,pass
 
    
 
    
@@ -354,34 +357,34 @@ forensic report that you can share publicly, please contact me!
    
 
    
 
    
 
    Installation
    
-
    parsedmarc works with Python 3 only.
    
+
    parsedmarc works with Python 3 only.
    
 
    On Debian or Ubuntu systems, run:
    
-
    $ sudo apt-get install python3-pip
+
    $ sudo apt-get install python3-pip
 
    
 
    
 
    Python 3 installers for Windows and macOS can be found at
 https://www.python.org/downloads/
    
-
    To install or upgrade to the latest stable release of parsedmarc on
+
    To install or upgrade to the latest stable release of parsedmarc on
 macOS or Linux, run
    
-
    $ sudo -H pip3 install -U parsedmarc
+
    $ sudo -H pip3 install -U parsedmarc
 
    
 
    
 
    Or, install the latest development release directly from GitHub:
    
-
    $ sudo -H pip3 install -U git+https://github.com/domainaware/parsedmarc.git
+
    $ sudo -H pip3 install -U git+https://github.com/domainaware/parsedmarc.git
 
    
 
    
 
    
 
    Note
    
-
    On Windows, pip3 is pip, even with Python 3. So on Windows, simply
-substitute pip as an administrator in place of sudo pip3, in the
+
    On Windows, pip3 is pip, even with Python 3. So on Windows, simply
+substitute pip as an administrator in place of sudo pip3, in the
 above commands.
    
 
    
 
    
 
    Optional dependencies
    
 
    If you would like to be able to parse emails saved from Microsoft Outlook
-(i.e. OLE .msg files), install msgconvert:
    
+(i.e. OLE .msg files), install msgconvert:
    
 
    On Debian or Ubuntu systems, run:
    
-
    $ sudo apt-get install libemail-outlook-message-perl
+
    $ sudo apt-get install libemail-outlook-message-perl
 
    
 
    
 
    
@@ -494,7 +497,7 @@ or bytes.
    Returns:

    Lists of aggregate_reports and forensic_reports

    +
    Returns:

    Lists of aggregate_reports and forensic_reports
    Return type:

    OrderedDict

    @@ -525,7 +528,7 @@ or bytes.

    parsedmarc.human_timestamp_to_datetime(human_timestamp)[source]
    -

    Converts a human-readable timestamp into a Python DateTime object

    +

    Converts a human-readable timestamp into a Python DateTime object

    @@ -596,7 +599,7 @@ aggregate DMARC report

    parsedmarc.parse_forensic_report(feedback_report, sample, sample_headers_only, nameservers=None, timeout=6.0)[source]
    -

    Converts a DMARC forensic report and sample to a OrderedDict

    +

    Converts a DMARC forensic report and sample to a OrderedDict

    @@ -637,8 +640,8 @@ aggregate DMARC report

    @@ -733,24 +736,6 @@ headers

    Returns:

      -
    • report_type: aggregate or forensic
      • -
    • report: The parsed report
      • +
    • report_type: aggregate or forensic
      • +
    • report: The parsed report
    -
    -
    -parsedmarc.timestamp_to_human(timestamp)[source]
    -

    Converts a UNIX/DMARC timestamp to a human-readable string

    - --- - - - - - - - -
    Parameters:timestamp – The timestamp
    Returns:The converted timestamp in YYYY-MM-DD HH:MM:SS format
    Return type:str
    -
    -
    parsedmarc.watch_inbox(host, username, password, callback, reports_folder='INBOX', archive_folder='Archive', delete=False, test=False, wait=30, nameservers=None, dns_timeout=6.0)[source]
    @@ -780,9 +765,70 @@ to a callback function

    +
    +

    parsedmarc.elastic

    +
    +
    +exception parsedmarc.elastic.AlreadySaved[source]
    +

    Raised when a report to be saved matches an existing report

    +
    + +
    +
    +parsedmarc.elastic.create_indexes()[source]
    +

    Creates the required indexes

    +
    + +
    +
    +parsedmarc.elastic.save_aggregate_report_to_elasticsearch(aggregate_report)[source]
    +

    Saves a parsed DMARC aggregate report to ElasticSearch

    + +++ + + + + + +
    Parameters:aggregate_report (OrderedDict) – A parsed forensic report
    Raises:AlreadySaved
    +
    + +
    +
    +parsedmarc.elastic.save_forensic_report_to_elasticsearch(forensic_report)[source]
    +

    Saves a parsed DMARC forensic report to ElasticSearch

    + +++ + + + + + +
    Parameters:forensic_report (OrderedDict) – A parsed forensic report
    Raises:AlreadySaved
    +
    + +
    +
    +parsedmarc.elastic.set_hosts(hosts)[source]
    +

    Sets the Elasticsearch hosts to use

    + +++ + + + +
    Parameters:hosts – A single hostname or URL, or list of hostnames or URLs
    +
    +
    +

    Indices and tables