Update docs

Avoid polluting the system Python installation

.buildinfo

@@ -0,0 +1,4 @@
+# Sphinx build info version 1
+# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
+config: a1168b776b620d01471713ec4f5cfa87
+tags: 645f666f9bcd5a90fca523b33c5a78b7

.gitignore

@@ -1,120 +1 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-env/
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-.hypothesis/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# pyenv
-.python-version
-
-# celery beat schedule file
-celerybeat-schedule
-
-# SageMath parsed files
-*.sage.py
-
-# dotenv
-.env
-
-# virtualenv
-.venv
-venv/
-ENV/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-
-# PyCharm Project settings
-.idea/
-
-# I/O files
-
-*.xml
-*.zip
-*.gz
-*.json
-*.csv
-*.xls*
-
-# LibreOffice lock files
-.~*
-
-# ignore data files
 *.dat
-*.mmdb

.travis.yml

@@ -1,23 +0,0 @@
-language: python
-
-sudo: false
-
-python:
-  - '2.7'
-  - '3.4'
-  - '3.5'
-  - '3.6'
-
-# commands to install dependencies
-install:
-  - "pip install flake8 pytest-cov pytest coveralls"
-  - "pip install -r requirements.txt"
-
-# commands to run samples
-script:
-  - "flake8 *.py"
-  - "cd docs"
-  - "make html"
-  - "cd .."
-  - "python tests.py"
-  - "python setup.py bdist_wheel"

CHANGELOG.md

@@ -1,3 +0,0 @@
-1.0.0
------
-- Initial release

LICENSE

@@ -1,201 +0,0 @@
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

README.rst

@@ -1,188 +0,0 @@
-checkdmarc
-==========
-
-|Build Status|
-
-``pasedmarc`` is a Python module and CLI utility for parsing aggregate DMARC reports.
-
-Features
-========
-
-* Parses draft and 1.0 standard aggregate reports
-* Transparently handles gzip or zip compressed reports
-* Consistent data structures
-* Simple JSON or CSV output
-* Python 2 and 3 support
-
-CLI help
-========
-
-::
-
-    usage: parsedmarc.py [-h] [-f FORMAT] [-o OUTPUT]
-                         [-n NAMESERVER [NAMESERVER ...]] [-t TIMEOUT] [-v]
-                         file_path [file_path ...]
-
-    Parses aggregate DMARC reports
-
-    positional arguments:
-      file_path             one or more paths of aggregate report files
-                            (compressed or uncompressed)
-
-    optional arguments:
-      -h, --help            show this help message and exit
-      -f FORMAT, --format FORMAT
-                            specify JSON or CSV output format
-      -o OUTPUT, --output OUTPUT
-                            output to a file path rather than printing to the
-                            screen
-      -n NAMESERVER [NAMESERVER ...], --nameserver NAMESERVER [NAMESERVER ...]
-                            nameservers to query
-      -t TIMEOUT, --timeout TIMEOUT
-                            number of seconds to wait for an answer from DNS
-                            (default 6.0)
-      -v, --version         show program's version number and exit
-
-
-Sample output
-=============
-
-Here are the results from parsing the `example <https://dmarc.org/wiki/FAQ#I_need_to_implement_aggregate_reports.2C_what_do_they_look_like.3F>`_
-report from the dmarc.org wiki. It's actually an older draft of the the 1.0
-report schema standardized in
-`RFC 7480 Appendix C <https://tools.ietf.org/html/rfc7489#appendix-C>`_.
-This draft schema is still in wide use.
-
-``parsedmarc`` produces consistent, normalized output, regardless of the report schema.
-
-JSON
-----
-
-.. code-block:: json
-
-    {
-      "xml_schema": "draft",
-      "report_metadata": {
-        "org_name": "acme.com",
-        "org_email": "noreply-dmarc-support@acme.com",
-        "org_extra_contact_info": "http://acme.com/dmarc/support",
-        "report_id": "9391651994964116463",
-        "begin_date": "2012-04-27 20:00:00",
-        "end_date": "2012-04-28 19:59:59",
-        "errors": []
-      },
-      "policy_published": {
-        "domain": "example.com",
-        "adkim": "r",
-        "aspf": "r",
-        "p": "none",
-        "sp": "none",
-        "pct": "100",
-        "fo": "0"
-      },
-      "records": [
-        {
-          "source": {
-            "ip_address": "72.150.241.94",
-            "country": "US",
-            "reverse_dns": "adsl-72-150-241-94.shv.bellsouth.net",
-            "base_domain": "bellsouth.net"
-          },
-          "count": 2,
-          "policy_evaluated": {
-            "disposition": "none",
-            "dkim": "fail",
-            "spf": "pass",
-            "policy_override_reasons": []
-          },
-          "identifiers": {
-            "header_from": "example.com",
-            "envelope_from": "example.com",
-            "envelope_to": null
-          },
-          "auth_results": {
-            "dkim": [
-              {
-                "domain": "example.com",
-                "selector": "none",
-                "result": "fail"
-              }
-            ],
-            "spf": [
-              {
-                "domain": "example.com",
-                "scope": "mfrom",
-                "result": "pass"
-              }
-            ]
-          }
-        }
-      ]
-    }
-
-CSV
----
-
-::
-
-    xml_schema,org_name,org_email,org_extra_contact_info,report_id,begin_date,end_date,errors,domain,adkim,aspf,p,sp,pct,fo,source_ip_address,source_country,source_reverse_dns,source_base_domain,count,disposition,dkim_alignment,spf_alignment,policy_override_reasons,policy_override_comments,envelope_from,header_from,envelope_to,dkim_domains,dkim_selectors,dkim_results,spf_domains,spf_scopes,spf_results
-    draft,acme.com,noreply-dmarc-support@acme.com,http://acme.com/dmarc/support,9391651994964116463,2012-04-27 20:00:00,2012-04-28 19:59:59,[],example.com,r,r,none,none,100,0,72.150.241.94,US,adsl-72-150-241-94.shv.bellsouth.net,bellsouth.net,2,none,fail,pass,,,example.com,example.com,,example.com,none,fail,example.com,mfrom,pass
-
-What about forensic DMARC reports?
-==================================
-
-Forensic DMARC reports are emails with an attached email sample that failed a
-DMARC check. You can parse them with any email message parser, such as
-`mail-parser <https://pypi.python.org/pypi/mail-parser/>`_.
-
-Very few recipients send forensic reports, and even those who do will often
-provide only the message headers, and not the message's content, for privacy
-reasons.
-
-Installation
-============
-
-``parsedmarc`` works with Python 2 or 3, but Python 3 is preferred.
-
-On Debian or Ubuntu systems, run:
-
-.. code-block:: bash
-
-    $ sudo apt-get install python3-pip
-
-
-Python 3 installers for Windows and macOS can be found at https://www.python.org/downloads/
-
-To install or upgrade to the latest stable release of ``parsedmarc`` on macOS or Linux, run
-
-.. code-block:: bash
-
-    $ sudo -H pip3 install -U checkdmarc
-
-Or, install the latest development release directly from GitHub:
-
-.. code-block:: bash
-
-    $ sudo -H pip3 install -U git+https://github.com/domainaware/parsedmarc.git
-
-.. note::
-
-    On Windows, ``pip3`` is ``pip``, even with Python 3. So on Windows, simply
-    substitute ``pip`` as an administrator in place of ``sudo pip3``, in the above commands.
-
-
-
-Documentation
-=============
-
-https://domainaware.github.io/parsedmarc
-
-Bug reports
-===========
-
-Please report bugs on the GitHub issue tracker
-
-https://github.com/domainaware/parsedmarc/issues
-
-.. |Build Status| image:: https://travis-ci.org/domainaware/parsedmarc.svg?branch=master
-   :target: https://travis-ci.org/domainaware/parsedmarc

_modules/elasticsearch_dsl/field.html

@@ -0,0 +1,579 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>elasticsearch_dsl.field &mdash; parsedmarc 3.3.1 documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../genindex.html"/>
+        <link rel="search" title="Search" href="../../search.html"/>
+    <link rel="top" title="parsedmarc 3.3.1 documentation" href="../../index.html"/>
+        <link rel="up" title="Module code" href="../index.html"/> 
+
+  
+  <script src="../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../index.html" class="icon icon-home"> parsedmarc
+          
+
+          
+          </a>
+
+          
+            
+            
+              <div class="version">
+                3.3.1
+              </div>
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <!-- Local TOC -->
+              <div class="local-toc"></div>
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../index.html">parsedmarc</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../index.html">Module code</a> &raquo;</li>
+        
+      <li>elasticsearch_dsl.field</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for elasticsearch_dsl.field</h1><div class="highlight"><pre>
+<span></span><span class="kn">import</span> <span class="nn">base64</span>
+<span class="kn">import</span> <span class="nn">ipaddress</span>
+
+<span class="kn">import</span> <span class="nn">collections</span>
+
+<span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">date</span><span class="p">,</span> <span class="n">datetime</span>
+
+<span class="kn">from</span> <span class="nn">dateutil</span> <span class="k">import</span> <span class="n">parser</span><span class="p">,</span> <span class="n">tz</span>
+<span class="kn">from</span> <span class="nn">six</span> <span class="k">import</span> <span class="n">itervalues</span><span class="p">,</span> <span class="n">string_types</span><span class="p">,</span> <span class="n">iteritems</span>
+<span class="kn">from</span> <span class="nn">six.moves</span> <span class="k">import</span> <span class="nb">map</span>
+
+<span class="kn">from</span> <span class="nn">.utils</span> <span class="k">import</span> <span class="n">DslBase</span><span class="p">,</span> <span class="n">ObjectBase</span><span class="p">,</span> <span class="n">AttrDict</span><span class="p">,</span> <span class="n">AttrList</span>
+<span class="kn">from</span> <span class="nn">.exceptions</span> <span class="k">import</span> <span class="n">ValidationException</span>
+
+<span class="n">unicode</span> <span class="o">=</span> <span class="nb">type</span><span class="p">(</span><span class="sa">u</span><span class="s1">&#39;&#39;</span><span class="p">)</span>
+
+<span class="k">def</span> <span class="nf">construct_field</span><span class="p">(</span><span class="n">name_or_field</span><span class="p">,</span> <span class="o">**</span><span class="n">params</span><span class="p">):</span>
+    <span class="c1"># {&quot;type&quot;: &quot;text&quot;, &quot;analyzer&quot;: &quot;snowball&quot;}</span>
+    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">name_or_field</span><span class="p">,</span> <span class="n">collections</span><span class="o">.</span><span class="n">Mapping</span><span class="p">):</span>
+        <span class="k">if</span> <span class="n">params</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;construct_field() cannot accept parameters when passing in a dict.&#39;</span><span class="p">)</span>
+        <span class="n">params</span> <span class="o">=</span> <span class="n">name_or_field</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+        <span class="k">if</span> <span class="s1">&#39;type&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">params</span><span class="p">:</span>
+            <span class="c1"># inner object can be implicitly defined</span>
+            <span class="k">if</span> <span class="s1">&#39;properties&#39;</span> <span class="ow">in</span> <span class="n">params</span><span class="p">:</span>
+                <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;object&#39;</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;construct_field() needs to have a &quot;type&quot; key.&#39;</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">name</span> <span class="o">=</span> <span class="n">params</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s1">&#39;type&#39;</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">Field</span><span class="o">.</span><span class="n">get_dsl_class</span><span class="p">(</span><span class="n">name</span><span class="p">)(</span><span class="o">**</span><span class="n">params</span><span class="p">)</span>
+
+    <span class="c1"># Text()</span>
+    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">name_or_field</span><span class="p">,</span> <span class="n">Field</span><span class="p">):</span>
+        <span class="k">if</span> <span class="n">params</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;construct_field() cannot accept parameters when passing in a construct_field object.&#39;</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">name_or_field</span>
+
+    <span class="c1"># &quot;text&quot;, analyzer=&quot;snowball&quot;</span>
+    <span class="k">return</span> <span class="n">Field</span><span class="o">.</span><span class="n">get_dsl_class</span><span class="p">(</span><span class="n">name_or_field</span><span class="p">)(</span><span class="o">**</span><span class="n">params</span><span class="p">)</span>
+
+<span class="k">class</span> <span class="nc">Field</span><span class="p">(</span><span class="n">DslBase</span><span class="p">):</span>
+    <span class="n">_type_name</span> <span class="o">=</span> <span class="s1">&#39;field&#39;</span>
+    <span class="n">_type_shortcut</span> <span class="o">=</span> <span class="nb">staticmethod</span><span class="p">(</span><span class="n">construct_field</span><span class="p">)</span>
+    <span class="c1"># all fields can be multifields</span>
+    <span class="n">_param_defs</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;fields&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;type&#39;</span><span class="p">:</span> <span class="s1">&#39;field&#39;</span><span class="p">,</span> <span class="s1">&#39;hash&#39;</span><span class="p">:</span> <span class="kc">True</span><span class="p">}}</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="n">_coerce</span> <span class="o">=</span> <span class="kc">False</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_multi</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s1">&#39;multi&#39;</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_required</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s1">&#39;required&#39;</span><span class="p">,</span> <span class="kc">False</span><span class="p">)</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">Field</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">__getitem__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">subfield</span><span class="p">):</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_params</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;fields&#39;</span><span class="p">,</span> <span class="p">{})[</span><span class="n">subfield</span><span class="p">]</span>
+
+    <span class="k">def</span> <span class="nf">_serialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
+        <span class="k">return</span> <span class="n">data</span>
+
+    <span class="k">def</span> <span class="nf">_deserialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
+        <span class="k">return</span> <span class="n">data</span>
+
+    <span class="k">def</span> <span class="nf">_empty</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">return</span> <span class="kc">None</span>
+
+    <span class="k">def</span> <span class="nf">empty</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_multi</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">AttrList</span><span class="p">([])</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_empty</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">serialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="n">AttrList</span><span class="p">)):</span>
+            <span class="k">return</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_serialize</span><span class="p">,</span> <span class="n">data</span><span class="p">))</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_serialize</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">deserialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="n">AttrList</span><span class="p">)):</span>
+            <span class="n">data</span><span class="p">[:]</span> <span class="o">=</span> <span class="p">[</span>
+                <span class="kc">None</span> <span class="k">if</span> <span class="n">d</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">_deserialize</span><span class="p">(</span><span class="n">d</span><span class="p">)</span>
+                <span class="k">for</span> <span class="n">d</span> <span class="ow">in</span> <span class="n">data</span>
+            <span class="p">]</span>
+            <span class="k">return</span> <span class="n">data</span>
+        <span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">return</span> <span class="kc">None</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_deserialize</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">clean</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
+        <span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">deserialize</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">data</span> <span class="ow">in</span> <span class="p">(</span><span class="kc">None</span><span class="p">,</span> <span class="p">[],</span> <span class="p">{})</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_required</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">ValidationException</span><span class="p">(</span><span class="s2">&quot;Value required for this field.&quot;</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">data</span>
+
+    <span class="k">def</span> <span class="nf">to_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="n">d</span> <span class="o">=</span> <span class="nb">super</span><span class="p">(</span><span class="n">Field</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">to_dict</span><span class="p">()</span>
+        <span class="n">name</span><span class="p">,</span> <span class="n">value</span> <span class="o">=</span> <span class="n">d</span><span class="o">.</span><span class="n">popitem</span><span class="p">()</span>
+        <span class="n">value</span><span class="p">[</span><span class="s1">&#39;type&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">name</span>
+        <span class="k">return</span> <span class="n">value</span>
+
+<span class="k">class</span> <span class="nc">CustomField</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;custom&#39;</span>
+    <span class="n">_coerce</span> <span class="o">=</span> <span class="kc">True</span>
+
+    <span class="k">def</span> <span class="nf">to_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">builtin_type</span><span class="p">,</span> <span class="n">Field</span><span class="p">):</span>
+            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">builtin_type</span><span class="o">.</span><span class="n">to_dict</span><span class="p">()</span>
+
+        <span class="n">d</span> <span class="o">=</span> <span class="nb">super</span><span class="p">(</span><span class="n">CustomField</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">to_dict</span><span class="p">()</span>
+        <span class="n">d</span><span class="p">[</span><span class="s1">&#39;type&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">builtin_type</span>
+        <span class="k">return</span> <span class="n">d</span>
+
+<span class="k">class</span> <span class="nc">Object</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;object&#39;</span>
+    <span class="n">_coerce</span> <span class="o">=</span> <span class="kc">True</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">doc_class</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_doc_class</span> <span class="o">=</span> <span class="n">doc_class</span>
+        <span class="k">if</span> <span class="n">doc_class</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="c1"># FIXME import</span>
+            <span class="kn">from</span> <span class="nn">.document</span> <span class="k">import</span> <span class="n">InnerDoc</span>
+            <span class="c1"># no InnerDoc subclass, creating one instead...</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">_doc_class</span> <span class="o">=</span> <span class="nb">type</span><span class="p">(</span><span class="s1">&#39;InnerDoc&#39;</span><span class="p">,</span> <span class="p">(</span><span class="n">InnerDoc</span><span class="p">,</span> <span class="p">),</span> <span class="p">{})</span>
+            <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">field</span> <span class="ow">in</span> <span class="n">iteritems</span><span class="p">(</span><span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s1">&#39;properties&#39;</span><span class="p">,</span> <span class="p">{})):</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">_doc_class</span><span class="o">.</span><span class="n">_doc_type</span><span class="o">.</span><span class="n">mapping</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">field</span><span class="p">)</span>
+            <span class="k">if</span> <span class="s1">&#39;dynamic&#39;</span> <span class="ow">in</span> <span class="n">kwargs</span><span class="p">:</span>
+                <span class="bp">self</span><span class="o">.</span><span class="n">_doc_class</span><span class="o">.</span><span class="n">_doc_type</span><span class="o">.</span><span class="n">mapping</span><span class="o">.</span><span class="n">meta</span><span class="p">(</span><span class="s1">&#39;dynamic&#39;</span><span class="p">,</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s1">&#39;dynamic&#39;</span><span class="p">))</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">_mapping</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_doc_class</span><span class="o">.</span><span class="n">_doc_type</span><span class="o">.</span><span class="n">mapping</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">Object</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">__getitem__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mapping</span><span class="p">[</span><span class="n">name</span><span class="p">]</span>
+
+    <span class="k">def</span> <span class="nf">__contains__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">):</span>
+        <span class="k">return</span> <span class="n">name</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mapping</span>
+
+    <span class="k">def</span> <span class="nf">_empty</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_wrap</span><span class="p">({})</span>
+
+    <span class="k">def</span> <span class="nf">_wrap</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_doc_class</span><span class="o">.</span><span class="n">from_es</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">empty</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_multi</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">AttrList</span><span class="p">([],</span> <span class="bp">self</span><span class="o">.</span><span class="n">_wrap</span><span class="p">)</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_empty</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">to_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="n">d</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mapping</span><span class="o">.</span><span class="n">to_dict</span><span class="p">()</span>
+        <span class="n">_</span><span class="p">,</span> <span class="n">d</span> <span class="o">=</span> <span class="n">d</span><span class="o">.</span><span class="n">popitem</span><span class="p">()</span>
+        <span class="n">d</span><span class="p">[</span><span class="s2">&quot;type&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">name</span>
+        <span class="k">return</span> <span class="n">d</span>
+
+    <span class="k">def</span> <span class="nf">_collect_fields</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mapping</span><span class="o">.</span><span class="n">properties</span><span class="o">.</span><span class="n">_collect_fields</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">_deserialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
+        <span class="c1"># don&#39;t wrap already wrapped data</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_doc_class</span><span class="p">):</span>
+            <span class="k">return</span> <span class="n">data</span>
+
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">AttrDict</span><span class="p">):</span>
+            <span class="n">data</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">_d_</span>
+
+        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_wrap</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_serialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
+        <span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">return</span> <span class="kc">None</span>
+
+        <span class="c1"># somebody assigned raw dict to the field, we should tolerate that</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">collections</span><span class="o">.</span><span class="n">Mapping</span><span class="p">):</span>
+            <span class="k">return</span> <span class="n">data</span>
+
+        <span class="k">return</span> <span class="n">data</span><span class="o">.</span><span class="n">to_dict</span><span class="p">()</span>
+
+    <span class="k">def</span> <span class="nf">clean</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
+        <span class="n">data</span> <span class="o">=</span> <span class="nb">super</span><span class="p">(</span><span class="n">Object</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">clean</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">return</span> <span class="kc">None</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="n">AttrList</span><span class="p">)):</span>
+            <span class="k">for</span> <span class="n">d</span> <span class="ow">in</span> <span class="n">data</span><span class="p">:</span>
+                <span class="n">d</span><span class="o">.</span><span class="n">full_clean</span><span class="p">()</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">data</span><span class="o">.</span><span class="n">full_clean</span><span class="p">()</span>
+        <span class="k">return</span> <span class="n">data</span>
+
+    <span class="k">def</span> <span class="nf">update</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">Object</span><span class="p">):</span>
+            <span class="c1"># not an inner/nested object, no merge possible</span>
+            <span class="k">return</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">_mapping</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">other</span><span class="o">.</span><span class="n">_mapping</span><span class="p">)</span>
+
+<span class="k">class</span> <span class="nc">Nested</span><span class="p">(</span><span class="n">Object</span><span class="p">):</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;nested&#39;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="n">kwargs</span><span class="o">.</span><span class="n">setdefault</span><span class="p">(</span><span class="s1">&#39;multi&#39;</span><span class="p">,</span> <span class="kc">True</span><span class="p">)</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">Nested</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+<span class="k">class</span> <span class="nc">Date</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;date&#39;</span>
+    <span class="n">_coerce</span> <span class="o">=</span> <span class="kc">True</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_default_timezone</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s1">&#39;default_timezone&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_default_timezone</span><span class="p">,</span> <span class="n">string_types</span><span class="p">):</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">_default_timezone</span> <span class="o">=</span> <span class="n">tz</span><span class="o">.</span><span class="n">gettz</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_default_timezone</span><span class="p">)</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">Date</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_deserialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">string_types</span><span class="p">):</span>
+            <span class="k">try</span><span class="p">:</span>
+                <span class="n">data</span> <span class="o">=</span> <span class="n">parser</span><span class="o">.</span><span class="n">parse</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
+            <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+                <span class="k">raise</span> <span class="n">ValidationException</span><span class="p">(</span><span class="s1">&#39;Could not parse date from the value (</span><span class="si">%r</span><span class="s1">)&#39;</span> <span class="o">%</span> <span class="n">data</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
+
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">datetime</span><span class="p">):</span>
+            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_default_timezone</span> <span class="ow">and</span> <span class="n">data</span><span class="o">.</span><span class="n">tzinfo</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+                <span class="n">data</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="n">tzinfo</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_default_timezone</span><span class="p">)</span>
+            <span class="k">return</span> <span class="n">data</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">date</span><span class="p">):</span>
+            <span class="k">return</span> <span class="n">data</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="nb">int</span><span class="p">):</span>
+            <span class="c1"># Divide by a float to preserve milliseconds on the datetime.</span>
+            <span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">utcfromtimestamp</span><span class="p">(</span><span class="n">data</span> <span class="o">/</span> <span class="mf">1000.0</span><span class="p">)</span>
+
+        <span class="k">raise</span> <span class="n">ValidationException</span><span class="p">(</span><span class="s1">&#39;Could not parse date from the value (</span><span class="si">%r</span><span class="s1">)&#39;</span> <span class="o">%</span> <span class="n">data</span><span class="p">)</span>
+
+<span class="k">class</span> <span class="nc">Text</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
+    <span class="n">_param_defs</span> <span class="o">=</span> <span class="p">{</span>
+        <span class="s1">&#39;fields&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;type&#39;</span><span class="p">:</span> <span class="s1">&#39;field&#39;</span><span class="p">,</span> <span class="s1">&#39;hash&#39;</span><span class="p">:</span> <span class="kc">True</span><span class="p">},</span>
+        <span class="s1">&#39;analyzer&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;type&#39;</span><span class="p">:</span> <span class="s1">&#39;analyzer&#39;</span><span class="p">},</span>
+        <span class="s1">&#39;search_analyzer&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;type&#39;</span><span class="p">:</span> <span class="s1">&#39;analyzer&#39;</span><span class="p">},</span>
+        <span class="s1">&#39;search_quote_analyzer&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;type&#39;</span><span class="p">:</span> <span class="s1">&#39;analyzer&#39;</span><span class="p">},</span>
+    <span class="p">}</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;text&#39;</span>
+
+<span class="k">class</span> <span class="nc">Keyword</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
+    <span class="n">_param_defs</span> <span class="o">=</span> <span class="p">{</span>
+        <span class="s1">&#39;fields&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;type&#39;</span><span class="p">:</span> <span class="s1">&#39;field&#39;</span><span class="p">,</span> <span class="s1">&#39;hash&#39;</span><span class="p">:</span> <span class="kc">True</span><span class="p">},</span>
+        <span class="s1">&#39;search_analyzer&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;type&#39;</span><span class="p">:</span> <span class="s1">&#39;analyzer&#39;</span><span class="p">},</span>
+        <span class="s1">&#39;normalizer&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;type&#39;</span><span class="p">:</span> <span class="s1">&#39;normalizer&#39;</span><span class="p">}</span>
+    <span class="p">}</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;keyword&#39;</span>
+
+<span class="k">class</span> <span class="nc">Boolean</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;boolean&#39;</span>
+    <span class="n">_coerce</span> <span class="o">=</span> <span class="kc">True</span>
+
+    <span class="k">def</span> <span class="nf">_deserialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
+        <span class="k">if</span> <span class="n">data</span> <span class="o">==</span> <span class="s2">&quot;false&quot;</span><span class="p">:</span>
+            <span class="k">return</span> <span class="kc">False</span>
+        <span class="k">return</span> <span class="nb">bool</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">clean</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
+        <span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">deserialize</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_required</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">ValidationException</span><span class="p">(</span><span class="s2">&quot;Value required for this field.&quot;</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">data</span>
+
+<span class="k">class</span> <span class="nc">Float</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;float&#39;</span>
+    <span class="n">_coerce</span> <span class="o">=</span> <span class="kc">True</span>
+
+    <span class="k">def</span> <span class="nf">_deserialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
+        <span class="k">return</span> <span class="nb">float</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
+
+<span class="k">class</span> <span class="nc">HalfFloat</span><span class="p">(</span><span class="n">Float</span><span class="p">):</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;half_float&#39;</span>
+
+<span class="k">class</span> <span class="nc">ScaledFloat</span><span class="p">(</span><span class="n">Float</span><span class="p">):</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;scaled_float&#39;</span>
+
+    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">scaling_factor</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+        <span class="nb">super</span><span class="p">(</span><span class="n">ScaledFloat</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">scaling_factor</span><span class="o">=</span><span class="n">scaling_factor</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+<span class="k">class</span> <span class="nc">Double</span><span class="p">(</span><span class="n">Float</span><span class="p">):</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;double&#39;</span>
+
+<span class="k">class</span> <span class="nc">Integer</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;integer&#39;</span>
+    <span class="n">_coerce</span> <span class="o">=</span> <span class="kc">True</span>
+
+    <span class="k">def</span> <span class="nf">_deserialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
+        <span class="k">return</span> <span class="nb">int</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
+
+<span class="k">class</span> <span class="nc">Byte</span><span class="p">(</span><span class="n">Integer</span><span class="p">):</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;byte&#39;</span>
+
+<span class="k">class</span> <span class="nc">Short</span><span class="p">(</span><span class="n">Integer</span><span class="p">):</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;short&#39;</span>
+
+<span class="k">class</span> <span class="nc">Long</span><span class="p">(</span><span class="n">Integer</span><span class="p">):</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;long&#39;</span>
+
+<span class="k">class</span> <span class="nc">Ip</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;ip&#39;</span>
+    <span class="n">_coerce</span> <span class="o">=</span> <span class="kc">True</span>
+
+    <span class="k">def</span> <span class="nf">_deserialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
+        <span class="c1"># the ipaddress library for pypy, python2.5 and 2.6 only accepts unicode.</span>
+        <span class="k">return</span> <span class="n">ipaddress</span><span class="o">.</span><span class="n">ip_address</span><span class="p">(</span><span class="n">unicode</span><span class="p">(</span><span class="n">data</span><span class="p">))</span>
+
+    <span class="k">def</span> <span class="nf">_serialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
+        <span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">return</span> <span class="kc">None</span>
+        <span class="k">return</span> <span class="nb">str</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
+
+<span class="k">class</span> <span class="nc">Binary</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;binary&#39;</span>
+    <span class="n">_coerce</span> <span class="o">=</span> <span class="kc">True</span>
+
+    <span class="k">def</span> <span class="nf">_deserialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
+        <span class="k">return</span> <span class="n">base64</span><span class="o">.</span><span class="n">b64decode</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="nf">_serialize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
+        <span class="k">if</span> <span class="n">data</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">return</span> <span class="kc">None</span>
+        <span class="k">return</span> <span class="n">base64</span><span class="o">.</span><span class="n">b64encode</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
+
+<span class="k">class</span> <span class="nc">GeoPoint</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;geo_point&#39;</span>
+
+<span class="k">class</span> <span class="nc">GeoShape</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;geo_shape&#39;</span>
+
+<span class="k">class</span> <span class="nc">Completion</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;completion&#39;</span>
+
+<span class="k">class</span> <span class="nc">Percolator</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;percolator&#39;</span>
+
+<span class="k">class</span> <span class="nc">IntegerRange</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;integer_range&#39;</span>
+
+<span class="k">class</span> <span class="nc">FloatRange</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;float_range&#39;</span>
+
+<span class="k">class</span> <span class="nc">LongRange</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;long_range&#39;</span>
+
+<span class="k">class</span> <span class="nc">DoubleRange</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;double_ranged&#39;</span>
+
+<span class="k">class</span> <span class="nc">DateRange</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;date_range&#39;</span>
+
+<span class="k">class</span> <span class="nc">Join</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;join&#39;</span>
+
+<span class="k">class</span> <span class="nc">TokenCount</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;token_count&#39;</span>
+
+<span class="k">class</span> <span class="nc">Murmur3</span><span class="p">(</span><span class="n">Field</span><span class="p">):</span>
+    <span class="n">name</span> <span class="o">=</span> <span class="s1">&#39;murmur3&#39;</span>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+        &copy; Copyright 2018, Sean Whalen.
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../',
+            VERSION:'3.3.1',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>

_modules/index.html

@@ -0,0 +1,116 @@
+
+
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" data-content_root="../">
+<head>
+  <meta charset="utf-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>Overview: module code &mdash; parsedmarc 9.0.10 documentation</title>
+      <link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=b86133f3" />
+      <link rel="stylesheet" type="text/css" href="../_static/css/theme.css?v=9edc463e" />
+
+  
+      <script src="../_static/jquery.js?v=5d32c60e"></script>
+      <script src="../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
+      <script src="../_static/documentation_options.js?v=164cc7e6"></script>
+      <script src="../_static/doctools.js?v=fd6eb6e6"></script>
+      <script src="../_static/sphinx_highlight.js?v=6ffebe34"></script>
+    <script src="../_static/js/theme.js"></script>
+    <link rel="index" title="Index" href="../genindex.html" />
+    <link rel="search" title="Search" href="../search.html" /> 
+</head>
+
+<body class="wy-body-for-nav"> 
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search" >
+
+          
+          
+          <a href="../index.html" class="icon icon-home">
+            parsedmarc
+          </a>
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+        </div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <p class="caption" role="heading"><span class="caption-text">Contents</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../usage.html">Using parsedmarc</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../output.html">Sample outputs</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../elasticsearch.html">Elasticsearch and Kibana</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../opensearch.html">OpenSearch and Grafana</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../kibana.html">Using the Kibana dashboards</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../splunk.html">Splunk</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../davmail.html">Accessing an inbox using OWA/EWS</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../dmarc.html">Understanding DMARC</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../contributing.html">Contributing to parsedmarc</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../api.html">API reference</a></li>
+</ul>
+
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../index.html">parsedmarc</a>
+      </nav>
+
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          <div role="navigation" aria-label="Page navigation">
+  <ul class="wy-breadcrumbs">
+      <li><a href="../index.html" class="icon icon-home" aria-label="Home"></a></li>
+      <li class="breadcrumb-item active">Overview: module code</li>
+      <li class="wy-breadcrumbs-aside">
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+             
+  <h1>All modules for which code is available</h1>
+<ul><li><a href="parsedmarc.html">parsedmarc</a></li>
+<ul><li><a href="parsedmarc/elastic.html">parsedmarc.elastic</a></li>
+<li><a href="parsedmarc/opensearch.html">parsedmarc.opensearch</a></li>
+<li><a href="parsedmarc/splunk.html">parsedmarc.splunk</a></li>
+<li><a href="parsedmarc/types.html">parsedmarc.types</a></li>
+<li><a href="parsedmarc/utils.html">parsedmarc.utils</a></li>
+</ul></ul>
+
+           </div>
+          </div>
+          <footer>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>&#169; Copyright 2018 - 2025, Sean Whalen and contributors.</p>
+  </div>
+
+  Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
+    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
+    provided by <a href="https://readthedocs.org">Read the Docs</a>.
+   
+
+</footer>
+        </div>
+      </div>
+    </section>
+  </div>
+  <script>
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>

_modules/parsedmarc/elastic.html

@@ -0,0 +1,989 @@
+
+
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" data-content_root="../../">
+<head>
+  <meta charset="utf-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>parsedmarc.elastic &mdash; parsedmarc 9.0.10 documentation</title>
+      <link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
+      <link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=9edc463e" />
+
+  
+      <script src="../../_static/jquery.js?v=5d32c60e"></script>
+      <script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
+      <script src="../../_static/documentation_options.js?v=164cc7e6"></script>
+      <script src="../../_static/doctools.js?v=fd6eb6e6"></script>
+      <script src="../../_static/sphinx_highlight.js?v=6ffebe34"></script>
+    <script src="../../_static/js/theme.js"></script>
+    <link rel="index" title="Index" href="../../genindex.html" />
+    <link rel="search" title="Search" href="../../search.html" /> 
+</head>
+
+<body class="wy-body-for-nav"> 
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search" >
+
+          
+          
+          <a href="../../index.html" class="icon icon-home">
+            parsedmarc
+          </a>
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+        </div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <p class="caption" role="heading"><span class="caption-text">Contents</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../usage.html">Using parsedmarc</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../output.html">Sample outputs</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../elasticsearch.html">Elasticsearch and Kibana</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../opensearch.html">OpenSearch and Grafana</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../kibana.html">Using the Kibana dashboards</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../splunk.html">Splunk</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../davmail.html">Accessing an inbox using OWA/EWS</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../dmarc.html">Understanding DMARC</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../contributing.html">Contributing to parsedmarc</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../api.html">API reference</a></li>
+</ul>
+
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../index.html">parsedmarc</a>
+      </nav>
+
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          <div role="navigation" aria-label="Page navigation">
+  <ul class="wy-breadcrumbs">
+      <li><a href="../../index.html" class="icon icon-home" aria-label="Home"></a></li>
+          <li class="breadcrumb-item"><a href="../index.html">Module code</a></li>
+          <li class="breadcrumb-item"><a href="../parsedmarc.html">parsedmarc</a></li>
+      <li class="breadcrumb-item active">parsedmarc.elastic</li>
+      <li class="wy-breadcrumbs-aside">
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+             
+  <h1>Source code for parsedmarc.elastic</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+
+<span class="kn">from</span><span class="w"> </span><span class="nn">__future__</span><span class="w"> </span><span class="kn">import</span> <span class="n">annotations</span>
+
+<span class="kn">from</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="kn">import</span> <span class="n">Any</span><span class="p">,</span> <span class="n">Optional</span><span class="p">,</span> <span class="n">Union</span>
+
+<span class="kn">from</span><span class="w"> </span><span class="nn">elasticsearch.helpers</span><span class="w"> </span><span class="kn">import</span> <span class="n">reindex</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">elasticsearch_dsl</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
+    <span class="n">Boolean</span><span class="p">,</span>
+    <span class="n">Date</span><span class="p">,</span>
+    <span class="n">Document</span><span class="p">,</span>
+    <span class="n">Index</span><span class="p">,</span>
+    <span class="n">InnerDoc</span><span class="p">,</span>
+    <span class="n">Integer</span><span class="p">,</span>
+    <span class="n">Ip</span><span class="p">,</span>
+    <span class="n">Nested</span><span class="p">,</span>
+    <span class="n">Object</span><span class="p">,</span>
+    <span class="n">Search</span><span class="p">,</span>
+    <span class="n">Text</span><span class="p">,</span>
+    <span class="n">connections</span><span class="p">,</span>
+<span class="p">)</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">elasticsearch_dsl.search</span><span class="w"> </span><span class="kn">import</span> <span class="n">Q</span>
+
+<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc</span><span class="w"> </span><span class="kn">import</span> <span class="n">InvalidForensicReport</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.log</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">human_timestamp_to_datetime</span>
+
+
+<div class="viewcode-block" id="ElasticsearchError">
+<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.ElasticsearchError">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">ElasticsearchError</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Raised when an Elasticsearch error occurs&quot;&quot;&quot;</span></div>
+
+
+
+<span class="k">class</span><span class="w"> </span><span class="nc">_PolicyOverride</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
+    <span class="nb">type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">comment</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+
+
+<span class="k">class</span><span class="w"> </span><span class="nc">_PublishedPolicy</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
+    <span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">adkim</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">aspf</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">p</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">sp</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">pct</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
+    <span class="n">fo</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+
+
+<span class="k">class</span><span class="w"> </span><span class="nc">_DKIMResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
+    <span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">selector</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">result</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+
+
+<span class="k">class</span><span class="w"> </span><span class="nc">_SPFResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
+    <span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">scope</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">results</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+
+
+<span class="k">class</span><span class="w"> </span><span class="nc">_AggregateReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
+    <span class="k">class</span><span class="w"> </span><span class="nc">Index</span><span class="p">:</span>
+        <span class="n">name</span> <span class="o">=</span> <span class="s2">&quot;dmarc_aggregate&quot;</span>
+
+    <span class="n">xml_schema</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">org_name</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">org_email</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">org_extra_contact_info</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">report_id</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">date_range</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
+    <span class="n">date_begin</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
+    <span class="n">date_end</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
+    <span class="n">normalized_timespan</span> <span class="o">=</span> <span class="n">Boolean</span><span class="p">()</span>
+    <span class="n">original_timespan_seconds</span> <span class="o">=</span> <span class="n">Integer</span>
+    <span class="n">errors</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">published_policy</span> <span class="o">=</span> <span class="n">Object</span><span class="p">(</span><span class="n">_PublishedPolicy</span><span class="p">)</span>
+    <span class="n">source_ip_address</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
+    <span class="n">source_country</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">source_reverse_dns</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">source_base_domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">source_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">source_name</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">message_count</span> <span class="o">=</span> <span class="n">Integer</span>
+    <span class="n">disposition</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">dkim_aligned</span> <span class="o">=</span> <span class="n">Boolean</span><span class="p">()</span>
+    <span class="n">spf_aligned</span> <span class="o">=</span> <span class="n">Boolean</span><span class="p">()</span>
+    <span class="n">passed_dmarc</span> <span class="o">=</span> <span class="n">Boolean</span><span class="p">()</span>
+    <span class="n">policy_overrides</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_PolicyOverride</span><span class="p">)</span>
+    <span class="n">header_from</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">envelope_from</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">envelope_to</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">dkim_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_DKIMResult</span><span class="p">)</span>
+    <span class="n">spf_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">)</span>
+
+    <span class="k">def</span><span class="w"> </span><span class="nf">add_policy_override</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">type_</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">comment</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">policy_overrides</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_PolicyOverride</span><span class="p">(</span><span class="nb">type</span><span class="o">=</span><span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="o">=</span><span class="n">comment</span><span class="p">))</span>  <span class="c1"># pyright: ignore[reportCallIssue]</span>
+
+    <span class="k">def</span><span class="w"> </span><span class="nf">add_dkim_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">selector</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">result</span><span class="p">:</span> <span class="n">_DKIMResult</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">dkim_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+            <span class="n">_DKIMResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="o">=</span><span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">)</span>
+        <span class="p">)</span>  <span class="c1"># pyright: ignore[reportCallIssue]</span>
+
+    <span class="k">def</span><span class="w"> </span><span class="nf">add_spf_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">scope</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">result</span><span class="p">:</span> <span class="n">_SPFResult</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">spf_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="o">=</span><span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">))</span>  <span class="c1"># pyright: ignore[reportCallIssue]</span>
+
+    <span class="k">def</span><span class="w"> </span><span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>  <span class="c1"># pyright: ignore[reportIncompatibleMethodOverride]</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="kc">False</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">spf_aligned</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">dkim_aligned</span>
+
+        <span class="k">return</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+
+<span class="k">class</span><span class="w"> </span><span class="nc">_EmailAddressDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
+    <span class="n">display_name</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">address</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+
+
+<span class="k">class</span><span class="w"> </span><span class="nc">_EmailAttachmentDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
+    <span class="n">filename</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">content_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">sha256</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+
+
+<span class="k">class</span><span class="w"> </span><span class="nc">_ForensicSampleDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
+    <span class="n">raw</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">headers</span> <span class="o">=</span> <span class="n">Object</span><span class="p">()</span>
+    <span class="n">headers_only</span> <span class="o">=</span> <span class="n">Boolean</span><span class="p">()</span>
+    <span class="n">to</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">)</span>
+    <span class="n">subject</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">filename_safe_subject</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">_from</span> <span class="o">=</span> <span class="n">Object</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">)</span>
+    <span class="n">date</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
+    <span class="n">reply_to</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">)</span>
+    <span class="n">cc</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">)</span>
+    <span class="n">bcc</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">)</span>
+    <span class="n">body</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">attachments</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAttachmentDoc</span><span class="p">)</span>
+
+    <span class="k">def</span><span class="w"> </span><span class="nf">add_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">address</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>  <span class="c1"># pyright: ignore[reportCallIssue]</span>
+
+    <span class="k">def</span><span class="w"> </span><span class="nf">add_reply_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">address</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">reply_to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+            <span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">)</span>
+        <span class="p">)</span>  <span class="c1"># pyright: ignore[reportCallIssue]</span>
+
+    <span class="k">def</span><span class="w"> </span><span class="nf">add_cc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">address</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>  <span class="c1"># pyright: ignore[reportCallIssue]</span>
+
+    <span class="k">def</span><span class="w"> </span><span class="nf">add_bcc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">address</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">bcc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>  <span class="c1"># pyright: ignore[reportCallIssue]</span>
+
+    <span class="k">def</span><span class="w"> </span><span class="nf">add_attachment</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filename</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">content_type</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">sha256</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">attachments</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+            <span class="n">_EmailAttachmentDoc</span><span class="p">(</span>
+                <span class="n">filename</span><span class="o">=</span><span class="n">filename</span><span class="p">,</span> <span class="n">content_type</span><span class="o">=</span><span class="n">content_type</span><span class="p">,</span> <span class="n">sha256</span><span class="o">=</span><span class="n">sha256</span>
+            <span class="p">)</span>
+        <span class="p">)</span>  <span class="c1"># pyright: ignore[reportCallIssue]</span>
+
+
+<span class="k">class</span><span class="w"> </span><span class="nc">_ForensicReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
+    <span class="k">class</span><span class="w"> </span><span class="nc">Index</span><span class="p">:</span>
+        <span class="n">name</span> <span class="o">=</span> <span class="s2">&quot;dmarc_forensic&quot;</span>
+
+    <span class="n">feedback_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">user_agent</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">version</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">original_mail_from</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">arrival_date</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
+    <span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">original_envelope_id</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">authentication_results</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">delivery_results</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">source_ip_address</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
+    <span class="n">source_country</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">source_reverse_dns</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">source_authentication_mechanisms</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">source_auth_failures</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">dkim_domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">original_rcpt_to</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">sample</span> <span class="o">=</span> <span class="n">Object</span><span class="p">(</span><span class="n">_ForensicSampleDoc</span><span class="p">)</span>
+
+
+<span class="k">class</span><span class="w"> </span><span class="nc">_SMTPTLSFailureDetailsDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
+    <span class="n">result_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">sending_mta_ip</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
+    <span class="n">receiving_mx_helo</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">receiving_ip</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
+    <span class="n">failed_session_count</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
+    <span class="n">additional_information_uri</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">failure_reason_code</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+
+
+<span class="k">class</span><span class="w"> </span><span class="nc">_SMTPTLSPolicyDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
+    <span class="n">policy_domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">policy_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">policy_strings</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">mx_host_patterns</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">successful_session_count</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
+    <span class="n">failed_session_count</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
+    <span class="n">failure_details</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSFailureDetailsDoc</span><span class="p">)</span>
+
+    <span class="k">def</span><span class="w"> </span><span class="nf">add_failure_details</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">result_type</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">ip_address</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">receiving_ip</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">receiving_mx_helo</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">failed_session_count</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">sending_mta_ip</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">receiving_mx_hostname</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">additional_information_uri</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">failure_reason_code</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="kc">None</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="n">_details</span> <span class="o">=</span> <span class="n">_SMTPTLSFailureDetailsDoc</span><span class="p">(</span>
+            <span class="n">result_type</span><span class="o">=</span><span class="n">result_type</span><span class="p">,</span>
+            <span class="n">ip_address</span><span class="o">=</span><span class="n">ip_address</span><span class="p">,</span>
+            <span class="n">sending_mta_ip</span><span class="o">=</span><span class="n">sending_mta_ip</span><span class="p">,</span>
+            <span class="n">receiving_mx_hostname</span><span class="o">=</span><span class="n">receiving_mx_hostname</span><span class="p">,</span>
+            <span class="n">receiving_mx_helo</span><span class="o">=</span><span class="n">receiving_mx_helo</span><span class="p">,</span>
+            <span class="n">receiving_ip</span><span class="o">=</span><span class="n">receiving_ip</span><span class="p">,</span>
+            <span class="n">failed_session_count</span><span class="o">=</span><span class="n">failed_session_count</span><span class="p">,</span>
+            <span class="n">additional_information</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
+            <span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span><span class="p">,</span>
+        <span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">failure_details</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_details</span><span class="p">)</span>  <span class="c1"># pyright: ignore[reportCallIssue]</span>
+
+
+<span class="k">class</span><span class="w"> </span><span class="nc">_SMTPTLSReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
+    <span class="k">class</span><span class="w"> </span><span class="nc">Index</span><span class="p">:</span>
+        <span class="n">name</span> <span class="o">=</span> <span class="s2">&quot;smtp_tls&quot;</span>
+
+    <span class="n">organization_name</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">date_range</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
+    <span class="n">date_begin</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
+    <span class="n">date_end</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
+    <span class="n">contact_info</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">report_id</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">policies</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSPolicyDoc</span><span class="p">)</span>
+
+    <span class="k">def</span><span class="w"> </span><span class="nf">add_policy</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">policy_type</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+        <span class="n">policy_domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+        <span class="n">successful_session_count</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+        <span class="n">failed_session_count</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+        <span class="o">*</span><span class="p">,</span>
+        <span class="n">policy_string</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mx_host_patterns</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">failure_details</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+            <span class="n">policy_type</span><span class="o">=</span><span class="n">policy_type</span><span class="p">,</span>
+            <span class="n">policy_domain</span><span class="o">=</span><span class="n">policy_domain</span><span class="p">,</span>
+            <span class="n">successful_session_count</span><span class="o">=</span><span class="n">successful_session_count</span><span class="p">,</span>
+            <span class="n">failed_session_count</span><span class="o">=</span><span class="n">failed_session_count</span><span class="p">,</span>
+            <span class="n">policy_string</span><span class="o">=</span><span class="n">policy_string</span><span class="p">,</span>
+            <span class="n">mx_host_patterns</span><span class="o">=</span><span class="n">mx_host_patterns</span><span class="p">,</span>
+            <span class="n">failure_details</span><span class="o">=</span><span class="n">failure_details</span><span class="p">,</span>
+        <span class="p">)</span>  <span class="c1"># pyright: ignore[reportCallIssue]</span>
+
+
+<div class="viewcode-block" id="AlreadySaved">
+<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.AlreadySaved">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">AlreadySaved</span><span class="p">(</span><span class="ne">ValueError</span><span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Raised when a report to be saved matches an existing report&quot;&quot;&quot;</span></div>
+
+
+
+<div class="viewcode-block" id="set_hosts">
+<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.set_hosts">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">set_hosts</span><span class="p">(</span>
+    <span class="n">hosts</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]],</span>
+    <span class="o">*</span><span class="p">,</span>
+    <span class="n">use_ssl</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="n">ssl_cert_path</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">username</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">password</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">api_key</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">timeout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">60.0</span><span class="p">,</span>
+<span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Sets the Elasticsearch hosts to use</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        hosts (str | list[str]): A single hostname or URL, or list of hostnames or URLs</span>
+<span class="sd">        use_ssl (bool): Use an HTTPS connection to the server</span>
+<span class="sd">        ssl_cert_path (str): Path to the certificate chain</span>
+<span class="sd">        username (str): The username to use for authentication</span>
+<span class="sd">        password (str): The password to use for authentication</span>
+<span class="sd">        api_key (str): The Base64 encoded API key to use for authentication</span>
+<span class="sd">        timeout (float): Timeout in seconds</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">hosts</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+        <span class="n">hosts</span> <span class="o">=</span> <span class="p">[</span><span class="n">hosts</span><span class="p">]</span>
+    <span class="n">conn_params</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;hosts&quot;</span><span class="p">:</span> <span class="n">hosts</span><span class="p">,</span> <span class="s2">&quot;timeout&quot;</span><span class="p">:</span> <span class="n">timeout</span><span class="p">}</span>
+    <span class="k">if</span> <span class="n">use_ssl</span><span class="p">:</span>
+        <span class="n">conn_params</span><span class="p">[</span><span class="s2">&quot;use_ssl&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
+        <span class="k">if</span> <span class="n">ssl_cert_path</span><span class="p">:</span>
+            <span class="n">conn_params</span><span class="p">[</span><span class="s2">&quot;verify_certs&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
+            <span class="n">conn_params</span><span class="p">[</span><span class="s2">&quot;ca_certs&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">ssl_cert_path</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">conn_params</span><span class="p">[</span><span class="s2">&quot;verify_certs&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
+    <span class="k">if</span> <span class="n">username</span> <span class="ow">and</span> <span class="n">password</span><span class="p">:</span>
+        <span class="n">conn_params</span><span class="p">[</span><span class="s2">&quot;http_auth&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">username</span> <span class="o">+</span> <span class="s2">&quot;:&quot;</span> <span class="o">+</span> <span class="n">password</span>
+    <span class="k">if</span> <span class="n">api_key</span><span class="p">:</span>
+        <span class="n">conn_params</span><span class="p">[</span><span class="s2">&quot;api_key&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">api_key</span>
+    <span class="n">connections</span><span class="o">.</span><span class="n">create_connection</span><span class="p">(</span><span class="o">**</span><span class="n">conn_params</span><span class="p">)</span></div>
+
+
+
+<div class="viewcode-block" id="create_indexes">
+<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.create_indexes">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">create_indexes</span><span class="p">(</span><span class="n">names</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">settings</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Create Elasticsearch indexes</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        names (list): A list of index names</span>
+<span class="sd">        settings (dict): Index settings</span>
+
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">names</span><span class="p">:</span>
+        <span class="n">index</span> <span class="o">=</span> <span class="n">Index</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="k">if</span> <span class="ow">not</span> <span class="n">index</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
+                <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Creating Elasticsearch index: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">name</span><span class="p">))</span>
+                <span class="k">if</span> <span class="n">settings</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+                    <span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="o">**</span><span class="n">settings</span><span class="p">)</span>
+                <span class="n">index</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
+        <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">&quot;Elasticsearch error: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
+
+
+
+<div class="viewcode-block" id="migrate_indexes">
+<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.migrate_indexes">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">migrate_indexes</span><span class="p">(</span>
+    <span class="n">aggregate_indexes</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">forensic_indexes</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+<span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Updates index mappings</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        aggregate_indexes (list): A list of aggregate index names</span>
+<span class="sd">        forensic_indexes (list): A list of forensic index names</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">version</span> <span class="o">=</span> <span class="mi">2</span>
+    <span class="k">if</span> <span class="n">aggregate_indexes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">aggregate_indexes</span> <span class="o">=</span> <span class="p">[]</span>
+    <span class="k">if</span> <span class="n">forensic_indexes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">forensic_indexes</span> <span class="o">=</span> <span class="p">[]</span>
+    <span class="k">for</span> <span class="n">aggregate_index_name</span> <span class="ow">in</span> <span class="n">aggregate_indexes</span><span class="p">:</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
+            <span class="k">continue</span>
+        <span class="n">aggregate_index</span> <span class="o">=</span> <span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span>
+        <span class="n">doc</span> <span class="o">=</span> <span class="s2">&quot;doc&quot;</span>
+        <span class="n">fo_field</span> <span class="o">=</span> <span class="s2">&quot;published_policy.fo&quot;</span>
+        <span class="n">fo</span> <span class="o">=</span> <span class="s2">&quot;fo&quot;</span>
+        <span class="n">fo_mapping</span> <span class="o">=</span> <span class="n">aggregate_index</span><span class="o">.</span><span class="n">get_field_mapping</span><span class="p">(</span><span class="n">fields</span><span class="o">=</span><span class="p">[</span><span class="n">fo_field</span><span class="p">])</span>
+        <span class="n">fo_mapping</span> <span class="o">=</span> <span class="n">fo_mapping</span><span class="p">[</span><span class="nb">list</span><span class="p">(</span><span class="n">fo_mapping</span><span class="o">.</span><span class="n">keys</span><span class="p">())[</span><span class="mi">0</span><span class="p">]][</span><span class="s2">&quot;mappings&quot;</span><span class="p">]</span>
+        <span class="k">if</span> <span class="n">doc</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">fo_mapping</span><span class="p">:</span>
+            <span class="k">continue</span>
+
+        <span class="n">fo_mapping</span> <span class="o">=</span> <span class="n">fo_mapping</span><span class="p">[</span><span class="n">doc</span><span class="p">][</span><span class="n">fo_field</span><span class="p">][</span><span class="s2">&quot;mapping&quot;</span><span class="p">][</span><span class="n">fo</span><span class="p">]</span>
+        <span class="n">fo_type</span> <span class="o">=</span> <span class="n">fo_mapping</span><span class="p">[</span><span class="s2">&quot;type&quot;</span><span class="p">]</span>
+        <span class="k">if</span> <span class="n">fo_type</span> <span class="o">==</span> <span class="s2">&quot;long&quot;</span><span class="p">:</span>
+            <span class="n">new_index_name</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">-v</span><span class="si">{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">,</span> <span class="n">version</span><span class="p">)</span>
+            <span class="n">body</span> <span class="o">=</span> <span class="p">{</span>
+                <span class="s2">&quot;properties&quot;</span><span class="p">:</span> <span class="p">{</span>
+                    <span class="s2">&quot;published_policy.fo&quot;</span><span class="p">:</span> <span class="p">{</span>
+                        <span class="s2">&quot;type&quot;</span><span class="p">:</span> <span class="s2">&quot;text&quot;</span><span class="p">,</span>
+                        <span class="s2">&quot;fields&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;keyword&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;type&quot;</span><span class="p">:</span> <span class="s2">&quot;keyword&quot;</span><span class="p">,</span> <span class="s2">&quot;ignore_above&quot;</span><span class="p">:</span> <span class="mi">256</span><span class="p">}},</span>
+                    <span class="p">}</span>
+                <span class="p">}</span>
+            <span class="p">}</span>
+            <span class="n">Index</span><span class="p">(</span><span class="n">new_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
+            <span class="n">Index</span><span class="p">(</span><span class="n">new_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">put_mapping</span><span class="p">(</span><span class="n">doc_type</span><span class="o">=</span><span class="n">doc</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span>
+            <span class="n">reindex</span><span class="p">(</span><span class="n">connections</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(),</span> <span class="n">aggregate_index_name</span><span class="p">,</span> <span class="n">new_index_name</span><span class="p">)</span>  <span class="c1"># pyright: ignore[reportArgumentType]</span>
+            <span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">delete</span><span class="p">()</span>
+
+    <span class="k">for</span> <span class="n">forensic_index</span> <span class="ow">in</span> <span class="n">forensic_indexes</span><span class="p">:</span>
+        <span class="k">pass</span></div>
+
+
+
+<div class="viewcode-block" id="save_aggregate_report_to_elasticsearch">
+<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_aggregate_report_to_elasticsearch">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">save_aggregate_report_to_elasticsearch</span><span class="p">(</span>
+    <span class="n">aggregate_report</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
+    <span class="n">index_suffix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">index_prefix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">monthly_indexes</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="n">number_of_shards</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+    <span class="n">number_of_replicas</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
+<span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Saves a parsed DMARC aggregate report to Elasticsearch</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        aggregate_report (dict): A parsed forensic report</span>
+<span class="sd">        index_suffix (str): The suffix of the name of the index to save to</span>
+<span class="sd">        index_prefix (str): The prefix of the name of the index to save to</span>
+<span class="sd">        monthly_indexes (bool): Use monthly indexes instead of daily indexes</span>
+<span class="sd">        number_of_shards (int): The number of shards to use in the index</span>
+<span class="sd">        number_of_replicas (int): The number of replicas to use in the index</span>
+
+<span class="sd">    Raises:</span>
+<span class="sd">            AlreadySaved</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Saving aggregate report to Elasticsearch&quot;</span><span class="p">)</span>
+    <span class="n">aggregate_report</span> <span class="o">=</span> <span class="n">aggregate_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+    <span class="n">metadata</span> <span class="o">=</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;report_metadata&quot;</span><span class="p">]</span>
+    <span class="n">org_name</span> <span class="o">=</span> <span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;org_name&quot;</span><span class="p">]</span>
+    <span class="n">report_id</span> <span class="o">=</span> <span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;report_id&quot;</span><span class="p">]</span>
+    <span class="n">domain</span> <span class="o">=</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;policy_published&quot;</span><span class="p">][</span><span class="s2">&quot;domain&quot;</span><span class="p">]</span>
+    <span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+    <span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;end_date&quot;</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+
+    <span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
+        <span class="n">index_date</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m&quot;</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">index_date</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2">&quot;</span><span class="p">)</span>
+
+    <span class="n">org_name_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">org_name</span><span class="o">=</span><span class="n">org_name</span><span class="p">)))</span>  <span class="c1"># type: ignore</span>
+    <span class="n">report_id_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">report_id</span><span class="o">=</span><span class="n">report_id</span><span class="p">)))</span>  <span class="c1"># pyright: ignore[reportArgumentType]</span>
+    <span class="n">domain_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;published_policy.domain&quot;</span><span class="p">:</span> <span class="n">domain</span><span class="p">}))</span>  <span class="c1"># pyright: ignore[reportArgumentType]</span>
+    <span class="n">begin_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_begin</span><span class="o">=</span><span class="n">begin_date</span><span class="p">)))</span>  <span class="c1"># pyright: ignore[reportArgumentType]</span>
+    <span class="n">end_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_end</span><span class="o">=</span><span class="n">end_date</span><span class="p">)))</span>  <span class="c1"># pyright: ignore[reportArgumentType]</span>
+
+    <span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">search_index</span> <span class="o">=</span> <span class="s2">&quot;dmarc_aggregate_</span><span class="si">{0}</span><span class="s2">*&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">search_index</span> <span class="o">=</span> <span class="s2">&quot;dmarc_aggregate*&quot;</span>
+    <span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">search_index</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
+    <span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
+    <span class="n">query</span> <span class="o">=</span> <span class="n">org_name_query</span> <span class="o">&amp;</span> <span class="n">report_id_query</span> <span class="o">&amp;</span> <span class="n">domain_query</span>
+    <span class="n">query</span> <span class="o">=</span> <span class="n">query</span> <span class="o">&amp;</span> <span class="n">begin_date_query</span> <span class="o">&amp;</span> <span class="n">end_date_query</span>
+    <span class="n">search</span><span class="o">.</span><span class="n">query</span> <span class="o">=</span> <span class="n">query</span>
+    <span class="n">begin_date_human</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ&quot;</span><span class="p">)</span>
+    <span class="n">end_date_human</span> <span class="o">=</span> <span class="n">end_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ&quot;</span><span class="p">)</span>
+
+    <span class="k">try</span><span class="p">:</span>
+        <span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+    <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error_</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span>
+            <span class="s2">&quot;Elasticsearch&#39;s search for existing report </span><span class="se">\</span>
+<span class="s2">            error: </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
+        <span class="p">)</span>
+
+    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
+            <span class="s2">&quot;An aggregate report ID </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> about </span><span class="si">{2}</span><span class="s2"> &quot;</span>
+            <span class="s2">&quot;with a date range of </span><span class="si">{3}</span><span class="s2"> UTC to </span><span class="si">{4}</span><span class="s2"> UTC already &quot;</span>
+            <span class="s2">&quot;exists in &quot;</span>
+            <span class="s2">&quot;Elasticsearch&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                <span class="n">report_id</span><span class="p">,</span> <span class="n">org_name</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">begin_date_human</span><span class="p">,</span> <span class="n">end_date_human</span>
+            <span class="p">)</span>
+        <span class="p">)</span>
+    <span class="n">published_policy</span> <span class="o">=</span> <span class="n">_PublishedPolicy</span><span class="p">(</span>
+        <span class="n">domain</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;policy_published&quot;</span><span class="p">][</span><span class="s2">&quot;domain&quot;</span><span class="p">],</span>
+        <span class="n">adkim</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;policy_published&quot;</span><span class="p">][</span><span class="s2">&quot;adkim&quot;</span><span class="p">],</span>
+        <span class="n">aspf</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;policy_published&quot;</span><span class="p">][</span><span class="s2">&quot;aspf&quot;</span><span class="p">],</span>
+        <span class="n">p</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;policy_published&quot;</span><span class="p">][</span><span class="s2">&quot;p&quot;</span><span class="p">],</span>
+        <span class="n">sp</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;policy_published&quot;</span><span class="p">][</span><span class="s2">&quot;sp&quot;</span><span class="p">],</span>
+        <span class="n">pct</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;policy_published&quot;</span><span class="p">][</span><span class="s2">&quot;pct&quot;</span><span class="p">],</span>
+        <span class="n">fo</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;policy_published&quot;</span><span class="p">][</span><span class="s2">&quot;fo&quot;</span><span class="p">],</span>
+    <span class="p">)</span>
+
+    <span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;records&quot;</span><span class="p">]:</span>
+        <span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;interval_begin&quot;</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+        <span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;interval_end&quot;</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+        <span class="n">normalized_timespan</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;normalized_timespan&quot;</span><span class="p">]</span>
+
+        <span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
+            <span class="n">index_date</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m&quot;</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">index_date</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">begin_date</span>
+        <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;end_date&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">end_date</span>
+        <span class="n">date_range</span> <span class="o">=</span> <span class="p">[</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">],</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;end_date&quot;</span><span class="p">]]</span>
+        <span class="n">agg_doc</span> <span class="o">=</span> <span class="n">_AggregateReportDoc</span><span class="p">(</span>
+            <span class="n">xml_schema</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;xml_schema&quot;</span><span class="p">],</span>
+            <span class="n">org_name</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;org_name&quot;</span><span class="p">],</span>
+            <span class="n">org_email</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;org_email&quot;</span><span class="p">],</span>
+            <span class="n">org_extra_contact_info</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;org_extra_contact_info&quot;</span><span class="p">],</span>
+            <span class="n">report_id</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;report_id&quot;</span><span class="p">],</span>
+            <span class="n">date_range</span><span class="o">=</span><span class="n">date_range</span><span class="p">,</span>
+            <span class="n">date_begin</span><span class="o">=</span><span class="n">begin_date</span><span class="p">,</span>
+            <span class="n">date_end</span><span class="o">=</span><span class="n">end_date</span><span class="p">,</span>
+            <span class="n">normalized_timespan</span><span class="o">=</span><span class="n">normalized_timespan</span><span class="p">,</span>
+            <span class="n">errors</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;errors&quot;</span><span class="p">],</span>
+            <span class="n">published_policy</span><span class="o">=</span><span class="n">published_policy</span><span class="p">,</span>
+            <span class="n">source_ip_address</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;ip_address&quot;</span><span class="p">],</span>
+            <span class="n">source_country</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;country&quot;</span><span class="p">],</span>
+            <span class="n">source_reverse_dns</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;reverse_dns&quot;</span><span class="p">],</span>
+            <span class="n">source_base_domain</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;base_domain&quot;</span><span class="p">],</span>
+            <span class="n">source_type</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;type&quot;</span><span class="p">],</span>
+            <span class="n">source_name</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;name&quot;</span><span class="p">],</span>
+            <span class="n">message_count</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;count&quot;</span><span class="p">],</span>
+            <span class="n">disposition</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;policy_evaluated&quot;</span><span class="p">][</span><span class="s2">&quot;disposition&quot;</span><span class="p">],</span>
+            <span class="n">dkim_aligned</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;policy_evaluated&quot;</span><span class="p">][</span><span class="s2">&quot;dkim&quot;</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="ow">and</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;policy_evaluated&quot;</span><span class="p">][</span><span class="s2">&quot;dkim&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">&quot;pass&quot;</span><span class="p">,</span>
+            <span class="n">spf_aligned</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;policy_evaluated&quot;</span><span class="p">][</span><span class="s2">&quot;spf&quot;</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="ow">and</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;policy_evaluated&quot;</span><span class="p">][</span><span class="s2">&quot;spf&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">&quot;pass&quot;</span><span class="p">,</span>
+            <span class="n">header_from</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;identifiers&quot;</span><span class="p">][</span><span class="s2">&quot;header_from&quot;</span><span class="p">],</span>
+            <span class="n">envelope_from</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;identifiers&quot;</span><span class="p">][</span><span class="s2">&quot;envelope_from&quot;</span><span class="p">],</span>
+            <span class="n">envelope_to</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;identifiers&quot;</span><span class="p">][</span><span class="s2">&quot;envelope_to&quot;</span><span class="p">],</span>
+        <span class="p">)</span>
+
+        <span class="k">for</span> <span class="n">override</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;policy_evaluated&quot;</span><span class="p">][</span><span class="s2">&quot;policy_override_reasons&quot;</span><span class="p">]:</span>
+            <span class="n">agg_doc</span><span class="o">.</span><span class="n">add_policy_override</span><span class="p">(</span>
+                <span class="n">type_</span><span class="o">=</span><span class="n">override</span><span class="p">[</span><span class="s2">&quot;type&quot;</span><span class="p">],</span> <span class="n">comment</span><span class="o">=</span><span class="n">override</span><span class="p">[</span><span class="s2">&quot;comment&quot;</span><span class="p">]</span>
+            <span class="p">)</span>
+
+        <span class="k">for</span> <span class="n">dkim_result</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;auth_results&quot;</span><span class="p">][</span><span class="s2">&quot;dkim&quot;</span><span class="p">]:</span>
+            <span class="n">agg_doc</span><span class="o">.</span><span class="n">add_dkim_result</span><span class="p">(</span>
+                <span class="n">domain</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">&quot;domain&quot;</span><span class="p">],</span>
+                <span class="n">selector</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">&quot;selector&quot;</span><span class="p">],</span>
+                <span class="n">result</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">&quot;result&quot;</span><span class="p">],</span>
+            <span class="p">)</span>
+
+        <span class="k">for</span> <span class="n">spf_result</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;auth_results&quot;</span><span class="p">][</span><span class="s2">&quot;spf&quot;</span><span class="p">]:</span>
+            <span class="n">agg_doc</span><span class="o">.</span><span class="n">add_spf_result</span><span class="p">(</span>
+                <span class="n">domain</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">&quot;domain&quot;</span><span class="p">],</span>
+                <span class="n">scope</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">&quot;scope&quot;</span><span class="p">],</span>
+                <span class="n">result</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">&quot;result&quot;</span><span class="p">],</span>
+            <span class="p">)</span>
+
+        <span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;dmarc_aggregate&quot;</span>
+        <span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
+            <span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
+            <span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
+
+        <span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
+        <span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
+            <span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
+        <span class="p">)</span>
+        <span class="n">create_indexes</span><span class="p">([</span><span class="n">index</span><span class="p">],</span> <span class="n">index_settings</span><span class="p">)</span>
+        <span class="n">agg_doc</span><span class="o">.</span><span class="n">meta</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>  <span class="c1"># pyright: ignore[reportOptionalMemberAccess, reportAttributeAccessIssue]</span>
+
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">agg_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
+        <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">&quot;Elasticsearch error: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
+
+
+
+<div class="viewcode-block" id="save_forensic_report_to_elasticsearch">
+<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_forensic_report_to_elasticsearch">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">save_forensic_report_to_elasticsearch</span><span class="p">(</span>
+    <span class="n">forensic_report</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
+    <span class="n">index_suffix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">index_prefix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">monthly_indexes</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="n">number_of_shards</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+    <span class="n">number_of_replicas</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
+<span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Saves a parsed DMARC forensic report to Elasticsearch</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        forensic_report (dict): A parsed forensic report</span>
+<span class="sd">        index_suffix (str): The suffix of the name of the index to save to</span>
+<span class="sd">        index_prefix (str): The prefix of the name of the index to save to</span>
+<span class="sd">        monthly_indexes (bool): Use monthly indexes instead of daily</span>
+<span class="sd">                                indexes</span>
+<span class="sd">        number_of_shards (int): The number of shards to use in the index</span>
+<span class="sd">        number_of_replicas (int): The number of replicas to use in the</span>
+<span class="sd">                                  index</span>
+
+<span class="sd">    Raises:</span>
+<span class="sd">        AlreadySaved</span>
+
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Saving forensic report to Elasticsearch&quot;</span><span class="p">)</span>
+    <span class="n">forensic_report</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+    <span class="n">sample_date</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="k">if</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;date&quot;</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">sample_date</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;date&quot;</span><span class="p">]</span>
+        <span class="n">sample_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">sample_date</span><span class="p">)</span>
+    <span class="n">original_headers</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;headers&quot;</span><span class="p">]</span>
+    <span class="n">headers</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
+    <span class="k">for</span> <span class="n">original_header</span> <span class="ow">in</span> <span class="n">original_headers</span><span class="p">:</span>
+        <span class="n">headers</span><span class="p">[</span><span class="n">original_header</span><span class="o">.</span><span class="n">lower</span><span class="p">()]</span> <span class="o">=</span> <span class="n">original_headers</span><span class="p">[</span><span class="n">original_header</span><span class="p">]</span>
+
+    <span class="n">arrival_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;arrival_date_utc&quot;</span><span class="p">])</span>
+    <span class="n">arrival_date_epoch_milliseconds</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">.</span><span class="n">timestamp</span><span class="p">()</span> <span class="o">*</span> <span class="mi">1000</span><span class="p">)</span>
+
+    <span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">search_index</span> <span class="o">=</span> <span class="s2">&quot;dmarc_forensic_</span><span class="si">{0}</span><span class="s2">*&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">search_index</span> <span class="o">=</span> <span class="s2">&quot;dmarc_forensic*&quot;</span>
+    <span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">search_index</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
+    <span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
+    <span class="n">q</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date_epoch_milliseconds</span><span class="p">)))</span>  <span class="c1"># pyright: ignore[reportArgumentType]</span>
+
+    <span class="n">from_</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="n">to_</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="n">subject</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="k">if</span> <span class="s2">&quot;from&quot;</span> <span class="ow">in</span> <span class="n">headers</span><span class="p">:</span>
+        <span class="c1"># We convert the FROM header from a string list to a flat string.</span>
+        <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
+        <span class="k">if</span> <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s2">&quot;&quot;</span><span class="p">:</span>
+            <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">][</span><span class="mi">1</span><span class="p">]</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot; &lt;&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">headers</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">])</span> <span class="o">+</span> <span class="s2">&quot;&gt;&quot;</span>
+
+        <span class="n">from_</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
+        <span class="n">from_</span><span class="p">[</span><span class="s2">&quot;sample.headers.from&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">]</span>
+        <span class="n">from_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="n">from_</span><span class="p">))</span>  <span class="c1"># pyright: ignore[reportArgumentType]</span>
+        <span class="n">q</span> <span class="o">=</span> <span class="n">q</span> <span class="o">&amp;</span> <span class="n">from_query</span>
+    <span class="k">if</span> <span class="s2">&quot;to&quot;</span> <span class="ow">in</span> <span class="n">headers</span><span class="p">:</span>
+        <span class="c1"># We convert the TO header from a string list to a flat string.</span>
+        <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
+        <span class="k">if</span> <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s2">&quot;&quot;</span><span class="p">:</span>
+            <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">][</span><span class="mi">1</span><span class="p">]</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot; &lt;&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">headers</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">])</span> <span class="o">+</span> <span class="s2">&quot;&gt;&quot;</span>
+
+        <span class="n">to_</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
+        <span class="n">to_</span><span class="p">[</span><span class="s2">&quot;sample.headers.to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">]</span>
+        <span class="n">to_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="n">to_</span><span class="p">))</span>  <span class="c1"># pyright: ignore[reportArgumentType]</span>
+        <span class="n">q</span> <span class="o">=</span> <span class="n">q</span> <span class="o">&amp;</span> <span class="n">to_query</span>
+    <span class="k">if</span> <span class="s2">&quot;subject&quot;</span> <span class="ow">in</span> <span class="n">headers</span><span class="p">:</span>
+        <span class="n">subject</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;subject&quot;</span><span class="p">]</span>
+        <span class="n">subject_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;match_phrase&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;sample.headers.subject&quot;</span><span class="p">:</span> <span class="n">subject</span><span class="p">}}</span>
+        <span class="n">q</span> <span class="o">=</span> <span class="n">q</span> <span class="o">&amp;</span> <span class="n">Q</span><span class="p">(</span><span class="n">subject_query</span><span class="p">)</span>  <span class="c1"># pyright: ignore[reportArgumentType]</span>
+
+    <span class="n">search</span><span class="o">.</span><span class="n">query</span> <span class="o">=</span> <span class="n">q</span>
+    <span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+
+    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
+            <span class="s2">&quot;A forensic sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> &quot;</span>
+            <span class="s2">&quot;with a subject of </span><span class="si">{2}</span><span class="s2"> and arrival date of </span><span class="si">{3}</span><span class="s2"> &quot;</span>
+            <span class="s2">&quot;already exists in &quot;</span>
+            <span class="s2">&quot;Elasticsearch&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                <span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;arrival_date_utc&quot;</span><span class="p">]</span>
+            <span class="p">)</span>
+        <span class="p">)</span>
+
+    <span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">]</span>
+    <span class="n">sample</span> <span class="o">=</span> <span class="n">_ForensicSampleDoc</span><span class="p">(</span>
+        <span class="n">raw</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;sample&quot;</span><span class="p">],</span>
+        <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">,</span>
+        <span class="n">headers_only</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;sample_headers_only&quot;</span><span class="p">],</span>
+        <span class="n">date</span><span class="o">=</span><span class="n">sample_date</span><span class="p">,</span>
+        <span class="n">subject</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;subject&quot;</span><span class="p">],</span>
+        <span class="n">filename_safe_subject</span><span class="o">=</span><span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;filename_safe_subject&quot;</span><span class="p">],</span>
+        <span class="n">body</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;body&quot;</span><span class="p">],</span>
+    <span class="p">)</span>
+
+    <span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;to&quot;</span><span class="p">]:</span>
+        <span class="n">sample</span><span class="o">.</span><span class="n">add_to</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">&quot;display_name&quot;</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">&quot;address&quot;</span><span class="p">])</span>
+    <span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;reply_to&quot;</span><span class="p">]:</span>
+        <span class="n">sample</span><span class="o">.</span><span class="n">add_reply_to</span><span class="p">(</span>
+            <span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">&quot;display_name&quot;</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">&quot;address&quot;</span><span class="p">]</span>
+        <span class="p">)</span>
+    <span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;cc&quot;</span><span class="p">]:</span>
+        <span class="n">sample</span><span class="o">.</span><span class="n">add_cc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">&quot;display_name&quot;</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">&quot;address&quot;</span><span class="p">])</span>
+    <span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;bcc&quot;</span><span class="p">]:</span>
+        <span class="n">sample</span><span class="o">.</span><span class="n">add_bcc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">&quot;display_name&quot;</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">&quot;address&quot;</span><span class="p">])</span>
+    <span class="k">for</span> <span class="n">attachment</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;attachments&quot;</span><span class="p">]:</span>
+        <span class="n">sample</span><span class="o">.</span><span class="n">add_attachment</span><span class="p">(</span>
+            <span class="n">filename</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">&quot;filename&quot;</span><span class="p">],</span>
+            <span class="n">content_type</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">&quot;mail_content_type&quot;</span><span class="p">],</span>
+            <span class="n">sha256</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">&quot;sha256&quot;</span><span class="p">],</span>
+        <span class="p">)</span>
+    <span class="k">try</span><span class="p">:</span>
+        <span class="n">forensic_doc</span> <span class="o">=</span> <span class="n">_ForensicReportDoc</span><span class="p">(</span>
+            <span class="n">feedback_type</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;feedback_type&quot;</span><span class="p">],</span>
+            <span class="n">user_agent</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;user_agent&quot;</span><span class="p">],</span>
+            <span class="n">version</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;version&quot;</span><span class="p">],</span>
+            <span class="n">original_mail_from</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;original_mail_from&quot;</span><span class="p">],</span>
+            <span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date_epoch_milliseconds</span><span class="p">,</span>
+            <span class="n">domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;reported_domain&quot;</span><span class="p">],</span>
+            <span class="n">original_envelope_id</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;original_envelope_id&quot;</span><span class="p">],</span>
+            <span class="n">authentication_results</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;authentication_results&quot;</span><span class="p">],</span>
+            <span class="n">delivery_results</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;delivery_result&quot;</span><span class="p">],</span>
+            <span class="n">source_ip_address</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;ip_address&quot;</span><span class="p">],</span>
+            <span class="n">source_country</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;country&quot;</span><span class="p">],</span>
+            <span class="n">source_reverse_dns</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;reverse_dns&quot;</span><span class="p">],</span>
+            <span class="n">source_base_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;base_domain&quot;</span><span class="p">],</span>
+            <span class="n">authentication_mechanisms</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;authentication_mechanisms&quot;</span><span class="p">],</span>
+            <span class="n">auth_failure</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;auth_failure&quot;</span><span class="p">],</span>
+            <span class="n">dkim_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;dkim_domain&quot;</span><span class="p">],</span>
+            <span class="n">original_rcpt_to</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;original_rcpt_to&quot;</span><span class="p">],</span>
+            <span class="n">sample</span><span class="o">=</span><span class="n">sample</span><span class="p">,</span>
+        <span class="p">)</span>
+
+        <span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;dmarc_forensic&quot;</span>
+        <span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
+            <span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
+            <span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
+            <span class="n">index_date</span> <span class="o">=</span> <span class="n">arrival_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m&quot;</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">index_date</span> <span class="o">=</span> <span class="n">arrival_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
+        <span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
+            <span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
+        <span class="p">)</span>
+        <span class="n">create_indexes</span><span class="p">([</span><span class="n">index</span><span class="p">],</span> <span class="n">index_settings</span><span class="p">)</span>
+        <span class="n">forensic_doc</span><span class="o">.</span><span class="n">meta</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>  <span class="c1"># pyright: ignore[reportAttributeAccessIssue, reportOptionalMemberAccess]</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">forensic_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
+        <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">&quot;Elasticsearch error: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
+    <span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span>
+            <span class="s2">&quot;Forensic report missing required field: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
+        <span class="p">)</span></div>
+
+
+
+<div class="viewcode-block" id="save_smtp_tls_report_to_elasticsearch">
+<a class="viewcode-back" href="../../api.html#parsedmarc.elastic.save_smtp_tls_report_to_elasticsearch">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">save_smtp_tls_report_to_elasticsearch</span><span class="p">(</span>
+    <span class="n">report</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
+    <span class="n">index_suffix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">index_prefix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">monthly_indexes</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="n">number_of_shards</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+    <span class="n">number_of_replicas</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
+<span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Saves a parsed SMTP TLS report to Elasticsearch</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        report (dict): A parsed SMTP TLS report</span>
+<span class="sd">        index_suffix (str): The suffix of the name of the index to save to</span>
+<span class="sd">        index_prefix (str): The prefix of the name of the index to save to</span>
+<span class="sd">        monthly_indexes (bool): Use monthly indexes instead of daily indexes</span>
+<span class="sd">        number_of_shards (int): The number of shards to use in the index</span>
+<span class="sd">        number_of_replicas (int): The number of replicas to use in the index</span>
+
+<span class="sd">    Raises:</span>
+<span class="sd">            AlreadySaved</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Saving smtp tls report to Elasticsearch&quot;</span><span class="p">)</span>
+    <span class="n">org_name</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">&quot;organization_name&quot;</span><span class="p">]</span>
+    <span class="n">report_id</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">&quot;report_id&quot;</span><span class="p">]</span>
+    <span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+    <span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;end_date&quot;</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+    <span class="n">begin_date_human</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ&quot;</span><span class="p">)</span>
+    <span class="n">end_date_human</span> <span class="o">=</span> <span class="n">end_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ&quot;</span><span class="p">)</span>
+    <span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
+        <span class="n">index_date</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m&quot;</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">index_date</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2">&quot;</span><span class="p">)</span>
+    <span class="n">report</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">begin_date</span>
+    <span class="n">report</span><span class="p">[</span><span class="s2">&quot;end_date&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">end_date</span>
+
+    <span class="n">org_name_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">org_name</span><span class="o">=</span><span class="n">org_name</span><span class="p">)))</span>  <span class="c1"># pyright: ignore[reportArgumentType]</span>
+    <span class="n">report_id_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">report_id</span><span class="o">=</span><span class="n">report_id</span><span class="p">)))</span>  <span class="c1"># pyright: ignore[reportArgumentType]</span>
+    <span class="n">begin_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_begin</span><span class="o">=</span><span class="n">begin_date</span><span class="p">)))</span>  <span class="c1"># pyright: ignore[reportArgumentType]</span>
+    <span class="n">end_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_end</span><span class="o">=</span><span class="n">end_date</span><span class="p">)))</span>  <span class="c1"># pyright: ignore[reportArgumentType]</span>
+
+    <span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">search_index</span> <span class="o">=</span> <span class="s2">&quot;smtp_tls_</span><span class="si">{0}</span><span class="s2">*&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">search_index</span> <span class="o">=</span> <span class="s2">&quot;smtp_tls*&quot;</span>
+    <span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">search_index</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
+    <span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
+    <span class="n">query</span> <span class="o">=</span> <span class="n">org_name_query</span> <span class="o">&amp;</span> <span class="n">report_id_query</span>
+    <span class="n">query</span> <span class="o">=</span> <span class="n">query</span> <span class="o">&amp;</span> <span class="n">begin_date_query</span> <span class="o">&amp;</span> <span class="n">end_date_query</span>
+    <span class="n">search</span><span class="o">.</span><span class="n">query</span> <span class="o">=</span> <span class="n">query</span>
+
+    <span class="k">try</span><span class="p">:</span>
+        <span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+    <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error_</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span>
+            <span class="s2">&quot;Elasticsearch&#39;s search for existing report </span><span class="se">\</span>
+<span class="s2">            error: </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
+        <span class="p">)</span>
+
+    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
+            <span class="sa">f</span><span class="s2">&quot;An SMTP TLS report ID </span><span class="si">{</span><span class="n">report_id</span><span class="si">}</span><span class="s2"> from &quot;</span>
+            <span class="sa">f</span><span class="s2">&quot; </span><span class="si">{</span><span class="n">org_name</span><span class="si">}</span><span class="s2"> with a date range of &quot;</span>
+            <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">begin_date_human</span><span class="si">}</span><span class="s2"> UTC to &quot;</span>
+            <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">end_date_human</span><span class="si">}</span><span class="s2"> UTC already &quot;</span>
+            <span class="s2">&quot;exists in Elasticsearch&quot;</span>
+        <span class="p">)</span>
+
+    <span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;smtp_tls&quot;</span>
+    <span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
+        <span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
+    <span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
+        <span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
+    <span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
+    <span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
+        <span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
+    <span class="p">)</span>
+
+    <span class="n">smtp_tls_doc</span> <span class="o">=</span> <span class="n">_SMTPTLSReportDoc</span><span class="p">(</span>
+        <span class="n">org_name</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;organization_name&quot;</span><span class="p">],</span>
+        <span class="n">date_range</span><span class="o">=</span><span class="p">[</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">],</span> <span class="n">report</span><span class="p">[</span><span class="s2">&quot;end_date&quot;</span><span class="p">]],</span>
+        <span class="n">date_begin</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">],</span>
+        <span class="n">date_end</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;end_date&quot;</span><span class="p">],</span>
+        <span class="n">contact_info</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;contact_info&quot;</span><span class="p">],</span>
+        <span class="n">report_id</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;report_id&quot;</span><span class="p">],</span>
+    <span class="p">)</span>
+
+    <span class="k">for</span> <span class="n">policy</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">&quot;policies&quot;</span><span class="p">]:</span>
+        <span class="n">policy_strings</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="n">mx_host_patterns</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="k">if</span> <span class="s2">&quot;policy_strings&quot;</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
+            <span class="n">policy_strings</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">&quot;policy_strings&quot;</span><span class="p">]</span>
+        <span class="k">if</span> <span class="s2">&quot;mx_host_patterns&quot;</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
+            <span class="n">mx_host_patterns</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">&quot;mx_host_patterns&quot;</span><span class="p">]</span>
+        <span class="n">policy_doc</span> <span class="o">=</span> <span class="n">_SMTPTLSPolicyDoc</span><span class="p">(</span>
+            <span class="n">policy_domain</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">&quot;policy_domain&quot;</span><span class="p">],</span>
+            <span class="n">policy_type</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">&quot;policy_type&quot;</span><span class="p">],</span>
+            <span class="n">successful_session_count</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">&quot;successful_session_count&quot;</span><span class="p">],</span>
+            <span class="n">failed_session_count</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">&quot;failed_session_count&quot;</span><span class="p">],</span>
+            <span class="n">policy_string</span><span class="o">=</span><span class="n">policy_strings</span><span class="p">,</span>
+            <span class="n">mx_host_patterns</span><span class="o">=</span><span class="n">mx_host_patterns</span><span class="p">,</span>
+        <span class="p">)</span>
+        <span class="k">if</span> <span class="s2">&quot;failure_details&quot;</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
+            <span class="k">for</span> <span class="n">failure_detail</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">[</span><span class="s2">&quot;failure_details&quot;</span><span class="p">]:</span>
+                <span class="n">receiving_mx_hostname</span> <span class="o">=</span> <span class="kc">None</span>
+                <span class="n">additional_information_uri</span> <span class="o">=</span> <span class="kc">None</span>
+                <span class="n">failure_reason_code</span> <span class="o">=</span> <span class="kc">None</span>
+                <span class="n">ip_address</span> <span class="o">=</span> <span class="kc">None</span>
+                <span class="n">receiving_ip</span> <span class="o">=</span> <span class="kc">None</span>
+                <span class="n">receiving_mx_helo</span> <span class="o">=</span> <span class="kc">None</span>
+                <span class="n">sending_mta_ip</span> <span class="o">=</span> <span class="kc">None</span>
+
+                <span class="k">if</span> <span class="s2">&quot;receiving_mx_hostname&quot;</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
+                    <span class="n">receiving_mx_hostname</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">&quot;receiving_mx_hostname&quot;</span><span class="p">]</span>
+                <span class="k">if</span> <span class="s2">&quot;additional_information_uri&quot;</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
+                    <span class="n">additional_information_uri</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span>
+                        <span class="s2">&quot;additional_information_uri&quot;</span>
+                    <span class="p">]</span>
+                <span class="k">if</span> <span class="s2">&quot;failure_reason_code&quot;</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
+                    <span class="n">failure_reason_code</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">&quot;failure_reason_code&quot;</span><span class="p">]</span>
+                <span class="k">if</span> <span class="s2">&quot;ip_address&quot;</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
+                    <span class="n">ip_address</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">&quot;ip_address&quot;</span><span class="p">]</span>
+                <span class="k">if</span> <span class="s2">&quot;receiving_ip&quot;</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
+                    <span class="n">receiving_ip</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">&quot;receiving_ip&quot;</span><span class="p">]</span>
+                <span class="k">if</span> <span class="s2">&quot;receiving_mx_helo&quot;</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
+                    <span class="n">receiving_mx_helo</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">&quot;receiving_mx_helo&quot;</span><span class="p">]</span>
+                <span class="k">if</span> <span class="s2">&quot;sending_mta_ip&quot;</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
+                    <span class="n">sending_mta_ip</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">&quot;sending_mta_ip&quot;</span><span class="p">]</span>
+                <span class="n">policy_doc</span><span class="o">.</span><span class="n">add_failure_details</span><span class="p">(</span>
+                    <span class="n">result_type</span><span class="o">=</span><span class="n">failure_detail</span><span class="p">[</span><span class="s2">&quot;result_type&quot;</span><span class="p">],</span>
+                    <span class="n">ip_address</span><span class="o">=</span><span class="n">ip_address</span><span class="p">,</span>
+                    <span class="n">receiving_ip</span><span class="o">=</span><span class="n">receiving_ip</span><span class="p">,</span>
+                    <span class="n">receiving_mx_helo</span><span class="o">=</span><span class="n">receiving_mx_helo</span><span class="p">,</span>
+                    <span class="n">failed_session_count</span><span class="o">=</span><span class="n">failure_detail</span><span class="p">[</span><span class="s2">&quot;failed_session_count&quot;</span><span class="p">],</span>
+                    <span class="n">sending_mta_ip</span><span class="o">=</span><span class="n">sending_mta_ip</span><span class="p">,</span>
+                    <span class="n">receiving_mx_hostname</span><span class="o">=</span><span class="n">receiving_mx_hostname</span><span class="p">,</span>
+                    <span class="n">additional_information_uri</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
+                    <span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span><span class="p">,</span>
+                <span class="p">)</span>
+        <span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">policy_doc</span><span class="p">)</span>  <span class="c1"># pyright: ignore[reportCallIssue]</span>
+
+    <span class="n">create_indexes</span><span class="p">([</span><span class="n">index</span><span class="p">],</span> <span class="n">index_settings</span><span class="p">)</span>
+    <span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">meta</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>  <span class="c1"># pyright: ignore[reportOptionalMemberAccess, reportAttributeAccessIssue]</span>
+
+    <span class="k">try</span><span class="p">:</span>
+        <span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
+    <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="n">ElasticsearchError</span><span class="p">(</span><span class="s2">&quot;Elasticsearch error: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
+
+</pre></div>
+
+           </div>
+          </div>
+          <footer>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>&#169; Copyright 2018 - 2025, Sean Whalen and contributors.</p>
+  </div>
+
+  Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
+    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
+    provided by <a href="https://readthedocs.org">Read the Docs</a>.
+   
+
+</footer>
+        </div>
+      </div>
+    </section>
+  </div>
+  <script>
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>

_modules/parsedmarc/opensearch.html

@@ -0,0 +1,989 @@
+
+
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" data-content_root="../../">
+<head>
+  <meta charset="utf-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>parsedmarc.opensearch &mdash; parsedmarc 9.0.10 documentation</title>
+      <link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
+      <link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=9edc463e" />
+
+  
+      <script src="../../_static/jquery.js?v=5d32c60e"></script>
+      <script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
+      <script src="../../_static/documentation_options.js?v=164cc7e6"></script>
+      <script src="../../_static/doctools.js?v=fd6eb6e6"></script>
+      <script src="../../_static/sphinx_highlight.js?v=6ffebe34"></script>
+    <script src="../../_static/js/theme.js"></script>
+    <link rel="index" title="Index" href="../../genindex.html" />
+    <link rel="search" title="Search" href="../../search.html" /> 
+</head>
+
+<body class="wy-body-for-nav"> 
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search" >
+
+          
+          
+          <a href="../../index.html" class="icon icon-home">
+            parsedmarc
+          </a>
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+        </div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <p class="caption" role="heading"><span class="caption-text">Contents</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../usage.html">Using parsedmarc</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../output.html">Sample outputs</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../elasticsearch.html">Elasticsearch and Kibana</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../opensearch.html">OpenSearch and Grafana</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../kibana.html">Using the Kibana dashboards</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../splunk.html">Splunk</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../davmail.html">Accessing an inbox using OWA/EWS</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../dmarc.html">Understanding DMARC</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../contributing.html">Contributing to parsedmarc</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../api.html">API reference</a></li>
+</ul>
+
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../index.html">parsedmarc</a>
+      </nav>
+
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          <div role="navigation" aria-label="Page navigation">
+  <ul class="wy-breadcrumbs">
+      <li><a href="../../index.html" class="icon icon-home" aria-label="Home"></a></li>
+          <li class="breadcrumb-item"><a href="../index.html">Module code</a></li>
+          <li class="breadcrumb-item"><a href="../parsedmarc.html">parsedmarc</a></li>
+      <li class="breadcrumb-item active">parsedmarc.opensearch</li>
+      <li class="wy-breadcrumbs-aside">
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+             
+  <h1>Source code for parsedmarc.opensearch</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+
+<span class="kn">from</span><span class="w"> </span><span class="nn">__future__</span><span class="w"> </span><span class="kn">import</span> <span class="n">annotations</span>
+
+<span class="kn">from</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="kn">import</span> <span class="n">Any</span><span class="p">,</span> <span class="n">Optional</span><span class="p">,</span> <span class="n">Union</span>
+
+<span class="kn">from</span><span class="w"> </span><span class="nn">opensearchpy</span><span class="w"> </span><span class="kn">import</span> <span class="p">(</span>
+    <span class="n">Boolean</span><span class="p">,</span>
+    <span class="n">Date</span><span class="p">,</span>
+    <span class="n">Document</span><span class="p">,</span>
+    <span class="n">Index</span><span class="p">,</span>
+    <span class="n">InnerDoc</span><span class="p">,</span>
+    <span class="n">Integer</span><span class="p">,</span>
+    <span class="n">Ip</span><span class="p">,</span>
+    <span class="n">Nested</span><span class="p">,</span>
+    <span class="n">Object</span><span class="p">,</span>
+    <span class="n">Q</span><span class="p">,</span>
+    <span class="n">Search</span><span class="p">,</span>
+    <span class="n">Text</span><span class="p">,</span>
+    <span class="n">connections</span><span class="p">,</span>
+<span class="p">)</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">opensearchpy.helpers</span><span class="w"> </span><span class="kn">import</span> <span class="n">reindex</span>
+
+<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc</span><span class="w"> </span><span class="kn">import</span> <span class="n">InvalidForensicReport</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.log</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">human_timestamp_to_datetime</span>
+
+
+<div class="viewcode-block" id="OpenSearchError">
+<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.OpenSearchError">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">OpenSearchError</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Raised when an OpenSearch error occurs&quot;&quot;&quot;</span></div>
+
+
+
+<span class="k">class</span><span class="w"> </span><span class="nc">_PolicyOverride</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
+    <span class="nb">type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">comment</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+
+
+<span class="k">class</span><span class="w"> </span><span class="nc">_PublishedPolicy</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
+    <span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">adkim</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">aspf</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">p</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">sp</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">pct</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
+    <span class="n">fo</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+
+
+<span class="k">class</span><span class="w"> </span><span class="nc">_DKIMResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
+    <span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">selector</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">result</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+
+
+<span class="k">class</span><span class="w"> </span><span class="nc">_SPFResult</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
+    <span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">scope</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">results</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+
+
+<span class="k">class</span><span class="w"> </span><span class="nc">_AggregateReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
+    <span class="k">class</span><span class="w"> </span><span class="nc">Index</span><span class="p">:</span>
+        <span class="n">name</span> <span class="o">=</span> <span class="s2">&quot;dmarc_aggregate&quot;</span>
+
+    <span class="n">xml_schema</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">org_name</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">org_email</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">org_extra_contact_info</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">report_id</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">date_range</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
+    <span class="n">date_begin</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
+    <span class="n">date_end</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
+    <span class="n">normalized_timespan</span> <span class="o">=</span> <span class="n">Boolean</span><span class="p">()</span>
+    <span class="n">original_timespan_seconds</span> <span class="o">=</span> <span class="n">Integer</span>
+    <span class="n">errors</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">published_policy</span> <span class="o">=</span> <span class="n">Object</span><span class="p">(</span><span class="n">_PublishedPolicy</span><span class="p">)</span>
+    <span class="n">source_ip_address</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
+    <span class="n">source_country</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">source_reverse_dns</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">source_base_domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">source_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">source_name</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">message_count</span> <span class="o">=</span> <span class="n">Integer</span>
+    <span class="n">disposition</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">dkim_aligned</span> <span class="o">=</span> <span class="n">Boolean</span><span class="p">()</span>
+    <span class="n">spf_aligned</span> <span class="o">=</span> <span class="n">Boolean</span><span class="p">()</span>
+    <span class="n">passed_dmarc</span> <span class="o">=</span> <span class="n">Boolean</span><span class="p">()</span>
+    <span class="n">policy_overrides</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_PolicyOverride</span><span class="p">)</span>
+    <span class="n">header_from</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">envelope_from</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">envelope_to</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">dkim_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_DKIMResult</span><span class="p">)</span>
+    <span class="n">spf_results</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">)</span>
+
+    <span class="k">def</span><span class="w"> </span><span class="nf">add_policy_override</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">type_</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">comment</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">policy_overrides</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_PolicyOverride</span><span class="p">(</span><span class="nb">type</span><span class="o">=</span><span class="n">type_</span><span class="p">,</span> <span class="n">comment</span><span class="o">=</span><span class="n">comment</span><span class="p">))</span>
+
+    <span class="k">def</span><span class="w"> </span><span class="nf">add_dkim_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">selector</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">result</span><span class="p">:</span> <span class="n">_DKIMResult</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">dkim_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+            <span class="n">_DKIMResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">selector</span><span class="o">=</span><span class="n">selector</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">)</span>
+        <span class="p">)</span>
+
+    <span class="k">def</span><span class="w"> </span><span class="nf">add_spf_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">scope</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">result</span><span class="p">:</span> <span class="n">_SPFResult</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">spf_results</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_SPFResult</span><span class="p">(</span><span class="n">domain</span><span class="o">=</span><span class="n">domain</span><span class="p">,</span> <span class="n">scope</span><span class="o">=</span><span class="n">scope</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">))</span>
+
+    <span class="k">def</span><span class="w"> </span><span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>  <span class="c1"># pyright: ignore[reportIncompatibleMethodOverride]</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="kc">False</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">passed_dmarc</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">spf_aligned</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">dkim_aligned</span>
+
+        <span class="k">return</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+
+
+<span class="k">class</span><span class="w"> </span><span class="nc">_EmailAddressDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
+    <span class="n">display_name</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">address</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+
+
+<span class="k">class</span><span class="w"> </span><span class="nc">_EmailAttachmentDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
+    <span class="n">filename</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">content_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">sha256</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+
+
+<span class="k">class</span><span class="w"> </span><span class="nc">_ForensicSampleDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
+    <span class="n">raw</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">headers</span> <span class="o">=</span> <span class="n">Object</span><span class="p">()</span>
+    <span class="n">headers_only</span> <span class="o">=</span> <span class="n">Boolean</span><span class="p">()</span>
+    <span class="n">to</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">)</span>
+    <span class="n">subject</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">filename_safe_subject</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">_from</span> <span class="o">=</span> <span class="n">Object</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">)</span>
+    <span class="n">date</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
+    <span class="n">reply_to</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">)</span>
+    <span class="n">cc</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">)</span>
+    <span class="n">bcc</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">)</span>
+    <span class="n">body</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">attachments</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_EmailAttachmentDoc</span><span class="p">)</span>
+
+    <span class="k">def</span><span class="w"> </span><span class="nf">add_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">address</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
+
+    <span class="k">def</span><span class="w"> </span><span class="nf">add_reply_to</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">address</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">reply_to</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+            <span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">)</span>
+        <span class="p">)</span>
+
+    <span class="k">def</span><span class="w"> </span><span class="nf">add_cc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">address</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">cc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
+
+    <span class="k">def</span><span class="w"> </span><span class="nf">add_bcc</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">display_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">address</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">bcc</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_EmailAddressDoc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">display_name</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">))</span>
+
+    <span class="k">def</span><span class="w"> </span><span class="nf">add_attachment</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">filename</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">content_type</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">sha256</span><span class="p">:</span> <span class="nb">str</span><span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">attachments</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+            <span class="n">_EmailAttachmentDoc</span><span class="p">(</span>
+                <span class="n">filename</span><span class="o">=</span><span class="n">filename</span><span class="p">,</span> <span class="n">content_type</span><span class="o">=</span><span class="n">content_type</span><span class="p">,</span> <span class="n">sha256</span><span class="o">=</span><span class="n">sha256</span>
+            <span class="p">)</span>
+        <span class="p">)</span>
+
+
+<span class="k">class</span><span class="w"> </span><span class="nc">_ForensicReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
+    <span class="k">class</span><span class="w"> </span><span class="nc">Index</span><span class="p">:</span>
+        <span class="n">name</span> <span class="o">=</span> <span class="s2">&quot;dmarc_forensic&quot;</span>
+
+    <span class="n">feedback_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">user_agent</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">version</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">original_mail_from</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">arrival_date</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
+    <span class="n">domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">original_envelope_id</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">authentication_results</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">delivery_results</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">source_ip_address</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
+    <span class="n">source_country</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">source_reverse_dns</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">source_authentication_mechanisms</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">source_auth_failures</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">dkim_domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">original_rcpt_to</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">sample</span> <span class="o">=</span> <span class="n">Object</span><span class="p">(</span><span class="n">_ForensicSampleDoc</span><span class="p">)</span>
+
+
+<span class="k">class</span><span class="w"> </span><span class="nc">_SMTPTLSFailureDetailsDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
+    <span class="n">result_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">sending_mta_ip</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
+    <span class="n">receiving_mx_helo</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">receiving_ip</span> <span class="o">=</span> <span class="n">Ip</span><span class="p">()</span>
+    <span class="n">failed_session_count</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
+    <span class="n">additional_information_uri</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">failure_reason_code</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+
+
+<span class="k">class</span><span class="w"> </span><span class="nc">_SMTPTLSPolicyDoc</span><span class="p">(</span><span class="n">InnerDoc</span><span class="p">):</span>
+    <span class="n">policy_domain</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">policy_type</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">policy_strings</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">mx_host_patterns</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">successful_session_count</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
+    <span class="n">failed_session_count</span> <span class="o">=</span> <span class="n">Integer</span><span class="p">()</span>
+    <span class="n">failure_details</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSFailureDetailsDoc</span><span class="p">)</span>
+
+    <span class="k">def</span><span class="w"> </span><span class="nf">add_failure_details</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">result_type</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">ip_address</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">receiving_ip</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">receiving_mx_helo</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">failed_session_count</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">sending_mta_ip</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">receiving_mx_hostname</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">additional_information_uri</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">failure_reason_code</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="kc">None</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="n">_details</span> <span class="o">=</span> <span class="n">_SMTPTLSFailureDetailsDoc</span><span class="p">(</span>
+            <span class="n">result_type</span><span class="o">=</span><span class="n">result_type</span><span class="p">,</span>
+            <span class="n">ip_address</span><span class="o">=</span><span class="n">ip_address</span><span class="p">,</span>
+            <span class="n">sending_mta_ip</span><span class="o">=</span><span class="n">sending_mta_ip</span><span class="p">,</span>
+            <span class="n">receiving_mx_hostname</span><span class="o">=</span><span class="n">receiving_mx_hostname</span><span class="p">,</span>
+            <span class="n">receiving_mx_helo</span><span class="o">=</span><span class="n">receiving_mx_helo</span><span class="p">,</span>
+            <span class="n">receiving_ip</span><span class="o">=</span><span class="n">receiving_ip</span><span class="p">,</span>
+            <span class="n">failed_session_count</span><span class="o">=</span><span class="n">failed_session_count</span><span class="p">,</span>
+            <span class="n">additional_information</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
+            <span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span><span class="p">,</span>
+        <span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">failure_details</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">_details</span><span class="p">)</span>
+
+
+<span class="k">class</span><span class="w"> </span><span class="nc">_SMTPTLSReportDoc</span><span class="p">(</span><span class="n">Document</span><span class="p">):</span>
+    <span class="k">class</span><span class="w"> </span><span class="nc">Index</span><span class="p">:</span>
+        <span class="n">name</span> <span class="o">=</span> <span class="s2">&quot;smtp_tls&quot;</span>
+
+    <span class="n">organization_name</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">date_range</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
+    <span class="n">date_begin</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
+    <span class="n">date_end</span> <span class="o">=</span> <span class="n">Date</span><span class="p">()</span>
+    <span class="n">contact_info</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">report_id</span> <span class="o">=</span> <span class="n">Text</span><span class="p">()</span>
+    <span class="n">policies</span> <span class="o">=</span> <span class="n">Nested</span><span class="p">(</span><span class="n">_SMTPTLSPolicyDoc</span><span class="p">)</span>
+
+    <span class="k">def</span><span class="w"> </span><span class="nf">add_policy</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">policy_type</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+        <span class="n">policy_domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+        <span class="n">successful_session_count</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+        <span class="n">failed_session_count</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
+        <span class="o">*</span><span class="p">,</span>
+        <span class="n">policy_string</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">mx_host_patterns</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="n">failure_details</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="p">):</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+            <span class="n">policy_type</span><span class="o">=</span><span class="n">policy_type</span><span class="p">,</span>
+            <span class="n">policy_domain</span><span class="o">=</span><span class="n">policy_domain</span><span class="p">,</span>
+            <span class="n">successful_session_count</span><span class="o">=</span><span class="n">successful_session_count</span><span class="p">,</span>
+            <span class="n">failed_session_count</span><span class="o">=</span><span class="n">failed_session_count</span><span class="p">,</span>
+            <span class="n">policy_string</span><span class="o">=</span><span class="n">policy_string</span><span class="p">,</span>
+            <span class="n">mx_host_patterns</span><span class="o">=</span><span class="n">mx_host_patterns</span><span class="p">,</span>
+            <span class="n">failure_details</span><span class="o">=</span><span class="n">failure_details</span><span class="p">,</span>
+        <span class="p">)</span>
+
+
+<div class="viewcode-block" id="AlreadySaved">
+<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.AlreadySaved">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">AlreadySaved</span><span class="p">(</span><span class="ne">ValueError</span><span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Raised when a report to be saved matches an existing report&quot;&quot;&quot;</span></div>
+
+
+
+<div class="viewcode-block" id="set_hosts">
+<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.set_hosts">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">set_hosts</span><span class="p">(</span>
+    <span class="n">hosts</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]],</span>
+    <span class="o">*</span><span class="p">,</span>
+    <span class="n">use_ssl</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="n">ssl_cert_path</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">username</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">password</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">api_key</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">timeout</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">float</span><span class="p">]</span> <span class="o">=</span> <span class="mf">60.0</span><span class="p">,</span>
+<span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Sets the OpenSearch hosts to use</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        hosts (str|list[str]): A single hostname or URL, or list of hostnames or URLs</span>
+<span class="sd">        use_ssl (bool): Use an HTTPS connection to the server</span>
+<span class="sd">        ssl_cert_path (str): Path to the certificate chain</span>
+<span class="sd">        username (str): The username to use for authentication</span>
+<span class="sd">        password (str): The password to use for authentication</span>
+<span class="sd">        api_key (str): The Base64 encoded API key to use for authentication</span>
+<span class="sd">        timeout (float): Timeout in seconds</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">hosts</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+        <span class="n">hosts</span> <span class="o">=</span> <span class="p">[</span><span class="n">hosts</span><span class="p">]</span>
+    <span class="n">conn_params</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;hosts&quot;</span><span class="p">:</span> <span class="n">hosts</span><span class="p">,</span> <span class="s2">&quot;timeout&quot;</span><span class="p">:</span> <span class="n">timeout</span><span class="p">}</span>
+    <span class="k">if</span> <span class="n">use_ssl</span><span class="p">:</span>
+        <span class="n">conn_params</span><span class="p">[</span><span class="s2">&quot;use_ssl&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
+        <span class="k">if</span> <span class="n">ssl_cert_path</span><span class="p">:</span>
+            <span class="n">conn_params</span><span class="p">[</span><span class="s2">&quot;verify_certs&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
+            <span class="n">conn_params</span><span class="p">[</span><span class="s2">&quot;ca_certs&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">ssl_cert_path</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">conn_params</span><span class="p">[</span><span class="s2">&quot;verify_certs&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
+    <span class="k">if</span> <span class="n">username</span> <span class="ow">and</span> <span class="n">password</span><span class="p">:</span>
+        <span class="n">conn_params</span><span class="p">[</span><span class="s2">&quot;http_auth&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">username</span> <span class="o">+</span> <span class="s2">&quot;:&quot;</span> <span class="o">+</span> <span class="n">password</span>
+    <span class="k">if</span> <span class="n">api_key</span><span class="p">:</span>
+        <span class="n">conn_params</span><span class="p">[</span><span class="s2">&quot;api_key&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">api_key</span>
+    <span class="n">connections</span><span class="o">.</span><span class="n">create_connection</span><span class="p">(</span><span class="o">**</span><span class="n">conn_params</span><span class="p">)</span></div>
+
+
+
+<div class="viewcode-block" id="create_indexes">
+<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.create_indexes">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">create_indexes</span><span class="p">(</span><span class="n">names</span><span class="p">:</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">settings</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Create OpenSearch indexes</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        names (list): A list of index names</span>
+<span class="sd">        settings (dict): Index settings</span>
+
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="n">names</span><span class="p">:</span>
+        <span class="n">index</span> <span class="o">=</span> <span class="n">Index</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="k">if</span> <span class="ow">not</span> <span class="n">index</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
+                <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Creating OpenSearch index: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">name</span><span class="p">))</span>
+                <span class="k">if</span> <span class="n">settings</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+                    <span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="n">number_of_shards</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="n">index</span><span class="o">.</span><span class="n">settings</span><span class="p">(</span><span class="o">**</span><span class="n">settings</span><span class="p">)</span>
+                <span class="n">index</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
+        <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">&quot;OpenSearch error: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
+
+
+
+<div class="viewcode-block" id="migrate_indexes">
+<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.migrate_indexes">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">migrate_indexes</span><span class="p">(</span>
+    <span class="n">aggregate_indexes</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">forensic_indexes</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+<span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Updates index mappings</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        aggregate_indexes (list): A list of aggregate index names</span>
+<span class="sd">        forensic_indexes (list): A list of forensic index names</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">version</span> <span class="o">=</span> <span class="mi">2</span>
+    <span class="k">if</span> <span class="n">aggregate_indexes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">aggregate_indexes</span> <span class="o">=</span> <span class="p">[]</span>
+    <span class="k">if</span> <span class="n">forensic_indexes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">forensic_indexes</span> <span class="o">=</span> <span class="p">[]</span>
+    <span class="k">for</span> <span class="n">aggregate_index_name</span> <span class="ow">in</span> <span class="n">aggregate_indexes</span><span class="p">:</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
+            <span class="k">continue</span>
+        <span class="n">aggregate_index</span> <span class="o">=</span> <span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span>
+        <span class="n">doc</span> <span class="o">=</span> <span class="s2">&quot;doc&quot;</span>
+        <span class="n">fo_field</span> <span class="o">=</span> <span class="s2">&quot;published_policy.fo&quot;</span>
+        <span class="n">fo</span> <span class="o">=</span> <span class="s2">&quot;fo&quot;</span>
+        <span class="n">fo_mapping</span> <span class="o">=</span> <span class="n">aggregate_index</span><span class="o">.</span><span class="n">get_field_mapping</span><span class="p">(</span><span class="n">fields</span><span class="o">=</span><span class="p">[</span><span class="n">fo_field</span><span class="p">])</span>
+        <span class="n">fo_mapping</span> <span class="o">=</span> <span class="n">fo_mapping</span><span class="p">[</span><span class="nb">list</span><span class="p">(</span><span class="n">fo_mapping</span><span class="o">.</span><span class="n">keys</span><span class="p">())[</span><span class="mi">0</span><span class="p">]][</span><span class="s2">&quot;mappings&quot;</span><span class="p">]</span>
+        <span class="k">if</span> <span class="n">doc</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">fo_mapping</span><span class="p">:</span>
+            <span class="k">continue</span>
+
+        <span class="n">fo_mapping</span> <span class="o">=</span> <span class="n">fo_mapping</span><span class="p">[</span><span class="n">doc</span><span class="p">][</span><span class="n">fo_field</span><span class="p">][</span><span class="s2">&quot;mapping&quot;</span><span class="p">][</span><span class="n">fo</span><span class="p">]</span>
+        <span class="n">fo_type</span> <span class="o">=</span> <span class="n">fo_mapping</span><span class="p">[</span><span class="s2">&quot;type&quot;</span><span class="p">]</span>
+        <span class="k">if</span> <span class="n">fo_type</span> <span class="o">==</span> <span class="s2">&quot;long&quot;</span><span class="p">:</span>
+            <span class="n">new_index_name</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">-v</span><span class="si">{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">,</span> <span class="n">version</span><span class="p">)</span>
+            <span class="n">body</span> <span class="o">=</span> <span class="p">{</span>
+                <span class="s2">&quot;properties&quot;</span><span class="p">:</span> <span class="p">{</span>
+                    <span class="s2">&quot;published_policy.fo&quot;</span><span class="p">:</span> <span class="p">{</span>
+                        <span class="s2">&quot;type&quot;</span><span class="p">:</span> <span class="s2">&quot;text&quot;</span><span class="p">,</span>
+                        <span class="s2">&quot;fields&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;keyword&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;type&quot;</span><span class="p">:</span> <span class="s2">&quot;keyword&quot;</span><span class="p">,</span> <span class="s2">&quot;ignore_above&quot;</span><span class="p">:</span> <span class="mi">256</span><span class="p">}},</span>
+                    <span class="p">}</span>
+                <span class="p">}</span>
+            <span class="p">}</span>
+            <span class="n">Index</span><span class="p">(</span><span class="n">new_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">create</span><span class="p">()</span>
+            <span class="n">Index</span><span class="p">(</span><span class="n">new_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">put_mapping</span><span class="p">(</span><span class="n">doc_type</span><span class="o">=</span><span class="n">doc</span><span class="p">,</span> <span class="n">body</span><span class="o">=</span><span class="n">body</span><span class="p">)</span>
+            <span class="n">reindex</span><span class="p">(</span><span class="n">connections</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(),</span> <span class="n">aggregate_index_name</span><span class="p">,</span> <span class="n">new_index_name</span><span class="p">)</span>
+            <span class="n">Index</span><span class="p">(</span><span class="n">aggregate_index_name</span><span class="p">)</span><span class="o">.</span><span class="n">delete</span><span class="p">()</span>
+
+    <span class="k">for</span> <span class="n">forensic_index</span> <span class="ow">in</span> <span class="n">forensic_indexes</span><span class="p">:</span>
+        <span class="k">pass</span></div>
+
+
+
+<div class="viewcode-block" id="save_aggregate_report_to_opensearch">
+<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_aggregate_report_to_opensearch">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">save_aggregate_report_to_opensearch</span><span class="p">(</span>
+    <span class="n">aggregate_report</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
+    <span class="n">index_suffix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">index_prefix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">monthly_indexes</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="n">number_of_shards</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+    <span class="n">number_of_replicas</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
+<span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Saves a parsed DMARC aggregate report to OpenSearch</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        aggregate_report (dict): A parsed forensic report</span>
+<span class="sd">        index_suffix (str): The suffix of the name of the index to save to</span>
+<span class="sd">        index_prefix (str): The prefix of the name of the index to save to</span>
+<span class="sd">        monthly_indexes (bool): Use monthly indexes instead of daily indexes</span>
+<span class="sd">        number_of_shards (int): The number of shards to use in the index</span>
+<span class="sd">        number_of_replicas (int): The number of replicas to use in the index</span>
+
+<span class="sd">    Raises:</span>
+<span class="sd">            AlreadySaved</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Saving aggregate report to OpenSearch&quot;</span><span class="p">)</span>
+    <span class="n">aggregate_report</span> <span class="o">=</span> <span class="n">aggregate_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+    <span class="n">metadata</span> <span class="o">=</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;report_metadata&quot;</span><span class="p">]</span>
+    <span class="n">org_name</span> <span class="o">=</span> <span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;org_name&quot;</span><span class="p">]</span>
+    <span class="n">report_id</span> <span class="o">=</span> <span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;report_id&quot;</span><span class="p">]</span>
+    <span class="n">domain</span> <span class="o">=</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;policy_published&quot;</span><span class="p">][</span><span class="s2">&quot;domain&quot;</span><span class="p">]</span>
+    <span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+    <span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;end_date&quot;</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+
+    <span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
+        <span class="n">index_date</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m&quot;</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">index_date</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2">&quot;</span><span class="p">)</span>
+
+    <span class="n">org_name_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">org_name</span><span class="o">=</span><span class="n">org_name</span><span class="p">)))</span>
+    <span class="n">report_id_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">report_id</span><span class="o">=</span><span class="n">report_id</span><span class="p">)))</span>
+    <span class="n">domain_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;published_policy.domain&quot;</span><span class="p">:</span> <span class="n">domain</span><span class="p">}))</span>
+    <span class="n">begin_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_begin</span><span class="o">=</span><span class="n">begin_date</span><span class="p">)))</span>
+    <span class="n">end_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_end</span><span class="o">=</span><span class="n">end_date</span><span class="p">)))</span>
+
+    <span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">search_index</span> <span class="o">=</span> <span class="s2">&quot;dmarc_aggregate_</span><span class="si">{0}</span><span class="s2">*&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">search_index</span> <span class="o">=</span> <span class="s2">&quot;dmarc_aggregate*&quot;</span>
+    <span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">search_index</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
+    <span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
+    <span class="n">query</span> <span class="o">=</span> <span class="n">org_name_query</span> <span class="o">&amp;</span> <span class="n">report_id_query</span> <span class="o">&amp;</span> <span class="n">domain_query</span>
+    <span class="n">query</span> <span class="o">=</span> <span class="n">query</span> <span class="o">&amp;</span> <span class="n">begin_date_query</span> <span class="o">&amp;</span> <span class="n">end_date_query</span>
+    <span class="n">search</span><span class="o">.</span><span class="n">query</span> <span class="o">=</span> <span class="n">query</span>
+    <span class="n">begin_date_human</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ&quot;</span><span class="p">)</span>
+    <span class="n">end_date_human</span> <span class="o">=</span> <span class="n">end_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ&quot;</span><span class="p">)</span>
+
+    <span class="k">try</span><span class="p">:</span>
+        <span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+    <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error_</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span>
+            <span class="s2">&quot;OpenSearch&#39;s search for existing report </span><span class="se">\</span>
+<span class="s2">            error: </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
+        <span class="p">)</span>
+
+    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
+            <span class="s2">&quot;An aggregate report ID </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> about </span><span class="si">{2}</span><span class="s2"> &quot;</span>
+            <span class="s2">&quot;with a date range of </span><span class="si">{3}</span><span class="s2"> UTC to </span><span class="si">{4}</span><span class="s2"> UTC already &quot;</span>
+            <span class="s2">&quot;exists in &quot;</span>
+            <span class="s2">&quot;OpenSearch&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                <span class="n">report_id</span><span class="p">,</span> <span class="n">org_name</span><span class="p">,</span> <span class="n">domain</span><span class="p">,</span> <span class="n">begin_date_human</span><span class="p">,</span> <span class="n">end_date_human</span>
+            <span class="p">)</span>
+        <span class="p">)</span>
+    <span class="n">published_policy</span> <span class="o">=</span> <span class="n">_PublishedPolicy</span><span class="p">(</span>
+        <span class="n">domain</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;policy_published&quot;</span><span class="p">][</span><span class="s2">&quot;domain&quot;</span><span class="p">],</span>
+        <span class="n">adkim</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;policy_published&quot;</span><span class="p">][</span><span class="s2">&quot;adkim&quot;</span><span class="p">],</span>
+        <span class="n">aspf</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;policy_published&quot;</span><span class="p">][</span><span class="s2">&quot;aspf&quot;</span><span class="p">],</span>
+        <span class="n">p</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;policy_published&quot;</span><span class="p">][</span><span class="s2">&quot;p&quot;</span><span class="p">],</span>
+        <span class="n">sp</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;policy_published&quot;</span><span class="p">][</span><span class="s2">&quot;sp&quot;</span><span class="p">],</span>
+        <span class="n">pct</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;policy_published&quot;</span><span class="p">][</span><span class="s2">&quot;pct&quot;</span><span class="p">],</span>
+        <span class="n">fo</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;policy_published&quot;</span><span class="p">][</span><span class="s2">&quot;fo&quot;</span><span class="p">],</span>
+    <span class="p">)</span>
+
+    <span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;records&quot;</span><span class="p">]:</span>
+        <span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;interval_begin&quot;</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+        <span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;interval_end&quot;</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+        <span class="n">normalized_timespan</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;normalized_timespan&quot;</span><span class="p">]</span>
+
+        <span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
+            <span class="n">index_date</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m&quot;</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">index_date</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">begin_date</span>
+        <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;end_date&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">end_date</span>
+        <span class="n">date_range</span> <span class="o">=</span> <span class="p">[</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">],</span> <span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;end_date&quot;</span><span class="p">]]</span>
+        <span class="n">agg_doc</span> <span class="o">=</span> <span class="n">_AggregateReportDoc</span><span class="p">(</span>
+            <span class="n">xml_schema</span><span class="o">=</span><span class="n">aggregate_report</span><span class="p">[</span><span class="s2">&quot;xml_schema&quot;</span><span class="p">],</span>
+            <span class="n">org_name</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;org_name&quot;</span><span class="p">],</span>
+            <span class="n">org_email</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;org_email&quot;</span><span class="p">],</span>
+            <span class="n">org_extra_contact_info</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;org_extra_contact_info&quot;</span><span class="p">],</span>
+            <span class="n">report_id</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;report_id&quot;</span><span class="p">],</span>
+            <span class="n">date_range</span><span class="o">=</span><span class="n">date_range</span><span class="p">,</span>
+            <span class="n">date_begin</span><span class="o">=</span><span class="n">begin_date</span><span class="p">,</span>
+            <span class="n">date_end</span><span class="o">=</span><span class="n">end_date</span><span class="p">,</span>
+            <span class="n">normalized_timespan</span><span class="o">=</span><span class="n">normalized_timespan</span><span class="p">,</span>
+            <span class="n">errors</span><span class="o">=</span><span class="n">metadata</span><span class="p">[</span><span class="s2">&quot;errors&quot;</span><span class="p">],</span>
+            <span class="n">published_policy</span><span class="o">=</span><span class="n">published_policy</span><span class="p">,</span>
+            <span class="n">source_ip_address</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;ip_address&quot;</span><span class="p">],</span>
+            <span class="n">source_country</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;country&quot;</span><span class="p">],</span>
+            <span class="n">source_reverse_dns</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;reverse_dns&quot;</span><span class="p">],</span>
+            <span class="n">source_base_domain</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;base_domain&quot;</span><span class="p">],</span>
+            <span class="n">source_type</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;type&quot;</span><span class="p">],</span>
+            <span class="n">source_name</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;name&quot;</span><span class="p">],</span>
+            <span class="n">message_count</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;count&quot;</span><span class="p">],</span>
+            <span class="n">disposition</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;policy_evaluated&quot;</span><span class="p">][</span><span class="s2">&quot;disposition&quot;</span><span class="p">],</span>
+            <span class="n">dkim_aligned</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;policy_evaluated&quot;</span><span class="p">][</span><span class="s2">&quot;dkim&quot;</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="ow">and</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;policy_evaluated&quot;</span><span class="p">][</span><span class="s2">&quot;dkim&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">&quot;pass&quot;</span><span class="p">,</span>
+            <span class="n">spf_aligned</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;policy_evaluated&quot;</span><span class="p">][</span><span class="s2">&quot;spf&quot;</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+            <span class="ow">and</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;policy_evaluated&quot;</span><span class="p">][</span><span class="s2">&quot;spf&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s2">&quot;pass&quot;</span><span class="p">,</span>
+            <span class="n">header_from</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;identifiers&quot;</span><span class="p">][</span><span class="s2">&quot;header_from&quot;</span><span class="p">],</span>
+            <span class="n">envelope_from</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;identifiers&quot;</span><span class="p">][</span><span class="s2">&quot;envelope_from&quot;</span><span class="p">],</span>
+            <span class="n">envelope_to</span><span class="o">=</span><span class="n">record</span><span class="p">[</span><span class="s2">&quot;identifiers&quot;</span><span class="p">][</span><span class="s2">&quot;envelope_to&quot;</span><span class="p">],</span>
+        <span class="p">)</span>
+
+        <span class="k">for</span> <span class="n">override</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;policy_evaluated&quot;</span><span class="p">][</span><span class="s2">&quot;policy_override_reasons&quot;</span><span class="p">]:</span>
+            <span class="n">agg_doc</span><span class="o">.</span><span class="n">add_policy_override</span><span class="p">(</span>
+                <span class="n">type_</span><span class="o">=</span><span class="n">override</span><span class="p">[</span><span class="s2">&quot;type&quot;</span><span class="p">],</span> <span class="n">comment</span><span class="o">=</span><span class="n">override</span><span class="p">[</span><span class="s2">&quot;comment&quot;</span><span class="p">]</span>
+            <span class="p">)</span>
+
+        <span class="k">for</span> <span class="n">dkim_result</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;auth_results&quot;</span><span class="p">][</span><span class="s2">&quot;dkim&quot;</span><span class="p">]:</span>
+            <span class="n">agg_doc</span><span class="o">.</span><span class="n">add_dkim_result</span><span class="p">(</span>
+                <span class="n">domain</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">&quot;domain&quot;</span><span class="p">],</span>
+                <span class="n">selector</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">&quot;selector&quot;</span><span class="p">],</span>
+                <span class="n">result</span><span class="o">=</span><span class="n">dkim_result</span><span class="p">[</span><span class="s2">&quot;result&quot;</span><span class="p">],</span>
+            <span class="p">)</span>
+
+        <span class="k">for</span> <span class="n">spf_result</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;auth_results&quot;</span><span class="p">][</span><span class="s2">&quot;spf&quot;</span><span class="p">]:</span>
+            <span class="n">agg_doc</span><span class="o">.</span><span class="n">add_spf_result</span><span class="p">(</span>
+                <span class="n">domain</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">&quot;domain&quot;</span><span class="p">],</span>
+                <span class="n">scope</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">&quot;scope&quot;</span><span class="p">],</span>
+                <span class="n">result</span><span class="o">=</span><span class="n">spf_result</span><span class="p">[</span><span class="s2">&quot;result&quot;</span><span class="p">],</span>
+            <span class="p">)</span>
+
+        <span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;dmarc_aggregate&quot;</span>
+        <span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
+            <span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
+            <span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
+
+        <span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
+        <span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
+            <span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
+        <span class="p">)</span>
+        <span class="n">create_indexes</span><span class="p">([</span><span class="n">index</span><span class="p">],</span> <span class="n">index_settings</span><span class="p">)</span>
+        <span class="n">agg_doc</span><span class="o">.</span><span class="n">meta</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>
+
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">agg_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
+        <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">&quot;OpenSearch error: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
+
+
+
+<div class="viewcode-block" id="save_forensic_report_to_opensearch">
+<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_forensic_report_to_opensearch">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">save_forensic_report_to_opensearch</span><span class="p">(</span>
+    <span class="n">forensic_report</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
+    <span class="n">index_suffix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">index_prefix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">monthly_indexes</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="n">number_of_shards</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+    <span class="n">number_of_replicas</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
+<span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Saves a parsed DMARC forensic report to OpenSearch</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        forensic_report (dict): A parsed forensic report</span>
+<span class="sd">        index_suffix (str): The suffix of the name of the index to save to</span>
+<span class="sd">        index_prefix (str): The prefix of the name of the index to save to</span>
+<span class="sd">        monthly_indexes (bool): Use monthly indexes instead of daily</span>
+<span class="sd">                                indexes</span>
+<span class="sd">        number_of_shards (int): The number of shards to use in the index</span>
+<span class="sd">        number_of_replicas (int): The number of replicas to use in the</span>
+<span class="sd">                                  index</span>
+
+<span class="sd">    Raises:</span>
+<span class="sd">        AlreadySaved</span>
+
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Saving forensic report to OpenSearch&quot;</span><span class="p">)</span>
+    <span class="n">forensic_report</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+    <span class="n">sample_date</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="k">if</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;date&quot;</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">sample_date</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;date&quot;</span><span class="p">]</span>
+        <span class="n">sample_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">sample_date</span><span class="p">)</span>
+    <span class="n">original_headers</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;headers&quot;</span><span class="p">]</span>
+    <span class="n">headers</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
+    <span class="k">for</span> <span class="n">original_header</span> <span class="ow">in</span> <span class="n">original_headers</span><span class="p">:</span>
+        <span class="n">headers</span><span class="p">[</span><span class="n">original_header</span><span class="o">.</span><span class="n">lower</span><span class="p">()]</span> <span class="o">=</span> <span class="n">original_headers</span><span class="p">[</span><span class="n">original_header</span><span class="p">]</span>
+
+    <span class="n">arrival_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;arrival_date_utc&quot;</span><span class="p">])</span>
+    <span class="n">arrival_date_epoch_milliseconds</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">.</span><span class="n">timestamp</span><span class="p">()</span> <span class="o">*</span> <span class="mi">1000</span><span class="p">)</span>
+
+    <span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">search_index</span> <span class="o">=</span> <span class="s2">&quot;dmarc_forensic_</span><span class="si">{0}</span><span class="s2">*&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">search_index</span> <span class="o">=</span> <span class="s2">&quot;dmarc_forensic*&quot;</span>
+    <span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">search_index</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
+    <span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
+    <span class="n">q</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date_epoch_milliseconds</span><span class="p">)))</span>
+
+    <span class="n">from_</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="n">to_</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="n">subject</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="k">if</span> <span class="s2">&quot;from&quot;</span> <span class="ow">in</span> <span class="n">headers</span><span class="p">:</span>
+        <span class="c1"># We convert the FROM header from a string list to a flat string.</span>
+        <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
+        <span class="k">if</span> <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s2">&quot;&quot;</span><span class="p">:</span>
+            <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">][</span><span class="mi">1</span><span class="p">]</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot; &lt;&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">headers</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">])</span> <span class="o">+</span> <span class="s2">&quot;&gt;&quot;</span>
+
+        <span class="n">from_</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
+        <span class="n">from_</span><span class="p">[</span><span class="s2">&quot;sample.headers.from&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">]</span>
+        <span class="n">from_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="n">from_</span><span class="p">))</span>
+        <span class="n">q</span> <span class="o">=</span> <span class="n">q</span> <span class="o">&amp;</span> <span class="n">from_query</span>
+    <span class="k">if</span> <span class="s2">&quot;to&quot;</span> <span class="ow">in</span> <span class="n">headers</span><span class="p">:</span>
+        <span class="c1"># We convert the TO header from a string list to a flat string.</span>
+        <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
+        <span class="k">if</span> <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s2">&quot;&quot;</span><span class="p">:</span>
+            <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">][</span><span class="mi">1</span><span class="p">]</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot; &lt;&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">headers</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">])</span> <span class="o">+</span> <span class="s2">&quot;&gt;&quot;</span>
+
+        <span class="n">to_</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
+        <span class="n">to_</span><span class="p">[</span><span class="s2">&quot;sample.headers.to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">]</span>
+        <span class="n">to_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="n">to_</span><span class="p">))</span>
+        <span class="n">q</span> <span class="o">=</span> <span class="n">q</span> <span class="o">&amp;</span> <span class="n">to_query</span>
+    <span class="k">if</span> <span class="s2">&quot;subject&quot;</span> <span class="ow">in</span> <span class="n">headers</span><span class="p">:</span>
+        <span class="n">subject</span> <span class="o">=</span> <span class="n">headers</span><span class="p">[</span><span class="s2">&quot;subject&quot;</span><span class="p">]</span>
+        <span class="n">subject_query</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;match_phrase&quot;</span><span class="p">:</span> <span class="p">{</span><span class="s2">&quot;sample.headers.subject&quot;</span><span class="p">:</span> <span class="n">subject</span><span class="p">}}</span>
+        <span class="n">q</span> <span class="o">=</span> <span class="n">q</span> <span class="o">&amp;</span> <span class="n">Q</span><span class="p">(</span><span class="n">subject_query</span><span class="p">)</span>
+
+    <span class="n">search</span><span class="o">.</span><span class="n">query</span> <span class="o">=</span> <span class="n">q</span>
+    <span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+
+    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
+            <span class="s2">&quot;A forensic sample to </span><span class="si">{0}</span><span class="s2"> from </span><span class="si">{1}</span><span class="s2"> &quot;</span>
+            <span class="s2">&quot;with a subject of </span><span class="si">{2}</span><span class="s2"> and arrival date of </span><span class="si">{3}</span><span class="s2"> &quot;</span>
+            <span class="s2">&quot;already exists in &quot;</span>
+            <span class="s2">&quot;OpenSearch&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                <span class="n">to_</span><span class="p">,</span> <span class="n">from_</span><span class="p">,</span> <span class="n">subject</span><span class="p">,</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;arrival_date_utc&quot;</span><span class="p">]</span>
+            <span class="p">)</span>
+        <span class="p">)</span>
+
+    <span class="n">parsed_sample</span> <span class="o">=</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">]</span>
+    <span class="n">sample</span> <span class="o">=</span> <span class="n">_ForensicSampleDoc</span><span class="p">(</span>
+        <span class="n">raw</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;sample&quot;</span><span class="p">],</span>
+        <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">,</span>
+        <span class="n">headers_only</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;sample_headers_only&quot;</span><span class="p">],</span>
+        <span class="n">date</span><span class="o">=</span><span class="n">sample_date</span><span class="p">,</span>
+        <span class="n">subject</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;subject&quot;</span><span class="p">],</span>
+        <span class="n">filename_safe_subject</span><span class="o">=</span><span class="n">parsed_sample</span><span class="p">[</span><span class="s2">&quot;filename_safe_subject&quot;</span><span class="p">],</span>
+        <span class="n">body</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;body&quot;</span><span class="p">],</span>
+    <span class="p">)</span>
+
+    <span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;to&quot;</span><span class="p">]:</span>
+        <span class="n">sample</span><span class="o">.</span><span class="n">add_to</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">&quot;display_name&quot;</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">&quot;address&quot;</span><span class="p">])</span>
+    <span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;reply_to&quot;</span><span class="p">]:</span>
+        <span class="n">sample</span><span class="o">.</span><span class="n">add_reply_to</span><span class="p">(</span>
+            <span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">&quot;display_name&quot;</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">&quot;address&quot;</span><span class="p">]</span>
+        <span class="p">)</span>
+    <span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;cc&quot;</span><span class="p">]:</span>
+        <span class="n">sample</span><span class="o">.</span><span class="n">add_cc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">&quot;display_name&quot;</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">&quot;address&quot;</span><span class="p">])</span>
+    <span class="k">for</span> <span class="n">address</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;bcc&quot;</span><span class="p">]:</span>
+        <span class="n">sample</span><span class="o">.</span><span class="n">add_bcc</span><span class="p">(</span><span class="n">display_name</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">&quot;display_name&quot;</span><span class="p">],</span> <span class="n">address</span><span class="o">=</span><span class="n">address</span><span class="p">[</span><span class="s2">&quot;address&quot;</span><span class="p">])</span>
+    <span class="k">for</span> <span class="n">attachment</span> <span class="ow">in</span> <span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;parsed_sample&quot;</span><span class="p">][</span><span class="s2">&quot;attachments&quot;</span><span class="p">]:</span>
+        <span class="n">sample</span><span class="o">.</span><span class="n">add_attachment</span><span class="p">(</span>
+            <span class="n">filename</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">&quot;filename&quot;</span><span class="p">],</span>
+            <span class="n">content_type</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">&quot;mail_content_type&quot;</span><span class="p">],</span>
+            <span class="n">sha256</span><span class="o">=</span><span class="n">attachment</span><span class="p">[</span><span class="s2">&quot;sha256&quot;</span><span class="p">],</span>
+        <span class="p">)</span>
+    <span class="k">try</span><span class="p">:</span>
+        <span class="n">forensic_doc</span> <span class="o">=</span> <span class="n">_ForensicReportDoc</span><span class="p">(</span>
+            <span class="n">feedback_type</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;feedback_type&quot;</span><span class="p">],</span>
+            <span class="n">user_agent</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;user_agent&quot;</span><span class="p">],</span>
+            <span class="n">version</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;version&quot;</span><span class="p">],</span>
+            <span class="n">original_mail_from</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;original_mail_from&quot;</span><span class="p">],</span>
+            <span class="n">arrival_date</span><span class="o">=</span><span class="n">arrival_date_epoch_milliseconds</span><span class="p">,</span>
+            <span class="n">domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;reported_domain&quot;</span><span class="p">],</span>
+            <span class="n">original_envelope_id</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;original_envelope_id&quot;</span><span class="p">],</span>
+            <span class="n">authentication_results</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;authentication_results&quot;</span><span class="p">],</span>
+            <span class="n">delivery_results</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;delivery_result&quot;</span><span class="p">],</span>
+            <span class="n">source_ip_address</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;ip_address&quot;</span><span class="p">],</span>
+            <span class="n">source_country</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;country&quot;</span><span class="p">],</span>
+            <span class="n">source_reverse_dns</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;reverse_dns&quot;</span><span class="p">],</span>
+            <span class="n">source_base_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;base_domain&quot;</span><span class="p">],</span>
+            <span class="n">authentication_mechanisms</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;authentication_mechanisms&quot;</span><span class="p">],</span>
+            <span class="n">auth_failure</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;auth_failure&quot;</span><span class="p">],</span>
+            <span class="n">dkim_domain</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;dkim_domain&quot;</span><span class="p">],</span>
+            <span class="n">original_rcpt_to</span><span class="o">=</span><span class="n">forensic_report</span><span class="p">[</span><span class="s2">&quot;original_rcpt_to&quot;</span><span class="p">],</span>
+            <span class="n">sample</span><span class="o">=</span><span class="n">sample</span><span class="p">,</span>
+        <span class="p">)</span>
+
+        <span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;dmarc_forensic&quot;</span>
+        <span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
+            <span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
+            <span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
+            <span class="n">index_date</span> <span class="o">=</span> <span class="n">arrival_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m&quot;</span><span class="p">)</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">index_date</span> <span class="o">=</span> <span class="n">arrival_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
+        <span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
+            <span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
+        <span class="p">)</span>
+        <span class="n">create_indexes</span><span class="p">([</span><span class="n">index</span><span class="p">],</span> <span class="n">index_settings</span><span class="p">)</span>
+        <span class="n">forensic_doc</span><span class="o">.</span><span class="n">meta</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">forensic_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
+        <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">&quot;OpenSearch error: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
+    <span class="k">except</span> <span class="ne">KeyError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="n">InvalidForensicReport</span><span class="p">(</span>
+            <span class="s2">&quot;Forensic report missing required field: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
+        <span class="p">)</span></div>
+
+
+
+<div class="viewcode-block" id="save_smtp_tls_report_to_opensearch">
+<a class="viewcode-back" href="../../api.html#parsedmarc.opensearch.save_smtp_tls_report_to_opensearch">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">save_smtp_tls_report_to_opensearch</span><span class="p">(</span>
+    <span class="n">report</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
+    <span class="n">index_suffix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">index_prefix</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">monthly_indexes</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="n">number_of_shards</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
+    <span class="n">number_of_replicas</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
+<span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Saves a parsed SMTP TLS report to OpenSearch</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        report (dict): A parsed SMTP TLS report</span>
+<span class="sd">        index_suffix (str): The suffix of the name of the index to save to</span>
+<span class="sd">        index_prefix (str): The prefix of the name of the index to save to</span>
+<span class="sd">        monthly_indexes (bool): Use monthly indexes instead of daily indexes</span>
+<span class="sd">        number_of_shards (int): The number of shards to use in the index</span>
+<span class="sd">        number_of_replicas (int): The number of replicas to use in the index</span>
+
+<span class="sd">    Raises:</span>
+<span class="sd">            AlreadySaved</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Saving SMTP TLS report to OpenSearch&quot;</span><span class="p">)</span>
+    <span class="n">org_name</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">&quot;organization_name&quot;</span><span class="p">]</span>
+    <span class="n">report_id</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">&quot;report_id&quot;</span><span class="p">]</span>
+    <span class="n">begin_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+    <span class="n">end_date</span> <span class="o">=</span> <span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;end_date&quot;</span><span class="p">],</span> <span class="n">to_utc</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+    <span class="n">begin_date_human</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ&quot;</span><span class="p">)</span>
+    <span class="n">end_date_human</span> <span class="o">=</span> <span class="n">end_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%SZ&quot;</span><span class="p">)</span>
+    <span class="k">if</span> <span class="n">monthly_indexes</span><span class="p">:</span>
+        <span class="n">index_date</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m&quot;</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">index_date</span> <span class="o">=</span> <span class="n">begin_date</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2">&quot;</span><span class="p">)</span>
+    <span class="n">report</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">begin_date</span>
+    <span class="n">report</span><span class="p">[</span><span class="s2">&quot;end_date&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">end_date</span>
+
+    <span class="n">org_name_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">org_name</span><span class="o">=</span><span class="n">org_name</span><span class="p">)))</span>
+    <span class="n">report_id_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match_phrase</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">report_id</span><span class="o">=</span><span class="n">report_id</span><span class="p">)))</span>
+    <span class="n">begin_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_begin</span><span class="o">=</span><span class="n">begin_date</span><span class="p">)))</span>
+    <span class="n">end_date_query</span> <span class="o">=</span> <span class="n">Q</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">match</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span><span class="n">date_end</span><span class="o">=</span><span class="n">end_date</span><span class="p">)))</span>
+
+    <span class="k">if</span> <span class="n">index_suffix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">search_index</span> <span class="o">=</span> <span class="s2">&quot;smtp_tls_</span><span class="si">{0}</span><span class="s2">*&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_suffix</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">search_index</span> <span class="o">=</span> <span class="s2">&quot;smtp_tls*&quot;</span>
+    <span class="k">if</span> <span class="n">index_prefix</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">search_index</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">search_index</span><span class="p">)</span>
+    <span class="n">search</span> <span class="o">=</span> <span class="n">Search</span><span class="p">(</span><span class="n">index</span><span class="o">=</span><span class="n">search_index</span><span class="p">)</span>
+    <span class="n">query</span> <span class="o">=</span> <span class="n">org_name_query</span> <span class="o">&amp;</span> <span class="n">report_id_query</span>
+    <span class="n">query</span> <span class="o">=</span> <span class="n">query</span> <span class="o">&amp;</span> <span class="n">begin_date_query</span> <span class="o">&amp;</span> <span class="n">end_date_query</span>
+    <span class="n">search</span><span class="o">.</span><span class="n">query</span> <span class="o">=</span> <span class="n">query</span>
+
+    <span class="k">try</span><span class="p">:</span>
+        <span class="n">existing</span> <span class="o">=</span> <span class="n">search</span><span class="o">.</span><span class="n">execute</span><span class="p">()</span>
+    <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">error_</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span>
+            <span class="s2">&quot;OpenSearch&#39;s search for existing report </span><span class="se">\</span>
+<span class="s2">            error: </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">error_</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
+        <span class="p">)</span>
+
+    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">existing</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="n">AlreadySaved</span><span class="p">(</span>
+            <span class="sa">f</span><span class="s2">&quot;An SMTP TLS report ID </span><span class="si">{</span><span class="n">report_id</span><span class="si">}</span><span class="s2"> from &quot;</span>
+            <span class="sa">f</span><span class="s2">&quot; </span><span class="si">{</span><span class="n">org_name</span><span class="si">}</span><span class="s2"> with a date range of &quot;</span>
+            <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">begin_date_human</span><span class="si">}</span><span class="s2"> UTC to &quot;</span>
+            <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">end_date_human</span><span class="si">}</span><span class="s2"> UTC already &quot;</span>
+            <span class="s2">&quot;exists in OpenSearch&quot;</span>
+        <span class="p">)</span>
+
+    <span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;smtp_tls&quot;</span>
+    <span class="k">if</span> <span class="n">index_suffix</span><span class="p">:</span>
+        <span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_suffix</span><span class="p">)</span>
+    <span class="k">if</span> <span class="n">index_prefix</span><span class="p">:</span>
+        <span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index_prefix</span><span class="p">,</span> <span class="n">index</span><span class="p">)</span>
+    <span class="n">index</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">-</span><span class="si">{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">index</span><span class="p">,</span> <span class="n">index_date</span><span class="p">)</span>
+    <span class="n">index_settings</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
+        <span class="n">number_of_shards</span><span class="o">=</span><span class="n">number_of_shards</span><span class="p">,</span> <span class="n">number_of_replicas</span><span class="o">=</span><span class="n">number_of_replicas</span>
+    <span class="p">)</span>
+
+    <span class="n">smtp_tls_doc</span> <span class="o">=</span> <span class="n">_SMTPTLSReportDoc</span><span class="p">(</span>
+        <span class="n">org_name</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;organization_name&quot;</span><span class="p">],</span>
+        <span class="n">date_range</span><span class="o">=</span><span class="p">[</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">],</span> <span class="n">report</span><span class="p">[</span><span class="s2">&quot;end_date&quot;</span><span class="p">]],</span>
+        <span class="n">date_begin</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">],</span>
+        <span class="n">date_end</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;end_date&quot;</span><span class="p">],</span>
+        <span class="n">contact_info</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;contact_info&quot;</span><span class="p">],</span>
+        <span class="n">report_id</span><span class="o">=</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;report_id&quot;</span><span class="p">],</span>
+    <span class="p">)</span>
+
+    <span class="k">for</span> <span class="n">policy</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">&quot;policies&quot;</span><span class="p">]:</span>
+        <span class="n">policy_strings</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="n">mx_host_patterns</span> <span class="o">=</span> <span class="kc">None</span>
+        <span class="k">if</span> <span class="s2">&quot;policy_strings&quot;</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
+            <span class="n">policy_strings</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">&quot;policy_strings&quot;</span><span class="p">]</span>
+        <span class="k">if</span> <span class="s2">&quot;mx_host_patterns&quot;</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
+            <span class="n">mx_host_patterns</span> <span class="o">=</span> <span class="n">policy</span><span class="p">[</span><span class="s2">&quot;mx_host_patterns&quot;</span><span class="p">]</span>
+        <span class="n">policy_doc</span> <span class="o">=</span> <span class="n">_SMTPTLSPolicyDoc</span><span class="p">(</span>
+            <span class="n">policy_domain</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">&quot;policy_domain&quot;</span><span class="p">],</span>
+            <span class="n">policy_type</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">&quot;policy_type&quot;</span><span class="p">],</span>
+            <span class="n">successful_session_count</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">&quot;successful_session_count&quot;</span><span class="p">],</span>
+            <span class="n">failed_session_count</span><span class="o">=</span><span class="n">policy</span><span class="p">[</span><span class="s2">&quot;failed_session_count&quot;</span><span class="p">],</span>
+            <span class="n">policy_string</span><span class="o">=</span><span class="n">policy_strings</span><span class="p">,</span>
+            <span class="n">mx_host_patterns</span><span class="o">=</span><span class="n">mx_host_patterns</span><span class="p">,</span>
+        <span class="p">)</span>
+        <span class="k">if</span> <span class="s2">&quot;failure_details&quot;</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">:</span>
+            <span class="k">for</span> <span class="n">failure_detail</span> <span class="ow">in</span> <span class="n">policy</span><span class="p">[</span><span class="s2">&quot;failure_details&quot;</span><span class="p">]:</span>
+                <span class="n">receiving_mx_hostname</span> <span class="o">=</span> <span class="kc">None</span>
+                <span class="n">additional_information_uri</span> <span class="o">=</span> <span class="kc">None</span>
+                <span class="n">failure_reason_code</span> <span class="o">=</span> <span class="kc">None</span>
+                <span class="n">ip_address</span> <span class="o">=</span> <span class="kc">None</span>
+                <span class="n">receiving_ip</span> <span class="o">=</span> <span class="kc">None</span>
+                <span class="n">receiving_mx_helo</span> <span class="o">=</span> <span class="kc">None</span>
+                <span class="n">sending_mta_ip</span> <span class="o">=</span> <span class="kc">None</span>
+
+                <span class="k">if</span> <span class="s2">&quot;receiving_mx_hostname&quot;</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
+                    <span class="n">receiving_mx_hostname</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">&quot;receiving_mx_hostname&quot;</span><span class="p">]</span>
+                <span class="k">if</span> <span class="s2">&quot;additional_information_uri&quot;</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
+                    <span class="n">additional_information_uri</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span>
+                        <span class="s2">&quot;additional_information_uri&quot;</span>
+                    <span class="p">]</span>
+                <span class="k">if</span> <span class="s2">&quot;failure_reason_code&quot;</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
+                    <span class="n">failure_reason_code</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">&quot;failure_reason_code&quot;</span><span class="p">]</span>
+                <span class="k">if</span> <span class="s2">&quot;ip_address&quot;</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
+                    <span class="n">ip_address</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">&quot;ip_address&quot;</span><span class="p">]</span>
+                <span class="k">if</span> <span class="s2">&quot;receiving_ip&quot;</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
+                    <span class="n">receiving_ip</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">&quot;receiving_ip&quot;</span><span class="p">]</span>
+                <span class="k">if</span> <span class="s2">&quot;receiving_mx_helo&quot;</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
+                    <span class="n">receiving_mx_helo</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">&quot;receiving_mx_helo&quot;</span><span class="p">]</span>
+                <span class="k">if</span> <span class="s2">&quot;sending_mta_ip&quot;</span> <span class="ow">in</span> <span class="n">failure_detail</span><span class="p">:</span>
+                    <span class="n">sending_mta_ip</span> <span class="o">=</span> <span class="n">failure_detail</span><span class="p">[</span><span class="s2">&quot;sending_mta_ip&quot;</span><span class="p">]</span>
+                <span class="n">policy_doc</span><span class="o">.</span><span class="n">add_failure_details</span><span class="p">(</span>
+                    <span class="n">result_type</span><span class="o">=</span><span class="n">failure_detail</span><span class="p">[</span><span class="s2">&quot;result_type&quot;</span><span class="p">],</span>
+                    <span class="n">ip_address</span><span class="o">=</span><span class="n">ip_address</span><span class="p">,</span>
+                    <span class="n">receiving_ip</span><span class="o">=</span><span class="n">receiving_ip</span><span class="p">,</span>
+                    <span class="n">receiving_mx_helo</span><span class="o">=</span><span class="n">receiving_mx_helo</span><span class="p">,</span>
+                    <span class="n">failed_session_count</span><span class="o">=</span><span class="n">failure_detail</span><span class="p">[</span><span class="s2">&quot;failed_session_count&quot;</span><span class="p">],</span>
+                    <span class="n">sending_mta_ip</span><span class="o">=</span><span class="n">sending_mta_ip</span><span class="p">,</span>
+                    <span class="n">receiving_mx_hostname</span><span class="o">=</span><span class="n">receiving_mx_hostname</span><span class="p">,</span>
+                    <span class="n">additional_information_uri</span><span class="o">=</span><span class="n">additional_information_uri</span><span class="p">,</span>
+                    <span class="n">failure_reason_code</span><span class="o">=</span><span class="n">failure_reason_code</span><span class="p">,</span>
+                <span class="p">)</span>
+        <span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">policies</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">policy_doc</span><span class="p">)</span>
+
+    <span class="n">create_indexes</span><span class="p">([</span><span class="n">index</span><span class="p">],</span> <span class="n">index_settings</span><span class="p">)</span>
+    <span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">meta</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>
+
+    <span class="k">try</span><span class="p">:</span>
+        <span class="n">smtp_tls_doc</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
+    <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="n">OpenSearchError</span><span class="p">(</span><span class="s2">&quot;OpenSearch error: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span></div>
+
+</pre></div>
+
+           </div>
+          </div>
+          <footer>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>&#169; Copyright 2018 - 2025, Sean Whalen and contributors.</p>
+  </div>
+
+  Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
+    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
+    provided by <a href="https://readthedocs.org">Read the Docs</a>.
+   
+
+</footer>
+        </div>
+      </div>
+    </section>
+  </div>
+  <script>
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>

_modules/parsedmarc/splunk.html

@@ -0,0 +1,336 @@
+
+
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" data-content_root="../../">
+<head>
+  <meta charset="utf-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>parsedmarc.splunk &mdash; parsedmarc 9.0.10 documentation</title>
+      <link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
+      <link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=9edc463e" />
+
+  
+      <script src="../../_static/jquery.js?v=5d32c60e"></script>
+      <script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
+      <script src="../../_static/documentation_options.js?v=164cc7e6"></script>
+      <script src="../../_static/doctools.js?v=fd6eb6e6"></script>
+      <script src="../../_static/sphinx_highlight.js?v=6ffebe34"></script>
+    <script src="../../_static/js/theme.js"></script>
+    <link rel="index" title="Index" href="../../genindex.html" />
+    <link rel="search" title="Search" href="../../search.html" /> 
+</head>
+
+<body class="wy-body-for-nav"> 
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search" >
+
+          
+          
+          <a href="../../index.html" class="icon icon-home">
+            parsedmarc
+          </a>
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+        </div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <p class="caption" role="heading"><span class="caption-text">Contents</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../usage.html">Using parsedmarc</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../output.html">Sample outputs</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../elasticsearch.html">Elasticsearch and Kibana</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../opensearch.html">OpenSearch and Grafana</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../kibana.html">Using the Kibana dashboards</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../splunk.html">Splunk</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../davmail.html">Accessing an inbox using OWA/EWS</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../dmarc.html">Understanding DMARC</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../contributing.html">Contributing to parsedmarc</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../api.html">API reference</a></li>
+</ul>
+
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../index.html">parsedmarc</a>
+      </nav>
+
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          <div role="navigation" aria-label="Page navigation">
+  <ul class="wy-breadcrumbs">
+      <li><a href="../../index.html" class="icon icon-home" aria-label="Home"></a></li>
+          <li class="breadcrumb-item"><a href="../index.html">Module code</a></li>
+          <li class="breadcrumb-item"><a href="../parsedmarc.html">parsedmarc</a></li>
+      <li class="breadcrumb-item active">parsedmarc.splunk</li>
+      <li class="wy-breadcrumbs-aside">
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+             
+  <h1>Source code for parsedmarc.splunk</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+
+<span class="kn">from</span><span class="w"> </span><span class="nn">__future__</span><span class="w"> </span><span class="kn">import</span> <span class="n">annotations</span>
+
+<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">socket</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="kn">import</span> <span class="n">Any</span><span class="p">,</span> <span class="n">Union</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">urllib.parse</span><span class="w"> </span><span class="kn">import</span> <span class="n">urlparse</span>
+
+<span class="kn">import</span><span class="w"> </span><span class="nn">requests</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">urllib3</span>
+
+<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.constants</span><span class="w"> </span><span class="kn">import</span> <span class="n">USER_AGENT</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.log</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.utils</span><span class="w"> </span><span class="kn">import</span> <span class="n">human_timestamp_to_unix_timestamp</span>
+
+<span class="n">urllib3</span><span class="o">.</span><span class="n">disable_warnings</span><span class="p">(</span><span class="n">urllib3</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">InsecureRequestWarning</span><span class="p">)</span>
+
+
+<div class="viewcode-block" id="SplunkError">
+<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.SplunkError">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">SplunkError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Raised when a Splunk API error occurs&quot;&quot;&quot;</span></div>
+
+
+
+<div class="viewcode-block" id="HECClient">
+<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">HECClient</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;A client for a Splunk HTTP Events Collector (HEC)&quot;&quot;&quot;</span>
+
+    <span class="c1"># http://docs.splunk.com/Documentation/Splunk/latest/Data/AboutHEC</span>
+    <span class="c1"># http://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTinput#services.2Fcollector</span>
+
+    <span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">url</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+        <span class="n">access_token</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+        <span class="n">index</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+        <span class="n">source</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;parsedmarc&quot;</span><span class="p">,</span>
+        <span class="n">verify</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+        <span class="n">timeout</span><span class="o">=</span><span class="mi">60</span><span class="p">,</span>
+    <span class="p">):</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Initializes the HECClient</span>
+
+<span class="sd">        Args:</span>
+<span class="sd">            url (str): The URL of the HEC</span>
+<span class="sd">            access_token (str): The HEC access token</span>
+<span class="sd">            index (str): The name of the index</span>
+<span class="sd">            source (str): The source name</span>
+<span class="sd">            verify (bool): Verify SSL certificates</span>
+<span class="sd">            timeout (float): Number of seconds to wait for the server to send</span>
+<span class="sd">                data before giving up</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">parsed_url</span> <span class="o">=</span> <span class="n">urlparse</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">url</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">://</span><span class="si">{1}</span><span class="s2">/services/collector/event/1.0&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+            <span class="n">parsed_url</span><span class="o">.</span><span class="n">scheme</span><span class="p">,</span> <span class="n">parsed_url</span><span class="o">.</span><span class="n">netloc</span>
+        <span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">access_token</span> <span class="o">=</span> <span class="n">access_token</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s2">&quot;Splunk &quot;</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">index</span> <span class="o">=</span> <span class="n">index</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">host</span> <span class="o">=</span> <span class="n">socket</span><span class="o">.</span><span class="n">getfqdn</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">source</span> <span class="o">=</span> <span class="n">source</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">session</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">Session</span><span class="p">()</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">timeout</span> <span class="o">=</span> <span class="n">timeout</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">verify</span> <span class="o">=</span> <span class="n">verify</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_common_data</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">float</span><span class="p">,</span> <span class="nb">dict</span><span class="p">]]</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
+            <span class="n">host</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">host</span><span class="p">,</span> <span class="n">source</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">source</span><span class="p">,</span> <span class="n">index</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">index</span>
+        <span class="p">)</span>
+
+        <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">headers</span> <span class="o">=</span> <span class="p">{</span>
+            <span class="s2">&quot;User-Agent&quot;</span><span class="p">:</span> <span class="n">USER_AGENT</span><span class="p">,</span>
+            <span class="s2">&quot;Authorization&quot;</span><span class="p">:</span> <span class="s2">&quot;Splunk </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">access_token</span><span class="p">),</span>
+        <span class="p">}</span>
+
+<div class="viewcode-block" id="HECClient.save_aggregate_reports_to_splunk">
+<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_aggregate_reports_to_splunk">[docs]</a>
+    <span class="k">def</span><span class="w"> </span><span class="nf">save_aggregate_reports_to_splunk</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">aggregate_reports</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]],</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]],</span>
+    <span class="p">):</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Saves aggregate DMARC reports to Splunk</span>
+
+<span class="sd">        Args:</span>
+<span class="sd">            aggregate_reports: A list of aggregate report dictionaries</span>
+<span class="sd">                to save in Splunk</span>
+
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Saving aggregate reports to Splunk&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">aggregate_reports</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
+            <span class="n">aggregate_reports</span> <span class="o">=</span> <span class="p">[</span><span class="n">aggregate_reports</span><span class="p">]</span>
+
+        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">aggregate_reports</span><span class="p">)</span> <span class="o">&lt;</span> <span class="mi">1</span><span class="p">:</span>
+            <span class="k">return</span>
+
+        <span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_common_data</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+        <span class="n">json_str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
+        <span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">aggregate_reports</span><span class="p">:</span>
+            <span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">&quot;records&quot;</span><span class="p">]:</span>
+                <span class="n">new_report</span><span class="p">:</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">float</span><span class="p">,</span> <span class="nb">dict</span><span class="p">]]</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
+                <span class="k">for</span> <span class="n">metadata</span> <span class="ow">in</span> <span class="n">report</span><span class="p">[</span><span class="s2">&quot;report_metadata&quot;</span><span class="p">]:</span>
+                    <span class="n">new_report</span><span class="p">[</span><span class="n">metadata</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">&quot;report_metadata&quot;</span><span class="p">][</span><span class="n">metadata</span><span class="p">]</span>
+                <span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;interval_begin&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;interval_begin&quot;</span><span class="p">]</span>
+                <span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;interval_end&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;interval_end&quot;</span><span class="p">]</span>
+                <span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;normalized_timespan&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;normalized_timespan&quot;</span><span class="p">]</span>
+                <span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;published_policy&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="p">[</span><span class="s2">&quot;policy_published&quot;</span><span class="p">]</span>
+                <span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;source_ip_address&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;ip_address&quot;</span><span class="p">]</span>
+                <span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;source_country&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;country&quot;</span><span class="p">]</span>
+                <span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;source_reverse_dns&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;reverse_dns&quot;</span><span class="p">]</span>
+                <span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;source_base_domain&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;base_domain&quot;</span><span class="p">]</span>
+                <span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;source_type&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;type&quot;</span><span class="p">]</span>
+                <span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;source_name&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;source&quot;</span><span class="p">][</span><span class="s2">&quot;name&quot;</span><span class="p">]</span>
+                <span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;message_count&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;count&quot;</span><span class="p">]</span>
+                <span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;disposition&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;policy_evaluated&quot;</span><span class="p">][</span><span class="s2">&quot;disposition&quot;</span><span class="p">]</span>
+                <span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;spf_aligned&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;alignment&quot;</span><span class="p">][</span><span class="s2">&quot;spf&quot;</span><span class="p">]</span>
+                <span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;dkim_aligned&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;alignment&quot;</span><span class="p">][</span><span class="s2">&quot;dkim&quot;</span><span class="p">]</span>
+                <span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;passed_dmarc&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;alignment&quot;</span><span class="p">][</span><span class="s2">&quot;dmarc&quot;</span><span class="p">]</span>
+                <span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;header_from&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;identifiers&quot;</span><span class="p">][</span><span class="s2">&quot;header_from&quot;</span><span class="p">]</span>
+                <span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;envelope_from&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;identifiers&quot;</span><span class="p">][</span><span class="s2">&quot;envelope_from&quot;</span><span class="p">]</span>
+                <span class="k">if</span> <span class="s2">&quot;dkim&quot;</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;auth_results&quot;</span><span class="p">]:</span>
+                    <span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;dkim_results&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;auth_results&quot;</span><span class="p">][</span><span class="s2">&quot;dkim&quot;</span><span class="p">]</span>
+                <span class="k">if</span> <span class="s2">&quot;spf&quot;</span> <span class="ow">in</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;auth_results&quot;</span><span class="p">]:</span>
+                    <span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;spf_results&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">record</span><span class="p">[</span><span class="s2">&quot;auth_results&quot;</span><span class="p">][</span><span class="s2">&quot;spf&quot;</span><span class="p">]</span>
+
+                <span class="n">data</span><span class="p">[</span><span class="s2">&quot;sourcetype&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;dmarc:aggregate&quot;</span>
+                <span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_unix_timestamp</span><span class="p">(</span>
+                    <span class="n">new_report</span><span class="p">[</span><span class="s2">&quot;interval_begin&quot;</span><span class="p">]</span>
+                <span class="p">)</span>
+                <span class="n">data</span><span class="p">[</span><span class="s2">&quot;time&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp</span>
+                <span class="n">data</span><span class="p">[</span><span class="s2">&quot;event&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+                <span class="n">json_str</span> <span class="o">+=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="se">\n</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">data</span><span class="p">))</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">verify</span><span class="p">:</span>
+            <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Skipping certificate verification for Splunk HEC&quot;</span><span class="p">)</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">post</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">json_str</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">)</span>
+            <span class="n">response</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
+        <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
+        <span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="s2">&quot;code&quot;</span><span class="p">]</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s2">&quot;text&quot;</span><span class="p">])</span></div>
+
+
+<div class="viewcode-block" id="HECClient.save_forensic_reports_to_splunk">
+<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_forensic_reports_to_splunk">[docs]</a>
+    <span class="k">def</span><span class="w"> </span><span class="nf">save_forensic_reports_to_splunk</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span>
+        <span class="n">forensic_reports</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]],</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]],</span>
+    <span class="p">):</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Saves forensic DMARC reports to Splunk</span>
+
+<span class="sd">        Args:</span>
+<span class="sd">            forensic_reports (list): A list of forensic report dictionaries</span>
+<span class="sd">                to save in Splunk</span>
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Saving forensic reports to Splunk&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">forensic_reports</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
+            <span class="n">forensic_reports</span> <span class="o">=</span> <span class="p">[</span><span class="n">forensic_reports</span><span class="p">]</span>
+
+        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">forensic_reports</span><span class="p">)</span> <span class="o">&lt;</span> <span class="mi">1</span><span class="p">:</span>
+            <span class="k">return</span>
+
+        <span class="n">json_str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
+        <span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">forensic_reports</span><span class="p">:</span>
+            <span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_common_data</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+            <span class="n">data</span><span class="p">[</span><span class="s2">&quot;sourcetype&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;dmarc:forensic&quot;</span>
+            <span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_unix_timestamp</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;arrival_date_utc&quot;</span><span class="p">])</span>
+            <span class="n">data</span><span class="p">[</span><span class="s2">&quot;time&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp</span>
+            <span class="n">data</span><span class="p">[</span><span class="s2">&quot;event&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+            <span class="n">json_str</span> <span class="o">+=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="se">\n</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">data</span><span class="p">))</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">verify</span><span class="p">:</span>
+            <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Skipping certificate verification for Splunk HEC&quot;</span><span class="p">)</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">post</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">json_str</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">)</span>
+            <span class="n">response</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
+        <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
+        <span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="s2">&quot;code&quot;</span><span class="p">]</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s2">&quot;text&quot;</span><span class="p">])</span></div>
+
+
+<div class="viewcode-block" id="HECClient.save_smtp_tls_reports_to_splunk">
+<a class="viewcode-back" href="../../api.html#parsedmarc.splunk.HECClient.save_smtp_tls_reports_to_splunk">[docs]</a>
+    <span class="k">def</span><span class="w"> </span><span class="nf">save_smtp_tls_reports_to_splunk</span><span class="p">(</span>
+        <span class="bp">self</span><span class="p">,</span> <span class="n">reports</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]],</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span>
+    <span class="p">):</span>
+<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">        Saves aggregate DMARC reports to Splunk</span>
+
+<span class="sd">        Args:</span>
+<span class="sd">            reports: A list of SMTP TLS report dictionaries</span>
+<span class="sd">                to save in Splunk</span>
+
+<span class="sd">        &quot;&quot;&quot;</span>
+        <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Saving SMTP TLS reports to Splunk&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">reports</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
+            <span class="n">reports</span> <span class="o">=</span> <span class="p">[</span><span class="n">reports</span><span class="p">]</span>
+
+        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">reports</span><span class="p">)</span> <span class="o">&lt;</span> <span class="mi">1</span><span class="p">:</span>
+            <span class="k">return</span>
+
+        <span class="n">data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_common_data</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+        <span class="n">json_str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span>
+        <span class="k">for</span> <span class="n">report</span> <span class="ow">in</span> <span class="n">reports</span><span class="p">:</span>
+            <span class="n">data</span><span class="p">[</span><span class="s2">&quot;sourcetype&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;smtp:tls&quot;</span>
+            <span class="n">timestamp</span> <span class="o">=</span> <span class="n">human_timestamp_to_unix_timestamp</span><span class="p">(</span><span class="n">report</span><span class="p">[</span><span class="s2">&quot;begin_date&quot;</span><span class="p">])</span>
+            <span class="n">data</span><span class="p">[</span><span class="s2">&quot;time&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">timestamp</span>
+            <span class="n">data</span><span class="p">[</span><span class="s2">&quot;event&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">report</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+            <span class="n">json_str</span> <span class="o">+=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="se">\n</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">data</span><span class="p">))</span>
+
+        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">verify</span><span class="p">:</span>
+            <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Skipping certificate verification for Splunk HEC&quot;</span><span class="p">)</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">session</span><span class="o">.</span><span class="n">post</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">url</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">json_str</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">timeout</span><span class="p">)</span>
+            <span class="n">response</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
+        <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">())</span>
+        <span class="k">if</span> <span class="n">response</span><span class="p">[</span><span class="s2">&quot;code&quot;</span><span class="p">]</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">SplunkError</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s2">&quot;text&quot;</span><span class="p">])</span></div>
+</div>
+
+</pre></div>
+
+           </div>
+          </div>
+          <footer>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>&#169; Copyright 2018 - 2025, Sean Whalen and contributors.</p>
+  </div>
+
+  Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
+    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
+    provided by <a href="https://readthedocs.org">Read the Docs</a>.
+   
+
+</footer>
+        </div>
+      </div>
+    </section>
+  </div>
+  <script>
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>

_modules/parsedmarc/types.html

@@ -0,0 +1,404 @@
+
+
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" data-content_root="../../">
+<head>
+  <meta charset="utf-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>parsedmarc.types &mdash; parsedmarc 9.0.10 documentation</title>
+      <link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
+      <link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=9edc463e" />
+
+  
+      <script src="../../_static/jquery.js?v=5d32c60e"></script>
+      <script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
+      <script src="../../_static/documentation_options.js?v=164cc7e6"></script>
+      <script src="../../_static/doctools.js?v=fd6eb6e6"></script>
+      <script src="../../_static/sphinx_highlight.js?v=6ffebe34"></script>
+    <script src="../../_static/js/theme.js"></script>
+    <link rel="index" title="Index" href="../../genindex.html" />
+    <link rel="search" title="Search" href="../../search.html" /> 
+</head>
+
+<body class="wy-body-for-nav"> 
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search" >
+
+          
+          
+          <a href="../../index.html" class="icon icon-home">
+            parsedmarc
+          </a>
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+        </div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <p class="caption" role="heading"><span class="caption-text">Contents</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../usage.html">Using parsedmarc</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../output.html">Sample outputs</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../elasticsearch.html">Elasticsearch and Kibana</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../opensearch.html">OpenSearch and Grafana</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../kibana.html">Using the Kibana dashboards</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../splunk.html">Splunk</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../davmail.html">Accessing an inbox using OWA/EWS</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../dmarc.html">Understanding DMARC</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../contributing.html">Contributing to parsedmarc</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../api.html">API reference</a></li>
+</ul>
+
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../index.html">parsedmarc</a>
+      </nav>
+
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          <div role="navigation" aria-label="Page navigation">
+  <ul class="wy-breadcrumbs">
+      <li><a href="../../index.html" class="icon icon-home" aria-label="Home"></a></li>
+          <li class="breadcrumb-item"><a href="../index.html">Module code</a></li>
+          <li class="breadcrumb-item"><a href="../parsedmarc.html">parsedmarc</a></li>
+      <li class="breadcrumb-item active">parsedmarc.types</li>
+      <li class="wy-breadcrumbs-aside">
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+             
+  <h1>Source code for parsedmarc.types</h1><div class="highlight"><pre>
+<span></span><span class="kn">from</span><span class="w"> </span><span class="nn">__future__</span><span class="w"> </span><span class="kn">import</span> <span class="n">annotations</span>
+
+<span class="kn">from</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="kn">import</span> <span class="n">Any</span><span class="p">,</span> <span class="n">Dict</span><span class="p">,</span> <span class="n">List</span><span class="p">,</span> <span class="n">Literal</span><span class="p">,</span> <span class="n">Optional</span><span class="p">,</span> <span class="n">TypedDict</span><span class="p">,</span> <span class="n">Union</span>
+
+<span class="c1"># NOTE: This module is intentionally Python 3.9 compatible.</span>
+<span class="c1"># - No PEP 604 unions (A | B)</span>
+<span class="c1"># - No typing.NotRequired / Required (3.11+) to avoid an extra dependency.</span>
+<span class="c1">#   For optional keys, use total=False TypedDicts.</span>
+
+
+<span class="n">ReportType</span> <span class="o">=</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;aggregate&quot;</span><span class="p">,</span> <span class="s2">&quot;forensic&quot;</span><span class="p">,</span> <span class="s2">&quot;smtp_tls&quot;</span><span class="p">]</span>
+
+
+<div class="viewcode-block" id="AggregateReportMetadata">
+<a class="viewcode-back" href="../../api.html#parsedmarc.types.AggregateReportMetadata">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">AggregateReportMetadata</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
+    <span class="n">org_name</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">org_email</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">org_extra_contact_info</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="n">report_id</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">begin_date</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">end_date</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">timespan_requires_normalization</span><span class="p">:</span> <span class="nb">bool</span>
+    <span class="n">original_timespan_seconds</span><span class="p">:</span> <span class="nb">int</span>
+    <span class="n">errors</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span></div>
+
+
+
+<div class="viewcode-block" id="AggregatePolicyPublished">
+<a class="viewcode-back" href="../../api.html#parsedmarc.types.AggregatePolicyPublished">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">AggregatePolicyPublished</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
+    <span class="n">domain</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">adkim</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">aspf</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">p</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">sp</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">pct</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">fo</span><span class="p">:</span> <span class="nb">str</span></div>
+
+
+
+<div class="viewcode-block" id="IPSourceInfo">
+<a class="viewcode-back" href="../../api.html#parsedmarc.types.IPSourceInfo">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">IPSourceInfo</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
+    <span class="n">ip_address</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">country</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="n">reverse_dns</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="n">base_domain</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="n">name</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="nb">type</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span></div>
+
+
+
+<div class="viewcode-block" id="AggregateAlignment">
+<a class="viewcode-back" href="../../api.html#parsedmarc.types.AggregateAlignment">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">AggregateAlignment</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
+    <span class="n">spf</span><span class="p">:</span> <span class="nb">bool</span>
+    <span class="n">dkim</span><span class="p">:</span> <span class="nb">bool</span>
+    <span class="n">dmarc</span><span class="p">:</span> <span class="nb">bool</span></div>
+
+
+
+<div class="viewcode-block" id="AggregateIdentifiers">
+<a class="viewcode-back" href="../../api.html#parsedmarc.types.AggregateIdentifiers">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">AggregateIdentifiers</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
+    <span class="n">header_from</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">envelope_from</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="n">envelope_to</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span></div>
+
+
+
+<div class="viewcode-block" id="AggregatePolicyOverrideReason">
+<a class="viewcode-back" href="../../api.html#parsedmarc.types.AggregatePolicyOverrideReason">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">AggregatePolicyOverrideReason</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
+    <span class="nb">type</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="n">comment</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span></div>
+
+
+
+<div class="viewcode-block" id="AggregateAuthResultDKIM">
+<a class="viewcode-back" href="../../api.html#parsedmarc.types.AggregateAuthResultDKIM">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">AggregateAuthResultDKIM</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
+    <span class="n">domain</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">result</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">selector</span><span class="p">:</span> <span class="nb">str</span></div>
+
+
+
+<div class="viewcode-block" id="AggregateAuthResultSPF">
+<a class="viewcode-back" href="../../api.html#parsedmarc.types.AggregateAuthResultSPF">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">AggregateAuthResultSPF</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
+    <span class="n">domain</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">result</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">scope</span><span class="p">:</span> <span class="nb">str</span></div>
+
+
+
+<div class="viewcode-block" id="AggregateAuthResults">
+<a class="viewcode-back" href="../../api.html#parsedmarc.types.AggregateAuthResults">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">AggregateAuthResults</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
+    <span class="n">dkim</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">AggregateAuthResultDKIM</span><span class="p">]</span>
+    <span class="n">spf</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">AggregateAuthResultSPF</span><span class="p">]</span></div>
+
+
+
+<div class="viewcode-block" id="AggregatePolicyEvaluated">
+<a class="viewcode-back" href="../../api.html#parsedmarc.types.AggregatePolicyEvaluated">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">AggregatePolicyEvaluated</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
+    <span class="n">disposition</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">dkim</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">spf</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">policy_override_reasons</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">AggregatePolicyOverrideReason</span><span class="p">]</span></div>
+
+
+
+<div class="viewcode-block" id="AggregateRecord">
+<a class="viewcode-back" href="../../api.html#parsedmarc.types.AggregateRecord">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">AggregateRecord</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
+    <span class="n">interval_begin</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">interval_end</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">source</span><span class="p">:</span> <span class="n">IPSourceInfo</span>
+    <span class="n">count</span><span class="p">:</span> <span class="nb">int</span>
+    <span class="n">alignment</span><span class="p">:</span> <span class="n">AggregateAlignment</span>
+    <span class="n">policy_evaluated</span><span class="p">:</span> <span class="n">AggregatePolicyEvaluated</span>
+    <span class="n">disposition</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">identifiers</span><span class="p">:</span> <span class="n">AggregateIdentifiers</span>
+    <span class="n">auth_results</span><span class="p">:</span> <span class="n">AggregateAuthResults</span></div>
+
+
+
+<div class="viewcode-block" id="AggregateReport">
+<a class="viewcode-back" href="../../api.html#parsedmarc.types.AggregateReport">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">AggregateReport</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
+    <span class="n">xml_schema</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">report_metadata</span><span class="p">:</span> <span class="n">AggregateReportMetadata</span>
+    <span class="n">policy_published</span><span class="p">:</span> <span class="n">AggregatePolicyPublished</span>
+    <span class="n">records</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">AggregateRecord</span><span class="p">]</span></div>
+
+
+
+<div class="viewcode-block" id="EmailAddress">
+<a class="viewcode-back" href="../../api.html#parsedmarc.types.EmailAddress">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">EmailAddress</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
+    <span class="n">display_name</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="n">address</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">local</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="n">domain</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span></div>
+
+
+
+<div class="viewcode-block" id="EmailAttachment">
+<a class="viewcode-back" href="../../api.html#parsedmarc.types.EmailAttachment">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">EmailAttachment</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">,</span> <span class="n">total</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+    <span class="n">filename</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="n">mail_content_type</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="n">sha256</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span></div>
+
+
+
+<span class="n">ParsedEmail</span> <span class="o">=</span> <span class="n">TypedDict</span><span class="p">(</span>
+    <span class="s2">&quot;ParsedEmail&quot;</span><span class="p">,</span>
+    <span class="p">{</span>
+        <span class="c1"># This is a lightly-specified version of mailsuite/mailparser JSON.</span>
+        <span class="c1"># It focuses on the fields parsedmarc uses in forensic handling.</span>
+        <span class="s2">&quot;headers&quot;</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
+        <span class="s2">&quot;subject&quot;</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span>
+        <span class="s2">&quot;filename_safe_subject&quot;</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span>
+        <span class="s2">&quot;date&quot;</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span>
+        <span class="s2">&quot;from&quot;</span><span class="p">:</span> <span class="n">EmailAddress</span><span class="p">,</span>
+        <span class="s2">&quot;to&quot;</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">EmailAddress</span><span class="p">],</span>
+        <span class="s2">&quot;cc&quot;</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">EmailAddress</span><span class="p">],</span>
+        <span class="s2">&quot;bcc&quot;</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">EmailAddress</span><span class="p">],</span>
+        <span class="s2">&quot;attachments&quot;</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">EmailAttachment</span><span class="p">],</span>
+        <span class="s2">&quot;body&quot;</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span>
+        <span class="s2">&quot;has_defects&quot;</span><span class="p">:</span> <span class="nb">bool</span><span class="p">,</span>
+        <span class="s2">&quot;defects&quot;</span><span class="p">:</span> <span class="n">Any</span><span class="p">,</span>
+        <span class="s2">&quot;defects_categories&quot;</span><span class="p">:</span> <span class="n">Any</span><span class="p">,</span>
+    <span class="p">},</span>
+    <span class="n">total</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+<span class="p">)</span>
+
+
+<div class="viewcode-block" id="ForensicReport">
+<a class="viewcode-back" href="../../api.html#parsedmarc.types.ForensicReport">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">ForensicReport</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
+    <span class="n">feedback_type</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="n">user_agent</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="n">version</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="n">original_envelope_id</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="n">original_mail_from</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="n">original_rcpt_to</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="n">arrival_date</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">arrival_date_utc</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">authentication_results</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="n">delivery_result</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="n">auth_failure</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="n">authentication_mechanisms</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="n">dkim_domain</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="n">reported_domain</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">sample_headers_only</span><span class="p">:</span> <span class="nb">bool</span>
+    <span class="n">source</span><span class="p">:</span> <span class="n">IPSourceInfo</span>
+    <span class="n">sample</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">parsed_sample</span><span class="p">:</span> <span class="n">ParsedEmail</span></div>
+
+
+
+<div class="viewcode-block" id="SMTPTLSFailureDetails">
+<a class="viewcode-back" href="../../api.html#parsedmarc.types.SMTPTLSFailureDetails">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">SMTPTLSFailureDetails</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
+    <span class="n">result_type</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">failed_session_count</span><span class="p">:</span> <span class="nb">int</span></div>
+
+
+
+<div class="viewcode-block" id="SMTPTLSFailureDetailsOptional">
+<a class="viewcode-back" href="../../api.html#parsedmarc.types.SMTPTLSFailureDetailsOptional">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">SMTPTLSFailureDetailsOptional</span><span class="p">(</span><span class="n">SMTPTLSFailureDetails</span><span class="p">,</span> <span class="n">total</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+    <span class="n">sending_mta_ip</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">receiving_ip</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">receiving_mx_hostname</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">receiving_mx_helo</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">additional_info_uri</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">failure_reason_code</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">ip_address</span><span class="p">:</span> <span class="nb">str</span></div>
+
+
+
+<div class="viewcode-block" id="SMTPTLSPolicySummary">
+<a class="viewcode-back" href="../../api.html#parsedmarc.types.SMTPTLSPolicySummary">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">SMTPTLSPolicySummary</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
+    <span class="n">policy_domain</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">policy_type</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">successful_session_count</span><span class="p">:</span> <span class="nb">int</span>
+    <span class="n">failed_session_count</span><span class="p">:</span> <span class="nb">int</span></div>
+
+
+
+<div class="viewcode-block" id="SMTPTLSPolicy">
+<a class="viewcode-back" href="../../api.html#parsedmarc.types.SMTPTLSPolicy">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">SMTPTLSPolicy</span><span class="p">(</span><span class="n">SMTPTLSPolicySummary</span><span class="p">,</span> <span class="n">total</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+    <span class="n">policy_strings</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="n">mx_host_patterns</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="n">failure_details</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">SMTPTLSFailureDetailsOptional</span><span class="p">]</span></div>
+
+
+
+<div class="viewcode-block" id="SMTPTLSReport">
+<a class="viewcode-back" href="../../api.html#parsedmarc.types.SMTPTLSReport">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">SMTPTLSReport</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
+    <span class="n">organization_name</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">begin_date</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">end_date</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">contact_info</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span>
+    <span class="n">report_id</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">policies</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">SMTPTLSPolicy</span><span class="p">]</span></div>
+
+
+
+<div class="viewcode-block" id="AggregateParsedReport">
+<a class="viewcode-back" href="../../api.html#parsedmarc.types.AggregateParsedReport">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">AggregateParsedReport</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
+    <span class="n">report_type</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;aggregate&quot;</span><span class="p">]</span>
+    <span class="n">report</span><span class="p">:</span> <span class="n">AggregateReport</span></div>
+
+
+
+<div class="viewcode-block" id="ForensicParsedReport">
+<a class="viewcode-back" href="../../api.html#parsedmarc.types.ForensicParsedReport">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">ForensicParsedReport</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
+    <span class="n">report_type</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;forensic&quot;</span><span class="p">]</span>
+    <span class="n">report</span><span class="p">:</span> <span class="n">ForensicReport</span></div>
+
+
+
+<div class="viewcode-block" id="SMTPTLSParsedReport">
+<a class="viewcode-back" href="../../api.html#parsedmarc.types.SMTPTLSParsedReport">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">SMTPTLSParsedReport</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
+    <span class="n">report_type</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;smtp_tls&quot;</span><span class="p">]</span>
+    <span class="n">report</span><span class="p">:</span> <span class="n">SMTPTLSReport</span></div>
+
+
+
+<span class="n">ParsedReport</span> <span class="o">=</span> <span class="n">Union</span><span class="p">[</span><span class="n">AggregateParsedReport</span><span class="p">,</span> <span class="n">ForensicParsedReport</span><span class="p">,</span> <span class="n">SMTPTLSParsedReport</span><span class="p">]</span>
+
+
+<div class="viewcode-block" id="ParsingResults">
+<a class="viewcode-back" href="../../api.html#parsedmarc.types.ParsingResults">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">ParsingResults</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
+    <span class="n">aggregate_reports</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">AggregateReport</span><span class="p">]</span>
+    <span class="n">forensic_reports</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">ForensicReport</span><span class="p">]</span>
+    <span class="n">smtp_tls_reports</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">SMTPTLSReport</span><span class="p">]</span></div>
+
+</pre></div>
+
+           </div>
+          </div>
+          <footer>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>&#169; Copyright 2018 - 2025, Sean Whalen and contributors.</p>
+  </div>
+
+  Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
+    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
+    provided by <a href="https://readthedocs.org">Read the Docs</a>.
+   
+
+</footer>
+        </div>
+      </div>
+    </section>
+  </div>
+  <script>
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>

_modules/parsedmarc/utils.html

@@ -0,0 +1,901 @@
+
+
+<!DOCTYPE html>
+<html class="writer-html5" lang="en" data-content_root="../../">
+<head>
+  <meta charset="utf-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>parsedmarc.utils &mdash; parsedmarc 9.0.10 documentation</title>
+      <link rel="stylesheet" type="text/css" href="../../_static/pygments.css?v=b86133f3" />
+      <link rel="stylesheet" type="text/css" href="../../_static/css/theme.css?v=9edc463e" />
+
+  
+      <script src="../../_static/jquery.js?v=5d32c60e"></script>
+      <script src="../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
+      <script src="../../_static/documentation_options.js?v=164cc7e6"></script>
+      <script src="../../_static/doctools.js?v=fd6eb6e6"></script>
+      <script src="../../_static/sphinx_highlight.js?v=6ffebe34"></script>
+    <script src="../../_static/js/theme.js"></script>
+    <link rel="index" title="Index" href="../../genindex.html" />
+    <link rel="search" title="Search" href="../../search.html" /> 
+</head>
+
+<body class="wy-body-for-nav"> 
+  <div class="wy-grid-for-nav">
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search" >
+
+          
+          
+          <a href="../../index.html" class="icon icon-home">
+            parsedmarc
+          </a>
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+        </div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
+              <p class="caption" role="heading"><span class="caption-text">Contents</span></p>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../usage.html">Using parsedmarc</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../output.html">Sample outputs</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../elasticsearch.html">Elasticsearch and Kibana</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../opensearch.html">OpenSearch and Grafana</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../kibana.html">Using the Kibana dashboards</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../splunk.html">Splunk</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../davmail.html">Accessing an inbox using OWA/EWS</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../dmarc.html">Understanding DMARC</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../contributing.html">Contributing to parsedmarc</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../api.html">API reference</a></li>
+</ul>
+
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../index.html">parsedmarc</a>
+      </nav>
+
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          <div role="navigation" aria-label="Page navigation">
+  <ul class="wy-breadcrumbs">
+      <li><a href="../../index.html" class="icon icon-home" aria-label="Home"></a></li>
+          <li class="breadcrumb-item"><a href="../index.html">Module code</a></li>
+          <li class="breadcrumb-item"><a href="../parsedmarc.html">parsedmarc</a></li>
+      <li class="breadcrumb-item active">parsedmarc.utils</li>
+      <li class="wy-breadcrumbs-aside">
+      </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+             
+  <h1>Source code for parsedmarc.utils</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+
+<span class="sd">&quot;&quot;&quot;Utility functions that might be useful for other projects&quot;&quot;&quot;</span>
+
+<span class="kn">from</span><span class="w"> </span><span class="nn">__future__</span><span class="w"> </span><span class="kn">import</span> <span class="n">annotations</span>
+
+<span class="kn">import</span><span class="w"> </span><span class="nn">base64</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">csv</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">hashlib</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">io</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">json</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">logging</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">mailbox</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">os</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">shutil</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">subprocess</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">tempfile</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">datetime</span><span class="w"> </span><span class="kn">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span><span class="p">,</span> <span class="n">timezone</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="kn">import</span> <span class="n">Optional</span><span class="p">,</span> <span class="n">TypedDict</span><span class="p">,</span> <span class="n">Union</span><span class="p">,</span> <span class="n">cast</span>
+
+<span class="kn">import</span><span class="w"> </span><span class="nn">mailparser</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">expiringdict</span><span class="w"> </span><span class="kn">import</span> <span class="n">ExpiringDict</span>
+
+<span class="k">try</span><span class="p">:</span>
+    <span class="kn">from</span><span class="w"> </span><span class="nn">importlib.resources</span><span class="w"> </span><span class="kn">import</span> <span class="n">files</span>
+<span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
+    <span class="c1"># Try backported to PY&lt;3 `importlib_resources`</span>
+    <span class="kn">from</span><span class="w"> </span><span class="nn">importlib.resources</span><span class="w"> </span><span class="kn">import</span> <span class="n">files</span>
+
+
+<span class="kn">import</span><span class="w"> </span><span class="nn">dns.exception</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">dns.resolver</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">dns.reversename</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">geoip2.database</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">geoip2.errors</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">publicsuffixlist</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">requests</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">dateutil.parser</span><span class="w"> </span><span class="kn">import</span> <span class="n">parse</span> <span class="k">as</span> <span class="n">parse_date</span>
+
+<span class="kn">import</span><span class="w"> </span><span class="nn">parsedmarc.resources.dbip</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">parsedmarc.resources.maps</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.constants</span><span class="w"> </span><span class="kn">import</span> <span class="n">USER_AGENT</span>
+<span class="kn">from</span><span class="w"> </span><span class="nn">parsedmarc.log</span><span class="w"> </span><span class="kn">import</span> <span class="n">logger</span>
+
+<span class="n">parenthesis_regex</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;\s*\(.*\)\s*&quot;</span><span class="p">)</span>
+
+<span class="n">null_file</span> <span class="o">=</span> <span class="nb">open</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">devnull</span><span class="p">,</span> <span class="s2">&quot;w&quot;</span><span class="p">)</span>
+<span class="n">mailparser_logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s2">&quot;mailparser&quot;</span><span class="p">)</span>
+<span class="n">mailparser_logger</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">CRITICAL</span><span class="p">)</span>
+<span class="n">psl</span> <span class="o">=</span> <span class="n">publicsuffixlist</span><span class="o">.</span><span class="n">PublicSuffixList</span><span class="p">()</span>
+<span class="n">psl_overrides_path</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">files</span><span class="p">(</span><span class="n">parsedmarc</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">maps</span><span class="p">)</span><span class="o">.</span><span class="n">joinpath</span><span class="p">(</span><span class="s2">&quot;psl_overrides.txt&quot;</span><span class="p">))</span>
+<span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">psl_overrides_path</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
+    <span class="n">psl_overrides</span> <span class="o">=</span> <span class="p">[</span><span class="n">line</span><span class="o">.</span><span class="n">rstrip</span><span class="p">()</span> <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="n">f</span><span class="o">.</span><span class="n">readlines</span><span class="p">()]</span>
+    <span class="k">while</span> <span class="s2">&quot;&quot;</span> <span class="ow">in</span> <span class="n">psl_overrides</span><span class="p">:</span>
+        <span class="n">psl_overrides</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="s2">&quot;&quot;</span><span class="p">)</span>
+
+
+<div class="viewcode-block" id="EmailParserError">
+<a class="viewcode-back" href="../../api.html#parsedmarc.utils.EmailParserError">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">EmailParserError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Raised when an error parsing the email occurs&quot;&quot;&quot;</span></div>
+
+
+
+<div class="viewcode-block" id="DownloadError">
+<a class="viewcode-back" href="../../api.html#parsedmarc.utils.DownloadError">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">DownloadError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;Raised when an error occurs when downloading a file&quot;&quot;&quot;</span></div>
+
+
+
+<div class="viewcode-block" id="ReverseDNSService">
+<a class="viewcode-back" href="../../api.html#parsedmarc.utils.ReverseDNSService">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">ReverseDNSService</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
+    <span class="n">name</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="nb">type</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span></div>
+
+
+
+<span class="n">ReverseDNSMap</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">ReverseDNSService</span><span class="p">]</span>
+
+
+<div class="viewcode-block" id="IPAddressInfo">
+<a class="viewcode-back" href="../../api.html#parsedmarc.utils.IPAddressInfo">[docs]</a>
+<span class="k">class</span><span class="w"> </span><span class="nc">IPAddressInfo</span><span class="p">(</span><span class="n">TypedDict</span><span class="p">):</span>
+    <span class="n">ip_address</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">reverse_dns</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="n">country</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="n">base_domain</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="n">name</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
+    <span class="nb">type</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span></div>
+
+
+
+<div class="viewcode-block" id="decode_base64">
+<a class="viewcode-back" href="../../api.html#parsedmarc.utils.decode_base64">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">decode_base64</span><span class="p">(</span><span class="n">data</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bytes</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Decodes a base64 string, with padding being optional</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        data (str): A base64 encoded string</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        bytes: The decoded bytes</span>
+
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">data_bytes</span> <span class="o">=</span> <span class="nb">bytes</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s2">&quot;ascii&quot;</span><span class="p">)</span>
+    <span class="n">missing_padding</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">data_bytes</span><span class="p">)</span> <span class="o">%</span> <span class="mi">4</span>
+    <span class="k">if</span> <span class="n">missing_padding</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
+        <span class="n">data_bytes</span> <span class="o">+=</span> <span class="sa">b</span><span class="s2">&quot;=&quot;</span> <span class="o">*</span> <span class="p">(</span><span class="mi">4</span> <span class="o">-</span> <span class="n">missing_padding</span><span class="p">)</span>
+    <span class="k">return</span> <span class="n">base64</span><span class="o">.</span><span class="n">b64decode</span><span class="p">(</span><span class="n">data_bytes</span><span class="p">)</span></div>
+
+
+
+<div class="viewcode-block" id="get_base_domain">
+<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_base_domain">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">get_base_domain</span><span class="p">(</span><span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Gets the base domain name for the given domain</span>
+
+<span class="sd">    .. note::</span>
+<span class="sd">        Results are based on a list of public domain suffixes at</span>
+<span class="sd">        https://publicsuffix.org/list/public_suffix_list.dat and overrides included in</span>
+<span class="sd">        parsedmarc.resources.maps.psl_overrides.txt</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        domain (str): A domain or subdomain</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        str: The base domain of the given domain</span>
+
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">domain</span> <span class="o">=</span> <span class="n">domain</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
+    <span class="n">publicsuffix</span> <span class="o">=</span> <span class="n">psl</span><span class="o">.</span><span class="n">privatesuffix</span><span class="p">(</span><span class="n">domain</span><span class="p">)</span>
+    <span class="k">for</span> <span class="n">override</span> <span class="ow">in</span> <span class="n">psl_overrides</span><span class="p">:</span>
+        <span class="k">if</span> <span class="n">domain</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="n">override</span><span class="p">):</span>
+            <span class="k">return</span> <span class="n">override</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s2">&quot;.&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s2">&quot;-&quot;</span><span class="p">)</span>
+    <span class="k">return</span> <span class="n">publicsuffix</span></div>
+
+
+
+<div class="viewcode-block" id="query_dns">
+<a class="viewcode-back" href="../../api.html#parsedmarc.utils.query_dns">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">query_dns</span><span class="p">(</span>
+    <span class="n">domain</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+    <span class="n">record_type</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
+    <span class="o">*</span><span class="p">,</span>
+    <span class="n">cache</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">ExpiringDict</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">nameservers</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">timeout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">2.0</span><span class="p">,</span>
+<span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Queries DNS</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        domain (str): The domain or subdomain to query about</span>
+<span class="sd">        record_type (str): The record type to query for</span>
+<span class="sd">        cache (ExpiringDict): Cache storage</span>
+<span class="sd">        nameservers (list): A list of one or more nameservers to use</span>
+<span class="sd">            (Cloudflare&#39;s public DNS resolvers by default)</span>
+<span class="sd">        timeout (float): Sets the DNS timeout in seconds</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        list: A list of answers</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">domain</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">domain</span><span class="p">)</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
+    <span class="n">record_type</span> <span class="o">=</span> <span class="n">record_type</span><span class="o">.</span><span class="n">upper</span><span class="p">()</span>
+    <span class="n">cache_key</span> <span class="o">=</span> <span class="s2">&quot;</span><span class="si">{0}</span><span class="s2">_</span><span class="si">{1}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">record_type</span><span class="p">)</span>
+    <span class="k">if</span> <span class="n">cache</span><span class="p">:</span>
+        <span class="n">cached_records</span> <span class="o">=</span> <span class="n">cache</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">cache_key</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">cached_records</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
+            <span class="k">return</span> <span class="n">cast</span><span class="p">(</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">cached_records</span><span class="p">)</span>
+
+    <span class="n">resolver</span> <span class="o">=</span> <span class="n">dns</span><span class="o">.</span><span class="n">resolver</span><span class="o">.</span><span class="n">Resolver</span><span class="p">()</span>
+    <span class="n">timeout</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">timeout</span><span class="p">)</span>
+    <span class="k">if</span> <span class="n">nameservers</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">nameservers</span> <span class="o">=</span> <span class="p">[</span>
+            <span class="s2">&quot;1.1.1.1&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;1.0.0.1&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;2606:4700:4700::1111&quot;</span><span class="p">,</span>
+            <span class="s2">&quot;2606:4700:4700::1001&quot;</span><span class="p">,</span>
+        <span class="p">]</span>
+    <span class="n">resolver</span><span class="o">.</span><span class="n">nameservers</span> <span class="o">=</span> <span class="n">nameservers</span>
+    <span class="n">resolver</span><span class="o">.</span><span class="n">timeout</span> <span class="o">=</span> <span class="n">timeout</span>
+    <span class="n">resolver</span><span class="o">.</span><span class="n">lifetime</span> <span class="o">=</span> <span class="n">timeout</span>
+    <span class="n">records</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span>
+        <span class="nb">map</span><span class="p">(</span>
+            <span class="k">lambda</span> <span class="n">r</span><span class="p">:</span> <span class="n">r</span><span class="o">.</span><span class="n">to_text</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">&#39;&quot;&#39;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">&quot;.&quot;</span><span class="p">),</span>
+            <span class="n">resolver</span><span class="o">.</span><span class="n">resolve</span><span class="p">(</span><span class="n">domain</span><span class="p">,</span> <span class="n">record_type</span><span class="p">,</span> <span class="n">lifetime</span><span class="o">=</span><span class="n">timeout</span><span class="p">),</span>
+        <span class="p">)</span>
+    <span class="p">)</span>
+    <span class="k">if</span> <span class="n">cache</span><span class="p">:</span>
+        <span class="n">cache</span><span class="p">[</span><span class="n">cache_key</span><span class="p">]</span> <span class="o">=</span> <span class="n">records</span>
+
+    <span class="k">return</span> <span class="n">records</span></div>
+
+
+
+<div class="viewcode-block" id="get_reverse_dns">
+<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_reverse_dns">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">get_reverse_dns</span><span class="p">(</span>
+    <span class="n">ip_address</span><span class="p">,</span>
+    <span class="o">*</span><span class="p">,</span>
+    <span class="n">cache</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">ExpiringDict</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">nameservers</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">timeout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">2.0</span><span class="p">,</span>
+<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Resolves an IP address to a hostname using a reverse DNS query</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        ip_address (str): The IP address to resolve</span>
+<span class="sd">        cache (ExpiringDict): Cache storage</span>
+<span class="sd">        nameservers (list): A list of one or more nameservers to use</span>
+<span class="sd">            (Cloudflare&#39;s public DNS resolvers by default)</span>
+<span class="sd">        timeout (float): Sets the DNS query timeout in seconds</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        str: The reverse DNS hostname (if any)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">hostname</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="k">try</span><span class="p">:</span>
+        <span class="n">address</span> <span class="o">=</span> <span class="n">dns</span><span class="o">.</span><span class="n">reversename</span><span class="o">.</span><span class="n">from_address</span><span class="p">(</span><span class="n">ip_address</span><span class="p">)</span>
+        <span class="n">hostname</span> <span class="o">=</span> <span class="n">query_dns</span><span class="p">(</span>
+            <span class="nb">str</span><span class="p">(</span><span class="n">address</span><span class="p">),</span> <span class="s2">&quot;PTR&quot;</span><span class="p">,</span> <span class="n">cache</span><span class="o">=</span><span class="n">cache</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span>
+        <span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
+
+    <span class="k">except</span> <span class="n">dns</span><span class="o">.</span><span class="n">exception</span><span class="o">.</span><span class="n">DNSException</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+        <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;get_reverse_dns(</span><span class="si">{</span><span class="n">ip_address</span><span class="si">}</span><span class="s2">) exception: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">pass</span>
+
+    <span class="k">return</span> <span class="n">hostname</span></div>
+
+
+
+<div class="viewcode-block" id="timestamp_to_datetime">
+<a class="viewcode-back" href="../../api.html#parsedmarc.utils.timestamp_to_datetime">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">timestamp_to_datetime</span><span class="p">(</span><span class="n">timestamp</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">datetime</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Converts a UNIX/DMARC timestamp to a Python ``datetime`` object</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        timestamp (int): The timestamp</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        datetime: The converted timestamp as a Python ``datetime`` object</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">timestamp</span><span class="p">))</span></div>
+
+
+
+<div class="viewcode-block" id="timestamp_to_human">
+<a class="viewcode-back" href="../../api.html#parsedmarc.utils.timestamp_to_human">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">timestamp_to_human</span><span class="p">(</span><span class="n">timestamp</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Converts a UNIX/DMARC timestamp to a human-readable string</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        timestamp: The timestamp</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        str: The converted timestamp in ``YYYY-MM-DD HH:MM:SS`` format</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">return</span> <span class="n">timestamp_to_datetime</span><span class="p">(</span><span class="n">timestamp</span><span class="p">)</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2"> %H:%M:%S&quot;</span><span class="p">)</span></div>
+
+
+
+<div class="viewcode-block" id="human_timestamp_to_datetime">
+<a class="viewcode-back" href="../../api.html#parsedmarc.utils.human_timestamp_to_datetime">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">human_timestamp_to_datetime</span><span class="p">(</span>
+    <span class="n">human_timestamp</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">to_utc</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
+<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">datetime</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Converts a human-readable timestamp into a Python ``datetime`` object</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        human_timestamp (str): A timestamp string</span>
+<span class="sd">        to_utc (bool): Convert the timestamp to UTC</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        datetime: The converted timestamp</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="n">human_timestamp</span> <span class="o">=</span> <span class="n">human_timestamp</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;-0000&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
+    <span class="n">human_timestamp</span> <span class="o">=</span> <span class="n">parenthesis_regex</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="s2">&quot;&quot;</span><span class="p">,</span> <span class="n">human_timestamp</span><span class="p">)</span>
+
+    <span class="n">dt</span> <span class="o">=</span> <span class="n">parse_date</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">)</span>
+    <span class="k">return</span> <span class="n">dt</span><span class="o">.</span><span class="n">astimezone</span><span class="p">(</span><span class="n">timezone</span><span class="o">.</span><span class="n">utc</span><span class="p">)</span> <span class="k">if</span> <span class="n">to_utc</span> <span class="k">else</span> <span class="n">dt</span></div>
+
+
+
+<div class="viewcode-block" id="human_timestamp_to_unix_timestamp">
+<a class="viewcode-back" href="../../api.html#parsedmarc.utils.human_timestamp_to_unix_timestamp">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">human_timestamp_to_unix_timestamp</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Converts a human-readable timestamp into a UNIX timestamp</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        human_timestamp (str): A timestamp in `YYYY-MM-DD HH:MM:SS`` format</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        float: The converted timestamp</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">human_timestamp</span> <span class="o">=</span> <span class="n">human_timestamp</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;T&quot;</span><span class="p">,</span> <span class="s2">&quot; &quot;</span><span class="p">)</span>
+    <span class="k">return</span> <span class="nb">int</span><span class="p">(</span><span class="n">human_timestamp_to_datetime</span><span class="p">(</span><span class="n">human_timestamp</span><span class="p">)</span><span class="o">.</span><span class="n">timestamp</span><span class="p">())</span></div>
+
+
+
+<div class="viewcode-block" id="get_ip_address_country">
+<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_ip_address_country">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">get_ip_address_country</span><span class="p">(</span>
+    <span class="n">ip_address</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">db_path</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
+<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Returns the ISO code for the country associated</span>
+<span class="sd">    with the given IPv4 or IPv6 address</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        ip_address (str): The IP address to query for</span>
+<span class="sd">        db_path (str): Path to a MMDB file from MaxMind or DBIP</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        str: And ISO country code associated with the given IP address</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">db_paths</span> <span class="o">=</span> <span class="p">[</span>
+        <span class="s2">&quot;GeoLite2-Country.mmdb&quot;</span><span class="p">,</span>
+        <span class="s2">&quot;/usr/local/share/GeoIP/GeoLite2-Country.mmdb&quot;</span><span class="p">,</span>
+        <span class="s2">&quot;/usr/share/GeoIP/GeoLite2-Country.mmdb&quot;</span><span class="p">,</span>
+        <span class="s2">&quot;/var/lib/GeoIP/GeoLite2-Country.mmdb&quot;</span><span class="p">,</span>
+        <span class="s2">&quot;/var/local/lib/GeoIP/GeoLite2-Country.mmdb&quot;</span><span class="p">,</span>
+        <span class="s2">&quot;/usr/local/var/GeoIP/GeoLite2-Country.mmdb&quot;</span><span class="p">,</span>
+        <span class="s2">&quot;%SystemDrive%</span><span class="se">\\</span><span class="s2">ProgramData</span><span class="se">\\</span><span class="s2">MaxMind</span><span class="se">\\</span><span class="s2">GeoIPUpdate</span><span class="se">\\</span><span class="s2">GeoIP</span><span class="se">\\</span><span class="s2">&quot;</span>
+        <span class="s2">&quot;GeoLite2-Country.mmdb&quot;</span><span class="p">,</span>
+        <span class="s2">&quot;C:</span><span class="se">\\</span><span class="s2">GeoIP</span><span class="se">\\</span><span class="s2">GeoLite2-Country.mmdb&quot;</span><span class="p">,</span>
+        <span class="s2">&quot;dbip-country-lite.mmdb&quot;</span><span class="p">,</span>
+        <span class="s2">&quot;dbip-country.mmdb&quot;</span><span class="p">,</span>
+    <span class="p">]</span>
+
+    <span class="k">if</span> <span class="n">db_path</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">isfile</span><span class="p">(</span><span class="n">db_path</span><span class="p">):</span>
+            <span class="n">db_path</span> <span class="o">=</span> <span class="kc">None</span>
+            <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
+                <span class="sa">f</span><span class="s2">&quot;No file exists at </span><span class="si">{</span><span class="n">db_path</span><span class="si">}</span><span class="s2">. Falling back to an &quot;</span>
+                <span class="s2">&quot;included copy of the IPDB IP to Country &quot;</span>
+                <span class="s2">&quot;Lite database.&quot;</span>
+            <span class="p">)</span>
+
+    <span class="k">if</span> <span class="n">db_path</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="k">for</span> <span class="n">system_path</span> <span class="ow">in</span> <span class="n">db_paths</span><span class="p">:</span>
+            <span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">system_path</span><span class="p">):</span>
+                <span class="n">db_path</span> <span class="o">=</span> <span class="n">system_path</span>
+                <span class="k">break</span>
+
+    <span class="k">if</span> <span class="n">db_path</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">db_path</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span>
+            <span class="n">files</span><span class="p">(</span><span class="n">parsedmarc</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">dbip</span><span class="p">)</span><span class="o">.</span><span class="n">joinpath</span><span class="p">(</span><span class="s2">&quot;dbip-country-lite.mmdb&quot;</span><span class="p">)</span>
+        <span class="p">)</span>
+
+    <span class="n">db_age</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span> <span class="o">-</span> <span class="n">datetime</span><span class="o">.</span><span class="n">fromtimestamp</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">stat</span><span class="p">(</span><span class="n">db_path</span><span class="p">)</span><span class="o">.</span><span class="n">st_mtime</span><span class="p">)</span>
+    <span class="k">if</span> <span class="n">db_age</span> <span class="o">&gt;</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">days</span><span class="o">=</span><span class="mi">30</span><span class="p">):</span>
+        <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;IP database is more than a month old&quot;</span><span class="p">)</span>
+
+    <span class="n">db_reader</span> <span class="o">=</span> <span class="n">geoip2</span><span class="o">.</span><span class="n">database</span><span class="o">.</span><span class="n">Reader</span><span class="p">(</span><span class="n">db_path</span><span class="p">)</span>
+
+    <span class="n">country</span> <span class="o">=</span> <span class="kc">None</span>
+
+    <span class="k">try</span><span class="p">:</span>
+        <span class="n">country</span> <span class="o">=</span> <span class="n">db_reader</span><span class="o">.</span><span class="n">country</span><span class="p">(</span><span class="n">ip_address</span><span class="p">)</span><span class="o">.</span><span class="n">country</span><span class="o">.</span><span class="n">iso_code</span>
+    <span class="k">except</span> <span class="n">geoip2</span><span class="o">.</span><span class="n">errors</span><span class="o">.</span><span class="n">AddressNotFoundError</span><span class="p">:</span>
+        <span class="k">pass</span>
+
+    <span class="k">return</span> <span class="n">country</span></div>
+
+
+
+<div class="viewcode-block" id="get_service_from_reverse_dns_base_domain">
+<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_service_from_reverse_dns_base_domain">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">get_service_from_reverse_dns_base_domain</span><span class="p">(</span>
+    <span class="n">base_domain</span><span class="p">,</span>
+    <span class="o">*</span><span class="p">,</span>
+    <span class="n">always_use_local_file</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="n">local_file_path</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">url</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">offline</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="n">reverse_dns_map</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">ReverseDNSMap</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">ReverseDNSService</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Returns the service name of a given base domain name from reverse DNS.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        base_domain (str): The base domain of the reverse DNS lookup</span>
+<span class="sd">        always_use_local_file (bool): Always use a local map file</span>
+<span class="sd">        local_file_path (str): Path to a local map file</span>
+<span class="sd">        url (str): URL ro a reverse DNS map</span>
+<span class="sd">        offline (bool): Use the built-in copy of the reverse DNS map</span>
+<span class="sd">        reverse_dns_map (dict): A reverse DNS map</span>
+<span class="sd">    Returns:</span>
+<span class="sd">        dict: A dictionary containing name and type.</span>
+<span class="sd">        If the service is unknown, the name will be</span>
+<span class="sd">        the supplied reverse_dns_base_domain and the type will be None</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="n">base_domain</span> <span class="o">=</span> <span class="n">base_domain</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
+    <span class="k">if</span> <span class="n">url</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">url</span> <span class="o">=</span> <span class="p">(</span>
+            <span class="s2">&quot;https://raw.githubusercontent.com/domainaware&quot;</span>
+            <span class="s2">&quot;/parsedmarc/master/parsedmarc/&quot;</span>
+            <span class="s2">&quot;resources/maps/base_reverse_dns_map.csv&quot;</span>
+        <span class="p">)</span>
+    <span class="n">reverse_dns_map_value</span><span class="p">:</span> <span class="n">ReverseDNSMap</span>
+    <span class="k">if</span> <span class="n">reverse_dns_map</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">reverse_dns_map_value</span> <span class="o">=</span> <span class="p">{}</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">reverse_dns_map_value</span> <span class="o">=</span> <span class="n">reverse_dns_map</span>
+
+    <span class="k">def</span><span class="w"> </span><span class="nf">load_csv</span><span class="p">(</span><span class="n">_csv_file</span><span class="p">):</span>
+        <span class="n">reader</span> <span class="o">=</span> <span class="n">csv</span><span class="o">.</span><span class="n">DictReader</span><span class="p">(</span><span class="n">_csv_file</span><span class="p">)</span>
+        <span class="k">for</span> <span class="n">row</span> <span class="ow">in</span> <span class="n">reader</span><span class="p">:</span>
+            <span class="n">key</span> <span class="o">=</span> <span class="n">row</span><span class="p">[</span><span class="s2">&quot;base_reverse_dns&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
+            <span class="n">reverse_dns_map_value</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
+                <span class="s2">&quot;name&quot;</span><span class="p">:</span> <span class="n">row</span><span class="p">[</span><span class="s2">&quot;name&quot;</span><span class="p">],</span>
+                <span class="s2">&quot;type&quot;</span><span class="p">:</span> <span class="n">row</span><span class="p">[</span><span class="s2">&quot;type&quot;</span><span class="p">],</span>
+            <span class="p">}</span>
+
+    <span class="n">csv_file</span> <span class="o">=</span> <span class="n">io</span><span class="o">.</span><span class="n">StringIO</span><span class="p">()</span>
+
+    <span class="k">if</span> <span class="ow">not</span> <span class="p">(</span><span class="n">offline</span> <span class="ow">or</span> <span class="n">always_use_local_file</span><span class="p">)</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">reverse_dns_map_value</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+        <span class="k">try</span><span class="p">:</span>
+            <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Trying to fetch reverse DNS map from </span><span class="si">{</span><span class="n">url</span><span class="si">}</span><span class="s2">...&quot;</span><span class="p">)</span>
+            <span class="n">headers</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;User-Agent&quot;</span><span class="p">:</span> <span class="n">USER_AGENT</span><span class="p">}</span>
+            <span class="n">response</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">)</span>
+            <span class="n">response</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span>
+            <span class="n">csv_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
+            <span class="n">csv_file</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
+            <span class="n">load_csv</span><span class="p">(</span><span class="n">csv_file</span><span class="p">)</span>
+        <span class="k">except</span> <span class="n">requests</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">RequestException</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+            <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Failed to fetch reverse DNS map: </span><span class="si">{</span><span class="n">e</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
+        <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
+            <span class="n">logger</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;Not a valid CSV file&quot;</span><span class="p">)</span>
+            <span class="n">csv_file</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
+            <span class="n">logging</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Response body:&quot;</span><span class="p">)</span>
+            <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="n">csv_file</span><span class="o">.</span><span class="n">read</span><span class="p">())</span>
+
+    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">reverse_dns_map_value</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+        <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Loading included reverse DNS map...&quot;</span><span class="p">)</span>
+        <span class="n">path</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span>
+            <span class="n">files</span><span class="p">(</span><span class="n">parsedmarc</span><span class="o">.</span><span class="n">resources</span><span class="o">.</span><span class="n">maps</span><span class="p">)</span><span class="o">.</span><span class="n">joinpath</span><span class="p">(</span><span class="s2">&quot;base_reverse_dns_map.csv&quot;</span><span class="p">)</span>
+        <span class="p">)</span>
+        <span class="k">if</span> <span class="n">local_file_path</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">path</span> <span class="o">=</span> <span class="n">local_file_path</span>
+        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">path</span><span class="p">)</span> <span class="k">as</span> <span class="n">csv_file</span><span class="p">:</span>
+            <span class="n">load_csv</span><span class="p">(</span><span class="n">csv_file</span><span class="p">)</span>
+    <span class="n">service</span><span class="p">:</span> <span class="n">ReverseDNSService</span>
+    <span class="k">try</span><span class="p">:</span>
+        <span class="n">service</span> <span class="o">=</span> <span class="n">reverse_dns_map_value</span><span class="p">[</span><span class="n">base_domain</span><span class="p">]</span>
+    <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
+        <span class="n">service</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;name&quot;</span><span class="p">:</span> <span class="n">base_domain</span><span class="p">,</span> <span class="s2">&quot;type&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">}</span>
+
+    <span class="k">return</span> <span class="n">service</span></div>
+
+
+
+<div class="viewcode-block" id="get_ip_address_info">
+<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_ip_address_info">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">get_ip_address_info</span><span class="p">(</span>
+    <span class="n">ip_address</span><span class="p">,</span>
+    <span class="o">*</span><span class="p">,</span>
+    <span class="n">ip_db_path</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">reverse_dns_map_path</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">always_use_local_files</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="n">reverse_dns_map_url</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">cache</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">ExpiringDict</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">reverse_dns_map</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">ReverseDNSMap</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">offline</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
+    <span class="n">nameservers</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="n">timeout</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mf">2.0</span><span class="p">,</span>
+<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">IPAddressInfo</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Returns reverse DNS and country information for the given IP address</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        ip_address (str): The IP address to check</span>
+<span class="sd">        ip_db_path (str): path to a MMDB file from MaxMind or DBIP</span>
+<span class="sd">        reverse_dns_map_path (str): Path to a reverse DNS map file</span>
+<span class="sd">        reverse_dns_map_url (str): URL to the reverse DNS map file</span>
+<span class="sd">        always_use_local_files (bool): Do not download files</span>
+<span class="sd">        cache (ExpiringDict): Cache storage</span>
+<span class="sd">        reverse_dns_map (dict): A reverse DNS map</span>
+<span class="sd">        offline (bool): Do not make online queries for geolocation or DNS</span>
+<span class="sd">        nameservers (list): A list of one or more nameservers to use</span>
+<span class="sd">            (Cloudflare&#39;s public DNS resolvers by default)</span>
+<span class="sd">        timeout (float): Sets the DNS timeout in seconds</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        dict: ``ip_address``, ``reverse_dns``, ``country``</span>
+
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">ip_address</span> <span class="o">=</span> <span class="n">ip_address</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
+    <span class="k">if</span> <span class="n">cache</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">cached_info</span> <span class="o">=</span> <span class="n">cache</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+        <span class="k">if</span> <span class="p">(</span>
+            <span class="n">cached_info</span>
+            <span class="ow">and</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">cached_info</span><span class="p">,</span> <span class="nb">dict</span><span class="p">)</span>
+            <span class="ow">and</span> <span class="s2">&quot;ip_address&quot;</span> <span class="ow">in</span> <span class="n">cached_info</span>
+        <span class="p">):</span>
+            <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;IP address </span><span class="si">{</span><span class="n">ip_address</span><span class="si">}</span><span class="s2"> was found in cache&quot;</span><span class="p">)</span>
+            <span class="k">return</span> <span class="n">cast</span><span class="p">(</span><span class="n">IPAddressInfo</span><span class="p">,</span> <span class="n">cached_info</span><span class="p">)</span>
+    <span class="n">info</span><span class="p">:</span> <span class="n">IPAddressInfo</span> <span class="o">=</span> <span class="p">{</span>
+        <span class="s2">&quot;ip_address&quot;</span><span class="p">:</span> <span class="n">ip_address</span><span class="p">,</span>
+        <span class="s2">&quot;reverse_dns&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="s2">&quot;country&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="s2">&quot;base_domain&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="s2">&quot;name&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
+        <span class="s2">&quot;type&quot;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
+    <span class="p">}</span>
+    <span class="k">if</span> <span class="n">offline</span><span class="p">:</span>
+        <span class="n">reverse_dns</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">reverse_dns</span> <span class="o">=</span> <span class="n">get_reverse_dns</span><span class="p">(</span>
+            <span class="n">ip_address</span><span class="p">,</span> <span class="n">nameservers</span><span class="o">=</span><span class="n">nameservers</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span>
+        <span class="p">)</span>
+    <span class="n">country</span> <span class="o">=</span> <span class="n">get_ip_address_country</span><span class="p">(</span><span class="n">ip_address</span><span class="p">,</span> <span class="n">db_path</span><span class="o">=</span><span class="n">ip_db_path</span><span class="p">)</span>
+    <span class="n">info</span><span class="p">[</span><span class="s2">&quot;country&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">country</span>
+    <span class="n">info</span><span class="p">[</span><span class="s2">&quot;reverse_dns&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">reverse_dns</span>
+    <span class="k">if</span> <span class="n">reverse_dns</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">base_domain</span> <span class="o">=</span> <span class="n">get_base_domain</span><span class="p">(</span><span class="n">reverse_dns</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">base_domain</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">service</span> <span class="o">=</span> <span class="n">get_service_from_reverse_dns_base_domain</span><span class="p">(</span>
+                <span class="n">base_domain</span><span class="p">,</span>
+                <span class="n">offline</span><span class="o">=</span><span class="n">offline</span><span class="p">,</span>
+                <span class="n">local_file_path</span><span class="o">=</span><span class="n">reverse_dns_map_path</span><span class="p">,</span>
+                <span class="n">url</span><span class="o">=</span><span class="n">reverse_dns_map_url</span><span class="p">,</span>
+                <span class="n">always_use_local_file</span><span class="o">=</span><span class="n">always_use_local_files</span><span class="p">,</span>
+                <span class="n">reverse_dns_map</span><span class="o">=</span><span class="n">reverse_dns_map</span><span class="p">,</span>
+            <span class="p">)</span>
+            <span class="n">info</span><span class="p">[</span><span class="s2">&quot;base_domain&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">base_domain</span>
+            <span class="n">info</span><span class="p">[</span><span class="s2">&quot;type&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">service</span><span class="p">[</span><span class="s2">&quot;type&quot;</span><span class="p">]</span>
+            <span class="n">info</span><span class="p">[</span><span class="s2">&quot;name&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">service</span><span class="p">[</span><span class="s2">&quot;name&quot;</span><span class="p">]</span>
+
+        <span class="k">if</span> <span class="n">cache</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="n">cache</span><span class="p">[</span><span class="n">ip_address</span><span class="p">]</span> <span class="o">=</span> <span class="n">info</span>
+            <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;IP address </span><span class="si">{</span><span class="n">ip_address</span><span class="si">}</span><span class="s2"> added to cache&quot;</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;IP address </span><span class="si">{</span><span class="n">ip_address</span><span class="si">}</span><span class="s2"> reverse_dns not found&quot;</span><span class="p">)</span>
+
+    <span class="k">return</span> <span class="n">info</span></div>
+
+
+
+<span class="k">def</span><span class="w"> </span><span class="nf">parse_email_address</span><span class="p">(</span><span class="n">original_address</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]]:</span>
+    <span class="k">if</span> <span class="n">original_address</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="s2">&quot;&quot;</span><span class="p">:</span>
+        <span class="n">display_name</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">display_name</span> <span class="o">=</span> <span class="n">original_address</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
+    <span class="n">address</span> <span class="o">=</span> <span class="n">original_address</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
+    <span class="n">address_parts</span> <span class="o">=</span> <span class="n">address</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s2">&quot;@&quot;</span><span class="p">)</span>
+    <span class="n">local</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="n">domain</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">address_parts</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
+        <span class="n">local</span> <span class="o">=</span> <span class="n">address_parts</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
+        <span class="n">domain</span> <span class="o">=</span> <span class="n">address_parts</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
+
+    <span class="k">return</span> <span class="p">{</span>
+        <span class="s2">&quot;display_name&quot;</span><span class="p">:</span> <span class="n">display_name</span><span class="p">,</span>
+        <span class="s2">&quot;address&quot;</span><span class="p">:</span> <span class="n">address</span><span class="p">,</span>
+        <span class="s2">&quot;local&quot;</span><span class="p">:</span> <span class="n">local</span><span class="p">,</span>
+        <span class="s2">&quot;domain&quot;</span><span class="p">:</span> <span class="n">domain</span><span class="p">,</span>
+    <span class="p">}</span>
+
+
+<div class="viewcode-block" id="get_filename_safe_string">
+<a class="viewcode-back" href="../../api.html#parsedmarc.utils.get_filename_safe_string">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">get_filename_safe_string</span><span class="p">(</span><span class="n">string</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Converts a string to a string that is safe for a filename</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        string (str): A string to make safe for a filename</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        str: A string safe for a filename</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">invalid_filename_chars</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;</span><span class="se">\\</span><span class="s2">&quot;</span><span class="p">,</span> <span class="s2">&quot;/&quot;</span><span class="p">,</span> <span class="s2">&quot;:&quot;</span><span class="p">,</span> <span class="s1">&#39;&quot;&#39;</span><span class="p">,</span> <span class="s2">&quot;*&quot;</span><span class="p">,</span> <span class="s2">&quot;?&quot;</span><span class="p">,</span> <span class="s2">&quot;|&quot;</span><span class="p">,</span> <span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">,</span> <span class="s2">&quot;</span><span class="se">\r</span><span class="s2">&quot;</span><span class="p">]</span>
+    <span class="k">if</span> <span class="n">string</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">string</span> <span class="o">=</span> <span class="s2">&quot;None&quot;</span>
+    <span class="k">for</span> <span class="n">char</span> <span class="ow">in</span> <span class="n">invalid_filename_chars</span><span class="p">:</span>
+        <span class="n">string</span> <span class="o">=</span> <span class="n">string</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="n">char</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">)</span>
+    <span class="n">string</span> <span class="o">=</span> <span class="n">string</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s2">&quot;.&quot;</span><span class="p">)</span>
+
+    <span class="n">string</span> <span class="o">=</span> <span class="p">(</span><span class="n">string</span><span class="p">[:</span><span class="mi">100</span><span class="p">])</span> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">string</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">100</span> <span class="k">else</span> <span class="n">string</span>
+
+    <span class="k">return</span> <span class="n">string</span></div>
+
+
+
+<div class="viewcode-block" id="is_mbox">
+<a class="viewcode-back" href="../../api.html#parsedmarc.utils.is_mbox">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">is_mbox</span><span class="p">(</span><span class="n">path</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Checks if the given content is an MBOX mailbox file</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        path: Content to check</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        bool: A flag that indicates if the file is an MBOX mailbox file</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">_is_mbox</span> <span class="o">=</span> <span class="kc">False</span>
+    <span class="k">try</span><span class="p">:</span>
+        <span class="n">mbox</span> <span class="o">=</span> <span class="n">mailbox</span><span class="o">.</span><span class="n">mbox</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
+        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">mbox</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="n">_is_mbox</span> <span class="o">=</span> <span class="kc">True</span>
+    <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+        <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Error checking for MBOX file: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
+
+    <span class="k">return</span> <span class="n">_is_mbox</span></div>
+
+
+
+<div class="viewcode-block" id="is_outlook_msg">
+<a class="viewcode-back" href="../../api.html#parsedmarc.utils.is_outlook_msg">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">is_outlook_msg</span><span class="p">(</span><span class="n">content</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bool</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Checks if the given content is an Outlook msg OLE/MSG file</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        content: Content to check</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        bool: A flag that indicates if the file is an Outlook MSG file</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">return</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">content</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">)</span> <span class="ow">and</span> <span class="n">content</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span>
+        <span class="sa">b</span><span class="s2">&quot;</span><span class="se">\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1</span><span class="s2">&quot;</span>
+    <span class="p">)</span></div>
+
+
+
+<div class="viewcode-block" id="convert_outlook_msg">
+<a class="viewcode-back" href="../../api.html#parsedmarc.utils.convert_outlook_msg">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">convert_outlook_msg</span><span class="p">(</span><span class="n">msg_bytes</span><span class="p">:</span> <span class="nb">bytes</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">bytes</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    Uses the ``msgconvert`` Perl utility to convert an Outlook MS file to</span>
+<span class="sd">    standard RFC 822 format</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        msg_bytes (bytes): the content of the .msg file</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        A RFC 822 bytes payload</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">if</span> <span class="ow">not</span> <span class="n">is_outlook_msg</span><span class="p">(</span><span class="n">msg_bytes</span><span class="p">):</span>
+        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;The supplied bytes are not an Outlook MSG file&quot;</span><span class="p">)</span>
+    <span class="n">orig_dir</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">getcwd</span><span class="p">()</span>
+    <span class="n">tmp_dir</span> <span class="o">=</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">mkdtemp</span><span class="p">()</span>
+    <span class="n">os</span><span class="o">.</span><span class="n">chdir</span><span class="p">(</span><span class="n">tmp_dir</span><span class="p">)</span>
+    <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="s2">&quot;sample.msg&quot;</span><span class="p">,</span> <span class="s2">&quot;wb&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">msg_file</span><span class="p">:</span>
+        <span class="n">msg_file</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">msg_bytes</span><span class="p">)</span>
+    <span class="k">try</span><span class="p">:</span>
+        <span class="n">subprocess</span><span class="o">.</span><span class="n">check_call</span><span class="p">(</span>
+            <span class="p">[</span><span class="s2">&quot;msgconvert&quot;</span><span class="p">,</span> <span class="s2">&quot;sample.msg&quot;</span><span class="p">],</span> <span class="n">stdout</span><span class="o">=</span><span class="n">null_file</span><span class="p">,</span> <span class="n">stderr</span><span class="o">=</span><span class="n">null_file</span>
+        <span class="p">)</span>
+        <span class="n">eml_path</span> <span class="o">=</span> <span class="s2">&quot;sample.eml&quot;</span>
+        <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">eml_path</span><span class="p">,</span> <span class="s2">&quot;rb&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">eml_file</span><span class="p">:</span>
+            <span class="n">rfc822</span> <span class="o">=</span> <span class="n">eml_file</span><span class="o">.</span><span class="n">read</span><span class="p">()</span>
+    <span class="k">except</span> <span class="ne">FileNotFoundError</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="n">EmailParserError</span><span class="p">(</span>
+            <span class="s2">&quot;Failed to convert Outlook MSG: msgconvert utility not found&quot;</span>
+        <span class="p">)</span>
+    <span class="k">finally</span><span class="p">:</span>
+        <span class="n">os</span><span class="o">.</span><span class="n">chdir</span><span class="p">(</span><span class="n">orig_dir</span><span class="p">)</span>
+        <span class="n">shutil</span><span class="o">.</span><span class="n">rmtree</span><span class="p">(</span><span class="n">tmp_dir</span><span class="p">)</span>
+
+    <span class="k">return</span> <span class="n">rfc822</span></div>
+
+
+
+<div class="viewcode-block" id="parse_email">
+<a class="viewcode-back" href="../../api.html#parsedmarc.utils.parse_email">[docs]</a>
+<span class="k">def</span><span class="w"> </span><span class="nf">parse_email</span><span class="p">(</span>
+    <span class="n">data</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">bytes</span><span class="p">,</span> <span class="nb">str</span><span class="p">],</span> <span class="o">*</span><span class="p">,</span> <span class="n">strip_attachment_payloads</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span>
+<span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">dict</span><span class="p">:</span>
+<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">    A simplified email parser</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">        data: The RFC 822 message string, or MSG binary</span>
+<span class="sd">        strip_attachment_payloads (bool): Remove attachment payloads</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">        dict: Parsed email data</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">):</span>
+        <span class="k">if</span> <span class="n">is_outlook_msg</span><span class="p">(</span><span class="n">data</span><span class="p">):</span>
+            <span class="n">data</span> <span class="o">=</span> <span class="n">convert_outlook_msg</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
+        <span class="n">data</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s2">&quot;utf-8&quot;</span><span class="p">,</span> <span class="n">errors</span><span class="o">=</span><span class="s2">&quot;replace&quot;</span><span class="p">)</span>
+    <span class="n">parsed_email</span> <span class="o">=</span> <span class="n">mailparser</span><span class="o">.</span><span class="n">parse_from_string</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
+    <span class="n">headers</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">parsed_email</span><span class="o">.</span><span class="n">headers_json</span><span class="p">)</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+    <span class="n">parsed_email</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">parsed_email</span><span class="o">.</span><span class="n">mail_json</span><span class="p">)</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
+    <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;headers&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">headers</span>
+
+    <span class="k">if</span> <span class="s2">&quot;received&quot;</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
+        <span class="k">for</span> <span class="n">received</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;received&quot;</span><span class="p">]:</span>
+            <span class="k">if</span> <span class="s2">&quot;date_utc&quot;</span> <span class="ow">in</span> <span class="n">received</span><span class="p">:</span>
+                <span class="k">if</span> <span class="n">received</span><span class="p">[</span><span class="s2">&quot;date_utc&quot;</span><span class="p">]</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+                    <span class="k">del</span> <span class="n">received</span><span class="p">[</span><span class="s2">&quot;date_utc&quot;</span><span class="p">]</span>
+                <span class="k">else</span><span class="p">:</span>
+                    <span class="n">received</span><span class="p">[</span><span class="s2">&quot;date_utc&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">received</span><span class="p">[</span><span class="s2">&quot;date_utc&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;T&quot;</span><span class="p">,</span> <span class="s2">&quot; &quot;</span><span class="p">)</span>
+
+    <span class="k">if</span> <span class="s2">&quot;from&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
+        <span class="k">if</span> <span class="s2">&quot;From&quot;</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;headers&quot;</span><span class="p">]:</span>
+            <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;Headers&quot;</span><span class="p">][</span><span class="s2">&quot;From&quot;</span><span class="p">]</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
+
+    <span class="k">if</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">]</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;from&quot;</span><span class="p">][</span><span class="mi">0</span><span class="p">])</span>
+
+    <span class="k">if</span> <span class="s2">&quot;date&quot;</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
+        <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;date&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;date&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;T&quot;</span><span class="p">,</span> <span class="s2">&quot; &quot;</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;date&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="k">if</span> <span class="s2">&quot;reply_to&quot;</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
+        <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;reply_to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span>
+            <span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;reply_to&quot;</span><span class="p">])</span>
+        <span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;reply_to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
+
+    <span class="k">if</span> <span class="s2">&quot;to&quot;</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
+        <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span>
+            <span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">])</span>
+        <span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
+
+    <span class="k">if</span> <span class="s2">&quot;cc&quot;</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
+        <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;cc&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span>
+            <span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;cc&quot;</span><span class="p">])</span>
+        <span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;cc&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
+
+    <span class="k">if</span> <span class="s2">&quot;bcc&quot;</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
+        <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;bcc&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span>
+            <span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;bcc&quot;</span><span class="p">])</span>
+        <span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;bcc&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
+
+    <span class="k">if</span> <span class="s2">&quot;delivered_to&quot;</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
+        <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;delivered_to&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span>
+            <span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">parse_email_address</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;delivered_to&quot;</span><span class="p">])</span>
+        <span class="p">)</span>
+
+    <span class="k">if</span> <span class="s2">&quot;attachments&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
+        <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;attachments&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
+    <span class="k">else</span><span class="p">:</span>
+        <span class="k">for</span> <span class="n">attachment</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;attachments&quot;</span><span class="p">]:</span>
+            <span class="k">if</span> <span class="s2">&quot;payload&quot;</span> <span class="ow">in</span> <span class="n">attachment</span><span class="p">:</span>
+                <span class="n">payload</span> <span class="o">=</span> <span class="n">attachment</span><span class="p">[</span><span class="s2">&quot;payload&quot;</span><span class="p">]</span>
+                <span class="k">try</span><span class="p">:</span>
+                    <span class="k">if</span> <span class="s2">&quot;content_transfer_encoding&quot;</span> <span class="ow">in</span> <span class="n">attachment</span><span class="p">:</span>
+                        <span class="k">if</span> <span class="n">attachment</span><span class="p">[</span><span class="s2">&quot;content_transfer_encoding&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="s2">&quot;base64&quot;</span><span class="p">:</span>
+                            <span class="n">payload</span> <span class="o">=</span> <span class="n">decode_base64</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
+                        <span class="k">else</span><span class="p">:</span>
+                            <span class="n">payload</span> <span class="o">=</span> <span class="nb">str</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span>
+                    <span class="n">attachment</span><span class="p">[</span><span class="s2">&quot;sha256&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">hashlib</span><span class="o">.</span><span class="n">sha256</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span><span class="o">.</span><span class="n">hexdigest</span><span class="p">()</span>
+                <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+                    <span class="n">logger</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s2">&quot;Unable to decode attachment: </span><span class="si">{0}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="fm">__str__</span><span class="p">()))</span>
+        <span class="k">if</span> <span class="n">strip_attachment_payloads</span><span class="p">:</span>
+            <span class="k">for</span> <span class="n">attachment</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;attachments&quot;</span><span class="p">]:</span>
+                <span class="k">if</span> <span class="s2">&quot;payload&quot;</span> <span class="ow">in</span> <span class="n">attachment</span><span class="p">:</span>
+                    <span class="k">del</span> <span class="n">attachment</span><span class="p">[</span><span class="s2">&quot;payload&quot;</span><span class="p">]</span>
+
+    <span class="k">if</span> <span class="s2">&quot;subject&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
+        <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;subject&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
+
+    <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;filename_safe_subject&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">get_filename_safe_string</span><span class="p">(</span>
+        <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;subject&quot;</span><span class="p">]</span>
+    <span class="p">)</span>
+
+    <span class="k">if</span> <span class="s2">&quot;body&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">parsed_email</span><span class="p">:</span>
+        <span class="n">parsed_email</span><span class="p">[</span><span class="s2">&quot;body&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
+
+    <span class="k">return</span> <span class="n">parsed_email</span></div>
+
+</pre></div>
+
+           </div>
+          </div>
+          <footer>
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>&#169; Copyright 2018 - 2025, Sean Whalen and contributors.</p>
+  </div>
+
+  Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
+    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
+    provided by <a href="https://readthedocs.org">Read the Docs</a>.
+   
+
+</footer>
+        </div>
+      </div>
+    </section>
+  </div>
+  <script>
+      jQuery(function () {
+          SphinxRtdTheme.Navigation.enable(true);
+      });
+  </script> 
+
+</body>
+</html>

_sources/api.md.txt

@@ -0,0 +1,48 @@
+# API reference
+
+## parsedmarc
+
+```{eval-rst}
+.. automodule:: parsedmarc
+   :members:
+```
+
+## parsedmarc.elastic
+
+```{eval-rst}
+.. automodule:: parsedmarc.elastic
+   :members:
+```
+
+## parsedmarc.opensearch
+
+```{eval-rst}
+.. automodule:: parsedmarc.opensearch
+   :members:
+```
+
+## parsedmarc.splunk
+
+```{eval-rst}
+.. automodule:: parsedmarc.splunk
+   :members:
+```
+
+## parsedmarc.types
+
+```{eval-rst}
+.. automodule:: parsedmarc.types
+   :members:
+```
+
+## parsedmarc.utils
+
+```{eval-rst}
+.. automodule:: parsedmarc.utils
+   :members:
+```
+
+## Indices and tables
+
+- {ref}`genindex`
+- {ref}`modindex`

_sources/contributing.md.txt

@@ -0,0 +1,7 @@
+# Contributing to parsedmarc
+
+## Bug reports
+
+Please report bugs on the GitHub issue tracker
+
+<https://github.com/domainaware/parsedmarc/issues>

_sources/davmail.md.txt

@@ -0,0 +1,189 @@
+# Accessing an inbox using OWA/EWS
+
+:::{note}
+Starting in 8.0.0, parsedmarc supports accessing Microsoft/Office 365
+inboxes via the Microsoft Graph API, which is preferred over Davmail.
+:::
+
+Some organizations do not allow IMAP or the Microsoft Graph API,
+and only support Exchange Web Services (EWS)/Outlook Web Access (OWA).
+In that case, Davmail will need to be set up
+as a local EWS/OWA IMAP gateway. It can even work where
+[Modern Auth/multi-factor authentication] is required.
+
+To do this, download the latest `davmail-version.zip` from
+<https://sourceforge.net/projects/davmail/files/>
+
+Extract the zip using the `unzip` command.
+
+Install Java:
+
+```bash
+sudo apt-get install default-jre-headless
+```
+
+Configure Davmail by creating a `davmail.properties` file
+
+```properties
+# DavMail settings, see http://davmail.sourceforge.net/ for documentation
+
+#############################################################
+# Basic settings
+
+# Server or workstation mode
+davmail.server=true
+
+# connection mode auto, EWS or WebDav
+davmail.enableEws=auto
+
+# base Exchange OWA or EWS url
+davmail.url=https://outlook.office365.com/EWS/Exchange.asmx
+
+# Listener ports
+davmail.imapPort=1143
+
+#############################################################
+# Network settings
+
+# Network proxy settings
+davmail.enableProxy=false
+davmail.useSystemProxies=false
+davmail.proxyHost=
+davmail.proxyPort=
+davmail.proxyUser=
+davmail.proxyPassword=
+
+# proxy exclude list
+davmail.noProxyFor=
+
+# block remote connection to DavMail
+davmail.allowRemote=false
+
+# bind server sockets to the loopback address
+davmail.bindAddress=127.0.0.1
+
+# disable SSL for specified listeners
+davmail.ssl.nosecureimap=true
+
+# Send keepalive character during large folder and messages download
+davmail.enableKeepalive=true
+
+# Message count limit on folder retrieval
+davmail.folderSizeLimit=0
+
+#############################################################
+# IMAP settings
+
+# Delete messages immediately on IMAP STORE \Deleted flag
+davmail.imapAutoExpunge=true
+
+# Enable IDLE support, set polling delay in minutes
+davmail.imapIdleDelay=1
+
+# Always reply to IMAP RFC822.SIZE requests with Exchange approximate
+# message size for performance reasons
+davmail.imapAlwaysApproxMsgSize=true
+
+# Client connection timeout in seconds - default 300, 0 to disable
+davmail.clientSoTimeout=0
+
+#############################################################
+```
+
+## Running DavMail as a systemd service
+
+Use systemd to run `davmail` as a service.
+
+Create a system user
+
+```bash
+sudo useradd davmail -r -s /bin/false
+```
+
+Protect the `davmail` configuration file from prying eyes
+
+```bash
+sudo chown root:davmail /opt/davmail/davmail.properties
+sudo chmod u=rw,g=r,o= /opt/davmail/davmail.properties
+```
+
+Create the service configuration file
+
+```bash
+sudo nano /etc/systemd/system/davmail.service
+```
+
+```ini
+[Unit]
+Description=DavMail gateway service
+Documentation=https://sourceforge.net/projects/davmail/
+Wants=network-online.target
+After=syslog.target network.target
+
+[Service]
+ExecStart=/opt/davmail/davmail /opt/davmail/davmail.properties
+User=davmail
+Group=davmail
+Restart=always
+RestartSec=5m
+
+[Install]
+WantedBy=multi-user.target
+```
+
+Then, enable the service
+
+```bash
+sudo systemctl daemon-reload
+sudo systemctl enable parsedmarc.service
+sudo service davmail restart
+```
+
+:::{note}
+You must also run the above commands whenever you edit
+`davmail.service`.
+:::
+
+:::{warning}
+Always restart the service every time you upgrade to a new version of
+`davmail`:
+
+```bash
+sudo service davmail restart
+```
+
+:::
+
+To check the status of the service, run:
+
+```bash
+service davmail status
+```
+
+:::{note}
+In the event of a crash, systemd will restart the service after 5
+minutes, but the `service davmail status` command will only show the
+logs for the current process. To vew the logs for previous runs as
+well as the current process (newest to oldest), run:
+
+```bash
+journalctl -u davmail.service -r
+```
+
+:::
+
+## Configuring parsedmarc for DavMail
+
+Because you are interacting with DavMail server over the loopback
+(i.e. `127.0.0.1`), add the following options to `parsedmarc.ini`
+config file:
+
+```ini
+[imap]
+host=127.0.0.1
+port=1143
+ssl=False
+watch=True
+```
+
+[modern auth/multi-factor authentication]: https://davmail.sourceforge.net/faq.html

_sources/dmarc.md.txt

@@ -0,0 +1,71 @@
+# Understanding DMARC
+
+## Resources
+
+### DMARC guides
+
+- [Demystifying DMARC] - A complete guide to SPF, DKIM, and DMARC
+
+[demystifying dmarc]: https://seanthegeek.net/459/demystifying-dmarc/
+
+### SPF and DMARC record validation
+
+If you are looking for SPF and DMARC record validation and parsing,
+check out the sister project,
+[checkdmarc](https://domainaware.github.io/checkdmarc/).
+
+### Lookalike domains
+
+DMARC protects against domain spoofing, not lookalike domains. for open source
+lookalike domain monitoring, check out [DomainAware](https://github.com/seanthegeek/domainaware).
+
+## DMARC Alignment Guide
+
+DMARC ensures that SPF and DKM authentication mechanisms actually authenticate
+against the same domain that the end user sees.
+
+A message passes a DMARC check by passing DKIM or SPF, **as long as the related
+indicators are also in alignment**.
+
+```{eval-rst}
++-----------------------+-----------------------+-----------------------+
+|                       | **DKIM**              | **SPF**               |
++-----------------------+-----------------------+-----------------------+
+| **Passing**           | The signature in the  | The mail server's IP  |
+|                       | DKIM header is        | address is listed in  |
+|                       | validated using a     | the SPF record of the |
+|                       | public key that is    | domain in the SMTP    |
+|                       | published as a DNS    | envelope's mail from  |
+|                       | record of the domain  | header                |
+|                       | name specified in the |                       |
+|                       | signature             |                       |
++-----------------------+-----------------------+-----------------------+
+| **Alignment**         | The signing domain    | The domain in the     |
+|                       | aligns with the       | SMTP envelope's mail  |
+|                       | domain in the         | from header aligns    |
+|                       | message's from header | with the domain in    |
+|                       |                       | the message's from    |
+|                       |                       | header                |
++-----------------------+-----------------------+-----------------------+
+```
+
+## What if a sender won't support DKIM/DMARC?
+
+1. Some vendors don't know about DMARC yet; ask about SPF and DKIM/email
+   authentication.
+2. Check if they can send through your email relays instead of theirs.
+3. Do they really need to spoof your domain? Why not use the display
+   name instead?
+4. Worst case, have that vendor send email as a specific subdomain of
+   your domain (e.g. `noreply@news.example.com`), and then create
+   separate SPF and DMARC records on `news.example.com`, and set
+   `p=none` in that DMARC record.
+
+:::{warning}
+Do not alter the `p` or `sp` values of the DMARC record on the
+Top-Level Domain (TLD) – that would leave you vulnerable to
+spoofing of your TLD and/or any subdomain.
+:::
+
+```{include} mailing-lists.md
+```

_sources/elasticsearch.md.txt

@@ -0,0 +1,236 @@
+# Elasticsearch and Kibana
+
+To set up visual dashboards of DMARC data, install Elasticsearch and Kibana.
+
+:::{note}
+Elasticsearch and Kibana 6 or later are required
+:::
+
+## Installation
+
+On Debian/Ubuntu based systems, run:
+
+```bash
+sudo apt-get install -y apt-transport-https
+wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo gpg --dearmor -o /usr/share/keyrings/elasticsearch-keyring.gpg
+echo "deb [signed-by=/usr/share/keyrings/elasticsearch-keyring.gpg] https://artifacts.elastic.co/packages/8.x/apt stable main" | sudo tee /etc/apt/sources.list.d/elastic-8.x.list
+sudo apt-get update
+sudo apt-get install -y elasticsearch kibana
+```
+
+For CentOS, RHEL, and other RPM systems, follow the Elastic RPM guides for
+[Elasticsearch] and [Kibana].
+
+:::{note}
+Previously, the default JVM heap size for Elasticsearch was very small (1g),
+which will cause it to crash under a heavy load. To fix this, increase the
+minimum and maximum JVM heap sizes in `/etc/elasticsearch/jvm.options` to
+more reasonable levels, depending on your server's resources.
+
+Make sure the system has at least 2 GB more RAM than the assigned JVM
+heap size.
+
+Always set the minimum and maximum JVM heap sizes to the same
+value.
+
+For example, to set a 4 GB heap size, set
+
+```bash
+-Xms4g
+-Xmx4g
+```
+
+See <https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#heap-size-settings>
+for more information.
+:::
+
+```bash
+sudo systemctl daemon-reload
+sudo systemctl enable elasticsearch.service
+sudo systemctl enable kibana.service
+sudo systemctl start elasticsearch.service
+sudo systemctl start kibana.service
+```
+
+As of Elasticsearch 8.7, activate secure mode (xpack.security.*.ssl)
+
+```bash
+sudo vim /etc/elasticsearch/elasticsearch.yml
+```
+
+Add the following configuration
+
+```text
+# Enable security features
+xpack.security.enabled: true
+xpack.security.enrollment.enabled: true
+# Enable encryption for HTTP API client connections, such as Kibana, Logstash, and Agents
+xpack.security.http.ssl:
+  enabled: true
+  keystore.path: certs/http.p12
+# Enable encryption and mutual authentication between cluster nodes
+xpack.security.transport.ssl:
+  enabled: true
+  verification_mode: certificate
+  keystore.path: certs/transport.p12
+  truststore.path: certs/transport.p12
+```
+
+```bash
+sudo systemctl restart elasticsearch
+```
+
+To create a self-signed certificate, run:
+
+```bash
+openssl req -x509 -nodes -days 365 -newkey rsa:4096 -keyout kibana.key -out kibana.crt
+```
+
+Or, to create a Certificate Signing Request (CSR) for a CA, run:
+
+```bash
+openssl req -newkey rsa:4096-nodes -keyout kibana.key -out kibana.csr
+```
+
+Fill in the prompts. Watch out for Common Name (e.g. server FQDN or YOUR
+domain name), which is the IP address or domain name that you will use to access Kibana. it is the most important field.
+
+If you generated a CSR, remove the CSR after you have your certs
+
+```bash
+rm -f kibana.csr
+```
+
+Move the keys into place and secure them:
+
+```bash
+sudo mv kibana.* /etc/kibana
+sudo chmod 660 /etc/kibana/kibana.key
+```
+
+Activate the HTTPS server in Kibana
+
+```bash
+sudo vim /etc/kibana/kibana.yml
+```
+
+Add the following configuration
+
+```text
+server.host: "SERVER_IP"
+server.publicBaseUrl: "https://SERVER_IP"
+server.ssl.enabled: true
+server.ssl.certificate: /etc/kibana/kibana.crt
+server.ssl.key: /etc/kibana/kibana.key
+```
+
+:::{note}
+For more security, you can configure Kibana to use a local network connexion
+to elasticsearch :
+```text
+elasticsearch.hosts: ['https://SERVER_IP:9200']
+```
+=> 
+```text
+elasticsearch.hosts: ['https://127.0.0.1:9200']
+```
+:::
+
+```bash
+sudo systemctl restart kibana
+```
+
+Enroll Kibana in Elasticsearch
+
+```bash
+sudo /usr/share/elasticsearch/bin/elasticsearch-create-enrollment-token -s kibana
+```
+
+Then access to your web server at `https://SERVER_IP:5601`, accept the self-signed
+certificate and paste the token in the "Enrollment token" field.
+
+```bash
+sudo /usr/share/kibana/bin/kibana-verification-code
+```
+
+Then put the verification code to your web browser.
+
+End Kibana configuration
+
+```bash
+sudo /usr/share/elasticsearch/bin/elasticsearch-setup-passwords interactive
+sudo /usr/share/kibana/bin/kibana-encryption-keys generate
+sudo vim /etc/kibana/kibana.yml
+```
+
+Add previously generated encryption keys
+
+```text
+xpack.encryptedSavedObjects.encryptionKey: xxxx...xxxx
+xpack.reporting.encryptionKey: xxxx...xxxx
+xpack.security.encryptionKey: xxxx...xxxx
+```
+
+```bash
+sudo systemctl restart kibana
+sudo systemctl restart elasticsearch
+```
+
+Now that Elasticsearch is up and running, use `parsedmarc` to send data to
+it.
+
+Download (right-click the link and click save as) [export.ndjson].
+
+Connect to kibana using the "elastic" user and the password you previously provide
+on the console ("End Kibana configuration" part).
+
+Import `export.ndjson` the Saved Objects tab of the Stack management
+page of Kibana. (Hamburger menu -> "Management" -> "Stack Management" ->
+"Kibana" -> "Saved Objects")
+
+It will give you the option to overwrite existing saved dashboards or
+visualizations, which could be used to restore them if you or someone else
+breaks them, as there are no permissions/access controls in Kibana without
+the commercial [X-Pack].
+
+```{image} _static/screenshots/saved-objects.png
+:align: center
+:alt: A screenshot of setting the Saved Objects Stack management UI in Kibana
+:target: _static/screenshots/saved-objects.png
+```
+
+```{image} _static/screenshots/confirm-overwrite.png
+:align: center
+:alt: A screenshot of the overwrite conformation prompt
+:target: _static/screenshots/confirm-overwrite.png
+```
+
+## Upgrading Kibana index patterns
+
+`parsedmarc` 5.0.0 makes some changes to the way data is indexed in
+Elasticsearch. if you are upgrading from a previous release of
+`parsedmarc`, you need to complete the following steps to replace the
+Kibana index patterns with versions that match the upgraded indexes:
+
+1. Login in to Kibana, and click on Management
+2. Under Kibana, click on Saved Objects
+3. Check the checkboxes for the `dmarc_aggregate` and `dmarc_forensic`
+   index patterns
+4. Click Delete
+5. Click Delete on the conformation message
+6. Download (right-click the link and click save as)
+   the latest version of [export.ndjson]
+7. Import `export.ndjson` by clicking Import from the Kibana
+   Saved Objects page
+
+## Records retention
+
+Starting in version 5.0.0, `parsedmarc` stores data in a separate
+index for each day to make it easy to comply with records
+retention regulations such as GDPR. For more information,
+check out the Elastic guide to [managing time-based indexes efficiently](https://www.elastic.co/blog/managing-time-based-indices-efficiently).
+
+[elasticsearch]: https://www.elastic.co/guide/en/elasticsearch/reference/current/rpm.html
+[export.ndjson]: https://raw.githubusercontent.com/domainaware/parsedmarc/master/kibana/export.ndjson
+[kibana]: https://www.elastic.co/guide/en/kibana/current/rpm.html
+[x-pack]: https://www.elastic.co/products/x-pack

_sources/index.md.txt

@@ -0,0 +1,84 @@
+# parsedmarc documentation - Open source DMARC report analyzer and visualizer
+
+[![Build
+Status](https://github.com/domainaware/parsedmarc/actions/workflows/python-tests.yml/badge.svg)](https://github.com/domainaware/parsedmarc/actions/workflows/python-tests.yml)
+[![Code
+Coverage](https://codecov.io/gh/domainaware/parsedmarc/branch/master/graph/badge.svg)](https://codecov.io/gh/domainaware/parsedmarc)
+[![PyPI
+Package](https://img.shields.io/pypi/v/parsedmarc.svg)](https://pypi.org/project/parsedmarc/)
+[![PyPI - Downloads](https://img.shields.io/pypi/dm/parsedmarc?color=blue)](https://pypistats.org/packages/parsedmarc)
+
+:::{note}
+**Help Wanted**
+
+This is a project is maintained by one developer.
+Please consider reviewing the open [issues] to see how you can contribute code, documentation, or user support.
+Assistance on the pinned issues would be particularly helpful.
+
+Thanks to all [contributors]!
+:::
+
+```{image} _static/screenshots/dmarc-summary-charts.png
+:align: center
+:alt: A screenshot of DMARC summary charts in Kibana
+:scale: 50 %
+:target: _static/screenshots/dmarc-summary-charts.png
+```
+
+`parsedmarc` is a Python module and CLI utility for parsing DMARC reports.
+When used with Elasticsearch and Kibana (or Splunk), or with OpenSearch and Grafana, it works as a self-hosted
+open source alternative to commercial DMARC report processing services such
+as Agari Brand Protection, Dmarcian, OnDMARC, ProofPoint Email Fraud Defense,
+and Valimail.
+
+## Features
+
+- Parses draft and 1.0 standard aggregate/rua DMARC reports
+- Parses forensic/failure/ruf DMARC reports
+- Parses reports from SMTP TLS Reporting
+- Can parse reports from an inbox over IMAP, Microsoft Graph, or Gmail API
+- Transparently handles gzip or zip compressed reports
+- Consistent data structures
+- Simple JSON and/or CSV output
+- Optionally email the results
+- Optionally send the results to Elasticsearch, Opensearch, and/or Splunk, for use
+    with premade dashboards
+- Optionally send reports to Apache Kafka
+
+## Python Compatibility
+
+This project supports the following Python versions, which are either actively maintained or are the default versions
+for RHEL or Debian.
+
+| Version | Supported | Reason                                                     |
+|---------|-----------|------------------------------------------------------------|
+| < 3.6   | ❌         | End of Life (EOL)                                          |
+| 3.6     | ❌         | Used in RHEL 8, but not supported by project dependencies |
+| 3.7     | ❌         | End of Life (EOL)                                          |
+| 3.8     | ❌         | End of Life (EOL)                                          |
+| 3.9     | ✅         | Supported until August 2026 (Debian 11); May 2032 (RHEL 9) |
+| 3.10    | ✅         | Actively maintained                                        |
+| 3.11    | ✅         | Actively maintained; supported until June 2028 (Debian 12) |
+| 3.12    | ✅         | Actively maintained; supported until May 2035 (RHEL 10)    |
+| 3.13    | ✅         | Actively maintained; supported until June 2030 (Debian 13) |
+| 3.14    | ✅         | Actively maintained                                        |
+
+```{toctree}
+:caption: 'Contents'
+:maxdepth: 2
+
+installation
+usage
+output
+elasticsearch
+opensearch
+kibana
+splunk
+davmail
+dmarc
+contributing
+api
+```
+
+[contributors]: https://github.com/domainaware/parsedmarc/graphs/contributors
+[issues]: https://github.com/domainaware/parsedmarc/issues

_sources/installation.md.txt

@@ -0,0 +1,205 @@
+# Installation
+
+## Prerequisites
+
+`parsedmarc` works with Python 3 only.
+
+### Testing multiple report analyzers
+
+If you would like to test parsedmarc and another report processing
+solution at the same time, you can have up to two `mailto` URIs in each of the rua and ruf
+tags in your DMARC record, separated by commas.
+
+### Using a web proxy
+
+If your system is behind a web proxy, you need to configure your system
+to use that proxy. To do this, edit `/etc/environment` and add your
+proxy details there, for example:
+
+```bash
+http_proxy=http://user:password@prox-server:3128
+https_proxy=https://user:password@prox-server:3128
+ftp_proxy=http://user:password@prox-server:3128
+```
+
+Or if no credentials are needed:
+
+```bash
+http_proxy=http://prox-server:3128
+https_proxy=https://prox-server:3128
+ftp_proxy=http://prox-server:3128
+```
+
+This will set the proxy up for use system-wide, including for `parsedmarc`.
+
+### Using Microsoft Exchange
+
+If your mail server is Microsoft Exchange, ensure that it is patched to at
+least:
+
+- Exchange Server 2010 Update Rollup 22 ([KB4295699])
+- Exchange Server 2013 Cumulative Update 21 ([KB4099855])
+- Exchange Server 2016 Cumulative Update 11 ([KB4134118])
+
+### geoipupdate setup
+
+:::{note}
+Starting in `parsedmarc` 7.1.0, a static copy of the
+[IP to Country Lite database] from IPDB is distributed with
+`parsedmarc`, under the terms of the
+[Creative Commons Attribution 4.0 International License].
+as a fallback if the [MaxMind GeoLite2 Country database] is not
+installed. However, `parsedmarc` cannot install updated versions of
+these databases as they are released, so MaxMind's databases and the
+[geoipupdate] tool is still the preferable solution.
+
+The location of the database file can be overridden by using the
+`ip_db_path` setting.
+:::
+
+On Debian 10 (Buster) or later, run:
+
+```bash
+sudo apt-get install -y geoipupdate
+```
+
+:::{note}
+[Component "contrib"] is required in your apt sources.
+:::
+
+On Ubuntu systems run:
+
+```bash
+sudo add-apt-repository ppa:maxmind/ppa
+sudo apt update
+sudo apt install -y geoipupdate
+```
+
+On CentOS or RHEL systems, run:
+
+```bash
+sudo dnf install -y geoipupdate
+```
+
+The latest builds for Linux, macOS, and Windows can be downloaded
+from the [geoipupdate releases page on GitHub].
+
+On December 30th, 2019, MaxMind started requiring free accounts to
+access the free Geolite2 databases, in order 
+[to comply with various privacy regulations].
+
+Start by [registering for a free GeoLite2 account], and signing in.
+
+Then, navigate to the [License Keys] page under your account,
+and create a new license key for the version of
+`geoipupdate` that was installed.
+
+:::{warning}
+The configuration file format is different for older (i.e. \<=3.1.1) and newer (i.e. >=3.1.1) versions
+of `geoipupdate`. Be sure to select the correct version for your system.
+:::
+
+:::{note}
+To check the version of `geoipupdate` that is installed, run:
+
+```bash
+geoipupdate -V
+```
+
+:::
+
+You can use `parsedmarc` as the description for the key.
+
+Once you have generated a key, download the config pre-filled
+configuration file. This file should be saved at `/etc/GeoIP.conf`
+on Linux or macOS systems, or at
+`%SystemDrive%\ProgramData\MaxMind\GeoIPUpdate\GeoIP.conf` on
+Windows systems.
+
+Then run
+
+```bash
+sudo geoipupdate
+```
+
+To download the databases for the first time.
+
+The GeoLite2 Country, City, and ASN databases are updated weekly,
+every Tuesday. `geoipupdate` can be run weekly by adding a cron
+job or scheduled task.
+
+More information about `geoipupdate` can be found at the
+[MaxMind geoipupdate page].
+
+## Installing parsedmarc
+
+On Debian or Ubuntu systems, run:
+
+```bash
+sudo apt-get install -y python3-pip python3-virtualenv python3-dev libxml2-dev libxslt-dev
+```
+
+On CentOS or RHEL systems, run:
+
+```bash
+sudo dnf install -y python39 python3-virtualenv python3-setuptools python3-devel libxml2-devel libxslt-devel
+```
+
+Python 3 installers for Windows and macOS can be found at
+<https://www.python.org/downloads/>.
+
+Create a system user
+
+```bash
+sudo mkdir /opt
+sudo useradd parsedmarc -r -s /bin/false -m -b /opt
+```
+
+Install parsedmarc in a virtualenv
+
+```bash
+sudo -u parsedmarc virtualenv /opt/parsedmarc/venv
+```
+
+CentOS/RHEL 8 systems use Python 3.6 by default, so on those systems
+explicitly tell `virtualenv` to use `python3.9` instead
+
+```bash
+sudo -u parsedmarc virtualenv -p python3.9  /opt/parsedmarc/venv
+```
+
+Activate the virtualenv
+
+```bash
+source /opt/parsedmarc/venv/bin/activate
+```
+
+To install or upgrade `parsedmarc` inside the virtualenv, run:
+
+```bash
+sudo -u parsedmarc /opt/parsedmarc/venv/bin/pip install -U parsedmarc
+```
+
+## Optional dependencies
+
+If you would like to be able to parse emails saved from Microsoft
+Outlook (i.e. OLE .msg files), install `msgconvert`:
+
+On Debian or Ubuntu systems, run:
+
+```bash
+sudo apt-get install libemail-outlook-message-perl
+```
+
+[KB4295699]: https://support.microsoft.com/KB/4295699
+[KB4099855]: https://support.microsoft.com/KB/4099855
+[KB4134118]: https://support.microsoft.com/kb/4134118
+[Component "contrib"]: https://wiki.debian.org/SourcesList#Component
+[geoipupdate]: https://github.com/maxmind/geoipupdate
+[geoipupdate releases page on github]: https://github.com/maxmind/geoipupdate/releases
+[ip to country lite database]: https://db-ip.com/db/download/ip-to-country-lite
+[license keys]: https://www.maxmind.com/en/accounts/current/license-key
+[maxmind geoipupdate page]: https://dev.maxmind.com/geoip/updating-databases/
+[maxmind geolite2 country database]: https://dev.maxmind.com/geoip/geolite2-free-geolocation-data
+[registering for a free geolite2 account]: https://www.maxmind.com/en/geolite2/signup
+[to comply with various privacy regulations]: https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-geolite2-databases/

_sources/kibana.md.txt

@@ -0,0 +1,87 @@
+
+# Using the Kibana dashboards
+
+The Kibana DMARC dashboards are a human-friendly way to understand the
+results from incoming DMARC reports.
+
+:::{note}
+The default dashboard is DMARC Summary. To switch between dashboards,
+click on the Dashboard link on the left side menu of Kibana.
+:::
+
+## DMARC Summary
+
+As the name suggests, this dashboard is the best place to start
+reviewing your aggregate DMARC data.
+
+Across the top of the dashboard, three pie charts display the percentage of
+alignment pass/fail for SPF, DKIM, and DMARC. Clicking on any chart segment
+will filter for that value.
+
+:::{note}
+Messages should not be considered malicious just because they failed to pass
+DMARC; especially if you have just started collecting data. It may be a
+legitimate service that needs SPF and DKIM configured correctly.
+:::
+
+Start by filtering the results to only show failed DKIM alignment. While DMARC
+passes if a message passes SPF or DKIM alignment, only DKIM alignment remains
+valid when a message is forwarded without changing the from address, which is
+often caused by a mailbox forwarding rule. This is because DKIM signatures are
+part of the message headers, whereas SPF relies on SMTP session headers.
+
+Underneath the pie charts. you can see graphs of DMARC passage and message
+disposition over time.
+
+Under the graphs you will find the most useful data tables on the dashboard. On
+the left, there is a list of organizations that are sending you DMARC reports.
+In the center, there is a list of sending servers grouped by the base domain
+in their reverse DNS. On the right, there is a list of email from domains,
+sorted by message volume.
+
+By hovering your mouse over a data table value and using the magnifying glass
+icons, you can filter on our filter out different values. Start by looking at
+the Message Sources by Reverse DNS table. Find a sender that you recognize,
+such as an email marketing service, hover over it, and click on the plus (+)
+magnifying glass icon, to add a filter that only shows results for that sender.
+Now, look at the Message From Header table to the right. That shows you the
+domains that a sender is sending as, which might tell you which brand/business
+is using a particular service. With that information, you can contact them and
+have them set up DKIM.
+
+:::{note}
+If you have a lot of B2C customers, you may see a high volume of emails as
+your domains coming from consumer email services, such as Google/Gmail and
+Yahoo! This occurs when customers have mailbox rules in place that forward
+emails from an old account to a new account, which is why DKIM
+authentication is so important, as mentioned earlier. Similar patterns may
+be observed with businesses who send from reverse DNS addressees of
+parent, subsidiary, and outdated brands.
+:::
+
+Further down the dashboard, you can filter by source country or source IP
+address.
+
+Tables showing SPF and DKIM alignment details are located under the IP address
+table.
+
+:::{note}
+Previously, the alignment tables were included in a separate dashboard
+called DMARC Alignment Failures. That dashboard has been consolidated into
+the DMARC Summary dashboard. To view failures only, use the pie chart.
+:::
+
+Any other filters work the same way. You can also add your own custom temporary
+filters by clicking on Add Filter at the upper right of the page.
+
+## DMARC Forensic Samples
+
+The DMARC Forensic Samples dashboard contains information on DMARC forensic
+reports (also known as failure reports or ruf reports). These reports contain
+samples of emails that have failed to pass DMARC.
+
+:::{note}
+Most recipients do not send forensic/failure/ruf reports at all to avoid
+privacy leaks. Some recipients (notably Chinese webmail services) will only
+supply the headers of sample emails. Very few provide the entire email.
+:::

_sources/mailing-lists.md.txt

@@ -0,0 +1,206 @@
+## What about mailing lists?
+
+When you deploy DMARC on your domain, you might find that messages
+relayed by mailing lists are failing DMARC, most likely because the mailing
+list is spoofing your from address, and modifying the subject,
+footer, or other part of the message, thereby breaking the
+DKIM signature.
+
+### Mailing list best practices
+
+Ideally, a mailing list should forward messages without altering the
+headers or body content at all. [Joe Nelson] does a fantastic job of
+explaining exactly what mailing lists should and shouldn't do to be
+fully DMARC compliant. Rather than repeat his fine work, here's a
+summary:
+
+#### Do
+
+- Retain headers from the original message
+
+- Add [RFC 2369] List-Unsubscribe headers to outgoing messages, instead of
+  adding unsubscribe links to the body
+
+> List-Unsubscribe: <https://list.example.com/unsubscribe-link>
+
+- Add [RFC 2919] List-Id headers instead of modifying the subject
+
+  > List-Id: Example Mailing List <list.example.com>
+
+Modern mail clients and webmail services generate unsubscribe buttons based on
+these headers.
+
+#### Do not
+
+- Remove or modify any existing headers from the original message, including
+  From, Date, Subject, etc.
+- Add to or remove content from the message body, **including traditional
+  disclaimers and unsubscribe footers**
+
+In addition to complying with DMARC, this configuration ensures that Reply
+and Reply All actions work like they would with any email message. Reply
+replies to the message sender, and Reply All replies to the sender and the
+list.
+
+Even without a subject prefix or body footer, mailing list users can still
+tell that a message came from the mailing list, because the message was sent
+to the mailing list post address, and not their email address.
+
+Configuration steps for common mailing list platforms are listed below.
+
+#### Mailman 2
+
+Navigate to General Settings, and configure the settings below
+
+```{eval-rst}
+============================ ==========
+**Setting**                  **Value**
+**subject_prefix**
+**from_is_list**             No
+**first_strip_reply_to**     No
+**reply_goes_to_list**       Poster
+**include_rfc2369_headers**  Yes
+**include_list_post_header** Yes
+**include_sender_header**    No
+============================ ==========
+```
+
+Navigate to Non-digest options, and configure the settings below
+
+```{eval-rst}
+=================== ==========
+**Setting**         **Value**
+**msg_header**
+**msg_footer**
+**scrub_nondigest**  No
+=================== ==========
+```
+
+Navigate to Privacy Options> Sending Filters, and configure the settings below
+
+```{eval-rst}
+====================================== ==========
+**Setting**                            **Value**
+**dmarc_moderation_action**            Accept
+**dmarc_quarantine_moderation_action** Yes
+**dmarc_none_moderation_action**       Yes
+====================================== ==========
+```
+
+#### Mailman 3
+
+Navigate to Settings> List Identity
+
+Make Subject prefix blank.
+
+Navigate to Settings> Alter Messages
+
+Configure the settings below
+
+```{eval-rst}
+====================================== ==========
+**Setting**                            **Value**
+**Convert html to plaintext**          No
+**Include RFC2369 headers**            Yes
+**Include the list post header**       Yes
+**Explicit reply-to address**
+**First strip replyto**                 No
+**Reply goes to list**                 No munging
+====================================== ==========
+```
+
+Navigate to Settings> DMARC Mitigation
+
+Configure the settings below
+
+```{eval-rst}
+================================== ===============================
+**Setting**                            **Value**
+**DMARC mitigation action**            No DMARC mitigations
+**DMARC mitigate unconditionally** No
+================================== ===============================
+```
+
+Create a blank footer template for your mailing list to remove the message
+footer. Unfortunately, the Postorius mailing list admin UI will not allow you
+to create an empty template, so you'll have to create one using the system's
+command line instead, for example:
+
+```bash
+touch var/templates/lists/list.example.com/en/list:member:regular:footer
+```
+
+Where `list.example.com` the list ID, and `en` is the language.
+
+Then restart mailman core.
+
+#### LISTSERV
+
+[LISTSERV 16.0-2017a] and higher will rewrite the From header for domains
+that enforce with a DMARC quarantine or reject policy.
+
+Some additional steps are needed for Linux hosts.
+
+#### Workarounds
+
+If a mailing list must go **against** best practices and
+modify the message (e.g. to add a required legal footer), the mailing
+list administrator must configure the list to replace the From address of the
+message (also known as munging) with the address of the mailing list, so they
+no longer spoof email addresses with domains protected by DMARC.
+
+Configuration steps for common mailing list platforms are listed below.
+
+##### Mailman 2
+
+Navigate to Privacy Options> Sending Filters, and configure the settings below
+
+```{eval-rst}
+====================================== ==========
+**Setting**                            **Value**
+**dmarc_moderation_action**            Munge From
+**dmarc_quarantine_moderation_action** Yes
+**dmarc_none_moderation_action**       Yes
+====================================== ==========
+```
+
+:::{note}
+Message wrapping could be used as the DMARC mitigation action instead. In
+that case, the original message is added as an attachment to the mailing
+list message, but that could interfere with inbox searching, or mobile
+clients.
+
+On the other hand, replacing the From address might cause users to
+accidentally reply to the entire list, when they only intended to reply to
+the original sender.
+
+Choose the option that best fits your community.
+:::
+
+##### Mailman 3
+
+In the DMARC Mitigations tab of the Settings page, configure the settings below
+
+```{eval-rst}
+================================== ===============================
+**Setting**                            **Value**
+**DMARC mitigation action**            Replace From: with list address
+**DMARC mitigate unconditionally** No
+================================== ===============================
+```
+
+:::{note}
+Message wrapping could be used as the DMARC mitigation action instead. In
+that case, the original message is added as an attachment to the mailing
+list message, but that could interfere with inbox searching, or mobile
+clients.
+
+On the other hand, replacing the From address might cause users to
+accidentally reply to the entire list, when they only intended to reply to
+the original sender.
+:::
+
+[joe nelson]: https://begriffs.com/posts/2018-09-18-dmarc-mailing-list.html
+[listserv 16.0-2017a]: https://www.lsoft.com/news/dmarc-issue1-2018.asp
+[rfc 2369]: https://tools.ietf.org/html/rfc2369
+[rfc 2919]: https://tools.ietf.org/html/rfc2919

_sources/opensearch.md.txt

@@ -0,0 +1,14 @@
+# OpenSearch and Grafana
+
+To set up visual dashboards of DMARC data, install OpenSearch and Grafana.
+
+## Installation
+
+OpenSearch: https://opensearch.org/docs/latest/install-and-configure/install-opensearch/index/
+Grafana: https://grafana.com/docs/grafana/latest/setup-grafana/installation/
+
+## Records retention
+
+Starting in version 5.0.0, `parsedmarc` stores data in a separate
+index for each day to make it easy to comply with records
+retention regulations such as GDPR.

_sources/output.md.txt

@@ -0,0 +1,241 @@
+# Sample outputs
+
+## Sample aggregate report output
+
+Here are the results from parsing the [example](https://dmarc.org/wiki/FAQ#I_need_to_implement_aggregate_reports.2C_what_do_they_look_like.3F)
+report from the dmarc.org wiki. It's actually an older draft of
+the 1.0 report schema standardized in
+[RFC 7480 Appendix C](https://tools.ietf.org/html/rfc7489#appendix-C).
+This draft schema is still in wide use.
+
+`parsedmarc` produces consistent, normalized output, regardless
+of the report schema.
+
+### JSON aggregate report
+
+```json
+{
+  "xml_schema": "draft",
+  "report_metadata": {
+    "org_name": "acme.com",
+    "org_email": "noreply-dmarc-support@acme.com",
+    "org_extra_contact_info": "http://acme.com/dmarc/support",
+    "report_id": "9391651994964116463",
+    "begin_date": "2012-04-27 20:00:00",
+    "end_date": "2012-04-28 19:59:59",
+    "timespan_requires_normalization": false,
+    "original_timespan_seconds": 86399,
+    "errors": []
+  },
+  "policy_published": {
+    "domain": "example.com",
+    "adkim": "r",
+    "aspf": "r",
+    "p": "none",
+    "sp": "none",
+    "pct": "100",
+    "fo": "0"
+  },
+  "records": [
+    {
+      "source": {
+        "ip_address": "72.150.241.94",
+        "country": "US",
+        "reverse_dns": null,
+        "base_domain": null,
+        "name": null,
+        "type": null
+      },
+      "count": 2,
+      "alignment": {
+        "spf": true,
+        "dkim": false,
+        "dmarc": true
+      },
+      "policy_evaluated": {
+        "disposition": "none",
+        "dkim": "fail",
+        "spf": "pass",
+        "policy_override_reasons": []
+      },
+      "identifiers": {
+        "header_from": "example.com",
+        "envelope_from": "example.com",
+        "envelope_to": null
+      },
+      "auth_results": {
+        "dkim": [
+          {
+            "domain": "example.com",
+            "selector": "none",
+            "result": "fail"
+          }
+        ],
+        "spf": [
+          {
+            "domain": "example.com",
+            "scope": "mfrom",
+            "result": "pass"
+          }
+        ]
+      },
+      "normalized_timespan": false,
+      "interval_begin": "2012-04-28 00:00:00",
+      "interval_end": "2012-04-28 23:59:59"
+    }
+  ]
+}
+```
+
+### CSV aggregate report
+
+```text
+xml_schema,org_name,org_email,org_extra_contact_info,report_id,begin_date,end_date,normalized_timespan,errors,domain,adkim,aspf,p,sp,pct,fo,source_ip_address,source_country,source_reverse_dns,source_base_domain,source_name,source_type,count,spf_aligned,dkim_aligned,dmarc_aligned,disposition,policy_override_reasons,policy_override_comments,envelope_from,header_from,envelope_to,dkim_domains,dkim_selectors,dkim_results,spf_domains,spf_scopes,spf_results
+draft,acme.com,noreply-dmarc-support@acme.com,http://acme.com/dmarc/support,9391651994964116463,2012-04-28 00:00:00,2012-04-28 23:59:59,False,,example.com,r,r,none,none,100,0,72.150.241.94,US,,,,,2,True,False,True,none,,,example.com,example.com,,example.com,none,fail,example.com,mfrom,pass
+draft,acme.com,noreply-dmarc-support@acme.com,http://acme.com/dmarc/support,9391651994964116463,2012-04-28 00:00:00,2012-04-28 23:59:59,False,,example.com,r,r,none,none,100,0,72.150.241.94,US,,,,,2,True,False,True,none,,,example.com,example.com,,example.com,none,fail,example.com,mfrom,pass
+
+```
+
+## Sample forensic report output
+
+Thanks to GitHub user [xennn](https://github.com/xennn) for the anonymized
+[forensic report email sample](<https://github.com/domainaware/parsedmarc/raw/master/samples/forensic/DMARC%20Failure%20Report%20for%20domain.de%20(mail-from%3Dsharepoint%40domain.de%2C%20ip%3D10.10.10.10).eml>).
+
+### JSON forensic report
+
+```json
+{
+     "feedback_type": "auth-failure",
+     "user_agent": "Lua/1.0",
+     "version": "1.0",
+     "original_mail_from": "sharepoint@domain.de",
+     "original_rcpt_to": "peter.pan@domain.de",
+     "arrival_date": "Mon, 01 Oct 2018 11:20:27 +0200",
+     "message_id": "<38.E7.30937.BD6E1BB5@ mailrelay.de>",
+     "authentication_results": "dmarc=fail (p=none, dis=none) header.from=domain.de",
+     "delivery_result": "policy",
+     "auth_failure": [
+       "dmarc"
+     ],
+     "reported_domain": "domain.de",
+     "arrival_date_utc": "2018-10-01 09:20:27",
+     "source": {
+       "ip_address": "10.10.10.10",
+       "country": null,
+       "reverse_dns": null,
+       "base_domain": null
+     },
+     "authentication_mechanisms": [],
+     "original_envelope_id": null,
+     "dkim_domain": null,
+     "sample_headers_only": false,
+     "sample": "Received: from Servernameone.domain.local (Servernameone.domain.local [10.10.10.10])\n\tby  mailrelay.de (mail.DOMAIN.de) with SMTP id 38.E7.30937.BD6E1BB5; Mon,  1 Oct 2018 11:20:27 +0200 (CEST)\nDate: 01 Oct 2018 11:20:27 +0200\nMessage-ID: <38.E7.30937.BD6E1BB5@ mailrelay.de>\nTo: <peter.pan@domain.de>\nfrom: \"=?utf-8?B?SW50ZXJha3RpdmUgV2V0dGJld2VyYmVyLcOcYmVyc2ljaHQ=?=\" <sharepoint@domain.de>\nSubject: Subject\nMIME-Version: 1.0\nX-Mailer: Microsoft SharePoint Foundation 2010\nContent-Type: text/html; charset=utf-8\nContent-Transfer-Encoding: quoted-printable\n\n<html><head><base href=3D'\nwettbewerb' /></head><body><!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 3.2//EN\"=\n><HTML><HEAD><META NAME=3D\"Generator\" CONTENT=3D\"MS Exchange Server version=\n 08.01.0240.003\"></html>\n",
+     "parsed_sample": {
+       "from": {
+         "display_name": "Interaktive Wettbewerber-Übersicht",
+         "address": "sharepoint@domain.de",
+         "local": "sharepoint",
+         "domain": "domain.de"
+       },
+       "to_domains": [
+         "domain.de"
+       ],
+       "to": [
+         {
+           "display_name": null,
+           "address": "peter.pan@domain.de",
+           "local": "peter.pan",
+           "domain": "domain.de"
+         }
+       ],
+       "subject": "Subject",
+       "timezone": "+2",
+       "mime-version": "1.0",
+       "date": "2018-10-01 09:20:27",
+       "content-type": "text/html; charset=utf-8",
+       "x-mailer": "Microsoft SharePoint Foundation 2010",
+       "body": "<html><head><base href='\nwettbewerb' /></head><body><!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 3.2//EN\"><HTML><HEAD><META NAME=\"Generator\" CONTENT=\"MS Exchange Server version 08.01.0240.003\"></html>",
+       "received": [
+         {
+           "from": "Servernameone.domain.local Servernameone.domain.local 10.10.10.10",
+           "by": "mailrelay.de mail.DOMAIN.de",
+           "with": "SMTP id 38.E7.30937.BD6E1BB5",
+           "date": "Mon, 1 Oct 2018 11:20:27 +0200 CEST",
+           "hop": 1,
+           "date_utc": "2018-10-01 09:20:27",
+           "delay": 0
+         }
+       ],
+       "content-transfer-encoding": "quoted-printable",
+       "message-id": "<38.E7.30937.BD6E1BB5@ mailrelay.de>",
+       "has_defects": false,
+       "headers": {
+         "Received": "from Servernameone.domain.local (Servernameone.domain.local [10.10.10.10])\n\tby  mailrelay.de (mail.DOMAIN.de) with SMTP id 38.E7.30937.BD6E1BB5; Mon,  1 Oct 2018 11:20:27 +0200 (CEST)",
+         "Date": "01 Oct 2018 11:20:27 +0200",
+         "Message-ID": "<38.E7.30937.BD6E1BB5@ mailrelay.de>",
+         "To": "<peter.pan@domain.de>",
+         "from": "\"Interaktive Wettbewerber-Übersicht\" <sharepoint@domain.de>",
+         "Subject": "Subject",
+         "MIME-Version": "1.0",
+         "X-Mailer": "Microsoft SharePoint Foundation 2010",
+         "Content-Type": "text/html; charset=utf-8",
+         "Content-Transfer-Encoding": "quoted-printable"
+       },
+       "reply_to": [],
+       "cc": [],
+       "bcc": [],
+       "attachments": [],
+       "filename_safe_subject": "Subject"
+     }
+   }
+```
+
+### CSV forensic report
+
+```text
+feedback_type,user_agent,version,original_envelope_id,original_mail_from,original_rcpt_to,arrival_date,arrival_date_utc,subject,message_id,authentication_results,dkim_domain,source_ip_address,source_country,source_reverse_dns,source_base_domain,delivery_result,auth_failure,reported_domain,authentication_mechanisms,sample_headers_only
+auth-failure,Lua/1.0,1.0,,sharepoint@domain.de,peter.pan@domain.de,"Mon, 01 Oct 2018 11:20:27 +0200",2018-10-01 09:20:27,Subject,<38.E7.30937.BD6E1BB5@ mailrelay.de>,"dmarc=fail (p=none, dis=none) header.from=domain.de",,10.10.10.10,,,,policy,dmarc,domain.de,,False
+```
+
+### JSON SMTP TLS report
+
+```json
+[
+  {
+    "organization_name": "Example Inc.",
+    "begin_date": "2024-01-09T00:00:00Z",
+    "end_date": "2024-01-09T23:59:59Z",
+    "report_id": "2024-01-09T00:00:00Z_example.com",
+    "policies": [
+      {
+        "policy_domain": "example.com",
+        "policy_type": "sts",
+        "policy_strings": [
+          "version: STSv1",
+          "mode: testing",
+          "mx: example.com",
+          "max_age: 86400"
+        ],
+        "successful_session_count": 0,
+        "failed_session_count": 3,
+        "failure_details": [
+          {
+            "result_type": "validation-failure",
+            "failed_session_count": 2,
+            "sending_mta_ip": "209.85.222.201",
+            "receiving_ip": "173.212.201.41",
+            "receiving_mx_hostname": "example.com"
+          },
+          {
+            "result_type": "validation-failure",
+            "failed_session_count": 1,
+            "sending_mta_ip": "209.85.208.176",
+            "receiving_ip": "173.212.201.41",
+            "receiving_mx_hostname": "example.com"
+          }
+        ]
+      }
+    ]
+  }
+]
+```

_sources/splunk.md.txt

@@ -0,0 +1,22 @@
+# Splunk
+
+Starting in version 4.3.0 `parsedmarc` supports sending aggregate and/or
+forensic DMARC data to a Splunk [HTTP Event collector (HEC)].
+
+The project repository contains [XML files] for premade Splunk
+dashboards for aggregate and forensic DMARC reports.
+
+Copy and paste the contents of each file into a separate Splunk
+dashboard XML editor.
+
+:::{warning}
+Change all occurrences of `index="email"` in the XML to
+match your own index name.
+:::
+
+The Splunk dashboards display the same content and layout as the
+Kibana dashboards, although the Kibana dashboards have slightly
+easier and more flexible filtering options.
+
+[xml files]: https://github.com/domainaware/parsedmarc/tree/master/splunk
+[http event collector (hec)]: http://docs.splunk.com/Documentation/Splunk/latest/Data/AboutHEC

_sources/usage.md.txt

@@ -0,0 +1,553 @@
+# Using parsedmarc
+
+## CLI help
+
+```text
+usage: parsedmarc [-h] [-c CONFIG_FILE] [--strip-attachment-payloads] [-o OUTPUT]
+                  [--aggregate-json-filename AGGREGATE_JSON_FILENAME] [--forensic-json-filename FORENSIC_JSON_FILENAME]
+                  [--smtp-tls-json-filename SMTP_TLS_JSON_FILENAME] [--aggregate-csv-filename AGGREGATE_CSV_FILENAME]
+                  [--forensic-csv-filename FORENSIC_CSV_FILENAME] [--smtp-tls-csv-filename SMTP_TLS_CSV_FILENAME]
+                  [-n NAMESERVERS [NAMESERVERS ...]] [-t DNS_TIMEOUT] [--offline] [-s] [-w] [--verbose] [--debug]
+                  [--log-file LOG_FILE] [--no-prettify-json] [-v]
+                  [file_path ...]
+
+Parses DMARC reports
+
+positional arguments:
+  file_path             one or more paths to aggregate or forensic report files, emails, or mbox files'
+
+options:
+  -h, --help            show this help message and exit
+  -c CONFIG_FILE, --config-file CONFIG_FILE
+                        a path to a configuration file (--silent implied)
+  --strip-attachment-payloads
+                        remove attachment payloads from forensic report output
+  -o OUTPUT, --output OUTPUT
+                        write output files to the given directory
+  --aggregate-json-filename AGGREGATE_JSON_FILENAME
+                        filename for the aggregate JSON output file
+  --forensic-json-filename FORENSIC_JSON_FILENAME
+                        filename for the forensic JSON output file
+  --smtp-tls-json-filename SMTP_TLS_JSON_FILENAME
+                        filename for the SMTP TLS JSON output file
+  --aggregate-csv-filename AGGREGATE_CSV_FILENAME
+                        filename for the aggregate CSV output file
+  --forensic-csv-filename FORENSIC_CSV_FILENAME
+                        filename for the forensic CSV output file
+  --smtp-tls-csv-filename SMTP_TLS_CSV_FILENAME
+                        filename for the SMTP TLS CSV output file
+  -n NAMESERVERS [NAMESERVERS ...], --nameservers NAMESERVERS [NAMESERVERS ...]
+                        nameservers to query
+  -t DNS_TIMEOUT, --dns_timeout DNS_TIMEOUT
+                        number of seconds to wait for an answer from DNS (default: 2.0)
+  --offline             do not make online queries for geolocation or DNS
+  -s, --silent          only print errors
+  -w, --warnings        print warnings in addition to errors
+  --verbose             more verbose output
+  --debug               print debugging information
+  --log-file LOG_FILE   output logging to a file
+  --no-prettify-json    output JSON in a single line without indentation
+  -v, --version         show program's version number and exit
+```
+
+:::{note}
+Starting in `parsedmarc` 6.0.0, most CLI options were moved to a
+configuration file, described below.
+:::
+
+## Configuration file
+
+`parsedmarc` can be configured by supplying the path to an INI file
+
+```bash
+parsedmarc -c /etc/parsedmarc.ini
+```
+
+For example
+
+```ini
+# This is an example comment
+
+[general]
+save_aggregate = True
+save_forensic = True
+
+[imap]
+host = imap.example.com
+user = dmarcresports@example.com
+password = $uperSecure
+
+[mailbox]
+watch = True
+delete = False
+
+[elasticsearch]
+hosts = 127.0.0.1:9200
+ssl = False
+
+[opensearch]
+hosts = https://admin:admin@127.0.0.1:9200
+ssl = True
+
+[splunk_hec]
+url = https://splunkhec.example.com
+token = HECTokenGoesHere
+index = email
+
+[s3]
+bucket = my-bucket
+path = parsedmarc
+
+[syslog]
+server = localhost
+port = 514
+
+[gelf]
+host = logger
+port = 12201
+mode = tcp
+
+[webhook]
+aggregate_url = https://aggregate_url.example.com
+forensic_url = https://forensic_url.example.com
+smtp_tls_url = https://smtp_tls_url.example.com
+timeout = 60
+```
+
+The full set of configuration options are:
+
+- `general`
+  - `save_aggregate` - bool: Save aggregate report data to
+      Elasticsearch, Splunk and/or S3
+  - `save_forensic` - bool: Save forensic report data to
+      Elasticsearch, Splunk and/or S3
+  - `save_smtp_tls` - bool: Save SMTP-STS report data to
+      Elasticsearch, Splunk and/or S3
+  - `index_prefix_domain_map` -  bool: A path mapping of Opensearch/Elasticsearch index prefixes to domain names
+  - `strip_attachment_payloads` - bool: Remove attachment
+      payloads from results
+  - `silent` - bool: Set this to `False` to output results to STDOUT
+  - `output` - str: Directory to place JSON and CSV files in.  This is required if you set either of the JSON output file options.
+  - `aggregate_json_filename` - str: filename for the aggregate
+      JSON output file
+  - `forensic_json_filename` - str: filename for the forensic
+      JSON output file
+  - `ip_db_path` - str: An optional custom path to a MMDB file
+      from MaxMind or DBIP
+  - `offline` - bool: Do not use online queries for geolocation
+      or DNS
+  - `always_use_local_files` - Disables the download of the reverse DNS map
+  - `local_reverse_dns_map_path` - Overrides the default local file path to use for the reverse DNS map
+  - `reverse_dns_map_url` - Overrides the default download URL for the reverse DNS map
+  - `nameservers` - str: A comma separated list of
+      DNS resolvers (Default: `[Cloudflare's public resolvers]`)
+  - `dns_test_address` - str: a dummy address used for DNS pre-flight checks
+      (Default: 1.1.1.1)
+  - `dns_timeout` - float: DNS timeout period
+  - `debug` - bool: Print debugging messages
+  - `silent` - bool: Only print errors (Default: `True`)
+  - `log_file` - str: Write log messages to a file at this path
+  - `n_procs` - int: Number of process to run in parallel when
+      parsing in CLI mode (Default: `1`)
+
+    :::{note}
+    Setting this to a number larger than one can improve
+    performance when processing thousands of files
+    :::
+
+- `mailbox`
+  - `reports_folder` - str: The mailbox folder (or label for
+      Gmail) where the incoming reports can be found
+      (Default: `INBOX`)
+  - `archive_folder` - str: The mailbox folder (or label for
+      Gmail) to sort processed emails into (Default: `Archive`)
+  - `watch` - bool: Use the IMAP `IDLE` command to process
+      messages as they arrive or poll MS Graph for new messages
+  - `delete` - bool: Delete messages after processing them,
+      instead of archiving them
+  - `test` - bool: Do not move or delete messages
+  - `batch_size` - int: Number of messages to read and process
+      before saving. Default `10`. Use `0` for no limit.
+  - `check_timeout` - int: Number of seconds to wait for a IMAP
+      IDLE response or the number of seconds until the next
+      mail check (Default: `30`)
+  - `since` - str: Search for messages since certain time. (Examples: `5m|3h|2d|1w`) 
+      Acceptable units - {"m":"minutes", "h":"hours", "d":"days", "w":"weeks"}. 
+      Defaults to `1d` if incorrect value is provided.
+- `imap`
+  - `host` - str: The IMAP server hostname or IP address
+  - `port` - int: The IMAP server port (Default: `993`)
+
+    :::{note}
+    `%` characters must be escaped with another `%` character,
+    so use `%%` wherever a `%` character is used.
+    :::
+
+    :::{note}
+    Starting in version 8.0.0, most options from the `imap`
+    section have been moved to the `mailbox` section.
+    :::
+
+    :::{note}
+    If your host recommends another port, still try 993
+    :::
+
+  - `ssl` - bool: Use an encrypted SSL/TLS connection
+      (Default: `True`)
+  - `skip_certificate_verification` - bool: Skip certificate
+      verification (not recommended)
+  - `user` - str: The IMAP user
+  - `password` - str: The IMAP password
+- `msgraph`
+  - `auth_method` - str: Authentication method, valid types are
+      `UsernamePassword`, `DeviceCode`, or `ClientSecret`
+      (Default: `UsernamePassword`).
+  - `user` - str: The M365 user, required when the auth method is
+      UsernamePassword
+  - `password` - str: The user password, required when the auth
+      method is UsernamePassword
+  - `client_id` - str: The app registration's client ID
+  - `client_secret` - str: The app registration's secret
+  - `tenant_id` - str: The Azure AD tenant ID. This is required
+      for all auth methods except UsernamePassword.
+  - `mailbox` - str: The mailbox name. This defaults to the
+      current user if using the UsernamePassword auth method, but
+      could be a shared mailbox if the user has access to the mailbox
+  - `graph_url` - str: Microsoft Graph URL.  Allows for use of National Clouds (ex Azure Gov)
+      (Default: https://graph.microsoft.com)
+  - `token_file` - str: Path to save the token file
+      (Default: `.token`)
+  - `allow_unencrypted_storage` - bool: Allows the Azure Identity
+      module to fall back to unencrypted token cache (Default: `False`).
+      Even if enabled, the cache will always try encrypted storage first.
+
+    :::{note}
+    You must create an app registration in Azure AD and have an
+    admin grant the Microsoft Graph `Mail.ReadWrite`
+    (delegated) permission to the app. If you are using
+    `UsernamePassword` auth and the mailbox is different from the
+    username, you must grant the app `Mail.ReadWrite.Shared`.
+    :::
+
+    :::{warning}
+    If you are using the `ClientSecret` auth method, you need to
+    grant the `Mail.ReadWrite` (application) permission to the
+    app. You must also restrict the application's access to a
+    specific mailbox since it allows all mailboxes by default.
+    Use the `New-ApplicationAccessPolicy` command in the
+    Exchange PowerShell module. If you need to scope the policy to
+    shared mailboxes, you can add them to a mail enabled security
+    group and use that as the group id.
+
+    ```powershell
+    New-ApplicationAccessPolicy -AccessRight RestrictAccess 
+    -AppId "<CLIENT_ID>" -PolicyScopeGroupId "<MAILBOX>"
+    -Description "Restrict access to dmarc reports mailbox."
+    ```
+
+    :::
+- `elasticsearch`
+  - `hosts` - str: A comma separated list of hostnames and ports
+      or URLs (e.g. `127.0.0.1:9200` or
+      `https://user:secret@localhost`)
+
+    :::{note}
+    Special characters in the username or password must be
+    [URL encoded].
+    :::
+  - `user` - str: Basic auth username
+  - `password` - str: Basic auth password
+  - `api_key` - str: API key
+  - `ssl` - bool: Use an encrypted SSL/TLS connection
+    (Default: `True`)
+  - `timeout` - float: Timeout in seconds (Default: 60)
+  - `cert_path` - str: Path to a trusted certificates
+  - `index_suffix` - str: A suffix to apply to the index names
+  - `index_prefix` - str: A prefix to apply to the index names
+  - `monthly_indexes` - bool: Use monthly indexes instead of daily indexes
+  - `number_of_shards` - int: The number of shards to use when
+    creating the index (Default: `1`)
+  - `number_of_replicas` - int: The number of replicas to use when
+    creating the index (Default: `0`)
+- `opensearch`
+  - `hosts` - str: A comma separated list of hostnames and ports
+    or URLs (e.g. `127.0.0.1:9200` or
+    `https://user:secret@localhost`)
+
+    :::{note}
+    Special characters in the username or password must be
+    [URL encoded].
+    :::
+  - `user` - str: Basic auth username
+  - `password` - str: Basic auth password
+  - `api_key` - str: API key
+  - `ssl` - bool: Use an encrypted SSL/TLS connection
+    (Default: `True`)
+  - `timeout` - float: Timeout in seconds (Default: 60)
+  - `cert_path` - str: Path to a trusted certificates
+  - `index_suffix` - str: A suffix to apply to the index names
+  - `index_prefix` - str: A prefix to apply to the index names
+  - `monthly_indexes` - bool: Use monthly indexes instead of daily indexes
+  - `number_of_shards` - int: The number of shards to use when
+    creating the index (Default: `1`)
+  - `number_of_replicas` - int: The number of replicas to use when
+    creating the index (Default: `0`)
+- `splunk_hec`
+  - `url` - str: The URL of the Splunk HTTP Events Collector (HEC)
+  - `token` - str: The HEC token
+  - `index` - str: The Splunk index to use
+  - `skip_certificate_verification` - bool: Skip certificate
+    verification (not recommended)
+- `kafka`
+  - `hosts` - str: A comma separated list of Kafka hosts
+  - `user` - str: The Kafka user
+  - `passsword` - str: The Kafka password
+  - `ssl` - bool: Use an encrypted SSL/TLS connection (Default: `True`)
+  - `skip_certificate_verification` - bool: Skip certificate
+    verification (not recommended)
+  - `aggregate_topic` - str: The Kafka topic for aggregate reports
+  - `forensic_topic` - str: The Kafka topic for forensic reports
+- `smtp`
+  - `host` - str: The SMTP hostname
+  - `port` - int: The SMTP port (Default: `25`)
+  - `ssl` - bool: Require SSL/TLS instead of using STARTTLS
+  - `skip_certificate_verification` - bool: Skip certificate
+    verification (not recommended)
+  - `user` - str: the SMTP username
+  - `password` - str: the SMTP password
+  - `from` - str: The From header to use in the email
+  - `to` - list: A list of email addresses to send to
+  - `subject` - str: The Subject header to use in the email
+    (Default: `parsedmarc report`)
+  - `attachment` - str: The ZIP attachment filenames
+  - `message` - str: The email message
+    (Default: `Please see the attached parsedmarc report.`)
+
+    :::{note}
+    `%` characters must be escaped with another `%` character,
+    so use `%%` wherever a `%` character is used.
+    :::
+- `s3`
+  - `bucket` - str: The S3 bucket name
+  - `path` - str: The path to upload reports to (Default: `/`)
+  - `region_name` - str: The region name (Optional)
+  - `endpoint_url` - str: The endpoint URL (Optional)
+  - `access_key_id` - str: The access key id (Optional)
+  - `secret_access_key` - str: The secret access key (Optional)
+- `syslog`
+  - `server` - str: The Syslog server name or IP address
+  - `port` - int: The UDP port to use (Default: `514`)
+- `gmail_api`
+  - `credentials_file` - str: Path to file containing the
+      credentials, None to disable (Default: `None`)
+  - `token_file` - str: Path to save the token file
+      (Default: `.token`)
+      
+    :::{note}
+    credentials_file and token_file can be got with [quickstart](https://developers.google.com/gmail/api/quickstart/python).Please change the scope to `https://www.googleapis.com/auth/gmail.modify`.
+    :::
+  - `include_spam_trash` - bool: Include messages in Spam and
+      Trash when searching reports (Default: `False`)
+  - `scopes` - str: Comma separated list of scopes to use when
+      acquiring credentials
+      (Default: `https://www.googleapis.com/auth/gmail.modify`)
+  - `oauth2_port` - int: The TCP port for the local server to
+      listen on for the OAuth2 response (Default: `8080`)
+  - `paginate_messages` - bool: When `True`, fetch all applicable Gmail messages.
+      When `False`, only fetch up to 100 new messages per run (Default: `True`)
+- `log_analytics`
+  - `client_id` - str: The app registration's client ID
+  - `client_secret` - str: The app registration's client secret
+  - `tenant_id` - str: The tenant id where the app registration resides
+  - `dce` - str: The Data Collection Endpoint (DCE). Example: `https://{DCE-NAME}.{REGION}.ingest.monitor.azure.com`.
+  - `dcr_immutable_id` - str: The immutable ID of the Data Collection Rule (DCR)
+  - `dcr_aggregate_stream` - str: The stream name for aggregate reports in the DCR
+  - `dcr_forensic_stream` - str: The stream name for the forensic reports in the DCR
+  - `dcr_smtp_tls_stream` - str: The stream name for the SMTP TLS reports in the DCR
+
+  :::{note}
+    Information regarding the setup of the Data Collection Rule can be found [here](https://learn.microsoft.com/en-us/azure/azure-monitor/logs/tutorial-logs-ingestion-portal).
+    :::
+- `gelf`
+  - `host` - str: The GELF server name or IP address
+  - `port` - int: The port to use
+  - `mode` - str: The GELF transport type to use. Valid modes: `tcp`, `udp`, `tls`
+
+- `maildir`
+  - `maildir_path` - str: Full path for mailbox maidir location (Default: `INBOX`)
+  - `maildir_create` - bool: Create maildir if not present (Default: False)
+
+- `webhook` - Post the individual reports to a webhook url with the report as the JSON body
+  - `aggregate_url` - str: URL of the webhook which should receive the aggregate reports
+  - `forensic_url` - str: URL of the webhook which should receive the forensic reports
+  - `smtp_tls_url` - str: URL of the webhook which should receive the smtp_tls reports
+  - `timeout` - int: Interval in which the webhook call should timeout
+
+:::{warning}
+It is **strongly recommended** to **not** use the `nameservers`
+setting. By default, `parsedmarc` uses
+[Cloudflare's public resolvers], which are much faster and more
+reliable than Google, Cisco OpenDNS, or even most local resolvers.
+
+The `nameservers` option should only be used if your network
+blocks DNS requests to outside resolvers.
+:::
+
+:::{note}
+`save_aggregate` and `save_forensic` are separate options
+because you may not want to save forensic reports
+(also known as failure reports) to your Elasticsearch instance,
+particularly if you are in a highly-regulated industry that
+handles sensitive data, such as healthcare or finance. If your
+legitimate outgoing email fails DMARC, it is possible
+that email may appear later in a forensic report.
+
+Forensic reports contain the original headers of an email that
+failed a DMARC check, and sometimes may also include the
+full message body, depending on the policy of the reporting
+organization.
+
+Most reporting organizations do not send forensic reports of any
+kind for privacy reasons. While aggregate DMARC reports are sent
+at least daily, it is normal to receive very few forensic reports.
+
+An alternative approach is to still collect forensic/failure/ruf
+reports in your DMARC inbox, but run `parsedmarc` with
+```save_forensic = True``` manually on a separate IMAP folder (using
+the ```reports_folder``` option), after you have manually moved
+known samples you want to save to that folder
+(e.g. malicious samples and non-sensitive legitimate samples).
+:::
+
+:::{warning}
+Elasticsearch 8 change limits policy for shards, restricting by
+default to 1000. parsedmarc use a shard per analyzed day. If you
+have more than ~3 years of data, you will need to update this
+limit.
+Check current usage (from Management -> Dev Tools -> Console):
+
+```text
+GET /_cluster/health?pretty
+{
+...
+  "active_primary_shards": 932,
+  "active_shards": 932,
+...
+}
+```
+
+Update the limit to 2k per example:
+
+```text
+PUT _cluster/settings
+{
+  "persistent" : {
+    "cluster.max_shards_per_node" : 2000 
+  }
+}
+```
+
+Increasing this value increases resource usage.
+:::
+
+## Multi-tenant support
+
+Starting in `8.19.0`, ParseDMARC provides multi-tenant support by placing data into separate OpenSearch or Elasticsearch index prefixes. To set this up, create a YAML file that is formatted where each key is a tenant name, and the value is a list of domains related to that tenant, not including subdomains, like this:
+
+```yaml
+example:
+  - example.com
+  - example.net
+  - example.org
+
+whalensolutions:
+  - whalensolutions.com
+```
+
+Save it to disk where the user running ParseDMARC can read it, then set `index_prefix_domain_map` to that filepath in the `[general]` section of the ParseDMARC configuration file and do not set an `index_prefix` option in the `[elasticsearch]` or `[opensearch]` sections.
+
+When configured correctly, if ParseDMARC finds that a report is related to a domain in the mapping, the report will be saved in an index name that has the tenant name prefixed to it with a trailing underscore. Then, you can use the security features of Opensearch or the ELK stack to only grant users access to the indexes that they need.
+
+ :::{note}
+ A domain cannot be used in multiple tenant lists. Only the first prefix list that contains the matching domain is used.
+:::
+
+## Running parsedmarc as a systemd service
+
+Use systemd to run `parsedmarc` as a service and process reports as
+they arrive.
+
+Protect the `parsedmarc` configuration file from prying eyes
+
+```bash
+sudo chown root:parsedmarc /etc/parsedmarc.ini
+sudo chmod u=rw,g=r,o= /etc/parsedmarc.ini
+```
+
+Create the service configuration file
+
+```bash
+sudo nano /etc/systemd/system/parsedmarc.service
+```
+
+```ini
+[Unit]
+Description=parsedmarc mailbox watcher
+Documentation=https://domainaware.github.io/parsedmarc/
+Wants=network-online.target
+After=network.target network-online.target elasticsearch.service
+
+[Service]
+ExecStart=/opt/parsedmarc/venv/bin/parsedmarc -c /etc/parsedmarc.ini
+User=parsedmarc
+Group=parsedmarc
+Restart=always
+RestartSec=5m
+
+[Install]
+WantedBy=multi-user.target
+```
+
+Then, enable the service
+
+```bash
+sudo systemctl daemon-reload
+sudo systemctl enable parsedmarc.service
+sudo service parsedmarc restart
+```
+
+:::{note}
+You must also run the above commands whenever you edit
+`parsedmarc.service`.
+:::
+
+:::{warning}
+Always restart the service every time you upgrade to a new version of
+`parsedmarc`:
+
+```bash
+sudo service parsedmarc restart
+```
+
+:::
+
+To check the status of the service, run:
+
+```bash
+service parsedmarc status
+```
+
+:::{note}
+In the event of a crash, systemd will restart the service after 10
+minutes, but the `service parsedmarc status` command will only show
+the logs for the current process. To view the logs for previous runs
+as well as the current process (newest to oldest), run:
+
+```bash
+journalctl -u parsedmarc.service -r
+```
+
+:::
+
+[cloudflare's public resolvers]: https://1.1.1.1/
+[url encoded]: https://en.wikipedia.org/wiki/Percent-encoding#Percent-encoding_reserved_characters

_static/_sphinx_javascript_frameworks_compat.js

@@ -0,0 +1,123 @@
+/* Compatability shim for jQuery and underscores.js.
+ *
+ * Copyright Sphinx contributors
+ * Released under the two clause BSD licence
+ */
+
+/**
+ * small helper function to urldecode strings
+ *
+ * See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/decodeURIComponent#Decoding_query_parameters_from_a_URL
+ */
+jQuery.urldecode = function(x) {
+    if (!x) {
+        return x
+    }
+    return decodeURIComponent(x.replace(/\+/g, ' '));
+};
+
+/**
+ * small helper function to urlencode strings
+ */
+jQuery.urlencode = encodeURIComponent;
+
+/**
+ * This function returns the parsed url parameters of the
+ * current request. Multiple values per key are supported,
+ * it will always return arrays of strings for the value parts.
+ */
+jQuery.getQueryParameters = function(s) {
+    if (typeof s === 'undefined')
+        s = document.location.search;
+    var parts = s.substr(s.indexOf('?') + 1).split('&');
+    var result = {};
+    for (var i = 0; i < parts.length; i++) {
+        var tmp = parts[i].split('=', 2);
+        var key = jQuery.urldecode(tmp[0]);
+        var value = jQuery.urldecode(tmp[1]);
+        if (key in result)
+            result[key].push(value);
+        else
+            result[key] = [value];
+    }
+    return result;
+};
+
+/**
+ * highlight a given string on a jquery object by wrapping it in
+ * span elements with the given class name.
+ */
+jQuery.fn.highlightText = function(text, className) {
+    function highlight(node, addItems) {
+        if (node.nodeType === 3) {
+            var val = node.nodeValue;
+            var pos = val.toLowerCase().indexOf(text);
+            if (pos >= 0 &&
+                !jQuery(node.parentNode).hasClass(className) &&
+                !jQuery(node.parentNode).hasClass("nohighlight")) {
+                var span;
+                var isInSVG = jQuery(node).closest("body, svg, foreignObject").is("svg");
+                if (isInSVG) {
+                    span = document.createElementNS("http://www.w3.org/2000/svg", "tspan");
+                } else {
+                    span = document.createElement("span");
+                    span.className = className;
+                }
+                span.appendChild(document.createTextNode(val.substr(pos, text.length)));
+                node.parentNode.insertBefore(span, node.parentNode.insertBefore(
+                    document.createTextNode(val.substr(pos + text.length)),
+                    node.nextSibling));
+                node.nodeValue = val.substr(0, pos);
+                if (isInSVG) {
+                    var rect = document.createElementNS("http://www.w3.org/2000/svg", "rect");
+                    var bbox = node.parentElement.getBBox();
+                    rect.x.baseVal.value = bbox.x;
+                    rect.y.baseVal.value = bbox.y;
+                    rect.width.baseVal.value = bbox.width;
+                    rect.height.baseVal.value = bbox.height;
+                    rect.setAttribute('class', className);
+                    addItems.push({
+                        "parent": node.parentNode,
+                        "target": rect});
+                }
+            }
+        }
+        else if (!jQuery(node).is("button, select, textarea")) {
+            jQuery.each(node.childNodes, function() {
+                highlight(this, addItems);
+            });
+        }
+    }
+    var addItems = [];
+    var result = this.each(function() {
+        highlight(this, addItems);
+    });
+    for (var i = 0; i < addItems.length; ++i) {
+        jQuery(addItems[i].parent).before(addItems[i].target);
+    }
+    return result;
+};
+
+/*
+ * backward compatibility for jQuery.browser
+ * This will be supported until firefox bug is fixed.
+ */
+if (!jQuery.browser) {
+    jQuery.uaMatch = function(ua) {
+        ua = ua.toLowerCase();
+
+        var match = /(chrome)[ \/]([\w.]+)/.exec(ua) ||
+            /(webkit)[ \/]([\w.]+)/.exec(ua) ||
+            /(opera)(?:.*version|)[ \/]([\w.]+)/.exec(ua) ||
+            /(msie) ([\w.]+)/.exec(ua) ||
+            ua.indexOf("compatible") < 0 && /(mozilla)(?:.*? rv:([\w.]+)|)/.exec(ua) ||
+            [];
+
+        return {
+            browser: match[ 1 ] || "",
+            version: match[ 2 ] || "0"
+        };
+    };
+    jQuery.browser = {};
+    jQuery.browser[jQuery.uaMatch(navigator.userAgent).browser] = true;
+}

_static/base-stemmer.js

@@ -0,0 +1,476 @@
+// @ts-check
+
+/**@constructor*/
+BaseStemmer = function() {
+    /** @protected */
+    this.current = '';
+    this.cursor = 0;
+    this.limit = 0;
+    this.limit_backward = 0;
+    this.bra = 0;
+    this.ket = 0;
+
+    /**
+     * @param {string} value
+     */
+    this.setCurrent = function(value) {
+        this.current = value;
+        this.cursor = 0;
+        this.limit = this.current.length;
+        this.limit_backward = 0;
+        this.bra = this.cursor;
+        this.ket = this.limit;
+    };
+
+    /**
+     * @return {string}
+     */
+    this.getCurrent = function() {
+        return this.current;
+    };
+
+    /**
+     * @param {BaseStemmer} other
+     */
+    this.copy_from = function(other) {
+        /** @protected */
+        this.current          = other.current;
+        this.cursor           = other.cursor;
+        this.limit            = other.limit;
+        this.limit_backward   = other.limit_backward;
+        this.bra              = other.bra;
+        this.ket              = other.ket;
+    };
+
+    /**
+     * @param {number[]} s
+     * @param {number} min
+     * @param {number} max
+     * @return {boolean}
+     */
+    this.in_grouping = function(s, min, max) {
+        /** @protected */
+        if (this.cursor >= this.limit) return false;
+        var ch = this.current.charCodeAt(this.cursor);
+        if (ch > max || ch < min) return false;
+        ch -= min;
+        if ((s[ch >>> 3] & (0x1 << (ch & 0x7))) == 0) return false;
+        this.cursor++;
+        return true;
+    };
+
+    /**
+     * @param {number[]} s
+     * @param {number} min
+     * @param {number} max
+     * @return {boolean}
+     */
+    this.go_in_grouping = function(s, min, max) {
+        /** @protected */
+        while (this.cursor < this.limit) {
+            var ch = this.current.charCodeAt(this.cursor);
+            if (ch > max || ch < min)
+                return true;
+            ch -= min;
+            if ((s[ch >>> 3] & (0x1 << (ch & 0x7))) == 0)
+                return true;
+            this.cursor++;
+        }
+        return false;
+    };
+
+    /**
+     * @param {number[]} s
+     * @param {number} min
+     * @param {number} max
+     * @return {boolean}
+     */
+    this.in_grouping_b = function(s, min, max) {
+        /** @protected */
+        if (this.cursor <= this.limit_backward) return false;
+        var ch = this.current.charCodeAt(this.cursor - 1);
+        if (ch > max || ch < min) return false;
+        ch -= min;
+        if ((s[ch >>> 3] & (0x1 << (ch & 0x7))) == 0) return false;
+        this.cursor--;
+        return true;
+    };
+
+    /**
+     * @param {number[]} s
+     * @param {number} min
+     * @param {number} max
+     * @return {boolean}
+     */
+    this.go_in_grouping_b = function(s, min, max) {
+        /** @protected */
+        while (this.cursor > this.limit_backward) {
+            var ch = this.current.charCodeAt(this.cursor - 1);
+            if (ch > max || ch < min) return true;
+            ch -= min;
+            if ((s[ch >>> 3] & (0x1 << (ch & 0x7))) == 0) return true;
+            this.cursor--;
+        }
+        return false;
+    };
+
+    /**
+     * @param {number[]} s
+     * @param {number} min
+     * @param {number} max
+     * @return {boolean}
+     */
+    this.out_grouping = function(s, min, max) {
+        /** @protected */
+        if (this.cursor >= this.limit) return false;
+        var ch = this.current.charCodeAt(this.cursor);
+        if (ch > max || ch < min) {
+            this.cursor++;
+            return true;
+        }
+        ch -= min;
+        if ((s[ch >>> 3] & (0X1 << (ch & 0x7))) == 0) {
+            this.cursor++;
+            return true;
+        }
+        return false;
+    };
+
+    /**
+     * @param {number[]} s
+     * @param {number} min
+     * @param {number} max
+     * @return {boolean}
+     */
+    this.go_out_grouping = function(s, min, max) {
+        /** @protected */
+        while (this.cursor < this.limit) {
+            var ch = this.current.charCodeAt(this.cursor);
+            if (ch <= max && ch >= min) {
+                ch -= min;
+                if ((s[ch >>> 3] & (0X1 << (ch & 0x7))) != 0) {
+                    return true;
+                }
+            }
+            this.cursor++;
+        }
+        return false;
+    };
+
+    /**
+     * @param {number[]} s
+     * @param {number} min
+     * @param {number} max
+     * @return {boolean}
+     */
+    this.out_grouping_b = function(s, min, max) {
+        /** @protected */
+        if (this.cursor <= this.limit_backward) return false;
+        var ch = this.current.charCodeAt(this.cursor - 1);
+        if (ch > max || ch < min) {
+            this.cursor--;
+            return true;
+        }
+        ch -= min;
+        if ((s[ch >>> 3] & (0x1 << (ch & 0x7))) == 0) {
+            this.cursor--;
+            return true;
+        }
+        return false;
+    };
+
+    /**
+     * @param {number[]} s
+     * @param {number} min
+     * @param {number} max
+     * @return {boolean}
+     */
+    this.go_out_grouping_b = function(s, min, max) {
+        /** @protected */
+        while (this.cursor > this.limit_backward) {
+            var ch = this.current.charCodeAt(this.cursor - 1);
+            if (ch <= max && ch >= min) {
+                ch -= min;
+                if ((s[ch >>> 3] & (0x1 << (ch & 0x7))) != 0) {
+                    return true;
+                }
+            }
+            this.cursor--;
+        }
+        return false;
+    };
+
+    /**
+     * @param {string} s
+     * @return {boolean}
+     */
+    this.eq_s = function(s)
+    {
+        /** @protected */
+        if (this.limit - this.cursor < s.length) return false;
+        if (this.current.slice(this.cursor, this.cursor + s.length) != s)
+        {
+            return false;
+        }
+        this.cursor += s.length;
+        return true;
+    };
+
+    /**
+     * @param {string} s
+     * @return {boolean}
+     */
+    this.eq_s_b = function(s)
+    {
+        /** @protected */
+        if (this.cursor - this.limit_backward < s.length) return false;
+        if (this.current.slice(this.cursor - s.length, this.cursor) != s)
+        {
+            return false;
+        }
+        this.cursor -= s.length;
+        return true;
+    };
+
+    /**
+     * @param {Among[]} v
+     * @return {number}
+     */
+    this.find_among = function(v)
+    {
+        /** @protected */
+        var i = 0;
+        var j = v.length;
+
+        var c = this.cursor;
+        var l = this.limit;
+
+        var common_i = 0;
+        var common_j = 0;
+
+        var first_key_inspected = false;
+
+        while (true)
+        {
+            var k = i + ((j - i) >>> 1);
+            var diff = 0;
+            var common = common_i < common_j ? common_i : common_j; // smaller
+            // w[0]: string, w[1]: substring_i, w[2]: result, w[3]: function (optional)
+            var w = v[k];
+            var i2;
+            for (i2 = common; i2 < w[0].length; i2++)
+            {
+                if (c + common == l)
+                {
+                    diff = -1;
+                    break;
+                }
+                diff = this.current.charCodeAt(c + common) - w[0].charCodeAt(i2);
+                if (diff != 0) break;
+                common++;
+            }
+            if (diff < 0)
+            {
+                j = k;
+                common_j = common;
+            }
+            else
+            {
+                i = k;
+                common_i = common;
+            }
+            if (j - i <= 1)
+            {
+                if (i > 0) break; // v->s has been inspected
+                if (j == i) break; // only one item in v
+
+                // - but now we need to go round once more to get
+                // v->s inspected. This looks messy, but is actually
+                // the optimal approach.
+
+                if (first_key_inspected) break;
+                first_key_inspected = true;
+            }
+        }
+        do {
+            var w = v[i];
+            if (common_i >= w[0].length)
+            {
+                this.cursor = c + w[0].length;
+                if (w.length < 4) return w[2];
+                var res = w[3](this);
+                this.cursor = c + w[0].length;
+                if (res) return w[2];
+            }
+            i = w[1];
+        } while (i >= 0);
+        return 0;
+    };
+
+    // find_among_b is for backwards processing. Same comments apply
+    /**
+     * @param {Among[]} v
+     * @return {number}
+     */
+    this.find_among_b = function(v)
+    {
+        /** @protected */
+        var i = 0;
+        var j = v.length
+
+        var c = this.cursor;
+        var lb = this.limit_backward;
+
+        var common_i = 0;
+        var common_j = 0;
+
+        var first_key_inspected = false;
+
+        while (true)
+        {
+            var k = i + ((j - i) >> 1);
+            var diff = 0;
+            var common = common_i < common_j ? common_i : common_j;
+            var w = v[k];
+            var i2;
+            for (i2 = w[0].length - 1 - common; i2 >= 0; i2--)
+            {
+                if (c - common == lb)
+                {
+                    diff = -1;
+                    break;
+                }
+                diff = this.current.charCodeAt(c - 1 - common) - w[0].charCodeAt(i2);
+                if (diff != 0) break;
+                common++;
+            }
+            if (diff < 0)
+            {
+                j = k;
+                common_j = common;
+            }
+            else
+            {
+                i = k;
+                common_i = common;
+            }
+            if (j - i <= 1)
+            {
+                if (i > 0) break;
+                if (j == i) break;
+                if (first_key_inspected) break;
+                first_key_inspected = true;
+            }
+        }
+        do {
+            var w = v[i];
+            if (common_i >= w[0].length)
+            {
+                this.cursor = c - w[0].length;
+                if (w.length < 4) return w[2];
+                var res = w[3](this);
+                this.cursor = c - w[0].length;
+                if (res) return w[2];
+            }
+            i = w[1];
+        } while (i >= 0);
+        return 0;
+    };
+
+    /* to replace chars between c_bra and c_ket in this.current by the
+     * chars in s.
+     */
+    /**
+     * @param {number} c_bra
+     * @param {number} c_ket
+     * @param {string} s
+     * @return {number}
+     */
+    this.replace_s = function(c_bra, c_ket, s)
+    {
+        /** @protected */
+        var adjustment = s.length - (c_ket - c_bra);
+        this.current = this.current.slice(0, c_bra) + s + this.current.slice(c_ket);
+        this.limit += adjustment;
+        if (this.cursor >= c_ket) this.cursor += adjustment;
+        else if (this.cursor > c_bra) this.cursor = c_bra;
+        return adjustment;
+    };
+
+    /**
+     * @return {boolean}
+     */
+    this.slice_check = function()
+    {
+        /** @protected */
+        if (this.bra < 0 ||
+            this.bra > this.ket ||
+            this.ket > this.limit ||
+            this.limit > this.current.length)
+        {
+            return false;
+        }
+        return true;
+    };
+
+    /**
+     * @param {number} c_bra
+     * @return {boolean}
+     */
+    this.slice_from = function(s)
+    {
+        /** @protected */
+        var result = false;
+        if (this.slice_check())
+        {
+            this.replace_s(this.bra, this.ket, s);
+            result = true;
+        }
+        return result;
+    };
+
+    /**
+     * @return {boolean}
+     */
+    this.slice_del = function()
+    {
+        /** @protected */
+        return this.slice_from("");
+    };
+
+    /**
+     * @param {number} c_bra
+     * @param {number} c_ket
+     * @param {string} s
+     */
+    this.insert = function(c_bra, c_ket, s)
+    {
+        /** @protected */
+        var adjustment = this.replace_s(c_bra, c_ket, s);
+        if (c_bra <= this.bra) this.bra += adjustment;
+        if (c_bra <= this.ket) this.ket += adjustment;
+    };
+
+    /**
+     * @return {string}
+     */
+    this.slice_to = function()
+    {
+        /** @protected */
+        var result = '';
+        if (this.slice_check())
+        {
+            result = this.current.slice(this.bra, this.ket);
+        }
+        return result;
+    };
+
+    /**
+     * @return {string}
+     */
+    this.assign_to = function()
+    {
+        /** @protected */
+        return this.current.slice(0, this.limit);
+    };
+};

_static/basic.css

@@ -0,0 +1,906 @@
+/*
+ * Sphinx stylesheet -- basic theme.
+ */
+
+/* -- main layout ----------------------------------------------------------- */
+
+div.clearer {
+    clear: both;
+}
+
+div.section::after {
+    display: block;
+    content: '';
+    clear: left;
+}
+
+/* -- relbar ---------------------------------------------------------------- */
+
+div.related {
+    width: 100%;
+    font-size: 90%;
+}
+
+div.related h3 {
+    display: none;
+}
+
+div.related ul {
+    margin: 0;
+    padding: 0 0 0 10px;
+    list-style: none;
+}
+
+div.related li {
+    display: inline;
+}
+
+div.related li.right {
+    float: right;
+    margin-right: 5px;
+}
+
+/* -- sidebar --------------------------------------------------------------- */
+
+div.sphinxsidebarwrapper {
+    padding: 10px 5px 0 10px;
+}
+
+div.sphinxsidebar {
+    float: left;
+    width: 230px;
+    margin-left: -100%;
+    font-size: 90%;
+    word-wrap: break-word;
+    overflow-wrap : break-word;
+}
+
+div.sphinxsidebar ul {
+    list-style: none;
+}
+
+div.sphinxsidebar ul ul,
+div.sphinxsidebar ul.want-points {
+    margin-left: 20px;
+    list-style: square;
+}
+
+div.sphinxsidebar ul ul {
+    margin-top: 0;
+    margin-bottom: 0;
+}
+
+div.sphinxsidebar form {
+    margin-top: 10px;
+}
+
+div.sphinxsidebar input {
+    border: 1px solid #98dbcc;
+    font-family: sans-serif;
+    font-size: 1em;
+}
+
+div.sphinxsidebar #searchbox form.search {
+    overflow: hidden;
+}
+
+div.sphinxsidebar #searchbox input[type="text"] {
+    float: left;
+    width: 80%;
+    padding: 0.25em;
+    box-sizing: border-box;
+}
+
+div.sphinxsidebar #searchbox input[type="submit"] {
+    float: left;
+    width: 20%;
+    border-left: none;
+    padding: 0.25em;
+    box-sizing: border-box;
+}
+
+
+img {
+    border: 0;
+    max-width: 100%;
+}
+
+/* -- search page ----------------------------------------------------------- */
+
+ul.search {
+    margin-top: 10px;
+}
+
+ul.search li {
+    padding: 5px 0;
+}
+
+ul.search li a {
+    font-weight: bold;
+}
+
+ul.search li p.context {
+    color: #888;
+    margin: 2px 0 0 30px;
+    text-align: left;
+}
+
+ul.keywordmatches li.goodmatch a {
+    font-weight: bold;
+}
+
+/* -- index page ------------------------------------------------------------ */
+
+table.contentstable {
+    width: 90%;
+    margin-left: auto;
+    margin-right: auto;
+}
+
+table.contentstable p.biglink {
+    line-height: 150%;
+}
+
+a.biglink {
+    font-size: 1.3em;
+}
+
+span.linkdescr {
+    font-style: italic;
+    padding-top: 5px;
+    font-size: 90%;
+}
+
+/* -- general index --------------------------------------------------------- */
+
+table.indextable {
+    width: 100%;
+}
+
+table.indextable td {
+    text-align: left;
+    vertical-align: top;
+}
+
+table.indextable ul {
+    margin-top: 0;
+    margin-bottom: 0;
+    list-style-type: none;
+}
+
+table.indextable > tbody > tr > td > ul {
+    padding-left: 0em;
+}
+
+table.indextable tr.pcap {
+    height: 10px;
+}
+
+table.indextable tr.cap {
+    margin-top: 10px;
+    background-color: #f2f2f2;
+}
+
+img.toggler {
+    margin-right: 3px;
+    margin-top: 3px;
+    cursor: pointer;
+}
+
+div.modindex-jumpbox {
+    border-top: 1px solid #ddd;
+    border-bottom: 1px solid #ddd;
+    margin: 1em 0 1em 0;
+    padding: 0.4em;
+}
+
+div.genindex-jumpbox {
+    border-top: 1px solid #ddd;
+    border-bottom: 1px solid #ddd;
+    margin: 1em 0 1em 0;
+    padding: 0.4em;
+}
+
+/* -- domain module index --------------------------------------------------- */
+
+table.modindextable td {
+    padding: 2px;
+    border-collapse: collapse;
+}
+
+/* -- general body styles --------------------------------------------------- */
+
+div.body {
+    min-width: 360px;
+    max-width: 800px;
+}
+
+div.body p, div.body dd, div.body li, div.body blockquote {
+    -moz-hyphens: auto;
+    -ms-hyphens: auto;
+    -webkit-hyphens: auto;
+    hyphens: auto;
+}
+
+a.headerlink {
+    visibility: hidden;
+}
+
+a:visited {
+    color: #551A8B;
+}
+
+h1:hover > a.headerlink,
+h2:hover > a.headerlink,
+h3:hover > a.headerlink,
+h4:hover > a.headerlink,
+h5:hover > a.headerlink,
+h6:hover > a.headerlink,
+dt:hover > a.headerlink,
+caption:hover > a.headerlink,
+p.caption:hover > a.headerlink,
+div.code-block-caption:hover > a.headerlink {
+    visibility: visible;
+}
+
+div.body p.caption {
+    text-align: inherit;
+}
+
+div.body td {
+    text-align: left;
+}
+
+.first {
+    margin-top: 0 !important;
+}
+
+p.rubric {
+    margin-top: 30px;
+    font-weight: bold;
+}
+
+img.align-left, figure.align-left, .figure.align-left, object.align-left {
+    clear: left;
+    float: left;
+    margin-right: 1em;
+}
+
+img.align-right, figure.align-right, .figure.align-right, object.align-right {
+    clear: right;
+    float: right;
+    margin-left: 1em;
+}
+
+img.align-center, figure.align-center, .figure.align-center, object.align-center {
+  display: block;
+  margin-left: auto;
+  margin-right: auto;
+}
+
+img.align-default, figure.align-default, .figure.align-default {
+  display: block;
+  margin-left: auto;
+  margin-right: auto;
+}
+
+.align-left {
+    text-align: left;
+}
+
+.align-center {
+    text-align: center;
+}
+
+.align-default {
+    text-align: center;
+}
+
+.align-right {
+    text-align: right;
+}
+
+/* -- sidebars -------------------------------------------------------------- */
+
+div.sidebar,
+aside.sidebar {
+    margin: 0 0 0.5em 1em;
+    border: 1px solid #ddb;
+    padding: 7px;
+    background-color: #ffe;
+    width: 40%;
+    float: right;
+    clear: right;
+    overflow-x: auto;
+}
+
+p.sidebar-title {
+    font-weight: bold;
+}
+
+nav.contents,
+aside.topic,
+div.admonition, div.topic, blockquote {
+    clear: left;
+}
+
+/* -- topics ---------------------------------------------------------------- */
+
+nav.contents,
+aside.topic,
+div.topic {
+    border: 1px solid #ccc;
+    padding: 7px;
+    margin: 10px 0 10px 0;
+}
+
+p.topic-title {
+    font-size: 1.1em;
+    font-weight: bold;
+    margin-top: 10px;
+}
+
+/* -- admonitions ----------------------------------------------------------- */
+
+div.admonition {
+    margin-top: 10px;
+    margin-bottom: 10px;
+    padding: 7px;
+}
+
+div.admonition dt {
+    font-weight: bold;
+}
+
+p.admonition-title {
+    margin: 0px 10px 5px 0px;
+    font-weight: bold;
+}
+
+div.body p.centered {
+    text-align: center;
+    margin-top: 25px;
+}
+
+/* -- content of sidebars/topics/admonitions -------------------------------- */
+
+div.sidebar > :last-child,
+aside.sidebar > :last-child,
+nav.contents > :last-child,
+aside.topic > :last-child,
+div.topic > :last-child,
+div.admonition > :last-child {
+    margin-bottom: 0;
+}
+
+div.sidebar::after,
+aside.sidebar::after,
+nav.contents::after,
+aside.topic::after,
+div.topic::after,
+div.admonition::after,
+blockquote::after {
+    display: block;
+    content: '';
+    clear: both;
+}
+
+/* -- tables ---------------------------------------------------------------- */
+
+table.docutils {
+    margin-top: 10px;
+    margin-bottom: 10px;
+    border: 0;
+    border-collapse: collapse;
+}
+
+table.align-center {
+    margin-left: auto;
+    margin-right: auto;
+}
+
+table.align-default {
+    margin-left: auto;
+    margin-right: auto;
+}
+
+table caption span.caption-number {
+    font-style: italic;
+}
+
+table caption span.caption-text {
+}
+
+table.docutils td, table.docutils th {
+    padding: 1px 8px 1px 5px;
+    border-top: 0;
+    border-left: 0;
+    border-right: 0;
+    border-bottom: 1px solid #aaa;
+}
+
+th {
+    text-align: left;
+    padding-right: 5px;
+}
+
+table.citation {
+    border-left: solid 1px gray;
+    margin-left: 1px;
+}
+
+table.citation td {
+    border-bottom: none;
+}
+
+th > :first-child,
+td > :first-child {
+    margin-top: 0px;
+}
+
+th > :last-child,
+td > :last-child {
+    margin-bottom: 0px;
+}
+
+/* -- figures --------------------------------------------------------------- */
+
+div.figure, figure {
+    margin: 0.5em;
+    padding: 0.5em;
+}
+
+div.figure p.caption, figcaption {
+    padding: 0.3em;
+}
+
+div.figure p.caption span.caption-number,
+figcaption span.caption-number {
+    font-style: italic;
+}
+
+div.figure p.caption span.caption-text,
+figcaption span.caption-text {
+}
+
+/* -- field list styles ----------------------------------------------------- */
+
+table.field-list td, table.field-list th {
+    border: 0 !important;
+}
+
+.field-list ul {
+    margin: 0;
+    padding-left: 1em;
+}
+
+.field-list p {
+    margin: 0;
+}
+
+.field-name {
+    -moz-hyphens: manual;
+    -ms-hyphens: manual;
+    -webkit-hyphens: manual;
+    hyphens: manual;
+}
+
+/* -- hlist styles ---------------------------------------------------------- */
+
+table.hlist {
+    margin: 1em 0;
+}
+
+table.hlist td {
+    vertical-align: top;
+}
+
+/* -- object description styles --------------------------------------------- */
+
+.sig {
+	font-family: 'Consolas', 'Menlo', 'DejaVu Sans Mono', 'Bitstream Vera Sans Mono', monospace;
+}
+
+.sig-name, code.descname {
+    background-color: transparent;
+    font-weight: bold;
+}
+
+.sig-name {
+	font-size: 1.1em;
+}
+
+code.descname {
+    font-size: 1.2em;
+}
+
+.sig-prename, code.descclassname {
+    background-color: transparent;
+}
+
+.optional {
+    font-size: 1.3em;
+}
+
+.sig-paren {
+    font-size: larger;
+}
+
+.sig-param.n {
+	font-style: italic;
+}
+
+/* C++ specific styling */
+
+.sig-inline.c-texpr,
+.sig-inline.cpp-texpr {
+	font-family: unset;
+}
+
+.sig.c   .k, .sig.c   .kt,
+.sig.cpp .k, .sig.cpp .kt {
+	color: #0033B3;
+}
+
+.sig.c   .m,
+.sig.cpp .m {
+	color: #1750EB;
+}
+
+.sig.c   .s, .sig.c   .sc,
+.sig.cpp .s, .sig.cpp .sc {
+	color: #067D17;
+}
+
+
+/* -- other body styles ----------------------------------------------------- */
+
+ol.arabic {
+    list-style: decimal;
+}
+
+ol.loweralpha {
+    list-style: lower-alpha;
+}
+
+ol.upperalpha {
+    list-style: upper-alpha;
+}
+
+ol.lowerroman {
+    list-style: lower-roman;
+}
+
+ol.upperroman {
+    list-style: upper-roman;
+}
+
+:not(li) > ol > li:first-child > :first-child,
+:not(li) > ul > li:first-child > :first-child {
+    margin-top: 0px;
+}
+
+:not(li) > ol > li:last-child > :last-child,
+:not(li) > ul > li:last-child > :last-child {
+    margin-bottom: 0px;
+}
+
+ol.simple ol p,
+ol.simple ul p,
+ul.simple ol p,
+ul.simple ul p {
+    margin-top: 0;
+}
+
+ol.simple > li:not(:first-child) > p,
+ul.simple > li:not(:first-child) > p {
+    margin-top: 0;
+}
+
+ol.simple p,
+ul.simple p {
+    margin-bottom: 0;
+}
+
+aside.footnote > span,
+div.citation > span {
+    float: left;
+}
+aside.footnote > span:last-of-type,
+div.citation > span:last-of-type {
+  padding-right: 0.5em;
+}
+aside.footnote > p {
+  margin-left: 2em;
+}
+div.citation > p {
+  margin-left: 4em;
+}
+aside.footnote > p:last-of-type,
+div.citation > p:last-of-type {
+    margin-bottom: 0em;
+}
+aside.footnote > p:last-of-type:after,
+div.citation > p:last-of-type:after {
+    content: "";
+    clear: both;
+}
+
+dl.field-list {
+    display: grid;
+    grid-template-columns: fit-content(30%) auto;
+}
+
+dl.field-list > dt {
+    font-weight: bold;
+    word-break: break-word;
+    padding-left: 0.5em;
+    padding-right: 5px;
+}
+
+dl.field-list > dd {
+    padding-left: 0.5em;
+    margin-top: 0em;
+    margin-left: 0em;
+    margin-bottom: 0em;
+}
+
+dl {
+    margin-bottom: 15px;
+}
+
+dd > :first-child {
+    margin-top: 0px;
+}
+
+dd ul, dd table {
+    margin-bottom: 10px;
+}
+
+dd {
+    margin-top: 3px;
+    margin-bottom: 10px;
+    margin-left: 30px;
+}
+
+.sig dd {
+    margin-top: 0px;
+    margin-bottom: 0px;
+}
+
+.sig dl {
+    margin-top: 0px;
+    margin-bottom: 0px;
+}
+
+dl > dd:last-child,
+dl > dd:last-child > :last-child {
+    margin-bottom: 0;
+}
+
+dt:target, span.highlighted {
+    background-color: #fbe54e;
+}
+
+rect.highlighted {
+    fill: #fbe54e;
+}
+
+dl.glossary dt {
+    font-weight: bold;
+    font-size: 1.1em;
+}
+
+.versionmodified {
+    font-style: italic;
+}
+
+.system-message {
+    background-color: #fda;
+    padding: 5px;
+    border: 3px solid red;
+}
+
+.footnote:target  {
+    background-color: #ffa;
+}
+
+.line-block {
+    display: block;
+    margin-top: 1em;
+    margin-bottom: 1em;
+}
+
+.line-block .line-block {
+    margin-top: 0;
+    margin-bottom: 0;
+    margin-left: 1.5em;
+}
+
+.guilabel, .menuselection {
+    font-family: sans-serif;
+}
+
+.accelerator {
+    text-decoration: underline;
+}
+
+.classifier {
+    font-style: oblique;
+}
+
+.classifier:before {
+    font-style: normal;
+    margin: 0 0.5em;
+    content: ":";
+    display: inline-block;
+}
+
+abbr, acronym {
+    border-bottom: dotted 1px;
+    cursor: help;
+}
+
+/* -- code displays --------------------------------------------------------- */
+
+pre {
+    overflow: auto;
+    overflow-y: hidden;  /* fixes display issues on Chrome browsers */
+}
+
+pre, div[class*="highlight-"] {
+    clear: both;
+}
+
+span.pre {
+    -moz-hyphens: none;
+    -ms-hyphens: none;
+    -webkit-hyphens: none;
+    hyphens: none;
+    white-space: nowrap;
+}
+
+div[class*="highlight-"] {
+    margin: 1em 0;
+}
+
+td.linenos pre {
+    border: 0;
+    background-color: transparent;
+    color: #aaa;
+}
+
+table.highlighttable {
+    display: block;
+}
+
+table.highlighttable tbody {
+    display: block;
+}
+
+table.highlighttable tr {
+    display: flex;
+}
+
+table.highlighttable td {
+    margin: 0;
+    padding: 0;
+}
+
+table.highlighttable td.linenos {
+    padding-right: 0.5em;
+}
+
+table.highlighttable td.code {
+    flex: 1;
+    overflow: hidden;
+}
+
+.highlight .hll {
+    display: block;
+}
+
+div.highlight pre,
+table.highlighttable pre {
+    margin: 0;
+}
+
+div.code-block-caption + div {
+    margin-top: 0;
+}
+
+div.code-block-caption {
+    margin-top: 1em;
+    padding: 2px 5px;
+    font-size: small;
+}
+
+div.code-block-caption code {
+    background-color: transparent;
+}
+
+table.highlighttable td.linenos,
+span.linenos,
+div.highlight span.gp {  /* gp: Generic.Prompt */
+  user-select: none;
+  -webkit-user-select: text; /* Safari fallback only */
+  -webkit-user-select: none; /* Chrome/Safari */
+  -moz-user-select: none; /* Firefox */
+  -ms-user-select: none; /* IE10+ */
+}
+
+div.code-block-caption span.caption-number {
+    padding: 0.1em 0.3em;
+    font-style: italic;
+}
+
+div.code-block-caption span.caption-text {
+}
+
+div.literal-block-wrapper {
+    margin: 1em 0;
+}
+
+code.xref, a code {
+    background-color: transparent;
+    font-weight: bold;
+}
+
+h1 code, h2 code, h3 code, h4 code, h5 code, h6 code {
+    background-color: transparent;
+}
+
+.viewcode-link {
+    float: right;
+}
+
+.viewcode-back {
+    float: right;
+    font-family: sans-serif;
+}
+
+div.viewcode-block:target {
+    margin: -1px -10px;
+    padding: 0 10px;
+}
+
+/* -- math display ---------------------------------------------------------- */
+
+img.math {
+    vertical-align: middle;
+}
+
+div.body div.math p {
+    text-align: center;
+}
+
+span.eqno {
+    float: right;
+}
+
+span.eqno a.headerlink {
+    position: absolute;
+    z-index: 1;
+}
+
+div.math:hover a.headerlink {
+    visibility: visible;
+}
+
+/* -- printout stylesheet --------------------------------------------------- */
+
+@media print {
+    div.document,
+    div.documentwrapper,
+    div.bodywrapper {
+        margin: 0 !important;
+        width: 100%;
+    }
+
+    div.sphinxsidebar,
+    div.related,
+    div.footer,
+    #top-link {
+        display: none;
+    }
+}

_static/css/badge_only.css

@@ -0,0 +1 @@
+.clearfix{*zoom:1}.clearfix:after,.clearfix:before{display:table;content:""}.clearfix:after{clear:both}@font-face{font-family:FontAwesome;font-style:normal;font-weight:400;src:url(fonts/fontawesome-webfont.eot?674f50d287a8c48dc19ba404d20fe713?#iefix) format("embedded-opentype"),url(fonts/fontawesome-webfont.woff2?af7ae505a9eed503f8b8e6982036873e) format("woff2"),url(fonts/fontawesome-webfont.woff?fee66e712a8a08eef5805a46892932ad) format("woff"),url(fonts/fontawesome-webfont.ttf?b06871f281fee6b241d60582ae9369b9) format("truetype"),url(fonts/fontawesome-webfont.svg?912ec66d7572ff821749319396470bde#FontAwesome) format("svg")}.fa:before{font-family:FontAwesome;font-style:normal;font-weight:400;line-height:1}.fa:before,a .fa{text-decoration:inherit}.fa:before,a .fa,li .fa{display:inline-block}li .fa-large:before{width:1.875em}ul.fas{list-style-type:none;margin-left:2em;text-indent:-.8em}ul.fas li .fa{width:.8em}ul.fas li .fa-large:before{vertical-align:baseline}.fa-book:before,.icon-book:before{content:"\f02d"}.fa-caret-down:before,.icon-caret-down:before{content:"\f0d7"}.fa-caret-up:before,.icon-caret-up:before{content:"\f0d8"}.fa-caret-left:before,.icon-caret-left:before{content:"\f0d9"}.fa-caret-right:before,.icon-caret-right:before{content:"\f0da"}.rst-versions{position:fixed;bottom:0;left:0;width:300px;color:#fcfcfc;background:#1f1d1d;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;z-index:400}.rst-versions a{color:#2980b9;text-decoration:none}.rst-versions .rst-badge-small{display:none}.rst-versions .rst-current-version{padding:12px;background-color:#272525;display:block;text-align:right;font-size:90%;cursor:pointer;color:#27ae60}.rst-versions .rst-current-version:after{clear:both;content:"";display:block}.rst-versions .rst-current-version .fa{color:#fcfcfc}.rst-versions .rst-current-version .fa-book,.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version.rst-out-of-date{background-color:#e74c3c;color:#fff}.rst-versions .rst-current-version.rst-active-old-version{background-color:#f1c40f;color:#000}.rst-versions.shift-up{height:auto;max-height:100%;overflow-y:scroll}.rst-versions.shift-up .rst-other-versions{display:block}.rst-versions .rst-other-versions{font-size:90%;padding:12px;color:grey;display:none}.rst-versions .rst-other-versions hr{display:block;height:1px;border:0;margin:20px 0;padding:0;border-top:1px solid #413d3d}.rst-versions .rst-other-versions dd{display:inline-block;margin:0}.rst-versions .rst-other-versions dd a{display:inline-block;padding:6px;color:#fcfcfc}.rst-versions .rst-other-versions .rtd-current-item{font-weight:700}.rst-versions.rst-badge{width:auto;bottom:20px;right:20px;left:auto;border:none;max-width:300px;max-height:90%}.rst-versions.rst-badge .fa-book,.rst-versions.rst-badge .icon-book{float:none;line-height:30px}.rst-versions.rst-badge.shift-up .rst-current-version{text-align:right}.rst-versions.rst-badge.shift-up .rst-current-version .fa-book,.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge>.rst-current-version{width:auto;height:30px;line-height:30px;padding:0 6px;display:block;text-align:center}@media screen and (max-width:768px){.rst-versions{width:85%;display:none}.rst-versions.shift{display:block}}#flyout-search-form{padding:6px}

_static/doctools.js

@@ -0,0 +1,150 @@
+/*
+ * Base JavaScript utilities for all Sphinx HTML documentation.
+ */
+"use strict";
+
+const BLACKLISTED_KEY_CONTROL_ELEMENTS = new Set([
+  "TEXTAREA",
+  "INPUT",
+  "SELECT",
+  "BUTTON",
+]);
+
+const _ready = (callback) => {
+  if (document.readyState !== "loading") {
+    callback();
+  } else {
+    document.addEventListener("DOMContentLoaded", callback);
+  }
+};
+
+/**
+ * Small JavaScript module for the documentation.
+ */
+const Documentation = {
+  init: () => {
+    Documentation.initDomainIndexTable();
+    Documentation.initOnKeyListeners();
+  },
+
+  /**
+   * i18n support
+   */
+  TRANSLATIONS: {},
+  PLURAL_EXPR: (n) => (n === 1 ? 0 : 1),
+  LOCALE: "unknown",
+
+  // gettext and ngettext don't access this so that the functions
+  // can safely bound to a different name (_ = Documentation.gettext)
+  gettext: (string) => {
+    const translated = Documentation.TRANSLATIONS[string];
+    switch (typeof translated) {
+      case "undefined":
+        return string; // no translation
+      case "string":
+        return translated; // translation exists
+      default:
+        return translated[0]; // (singular, plural) translation tuple exists
+    }
+  },
+
+  ngettext: (singular, plural, n) => {
+    const translated = Documentation.TRANSLATIONS[singular];
+    if (typeof translated !== "undefined")
+      return translated[Documentation.PLURAL_EXPR(n)];
+    return n === 1 ? singular : plural;
+  },
+
+  addTranslations: (catalog) => {
+    Object.assign(Documentation.TRANSLATIONS, catalog.messages);
+    Documentation.PLURAL_EXPR = new Function(
+      "n",
+      `return (${catalog.plural_expr})`,
+    );
+    Documentation.LOCALE = catalog.locale;
+  },
+
+  /**
+   * helper function to focus on search bar
+   */
+  focusSearchBar: () => {
+    document.querySelectorAll("input[name=q]")[0]?.focus();
+  },
+
+  /**
+   * Initialise the domain index toggle buttons
+   */
+  initDomainIndexTable: () => {
+    const toggler = (el) => {
+      const idNumber = el.id.substr(7);
+      const toggledRows = document.querySelectorAll(`tr.cg-${idNumber}`);
+      if (el.src.substr(-9) === "minus.png") {
+        el.src = `${el.src.substr(0, el.src.length - 9)}plus.png`;
+        toggledRows.forEach((el) => (el.style.display = "none"));
+      } else {
+        el.src = `${el.src.substr(0, el.src.length - 8)}minus.png`;
+        toggledRows.forEach((el) => (el.style.display = ""));
+      }
+    };
+
+    const togglerElements = document.querySelectorAll("img.toggler");
+    togglerElements.forEach((el) =>
+      el.addEventListener("click", (event) => toggler(event.currentTarget)),
+    );
+    togglerElements.forEach((el) => (el.style.display = ""));
+    if (DOCUMENTATION_OPTIONS.COLLAPSE_INDEX) togglerElements.forEach(toggler);
+  },
+
+  initOnKeyListeners: () => {
+    // only install a listener if it is really needed
+    if (
+      !DOCUMENTATION_OPTIONS.NAVIGATION_WITH_KEYS
+      && !DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS
+    )
+      return;
+
+    document.addEventListener("keydown", (event) => {
+      // bail for input elements
+      if (BLACKLISTED_KEY_CONTROL_ELEMENTS.has(document.activeElement.tagName))
+        return;
+      // bail with special keys
+      if (event.altKey || event.ctrlKey || event.metaKey) return;
+
+      if (!event.shiftKey) {
+        switch (event.key) {
+          case "ArrowLeft":
+            if (!DOCUMENTATION_OPTIONS.NAVIGATION_WITH_KEYS) break;
+
+            const prevLink = document.querySelector('link[rel="prev"]');
+            if (prevLink && prevLink.href) {
+              window.location.href = prevLink.href;
+              event.preventDefault();
+            }
+            break;
+          case "ArrowRight":
+            if (!DOCUMENTATION_OPTIONS.NAVIGATION_WITH_KEYS) break;
+
+            const nextLink = document.querySelector('link[rel="next"]');
+            if (nextLink && nextLink.href) {
+              window.location.href = nextLink.href;
+              event.preventDefault();
+            }
+            break;
+        }
+      }
+
+      // some keyboard layouts may need Shift to get /
+      switch (event.key) {
+        case "/":
+          if (!DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS) break;
+          Documentation.focusSearchBar();
+          event.preventDefault();
+      }
+    });
+  },
+};
+
+// quick alias for translations
+const _ = Documentation.gettext;
+
+_ready(Documentation.init);

_static/documentation_options.js

@@ -0,0 +1,13 @@
+const DOCUMENTATION_OPTIONS = {
+    VERSION: '9.0.10',
+    LANGUAGE: 'en',
+    COLLAPSE_INDEX: false,
+    BUILDER: 'html',
+    FILE_SUFFIX: '.html',
+    LINK_SUFFIX: '.html',
+    HAS_SOURCE: true,
+    SOURCELINK_SUFFIX: '.txt',
+    NAVIGATION_WITH_KEYS: false,
+    SHOW_SEARCH_SUMMARY: true,
+    ENABLE_SEARCH_SHORTCUTS: true,
+};