mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-08-02 13:42:19 +00:00
453 lines
32 KiB
HTML
453 lines
32 KiB
HTML
|
||
|
||
<!DOCTYPE html>
|
||
<html class="writer-html5" lang="en" data-content_root="./">
|
||
<head>
|
||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||
<title>Elasticsearch and Kibana — parsedmarc 10.4.0 documentation</title>
|
||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=9edc463e" />
|
||
|
||
|
||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||
<script src="_static/documentation_options.js?v=2f7da7c1"></script>
|
||
<script src="_static/doctools.js?v=fd6eb6e6"></script>
|
||
<script src="_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||
<script src="_static/js/theme.js"></script>
|
||
<link rel="index" title="Index" href="genindex.html" />
|
||
<link rel="search" title="Search" href="search.html" />
|
||
<link rel="next" title="OpenSearch and Grafana" href="opensearch.html" />
|
||
<link rel="prev" title="Sample outputs" href="output.html" />
|
||
</head>
|
||
|
||
<body class="wy-body-for-nav">
|
||
<div class="wy-grid-for-nav">
|
||
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
|
||
<div class="wy-side-scroll">
|
||
<div class="wy-side-nav-search" >
|
||
|
||
|
||
|
||
<a href="index.html" class="icon icon-home">
|
||
parsedmarc
|
||
</a>
|
||
<div role="search">
|
||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||
<input type="hidden" name="check_keywords" value="yes" />
|
||
<input type="hidden" name="area" value="default" />
|
||
</form>
|
||
</div>
|
||
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
|
||
<p class="caption" role="heading"><span class="caption-text">Contents</span></p>
|
||
<ul class="current">
|
||
<li class="toctree-l1"><a class="reference internal" href="installation.html">Installation</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="usage.html">Using parsedmarc</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="output.html">Sample outputs</a></li>
|
||
<li class="toctree-l1 current"><a class="current reference internal" href="#">Elasticsearch and Kibana</a><ul>
|
||
<li class="toctree-l2"><a class="reference internal" href="#installation">Installation</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="#upgrading-kibana-index-patterns">Upgrading Kibana index patterns</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="#backfilling-the-combined-dkim-spf-result-fields">Backfilling the combined DKIM/SPF result fields</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="#records-retention">Records retention</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l1"><a class="reference internal" href="opensearch.html">OpenSearch and Grafana</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="kibana.html">Using the Kibana dashboards</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="splunk.html">Splunk</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="davmail.html">Accessing an inbox using OWA/EWS</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="dmarc.html">Understanding DMARC</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="contributing.html">Contributing to parsedmarc</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="api.html">API reference</a></li>
|
||
</ul>
|
||
|
||
</div>
|
||
</div>
|
||
</nav>
|
||
|
||
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
|
||
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
||
<a href="index.html">parsedmarc</a>
|
||
</nav>
|
||
|
||
<div class="wy-nav-content">
|
||
<div class="rst-content">
|
||
<div role="navigation" aria-label="Page navigation">
|
||
<ul class="wy-breadcrumbs">
|
||
<li><a href="index.html" class="icon icon-home" aria-label="Home"></a></li>
|
||
<li class="breadcrumb-item active">Elasticsearch and Kibana</li>
|
||
<li class="wy-breadcrumbs-aside">
|
||
<a href="_sources/elasticsearch.md.txt" rel="nofollow"> View page source</a>
|
||
</li>
|
||
</ul>
|
||
<hr/>
|
||
</div>
|
||
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
|
||
<div itemprop="articleBody">
|
||
|
||
<section class="tex2jax_ignore mathjax_ignore" id="elasticsearch-and-kibana">
|
||
<h1>Elasticsearch and Kibana<a class="headerlink" href="#elasticsearch-and-kibana" title="Link to this heading"></a></h1>
|
||
<p>To set up visual dashboards of DMARC data, install Elasticsearch and Kibana.</p>
|
||
<div class="admonition note">
|
||
<p class="admonition-title">Note</p>
|
||
<p>Elasticsearch and Kibana 8 or later are required (parsedmarc’s 8.x Python
|
||
client also supports Elasticsearch 9). OpenSearch users must use the
|
||
<code class="docutils literal notranslate"><span class="pre">[opensearch]</span></code> configuration section instead — the Elasticsearch 8.x client
|
||
refuses to connect to non-Elasticsearch clusters.</p>
|
||
</div>
|
||
<section id="installation">
|
||
<h2>Installation<a class="headerlink" href="#installation" title="Link to this heading"></a></h2>
|
||
<p>On Debian/Ubuntu based systems, run:</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>sudo<span class="w"> </span>apt-get<span class="w"> </span>install<span class="w"> </span>-y<span class="w"> </span>apt-transport-https
|
||
wget<span class="w"> </span>-qO<span class="w"> </span>-<span class="w"> </span>https://artifacts.elastic.co/GPG-KEY-elasticsearch<span class="w"> </span><span class="p">|</span><span class="w"> </span>sudo<span class="w"> </span>gpg<span class="w"> </span>--dearmor<span class="w"> </span>-o<span class="w"> </span>/usr/share/keyrings/elasticsearch-keyring.gpg
|
||
<span class="nb">echo</span><span class="w"> </span><span class="s2">"deb [signed-by=/usr/share/keyrings/elasticsearch-keyring.gpg] https://artifacts.elastic.co/packages/8.x/apt stable main"</span><span class="w"> </span><span class="p">|</span><span class="w"> </span>sudo<span class="w"> </span>tee<span class="w"> </span>/etc/apt/sources.list.d/elastic-8.x.list
|
||
sudo<span class="w"> </span>apt-get<span class="w"> </span>update
|
||
sudo<span class="w"> </span>apt-get<span class="w"> </span>install<span class="w"> </span>-y<span class="w"> </span>elasticsearch<span class="w"> </span>kibana
|
||
</pre></div>
|
||
</div>
|
||
<p>For CentOS, RHEL, and other RPM systems, follow the Elastic RPM guides for
|
||
<a class="reference external" href="https://www.elastic.co/guide/en/elasticsearch/reference/current/rpm.html">Elasticsearch</a> and <a class="reference external" href="https://www.elastic.co/guide/en/kibana/current/rpm.html">Kibana</a>.</p>
|
||
<div class="admonition note">
|
||
<p class="admonition-title">Note</p>
|
||
<p>Previously, the default JVM heap size for Elasticsearch was very small (1g),
|
||
which will cause it to crash under a heavy load. To fix this, increase the
|
||
minimum and maximum JVM heap sizes in <code class="docutils literal notranslate"><span class="pre">/etc/elasticsearch/jvm.options</span></code> to
|
||
more reasonable levels, depending on your server’s resources.</p>
|
||
<p>Make sure the system has at least 2 GB more RAM than the assigned JVM
|
||
heap size.</p>
|
||
<p>Always set the minimum and maximum JVM heap sizes to the same
|
||
value.</p>
|
||
<p>For example, to set a 4 GB heap size, set</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>-Xms4g
|
||
-Xmx4g
|
||
</pre></div>
|
||
</div>
|
||
<p>See <a class="reference external" href="https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#heap-size-settings">https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#heap-size-settings</a>
|
||
for more information.</p>
|
||
</div>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>sudo<span class="w"> </span>systemctl<span class="w"> </span>daemon-reload
|
||
sudo<span class="w"> </span>systemctl<span class="w"> </span><span class="nb">enable</span><span class="w"> </span>elasticsearch.service
|
||
sudo<span class="w"> </span>systemctl<span class="w"> </span><span class="nb">enable</span><span class="w"> </span>kibana.service
|
||
sudo<span class="w"> </span>systemctl<span class="w"> </span>start<span class="w"> </span>elasticsearch.service
|
||
sudo<span class="w"> </span>systemctl<span class="w"> </span>start<span class="w"> </span>kibana.service
|
||
</pre></div>
|
||
</div>
|
||
<p>As of Elasticsearch 8.7, activate secure mode (xpack.security.*.ssl)</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>sudo<span class="w"> </span>vim<span class="w"> </span>/etc/elasticsearch/elasticsearch.yml
|
||
</pre></div>
|
||
</div>
|
||
<p>Add the following configuration</p>
|
||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span># Enable security features
|
||
xpack.security.enabled: true
|
||
xpack.security.enrollment.enabled: true
|
||
# Enable encryption for HTTP API client connections, such as Kibana, Logstash, and Agents
|
||
xpack.security.http.ssl:
|
||
enabled: true
|
||
keystore.path: certs/http.p12
|
||
# Enable encryption and mutual authentication between cluster nodes
|
||
xpack.security.transport.ssl:
|
||
enabled: true
|
||
verification_mode: certificate
|
||
keystore.path: certs/transport.p12
|
||
truststore.path: certs/transport.p12
|
||
</pre></div>
|
||
</div>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>sudo<span class="w"> </span>systemctl<span class="w"> </span>restart<span class="w"> </span>elasticsearch
|
||
</pre></div>
|
||
</div>
|
||
<p>To create a self-signed certificate, run:</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>openssl<span class="w"> </span>req<span class="w"> </span>-x509<span class="w"> </span>-nodes<span class="w"> </span>-days<span class="w"> </span><span class="m">365</span><span class="w"> </span>-newkey<span class="w"> </span>rsa:4096<span class="w"> </span>-keyout<span class="w"> </span>kibana.key<span class="w"> </span>-out<span class="w"> </span>kibana.crt
|
||
</pre></div>
|
||
</div>
|
||
<p>Or, to create a Certificate Signing Request (CSR) for a CA, run:</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>openssl<span class="w"> </span>req<span class="w"> </span>-newkey<span class="w"> </span>rsa:4096-nodes<span class="w"> </span>-keyout<span class="w"> </span>kibana.key<span class="w"> </span>-out<span class="w"> </span>kibana.csr
|
||
</pre></div>
|
||
</div>
|
||
<p>Fill in the prompts. Watch out for Common Name (e.g. server FQDN or YOUR
|
||
domain name), which is the IP address or domain name that you will use to access Kibana. it is the most important field.</p>
|
||
<p>If you generated a CSR, remove the CSR after you have your certs</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>rm<span class="w"> </span>-f<span class="w"> </span>kibana.csr
|
||
</pre></div>
|
||
</div>
|
||
<p>Move the keys into place and secure them:</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>sudo<span class="w"> </span>mv<span class="w"> </span>kibana.*<span class="w"> </span>/etc/kibana
|
||
sudo<span class="w"> </span>chmod<span class="w"> </span><span class="m">660</span><span class="w"> </span>/etc/kibana/kibana.key
|
||
</pre></div>
|
||
</div>
|
||
<p>Activate the HTTPS server in Kibana</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>sudo<span class="w"> </span>vim<span class="w"> </span>/etc/kibana/kibana.yml
|
||
</pre></div>
|
||
</div>
|
||
<p>Add the following configuration</p>
|
||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>server.host: "SERVER_IP"
|
||
server.publicBaseUrl: "https://SERVER_IP"
|
||
server.ssl.enabled: true
|
||
server.ssl.certificate: /etc/kibana/kibana.crt
|
||
server.ssl.key: /etc/kibana/kibana.key
|
||
</pre></div>
|
||
</div>
|
||
<div class="admonition note">
|
||
<p class="admonition-title">Note</p>
|
||
<p>For more security, you can configure Kibana to use a local network connection
|
||
to elasticsearch :</p>
|
||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>elasticsearch.hosts: ['https://SERVER_IP:9200']
|
||
</pre></div>
|
||
</div>
|
||
<p>=></p>
|
||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>elasticsearch.hosts: ['https://127.0.0.1:9200']
|
||
</pre></div>
|
||
</div>
|
||
</div>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>sudo<span class="w"> </span>systemctl<span class="w"> </span>restart<span class="w"> </span>kibana
|
||
</pre></div>
|
||
</div>
|
||
<p>Enroll Kibana in Elasticsearch</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>sudo<span class="w"> </span>/usr/share/elasticsearch/bin/elasticsearch-create-enrollment-token<span class="w"> </span>-s<span class="w"> </span>kibana
|
||
</pre></div>
|
||
</div>
|
||
<p>Then access to your web server at <code class="docutils literal notranslate"><span class="pre">https://SERVER_IP:5601</span></code>, accept the self-signed
|
||
certificate and paste the token in the “Enrollment token” field.</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>sudo<span class="w"> </span>/usr/share/kibana/bin/kibana-verification-code
|
||
</pre></div>
|
||
</div>
|
||
<p>Then put the verification code to your web browser.</p>
|
||
<p>End Kibana configuration</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>sudo<span class="w"> </span>/usr/share/elasticsearch/bin/elasticsearch-setup-passwords<span class="w"> </span>interactive
|
||
sudo<span class="w"> </span>/usr/share/kibana/bin/kibana-encryption-keys<span class="w"> </span>generate
|
||
sudo<span class="w"> </span>vim<span class="w"> </span>/etc/kibana/kibana.yml
|
||
</pre></div>
|
||
</div>
|
||
<p>Add previously generated encryption keys</p>
|
||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>xpack.encryptedSavedObjects.encryptionKey: xxxx...xxxx
|
||
xpack.reporting.encryptionKey: xxxx...xxxx
|
||
xpack.security.encryptionKey: xxxx...xxxx
|
||
</pre></div>
|
||
</div>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>sudo<span class="w"> </span>systemctl<span class="w"> </span>restart<span class="w"> </span>kibana
|
||
sudo<span class="w"> </span>systemctl<span class="w"> </span>restart<span class="w"> </span>elasticsearch
|
||
</pre></div>
|
||
</div>
|
||
<p>Now that Elasticsearch is up and running, use <code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> to send data to
|
||
it.</p>
|
||
<p>Download (right-click the link and click save as) <a class="reference external" href="https://raw.githubusercontent.com/domainaware/parsedmarc/master/dashboards/opensearch/opensearch_dashboards.ndjson">export.ndjson</a>.</p>
|
||
<p>Connect to kibana using the “elastic” user and the password you previously provide
|
||
on the console (“End Kibana configuration” part).</p>
|
||
<p>Import <code class="docutils literal notranslate"><span class="pre">export.ndjson</span></code> the Saved Objects tab of the Stack management
|
||
page of Kibana. (Hamburger menu -> “Management” -> “Stack Management” ->
|
||
“Kibana” -> “Saved Objects”)</p>
|
||
<p>It will give you the option to overwrite existing saved dashboards or
|
||
visualizations, which could be used to restore them if you or someone else
|
||
breaks them, as there are no permissions/access controls in Kibana without
|
||
the commercial <a class="reference external" href="https://www.elastic.co/products/x-pack">X-Pack</a>.</p>
|
||
<a class="reference external image-reference" href="_static/screenshots/saved-objects.png"><img alt="A screenshot of setting the Saved Objects Stack management UI in Kibana" class="align-center" src="_images/saved-objects.png" />
|
||
</a>
|
||
<a class="reference external image-reference" href="_static/screenshots/confirm-overwrite.png"><img alt="A screenshot of the overwrite conformation prompt" class="align-center" src="_images/confirm-overwrite.png" />
|
||
</a>
|
||
</section>
|
||
<section id="upgrading-kibana-index-patterns">
|
||
<h2>Upgrading Kibana index patterns<a class="headerlink" href="#upgrading-kibana-index-patterns" title="Link to this heading"></a></h2>
|
||
<p><code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> 5.0.0 makes some changes to the way data is indexed in
|
||
Elasticsearch. if you are upgrading from a previous release of
|
||
<code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code>, you need to complete the following steps to replace the
|
||
Kibana index patterns with versions that match the upgraded indexes:</p>
|
||
<ol class="arabic simple">
|
||
<li><p>Login in to Kibana, and click on Management</p></li>
|
||
<li><p>Under Kibana, click on Saved Objects</p></li>
|
||
<li><p>Check the checkboxes for the <code class="docutils literal notranslate"><span class="pre">dmarc_aggregate</span></code> and <code class="docutils literal notranslate"><span class="pre">dmarc_failure</span></code>
|
||
index patterns</p></li>
|
||
<li><p>Click Delete</p></li>
|
||
<li><p>Click Delete on the conformation message</p></li>
|
||
<li><p>Download (right-click the link and click save as)
|
||
the latest version of <a class="reference external" href="https://raw.githubusercontent.com/domainaware/parsedmarc/master/dashboards/opensearch/opensearch_dashboards.ndjson">export.ndjson</a></p></li>
|
||
<li><p>Import <code class="docutils literal notranslate"><span class="pre">export.ndjson</span></code> by clicking Import from the Kibana
|
||
Saved Objects page</p></li>
|
||
</ol>
|
||
</section>
|
||
<section id="backfilling-the-combined-dkim-spf-result-fields">
|
||
<h2>Backfilling the combined DKIM/SPF result fields<a class="headerlink" href="#backfilling-the-combined-dkim-spf-result-fields" title="Link to this heading"></a></h2>
|
||
<p>As of the version fixing <a class="reference external" href="https://github.com/domainaware/parsedmarc/issues/169">#169</a>,
|
||
aggregate documents include <code class="docutils literal notranslate"><span class="pre">dkim_results_combined</span></code> and <code class="docutils literal notranslate"><span class="pre">spf_results_combined</span></code> —
|
||
scalar string arrays that keep each auth result’s selector/scope, domain, and
|
||
result paired, which the dashboards’ alignment-detail tables aggregate on.
|
||
Reports saved by older versions lack these fields and will not appear in
|
||
those tables.</p>
|
||
<p>parsedmarc now backfills this automatically. On startup, it runs a cheap
|
||
count query against each configured aggregate index pattern to check for
|
||
documents that have DKIM or SPF results but are missing the corresponding
|
||
combined field. If any are found, it submits the backfill as a background
|
||
<code class="docutils literal notranslate"><span class="pre">_update_by_query</span></code> task (<code class="docutils literal notranslate"><span class="pre">wait_for_completion=false</span></code>), so startup is never
|
||
blocked on it; progress is logged, including the task ID. The check itself
|
||
is idempotent — once an index is fully backfilled, later startups see a
|
||
count of 0 and log nothing further — and it works the same way on
|
||
OpenSearch. Any error talking to the cluster (for example, no indexes yet
|
||
on a fresh install) is logged as a warning and retried on the next startup,
|
||
rather than aborting parsedmarc.</p>
|
||
<p>If you upgrade the dashboards without pointing the new parsedmarc version
|
||
at the cluster, or you’d rather control when the write load happens, you
|
||
can still run the backfill manually. It is idempotent (documents that
|
||
already have the fields are skipped), so it is safe to re-run. It works
|
||
identically on OpenSearch; just adjust the URL and credentials. The query
|
||
matches only documents that have at least one DKIM or SPF auth result and
|
||
lack the corresponding combined field; documents with no auth results are
|
||
skipped, because an <code class="docutils literal notranslate"><span class="pre">exists</span></code> query cannot see an empty array, and for
|
||
search purposes an empty <code class="docutils literal notranslate"><span class="pre">dkim_results_combined</span></code> is identical to an
|
||
absent one. Each result is matched on either its <code class="docutils literal notranslate"><span class="pre">domain</span></code> or its <code class="docutils literal notranslate"><span class="pre">result</span></code>
|
||
subfield as defense in depth: an empty string indexes no text tokens and
|
||
is invisible to <code class="docutils literal notranslate"><span class="pre">exists</span></code>, and the storage shape of every historical
|
||
parsedmarc version can’t be audited, so matching either subfield ensures
|
||
no backfillable document is skipped.</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>curl<span class="w"> </span>-X<span class="w"> </span>POST<span class="w"> </span><span class="s2">"http://localhost:9200/dmarc_aggregate*/_update_by_query?conflicts=proceed&wait_for_completion=false"</span><span class="w"> </span><span class="se">\</span>
|
||
<span class="w"> </span>-H<span class="w"> </span><span class="s2">"Content-Type: application/json"</span><span class="w"> </span>-d<span class="w"> </span><span class="s1">'</span>
|
||
<span class="s1">{</span>
|
||
<span class="s1"> "query": {</span>
|
||
<span class="s1"> "bool": {</span>
|
||
<span class="s1"> "minimum_should_match": 1,</span>
|
||
<span class="s1"> "should": [</span>
|
||
<span class="s1"> {</span>
|
||
<span class="s1"> "bool": {</span>
|
||
<span class="s1"> "must": [</span>
|
||
<span class="s1"> {</span>
|
||
<span class="s1"> "bool": {</span>
|
||
<span class="s1"> "minimum_should_match": 1,</span>
|
||
<span class="s1"> "should": [</span>
|
||
<span class="s1"> {"exists": {"field": "dkim_results.domain"}},</span>
|
||
<span class="s1"> {"exists": {"field": "dkim_results.result"}}</span>
|
||
<span class="s1"> ]</span>
|
||
<span class="s1"> }</span>
|
||
<span class="s1"> }</span>
|
||
<span class="s1"> ],</span>
|
||
<span class="s1"> "must_not": [{"exists": {"field": "dkim_results_combined"}}]</span>
|
||
<span class="s1"> }</span>
|
||
<span class="s1"> },</span>
|
||
<span class="s1"> {</span>
|
||
<span class="s1"> "bool": {</span>
|
||
<span class="s1"> "must": [</span>
|
||
<span class="s1"> {</span>
|
||
<span class="s1"> "bool": {</span>
|
||
<span class="s1"> "minimum_should_match": 1,</span>
|
||
<span class="s1"> "should": [</span>
|
||
<span class="s1"> {"exists": {"field": "spf_results.domain"}},</span>
|
||
<span class="s1"> {"exists": {"field": "spf_results.result"}}</span>
|
||
<span class="s1"> ]</span>
|
||
<span class="s1"> }</span>
|
||
<span class="s1"> }</span>
|
||
<span class="s1"> ],</span>
|
||
<span class="s1"> "must_not": [{"exists": {"field": "spf_results_combined"}}]</span>
|
||
<span class="s1"> }</span>
|
||
<span class="s1"> }</span>
|
||
<span class="s1"> ]</span>
|
||
<span class="s1"> }</span>
|
||
<span class="s1"> },</span>
|
||
<span class="s1"> "script": {</span>
|
||
<span class="s1"> "lang": "painless",</span>
|
||
<span class="s1"> "source": "List dk = new ArrayList(); def dr = ctx._source.dkim_results; if (dr != null) { if (!(dr instanceof List)) { dr = [dr]; } for (e in dr) { if (e == null) { continue; } def sel = e.selector != null ? e.selector : \"none\"; def dom = e.domain != null ? e.domain : \"none\"; def res = e.result != null ? e.result : \"none\"; dk.add(sel + \" / \" + dom + \" / \" + res); } } ctx._source.dkim_results_combined = dk; List sp = new ArrayList(); def sr = ctx._source.spf_results; if (sr != null) { if (!(sr instanceof List)) { sr = [sr]; } for (e in sr) { if (e == null) { continue; } def sc = e.scope != null ? e.scope : \"mfrom\"; def dom = e.domain != null ? e.domain : \"none\"; def res = e.result != null ? e.result : (e.results != null ? e.results : \"none\"); sp.add(sc + \" / \" + dom + \" / \" + res); } } ctx._source.spf_results_combined = sp;"</span>
|
||
<span class="s1"> }</span>
|
||
<span class="s1">}'</span>
|
||
</pre></div>
|
||
</div>
|
||
<p><code class="docutils literal notranslate"><span class="pre">wait_for_completion=false</span></code> returns a task ID — check progress with
|
||
<code class="docutils literal notranslate"><span class="pre">GET</span> <span class="pre">_tasks/<task</span> <span class="pre">id></span></code>. Adjust the index pattern if you use a custom
|
||
<code class="docutils literal notranslate"><span class="pre">index_prefix</span></code>/<code class="docutils literal notranslate"><span class="pre">index_suffix</span></code>. After backfilling, re-import the updated
|
||
dashboards ndjson (the index pattern saved object changed too) per the
|
||
import instructions above.</p>
|
||
<p>SMTP TLS documents have the same class of defect one level deeper:
|
||
<code class="docutils literal notranslate"><span class="pre">policies</span></code> is an object array, and each policy’s <code class="docutils literal notranslate"><span class="pre">failure_details</span></code> is an
|
||
object array inside it. SMTP TLS documents now also carry
|
||
<code class="docutils literal notranslate"><span class="pre">policies_combined</span></code> and <code class="docutils literal notranslate"><span class="pre">failure_details_combined</span></code>, backfilled
|
||
automatically at startup the same way, and the equivalent manual command
|
||
is:</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>curl<span class="w"> </span>-X<span class="w"> </span>POST<span class="w"> </span><span class="s2">"http://localhost:9200/smtp_tls*/_update_by_query?conflicts=proceed&wait_for_completion=false"</span><span class="w"> </span><span class="se">\</span>
|
||
<span class="w"> </span>-H<span class="w"> </span><span class="s2">"Content-Type: application/json"</span><span class="w"> </span>-d<span class="w"> </span><span class="s1">'</span>
|
||
<span class="s1">{</span>
|
||
<span class="s1"> "query": {</span>
|
||
<span class="s1"> "bool": {</span>
|
||
<span class="s1"> "minimum_should_match": 1,</span>
|
||
<span class="s1"> "should": [</span>
|
||
<span class="s1"> {</span>
|
||
<span class="s1"> "bool": {</span>
|
||
<span class="s1"> "must": [</span>
|
||
<span class="s1"> {</span>
|
||
<span class="s1"> "bool": {</span>
|
||
<span class="s1"> "minimum_should_match": 1,</span>
|
||
<span class="s1"> "should": [</span>
|
||
<span class="s1"> {"exists": {"field": "policies.policy_domain"}},</span>
|
||
<span class="s1"> {"exists": {"field": "policies.policy_type"}}</span>
|
||
<span class="s1"> ]</span>
|
||
<span class="s1"> }</span>
|
||
<span class="s1"> }</span>
|
||
<span class="s1"> ],</span>
|
||
<span class="s1"> "must_not": [{"exists": {"field": "policies_combined"}}]</span>
|
||
<span class="s1"> }</span>
|
||
<span class="s1"> },</span>
|
||
<span class="s1"> {</span>
|
||
<span class="s1"> "bool": {</span>
|
||
<span class="s1"> "must": [</span>
|
||
<span class="s1"> {</span>
|
||
<span class="s1"> "bool": {</span>
|
||
<span class="s1"> "minimum_should_match": 1,</span>
|
||
<span class="s1"> "should": [</span>
|
||
<span class="s1"> {"exists": {"field": "policies.failure_details.result_type"}},</span>
|
||
<span class="s1"> {"exists": {"field": "policies.failure_details.sending_mta_ip"}}</span>
|
||
<span class="s1"> ]</span>
|
||
<span class="s1"> }</span>
|
||
<span class="s1"> }</span>
|
||
<span class="s1"> ],</span>
|
||
<span class="s1"> "must_not": [{"exists": {"field": "failure_details_combined"}}]</span>
|
||
<span class="s1"> }</span>
|
||
<span class="s1"> }</span>
|
||
<span class="s1"> ]</span>
|
||
<span class="s1"> }</span>
|
||
<span class="s1"> },</span>
|
||
<span class="s1"> "script": {</span>
|
||
<span class="s1"> "lang": "painless",</span>
|
||
<span class="s1"> "source": "List pols = new ArrayList(); List dets = new ArrayList(); def ps = ctx._source.policies; if (ps != null) { if (!(ps instanceof List)) { ps = [ps]; } for (p in ps) { if (p == null) { continue; } def dom = p.policy_domain != null ? p.policy_domain : \"none\"; def typ = p.policy_type != null ? p.policy_type : \"none\"; pols.add(dom + \" / \" + typ); def fds = p.failure_details; if (fds != null) { if (!(fds instanceof List)) { fds = [fds]; } for (f in fds) { if (f == null) { continue; } def rt = f.result_type != null ? f.result_type : \"none\"; def smi = f.sending_mta_ip != null ? f.sending_mta_ip : \"none\"; def ri = f.receiving_ip != null ? f.receiving_ip : \"none\"; def rmh = f.receiving_mx_hostname != null ? f.receiving_mx_hostname : \"none\"; dets.add(dom + \" / \" + typ + \" / \" + rt + \" / \" + smi + \" / \" + ri + \" / \" + rmh); } } } } ctx._source.policies_combined = pols; ctx._source.failure_details_combined = dets;"</span>
|
||
<span class="s1"> }</span>
|
||
<span class="s1">}'</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>It works identically on OpenSearch; just adjust the URL and credentials, same
|
||
as the aggregate command above.</p>
|
||
</section>
|
||
<section id="records-retention">
|
||
<h2>Records retention<a class="headerlink" href="#records-retention" title="Link to this heading"></a></h2>
|
||
<p>Starting in version 5.0.0, <code class="docutils literal notranslate"><span class="pre">parsedmarc</span></code> stores data in a separate
|
||
index for each day to make it easy to comply with records
|
||
retention regulations such as GDPR. For more information,
|
||
check out the Elastic guide to <a class="reference external" href="https://www.elastic.co/blog/managing-time-based-indices-efficiently">managing time-based indexes efficiently</a>.</p>
|
||
</section>
|
||
</section>
|
||
|
||
|
||
</div>
|
||
</div>
|
||
<footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
|
||
<a href="output.html" class="btn btn-neutral float-left" title="Sample outputs" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
|
||
<a href="opensearch.html" class="btn btn-neutral float-right" title="OpenSearch and Grafana" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
|
||
</div>
|
||
|
||
<hr/>
|
||
|
||
<div role="contentinfo">
|
||
<p>© Copyright 2018 - 2025, Sean Whalen and contributors.</p>
|
||
</div>
|
||
|
||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||
<a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
|
||
provided by <a href="https://readthedocs.org">Read the Docs</a>.
|
||
|
||
|
||
</footer>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
</div>
|
||
<script>
|
||
jQuery(function () {
|
||
SphinxRtdTheme.Navigation.enable(true);
|
||
});
|
||
</script>
|
||
|
||
</body>
|
||
</html> |