diff --git a/.vscode/settings.json b/.vscode/settings.json index 23b9589..ceca003 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -23,6 +23,7 @@ "dateutil", "Davmail", "DBIP", + "dearmor", "deflist", "devel", "DMARC", @@ -48,6 +49,7 @@ "journalctl", "keepalive", "keyout", + "keyrings", "Leeman", "libemail", "linkify", @@ -93,11 +95,13 @@ "Servernameone", "setuptools", "smartquotes", + "sourcetype", "STARTTLS", "tasklist", "toctree", "TQDDM", "tqdm", + "truststore", "Übersicht", "uids", "uper", @@ -112,6 +116,7 @@ "whitespaces", "xennn", "xmltodict", + "xpack", "zscholl" ], } \ No newline at end of file diff --git a/docs/source/contributing.md b/docs/source/contributing.md index ad4a02e..74cb479 100644 --- a/docs/source/contributing.md +++ b/docs/source/contributing.md @@ -1,9 +1,7 @@ # Contributing to parsedmarc - ## Bug reports Please report bugs on the GitHub issue tracker - diff --git a/docs/source/davmail.md b/docs/source/davmail.md index 6249803..556b5f9 100644 --- a/docs/source/davmail.md +++ b/docs/source/davmail.md @@ -1,5 +1,4 @@ - -### Accessing an inbox using OWA/EWS +# Accessing an inbox using OWA/EWS :::{note} Starting in 8.0.0, parsedmarc supports accessing Microsoft/Office 365 @@ -91,7 +90,7 @@ davmail.clientSoTimeout=0 ############################################################# ``` -#### Running DavMail as a systemd service +## Running DavMail as a systemd service Use systemd to run `davmail` as a service. @@ -173,7 +172,7 @@ journalctl -u davmail.service -r ::: -#### Configuring parsedmarc for DavMail +## Configuring parsedmarc for DavMail Because you are interacting with DavMail server over the loopback (i.e. `127.0.0.1`), add the following options to `parsedmarc.ini` diff --git a/docs/source/dmarc.md b/docs/source/dmarc.md index 41dd48f..73cc897 100644 --- a/docs/source/dmarc.md +++ b/docs/source/dmarc.md @@ -1,4 +1,5 @@ # Understanding DMARC + ## Resources ### DMARC guides @@ -18,7 +19,6 @@ check out the sister project, DMARC protects against domain spoofing, not lookalike domains. for open source lookalike domain monitoring, check out [DomainAware](https://github.com/seanthegeek/domainaware). - ## DMARC Alignment Guide DMARC ensures that SPF and DKM authentication mechanisms actually authenticate diff --git a/docs/source/elasticsearch.md b/docs/source/elasticsearch.md index e035323..3517371 100644 --- a/docs/source/elasticsearch.md +++ b/docs/source/elasticsearch.md @@ -1,5 +1,4 @@ - -### Elasticsearch and Kibana +# Elasticsearch and Kibana :::{note} Splunk is also supported starting with `parsedmarc` 4.3.0 @@ -55,6 +54,34 @@ sudo service elasticsearch start sudo service kibana start ``` +As of Elasticsearch 8.7, activate secure mode (xpack.security.*.ssl) + +```bash +sudo vim /etc/elasticsearch/elasticsearch.yml +``` + +Add the following configuration + +```text +# Enable security features +xpack.security.enabled: true +xpack.security.enrollment.enabled: true +# Enable encryption for HTTP API client connections, such as Kibana, Logstash, and Agents +xpack.security.http.ssl: + enabled: true + keystore.path: certs/http.p12 +# Enable encryption and mutual authentication between cluster nodes +xpack.security.transport.ssl: + enabled: true + verification_mode: certificate + keystore.path: certs/transport.p12 + truststore.path: certs/transport.p12 +``` + +```bash +sudo systemctl restart elasticsearch +``` + To create a self-signed certificate, run: ```bash @@ -68,7 +95,7 @@ openssl req -newkey rsa:4096-nodes -keyout kibana.key -out kibana.csr ``` Fill in the prompts. Watch out for Common Name (e.g. server FQDN or YOUR -domain name), which is the IP address or domain name that you will bebana on. it is the most important field. +domain name), which is the IP address or domain name that you will use to access Kibana. it is the most important field. If you generated a CSR, remove the CSR after you have your certs @@ -84,44 +111,56 @@ sudo chmod 660 /etc/kibana/kibana.key ``` Activate the HTTPS server in Kibana + ```bash sudo vim /etc/kibana/kibana.yml ``` + Add the following configuration -``` + +```text server.host: "SERVER_IP" server.publicBaseUrl: "https://SERVER_IP" server.ssl.enabled: true server.ssl.certificate: /etc/kibana/kibana.crt server.ssl.key: /etc/kibana/kibana.key ``` + ```bash sudo systemctl restart kibana ``` Enroll Kibana in Elasticsearch + ```bash sudo /usr/share/elasticsearch/bin/elasticsearch-create-enrollment-token -s kibana ``` -Then access to your webserver at https://SERVER_IP:5601, accept the self-signed + +Then access to your web server at `https://SERVER_IP:5601`, accept the self-signed certificate and paste the token in the "Enrollment token" field. + ```bash sudo /usr/share/kibana/bin/kibana-verification-code ``` + Then put the verification code to your web browser. End Kibana configuration + ```bash sudo /usr/share/elasticsearch/bin/elasticsearch-setup-passwords interactive sudo /usr/share/kibana/bin/kibana-encryption-keys generate sudo vim /etc/kibana/kibana.yml ``` + Add previously generated encryption keys -``` + +```text xpack.encryptedSavedObjects.encryptionKey: xxxx...xxxx xpack.reporting.encryptionKey: xxxx...xxxx xpack.security.encryptionKey: xxxx...xxxx ``` + ```bash sudo systemctl restart kibana sudo systemctl restart elasticsearch @@ -136,7 +175,7 @@ Connect to kibana using the "elastic" user and the password you previously provi on the console ("End Kibana configuration" part). Import `export.ndjson` the Saved Objects tab of the Stack management -page of Kibana. (Hamburger menu -> "Management" -> "Stack Management" -> +page of Kibana. (Hamburger menu -> "Management" -> "Stack Management" -> "Kibana" -> "Saved Objects") It will give you the option to overwrite existing saved dashboards or @@ -156,7 +195,7 @@ the commercial [X-Pack]. :target: _static/screenshots/confirm-overwrite.png ``` -#### Upgrading Kibana index patterns +## Upgrading Kibana index patterns `parsedmarc` 5.0.0 makes some changes to the way data is indexed in Elasticsearch. if you are upgrading from a previous release of @@ -174,14 +213,13 @@ Kibana index patterns with versions that match the upgraded indexes: 7. Import `export.ndjson` by clicking Import from the Kibana Saved Objects page -#### Records retention +## Records retention Starting in version 5.0.0, `parsedmarc` stores data in a separate index for each day to make it easy to comply with records retention regulations such as GDPR. For fore information, check out the Elastic guide to [managing time-based indexes efficiently](https://www.elastic.co/blog/managing-time-based-indices-efficiently). - [elasticsearch]: https://www.elastic.co/guide/en/elasticsearch/reference/current/rpm.html [export.ndjson]: https://raw.githubusercontent.com/domainaware/parsedmarc/master/kibana/export.ndjson [kibana]: https://www.elastic.co/guide/en/kibana/current/rpm.html diff --git a/docs/source/index.md b/docs/source/index.md index 2db7c1d..8020d1d 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -59,8 +59,5 @@ contributing api ``` - [contributors]: https://github.com/domainaware/parsedmarc/graphs/contributors -[creative commons attribution 4.0 international license]: https://creativecommons.org/licenses/by/4.0/ [issues]: https://github.com/domainaware/parsedmarc/issues -[readonlyrest]: https://readonlyrest.com/ diff --git a/docs/source/installation.md b/docs/source/installation.md index a0b6a77..cea325d 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -1,6 +1,5 @@ # Installation -:::{note} ## Testing multiple report analyzers If you would like to test parsedmarc and another report processing @@ -63,6 +62,7 @@ On Debian 10 (Buster) or later, run: ```bash sudo apt-get install -y geoipupdate ``` + :::{note} [Component "contrib"] is required in your apt sources. ::: @@ -86,7 +86,7 @@ from the [geoipupdate releases page on GitHub]. On December 30th, 2019, MaxMind started requiring free accounts to access the free Geolite2 databases, in order [to -comply with various privacy +comply with various privacy regulations][to comply with various privacy regulations]. Start by [registering for a free GeoLite2 account], and signing in. diff --git a/docs/source/kibana.md b/docs/source/kibana.md index 0ff1f2a..67c224e 100644 --- a/docs/source/kibana.md +++ b/docs/source/kibana.md @@ -1,5 +1,5 @@ -## Using the Kibana dashboards +# Using the Kibana dashboards The Kibana DMARC dashboards are a human-friendly way to understand the results from incoming DMARC reports. @@ -9,7 +9,7 @@ The default dashboard is DMARC Summary. To switch between dashboards, click on the Dashboard link in the left side menu of Kibana. ::: -### DMARC Summary +## DMARC Summary As the name suggests, this dashboard is the best place to start reviewing your aggregate DMARC data. @@ -74,7 +74,7 @@ the DMARC Summary dashboard. To view failures only, use the pie chart. Any other filters work the same way. You can also add your own custom temporary filters by clicking on Add Filter at the upper right of the page. -### DMARC Forensic Samples +## DMARC Forensic Samples The DMARC Forensic Samples dashboard contains information on DMARC forensic reports (also known as failure reports or ruf reports). These reports contain diff --git a/docs/source/mailing-lists.md b/docs/source/mailing-lists.md index f314814..50440e4 100644 --- a/docs/source/mailing-lists.md +++ b/docs/source/mailing-lists.md @@ -1,4 +1,4 @@ -## What about mailing lists? +# What about mailing lists? When you deploy DMARC on your domain, you might find that messages relayed by mailing lists are failing DMARC, most likely because the mailing @@ -6,7 +6,7 @@ list is spoofing your from address, and modifying the subject, footer, or other part of the message, thereby breaking the DKIM signature. -### Mailing list list best practices +## Mailing list list best practices Ideally, a mailing list should forward messages without altering the headers or body content at all. [Joe Nelson] does a fantastic job of @@ -14,7 +14,7 @@ explaining exactly what mailing lists should and shouldn't do to be fully DMARC compliant. Rather than repeat his fine work, here's a summary: -#### Do +### Do - Retain headers from the original message @@ -30,7 +30,7 @@ summary: Modern mail clients and webmail services generate unsubscribe buttons based on these headers. -#### Do not +### Do not - Remove or modify any existing headers from the original message, including From, Date, Subject, etc. @@ -48,7 +48,7 @@ to the mailing list post address, and not their email address. Configuration steps for common mailing list platforms are listed below. -#### Mailman 2 +### Mailman 2 Navigate to General Settings, and configure the settings below @@ -87,7 +87,7 @@ Navigate to Privacy Options> Sending Filters, and configure the settings below ====================================== ========== ``` -#### Mailman 3 +### Mailman 3 Navigate to Settings> List Identity @@ -144,7 +144,7 @@ no longer spoof email addresses with domains protected by DMARC. Configuration steps for common mailing list platforms are listed below. -#### Mailman 2 +### Mailman 2 Navigate to Privacy Options> Sending Filters, and configure the settings below @@ -170,7 +170,7 @@ the original sender. Choose the option that best fits your community. ::: -#### Mailman 3 +### Mailman 3 In the DMARC Mitigations tab of the Settings page, configure the settings below @@ -193,7 +193,7 @@ accidentally reply to the entire list, when they only intended to reply to the original sender. ::: -#### LISTSERV +### LISTSERV [LISTSERV 16.0-2017a] and higher will rewrite the From header for domains that enforce with a DMARC quarantine or reject policy. diff --git a/docs/source/splunk.md b/docs/source/splunk.md index 0b32215..f21d24b 100644 --- a/docs/source/splunk.md +++ b/docs/source/splunk.md @@ -1,4 +1,4 @@ -### Splunk +# Splunk Starting in version 4.3.0 `parsedmarc` supports sending aggregate and/or forensic DMARC data to a Splunk [HTTP Event collector (HEC)]. diff --git a/docs/source/usage.md b/docs/source/usage.md index f954c4a..bac28fd 100644 --- a/docs/source/usage.md +++ b/docs/source/usage.md @@ -129,6 +129,7 @@ The full set of configuration options are: Setting this to a number larger than one can improve performance when processing thousands of files ::: + - `mailbox` - `reports_folder` - str: The mailbox folder (or label for Gmail) where the incoming reports can be found @@ -143,7 +144,7 @@ The full set of configuration options are: - `batch_size` - int: Number of messages to read and process before saving. Default `10`. Use `0` for no limit. - `check_timeout` - int: Number of seconds to wait for a IMAP - IDLE response or the number of seconds until the next + IDLE response or the number of seconds until the next mail check (Default: `30`) - `imap` - `host` - str: The IMAP server hostname or IP address @@ -291,6 +292,18 @@ The full set of configuration options are: (Default: `https://www.googleapis.com/auth/gmail.modify`) - `oauth2_port` - int: The TCP port for the local server to listen on for the OAuth2 response (Default: 8080) +- `log_analytics` + - `client_id` - str: The app registration's client ID + - `client_secret` - str: The app registration's client secret + - `tenant_id` - str: The tenant id where the app registration resides + - `dce` - str: The Data Collection Endpoint (DCE). Example: `https://{DCE-NAME}.{REGION}.ingest.monitor.azure.com`. + - `dcr_immutable_id` - str: The immutable ID of the Data Collection Rule (DCR) + - `dcr_aggregate_stream` - str: The stream name for aggregate reports in the DCR + - `dcr_forensic_stream` - str: The stream name for the forensic reports in the DCR + + :::{note} + Information regarding the setup of the Data Collection Rule can be found [here](https://learn.microsoft.com/en-us/azure/azure-monitor/logs/tutorial-logs-ingestion-portal). + ::: :::{warning} It is **strongly recommended** to **not** use the `nameservers` @@ -329,12 +342,13 @@ known samples you want to save to that folder ::: :::{warning} -Elasticsearch 8 change limits policy for shards, restricting by -default to 1000. parsedmarc use a shard per analyzed day. If you -have more than ~3 years of data, you will need to update this +Elasticsearch 8 change limits policy for shards, restricting by +default to 1000. parsedmarc use a shard per analyzed day. If you +have more than ~3 years of data, you will need to update this limit. Check current usage (from Management -> Dev Tools -> Console): -``` + +```http GET /_cluster/health?pretty ... "active_primary_shards": 932, @@ -342,8 +356,10 @@ GET /_cluster/health?pretty ... } ``` -Update the limit to 2k per exemple: -``` + +Update the limit to 2k per example: + +```http PUT _cluster/settings { "persistent" : { @@ -351,7 +367,8 @@ PUT _cluster/settings } } ``` -Be warned that increasing this value increase ressources usage. + +Increasing this value increases resource usage. ::: ## Running parsedmarc as a systemd service