From bd8f13796ed35ed22e06a346bde4eb0c9ae830bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matth=C3=A4us=20Wander?= Date: Sun, 6 Jun 2021 16:35:07 +0200 Subject: [PATCH 1/5] Added splunk installation guide --- splunk/README.rst | 73 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 68 insertions(+), 5 deletions(-) diff --git a/splunk/README.rst b/splunk/README.rst index 44a8ccf..42198f5 100644 --- a/splunk/README.rst +++ b/splunk/README.rst @@ -1,9 +1,53 @@ -================= -Splunk dashboards -================= +=================== +Splunk Installation +=================== -Setup guide ------------ +Install Splunk for use with Docker +---------------------------------- + +Download latest Splunk image:: + + docker pull splunk/splunk:latest + +Run Splunk with Docker +---------------------- + +Listen on all network interfaces:: + + docker run -d -p 8000:8000 -p 8088:8088 -e "SPLUNK_START_ARGS=--accept-license" -e "SPLUNK_PASSWORD=password1234" -e "SPLUNK_HEC_TOKEN=hec-token-1234" --name splunk splunk/splunk:latest + +Listen on localhost for use with reverse proxy with base URL `/splunk`:: + + docker run -d -p 127.0.0.1:8000:8000 -p 127.0.0.1:8088:8088 -e "SPLUNK_START_ARGS=--accept-license" -e "SPLUNK_PASSWORD=password1234" -e "SPLUNK_HEC_TOKEN=hec-token-1234" -e "SPLUNK_ROOT_ENDPOINT=/splunk" --name splunk splunk/splunk:latest + +Set up reverse proxy, e.g. Apache2:: + + ProxyPass /splunk http://127.0.0.1:8000/splunk + ProxyPassReverse /splunk http://127.0.0.1:8000/splunk + +Splunk Configuration +-------------------- + +Access web UI at http://127.0.0.1:8000 and log in with `admin:password1234`. + +Create App and Index +~~~~~~~~~~~~~~~~~~~~ + +- Settings > Data > Indexes: New Index + + - Index name: "email" + +- HEC token `hec-token-1234` should be already set up. + + - Check under Settings > Data > Data inputs: HTTP Event Collector + +- Apps > Manage Apps: Create app + + - Name: "parsedmarc" + - Folder name: "parsedmarc" + +Create Dashboards +~~~~~~~~~~~~~~~~~ 1. Navigate to the app you want to add the dashboards to, or create a new app called DMARC 2. Click Dashboards @@ -22,3 +66,22 @@ Setup guide 15. Paste the content of ''dmarc_forensic_dashboard.xml`` into the source editor 16. If the index storing the DMARC data is not named email, replace index="email" accordingly 17. Click Save + +============== +Example Config +============== + +parsedmarc.ini:: + + [splunk_hec] + url = https://127.0.0.1:8088/ + token = hec-token-1234 + index = email + skip_certificate_verification = True + +Note that `skip_certificate_verification = True` disables security checks. + +Run parsedmarc:: + + python3 -m parsedmarc.cli -c parsedmarc.ini + From bbc379aacae2ed8670048f84adfdc0058fea56fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matth=C3=A4us=20Wander?= Date: Sun, 6 Jun 2021 16:44:40 +0200 Subject: [PATCH 2/5] Minor formatting --- splunk/README.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/splunk/README.rst b/splunk/README.rst index 42198f5..acf7bce 100644 --- a/splunk/README.rst +++ b/splunk/README.rst @@ -16,7 +16,7 @@ Listen on all network interfaces:: docker run -d -p 8000:8000 -p 8088:8088 -e "SPLUNK_START_ARGS=--accept-license" -e "SPLUNK_PASSWORD=password1234" -e "SPLUNK_HEC_TOKEN=hec-token-1234" --name splunk splunk/splunk:latest -Listen on localhost for use with reverse proxy with base URL `/splunk`:: +Listen on localhost for use with reverse proxy with base URL ``/splunk``:: docker run -d -p 127.0.0.1:8000:8000 -p 127.0.0.1:8088:8088 -e "SPLUNK_START_ARGS=--accept-license" -e "SPLUNK_PASSWORD=password1234" -e "SPLUNK_HEC_TOKEN=hec-token-1234" -e "SPLUNK_ROOT_ENDPOINT=/splunk" --name splunk splunk/splunk:latest @@ -28,7 +28,7 @@ Set up reverse proxy, e.g. Apache2:: Splunk Configuration -------------------- -Access web UI at http://127.0.0.1:8000 and log in with `admin:password1234`. +Access web UI at http://127.0.0.1:8000 and log in with ``admin:password1234``. Create App and Index ~~~~~~~~~~~~~~~~~~~~ @@ -37,7 +37,7 @@ Create App and Index - Index name: "email" -- HEC token `hec-token-1234` should be already set up. +- HEC token ``hec-token-1234`` should be already set up. - Check under Settings > Data > Data inputs: HTTP Event Collector @@ -79,7 +79,7 @@ parsedmarc.ini:: index = email skip_certificate_verification = True -Note that `skip_certificate_verification = True` disables security checks. +Note that ``skip_certificate_verification = True`` disables security checks. Run parsedmarc:: From 9dab931f44c021657e053296b5691ed98f9693c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matth=C3=A4us=20Wander?= Date: Sun, 6 Jun 2021 16:57:40 +0200 Subject: [PATCH 3/5] convert unixtime into UTC datetime --- parsedmarc/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/parsedmarc/utils.py b/parsedmarc/utils.py index dbf525f..177a880 100644 --- a/parsedmarc/utils.py +++ b/parsedmarc/utils.py @@ -201,15 +201,15 @@ def get_reverse_dns(ip_address, cache=None, nameservers=None, timeout=2.0): def timestamp_to_datetime(timestamp): """ - Converts a UNIX/DMARC timestamp to a Python ``DateTime`` object + Converts a UNIX/DMARC timestamp to a Python ``DateTime`` object (UTC) Args: timestamp (int): The timestamp Returns: - DateTime: The converted timestamp as a Python ``DateTime`` object + DateTime: The converted timestamp as a Python ``DateTime`` object (UTC) """ - return datetime.fromtimestamp(int(timestamp)) + return datetime.utcfromtimestamp(int(timestamp)) def timestamp_to_human(timestamp): @@ -220,7 +220,7 @@ def timestamp_to_human(timestamp): timestamp: The timestamp Returns: - str: The converted timestamp in ``YYYY-MM-DD HH:MM:SS`` format + str: The converted timestamp in ``YYYY-MM-DD HH:MM:SS`` format (UTC) """ return timestamp_to_datetime(timestamp).strftime("%Y-%m-%d %H:%M:%S") From 7e9bbfc805eded3cbb283304f8202b4640aae816 Mon Sep 17 00:00:00 2001 From: mwander Date: Sun, 6 Jun 2021 19:04:45 +0200 Subject: [PATCH 4/5] Revert "convert unixtime into UTC datetime" This reverts commit 9dab931f44c021657e053296b5691ed98f9693c6. 9dab93 fixes Elastic output, but breaks Splunk output. See 4ad693 for a better fix. --- parsedmarc/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/parsedmarc/utils.py b/parsedmarc/utils.py index 177a880..dbf525f 100644 --- a/parsedmarc/utils.py +++ b/parsedmarc/utils.py @@ -201,15 +201,15 @@ def get_reverse_dns(ip_address, cache=None, nameservers=None, timeout=2.0): def timestamp_to_datetime(timestamp): """ - Converts a UNIX/DMARC timestamp to a Python ``DateTime`` object (UTC) + Converts a UNIX/DMARC timestamp to a Python ``DateTime`` object Args: timestamp (int): The timestamp Returns: - DateTime: The converted timestamp as a Python ``DateTime`` object (UTC) + DateTime: The converted timestamp as a Python ``DateTime`` object """ - return datetime.utcfromtimestamp(int(timestamp)) + return datetime.fromtimestamp(int(timestamp)) def timestamp_to_human(timestamp): @@ -220,7 +220,7 @@ def timestamp_to_human(timestamp): timestamp: The timestamp Returns: - str: The converted timestamp in ``YYYY-MM-DD HH:MM:SS`` format (UTC) + str: The converted timestamp in ``YYYY-MM-DD HH:MM:SS`` format """ return timestamp_to_datetime(timestamp).strftime("%Y-%m-%d %H:%M:%S") From d335d594f5ff719fe007f78c996ba959f2eea52c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matth=C3=A4us=20Wander?= Date: Sun, 22 Aug 2021 10:31:09 +0200 Subject: [PATCH 5/5] print tqdm progress bar only in interactive tty (as opposed to cronjob) --- parsedmarc/cli.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index be2026a..81aa175 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -13,6 +13,7 @@ import json from ssl import CERT_NONE, create_default_context from multiprocessing import Pool, Value from itertools import repeat +import sys import time from tqdm import tqdm @@ -626,11 +627,15 @@ def _main(): repeat(opts.offline), repeat(opts.n_procs >= 1)), opts.chunk_size) - pbar = tqdm(total=len(file_paths)) - while not results.ready(): - pbar.update(counter.value - pbar.n) - time.sleep(0.1) - pbar.close() + if sys.stdout.isatty(): + pbar = tqdm(total=len(file_paths)) + while not results.ready(): + pbar.update(counter.value - pbar.n) + time.sleep(0.1) + pbar.close() + else: + while not results.ready(): + time.sleep(0.1) results = results.get() pool.close() pool.join()