diff --git a/.vscode/settings.json b/.vscode/settings.json index 370d315d..131045dc 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -176,6 +176,7 @@ "IPFS", "ipinfo", "isinstance", + "isready", "journalctl", "junitxml", "kafkaclient", @@ -214,6 +215,7 @@ "mbox", "mcdlv", "mcsv", + "metacharacters", "mfrom", "mhdw", "Miasta", @@ -236,6 +238,7 @@ "myshopify", "namespaceless", "ndjson", + "Netease", "Newfold", "newkey", "Newswire", @@ -267,6 +270,7 @@ "pbar", "penyedia", "perfdrive", + "PGPASSWORD", "pharma", "pipefail", "plog", @@ -277,6 +281,7 @@ "prestataire", "privatesuffix", "procs", + "psql", "psycopg", "publicsuffix", "publicsuffixlist", @@ -327,6 +332,7 @@ "sourcetype", "splunkd", "sqls", + "sslmode", "STARTTLS", "subfolders", "subzones", diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e75bb23..c6e0ccb0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ Backwards compatibility to RFC 7489 is maintained. #### PostgreSQL storage backend -New optional PostgreSQL output backend as a lighter-weight alternative to Elasticsearch/OpenSearch, configured via a `[postgresql]` section (host/port/user/password/database or a libpq `connection_string`). Tables are created automatically on first run, and the schema captures the RFC 9990 aggregate fields (`np`, `testing`, `discovery_method`, `generator`, `xml_namespace`, and per-result `human_result`). A Grafana dashboard (`dashboards/grafana/Grafana-DMARC_Reports-PostgreSQL.json`) is included. Aggregate and SMTP-TLS reports are de-duplicated via `ON CONFLICT`; failure reports via an arrival-date / From / To / Subject check mirroring the Elasticsearch backend. +New optional PostgreSQL output backend as a lighter-weight alternative to Elasticsearch/OpenSearch, configured via a `[postgresql]` section (host/port/user/password/database or a libpq `connection_string`), or equivalently through `PARSEDMARC_POSTGRESQL_*` environment variables and their `_FILE` Docker-secret variants like every other backend. Tables are created automatically on first run, and the schema captures the RFC 9990 aggregate fields (`np`, `testing`, `discovery_method`, `generator`, `xml_namespace`, and per-result `human_result`). A Grafana dashboard (`dashboards/grafana/Grafana-DMARC_Reports-PostgreSQL.json`) is included. Aggregate and SMTP-TLS reports are de-duplicated via `ON CONFLICT`; failure reports via an arrival-date / From / To / Subject check mirroring the Elasticsearch backend. The backend is opt-in: install it with `pip install parsedmarc[postgresql]` (it pulls in `psycopg`). It is not a mandatory dependency because the prebuilt `psycopg` binary wheels are not available for every platform. @@ -48,6 +48,9 @@ New `[elasticsearch] serverless` config flag (env var `PARSEDMARC_ELASTICSEARCH_ - **`save_smtp_tls_report_to_s3` was completely broken.** `parsedmarc/s3.py:save_report_to_s3` unconditionally read `report["report_metadata"]` when assembling S3 object metadata, but SMTP TLS reports are flat per RFC 8460 §4.3 — they have no `report_metadata` sub-object — and `parse_smtp_tls_report_json` correctly stores `begin_date` as the raw ISO-8601 string from the report. The S3 path branch also assumed `begin_date` was a `datetime` and did `.year` / `.month` / `.day` on it. The CLI's surrounding `try/except` silently swallowed the resulting `KeyError`, so every SMTP-TLS report quietly failed to upload to S3 in production. Both issues are fixed: SMTP-TLS metadata is now built from the flat report fields directly, and the date is normalized via `human_timestamp_to_datetime`. - **`append_json` corrupted JSON output files on the second write.** The original implementation opened files in `"a+"` mode, then `seek()`ed backwards to overwrite the trailing `]` with `,\n` before appending more elements. [Python's docs are explicit](https://docs.python.org/3/library/functions.html#open): on POSIX, writes in `"a"`/`"a+"` mode always go to EOF regardless of seek position. The result was that every second call onto an existing file produced `[...]\n],\n[...]`-style corrupted output instead of a single merged JSON array. Anyone running parsedmarc in watch mode with JSON output enabled had `aggregate.json` / `failure.json` / `smtp_tls.json` quietly turning into invalid JSON after the first overlap. Replaced with a read-merge-write pattern: load the existing array (if any), append the new elements, rewrite the whole file. `append_csv` was not affected — it doesn't seek backwards. - **Removed redundant try/except in `parsedmarc/webhook.py`.** `save_aggregate_report_to_webhook` / `save_failure_report_to_webhook` / `save_smtp_tls_report_to_webhook` each wrapped `self._send_to_webhook(...)` in a try/except, but `_send_to_webhook` already catches every `Exception` itself, so the outer except blocks were unreachable dead code. +- **Report files whose names contain glob metacharacters were silently skipped.** The CLI expanded every file argument with `glob()` ([`parsedmarc/cli.py`](parsedmarc/cli.py)), which interprets `[`, `]`, `*`, and `?` as pattern syntax (see the [`glob` docs](https://docs.python.org/3/library/glob.html)). A literal path such as `[Netease DMARC Failure Report] Rent Reminder.eml` — the bracketed shape many providers use for emailed failure reports — was treated as a character class, matched nothing, and was dropped before reaching the parser, with no error. File arguments that already exist on disk are now taken literally; only non-existent paths are treated as glob patterns, so shell-style wildcards (`samples/*.xml`) still expand. +- **OpenSearch Dashboards reported a mapping conflict on the aggregate index pattern's `org_email` field.** The shipped `dashboards/opensearch/opensearch_dashboards.ndjson` froze a cached field-list snapshot in which `org_email` was a `text` / `object` conflict, alongside leftover `org_email.#text` and `org_email.#text.keyword` subfields — artifacts of a cluster that had once indexed a `langAttrString` `email` dict (`{"#text": …, "@lang": …}`) before the parser unwrapped it. `org_email` is mapped as `Text()` and the parser now unwraps a dict `email` to a plain string, so live data is consistent; cleared the stale conflict and the two artifact subfields from the index pattern, leaving `org_email` (text) and `org_email.keyword` so importers no longer see the warning. +- **`dashboard-dev-bootstrap.sh` imported the OpenSearch Dashboards saved objects into the wrong tenant.** The script sent `securitytenant: global_tenant`, but the OpenSearch security plugin reads that header as a tenant *name*, and `global_tenant` is a sample custom tenant shipped in the security demo config — not the shared **Global** tenant, whose token is the literal `global`. The import succeeded into a separate `global_tenant` tenant (its own `.kibana__globaltenant_1` index), so the dashboards were invisible to anyone viewing the Global tenant in OpenSearch Dashboards. Changed the default `OSD_TENANT` to `global`. (An empty/omitted `securitytenant` header is *not* equivalent — it falls back to the user's configured default tenant, not Global.) This affects the contributor dev stack only, not the shipped dashboards. ### Breaking changes diff --git a/README.md b/README.md index a1ca67e3..90a6be1f 100644 --- a/README.md +++ b/README.md @@ -39,9 +39,9 @@ Please consider [sponsoring my work](https://github.com/sponsors/seanthegeek) if - Consistent data structures - Simple JSON and/or CSV output - Optionally email the results -- Optionally send the results to Elasticsearch, OpenSearch, or Splunk, for use - with premade dashboards -- Optionally send the results to PostgreSQL, Apache Kafka, Amazon S3, Azure Log +- Optionally send the results to Elasticsearch, OpenSearch, Splunk, or + PostgreSQL, for use with premade dashboards +- Optionally send the results to Apache Kafka, Amazon S3, Azure Log Analytics (Microsoft Sentinel), a Graylog (GELF) endpoint, a syslog server, or an HTTP webhook diff --git a/dashboard-dev-bootstrap.sh b/dashboard-dev-bootstrap.sh index 41ab042d..3e554e6b 100755 --- a/dashboard-dev-bootstrap.sh +++ b/dashboard-dev-bootstrap.sh @@ -19,6 +19,12 @@ set +a GRAFANA_USER="${GRAFANA_USER:-admin}" GRAFANA_PASSWORD="${GRAFANA_PASSWORD:-admin}" +# PostgreSQL dev credentials. Defaults match docker-compose.dashboard-dev.yml's +# ${POSTGRESQL_*:-parsedmarc} fallbacks; override all four in lockstep via .env. +PG_USER="${POSTGRESQL_USER:-parsedmarc}" +PG_PASSWORD="${POSTGRESQL_PASSWORD:-parsedmarc}" +PG_DB="${POSTGRESQL_DB:-parsedmarc}" + log() { printf '\n\033[1;36m== %s\033[0m\n' "$*"; } wait_for() { @@ -59,6 +65,8 @@ wait_for "OpenSearch Dashboards" \ curl -ksf -u "admin:${OPENSEARCH_INITIAL_ADMIN_PASSWORD}" \ http://localhost:5602/api/status wait_for "Grafana" curl -sf http://localhost:3000/api/health +wait_for "PostgreSQL" \ + "${COMPOSE[@]}" exec -T postgresql pg_isready -U "$PG_USER" -d "$PG_DB" # Splunk's HEC port is healthy once management API is up too. wait_for "Splunk HEC" curl -ksf https://localhost:8088/services/collector/health # Splunkd management API (used for dashboard imports) lives inside the container. @@ -204,14 +212,52 @@ else splunk_curl -X POST \ "https://localhost:8089/servicesNS/admin/splunk_httpinput/data/inputs/http/splunk_hec_token" \ -d "indexes=email,main" -d "index=email" -d "disabled=0" >/dev/null + # PostgreSQL: drop and recreate the public schema. parsedmarc recreates + # its tables on the next seed run, so this is a clean wipe. + "${COMPOSE[@]}" exec -T -e PGPASSWORD="$PG_PASSWORD" postgresql \ + psql -U "$PG_USER" -d "$PG_DB" \ + -c 'DROP SCHEMA public CASCADE; CREATE SCHEMA public;' >/dev/null 2>&1 || true fi - PARSEDMARC_BIN="${PARSEDMARC_BIN:-$REPO_ROOT/venv/bin/parsedmarc}" - if [ ! -x "$PARSEDMARC_BIN" ]; then - PARSEDMARC_BIN="$(command -v parsedmarc || true)" + # Resolve a Python environment for the seed and make sure parsedmarc plus + # the PostgreSQL extra (psycopg) are installed in it, so the same run can + # populate Postgres. Precedence: + # 1. An explicit PARSEDMARC_BIN — used as-is, nothing installed. + # 2. An already-activated virtualenv ($VIRTUAL_ENV). + # 3. An existing repo venv/ or .venv/. + # 4. Otherwise a freshly created $REPO_ROOT/venv. + # Cases 2-4 run `pip install -e .[postgresql]` only when the CLI or psycopg + # is missing, so it's a no-op once the environment is set up. + if [ -n "${PARSEDMARC_BIN:-}" ]; then + if [ ! -x "$PARSEDMARC_BIN" ]; then + echo "ERROR: PARSEDMARC_BIN is set but not executable: $PARSEDMARC_BIN" >&2 + exit 1 + fi + echo " using PARSEDMARC_BIN: $PARSEDMARC_BIN" + else + if [ -n "${VIRTUAL_ENV:-}" ]; then + seed_venv="$VIRTUAL_ENV" + echo " using active virtualenv: $seed_venv" + elif [ -d "$REPO_ROOT/venv" ]; then + seed_venv="$REPO_ROOT/venv" + echo " using existing venv: $seed_venv" + elif [ -d "$REPO_ROOT/.venv" ]; then + seed_venv="$REPO_ROOT/.venv" + echo " using existing .venv: $seed_venv" + else + seed_venv="$REPO_ROOT/venv" + echo " creating virtualenv: $seed_venv" + python3 -m venv "$seed_venv" + fi + PARSEDMARC_BIN="$seed_venv/bin/parsedmarc" + if [ ! -x "$PARSEDMARC_BIN" ] || + ! "$seed_venv/bin/python" -c 'import psycopg' >/dev/null 2>&1; then + echo " installing parsedmarc[postgresql] into $seed_venv" + "$seed_venv/bin/python" -m pip install -q -e "${REPO_ROOT}[postgresql]" + fi fi - if [ -z "$PARSEDMARC_BIN" ] || [ ! -x "$PARSEDMARC_BIN" ]; then - echo "ERROR: parsedmarc CLI not found. Install with 'pip install -e .[build]' or set PARSEDMARC_BIN." >&2 + if [ ! -x "$PARSEDMARC_BIN" ]; then + echo "ERROR: parsedmarc CLI not found at $PARSEDMARC_BIN" >&2 exit 1 fi @@ -232,11 +278,36 @@ else samples/aggregate/protection.outlook.com!example.com!1711756800!1711843200.xml samples/aggregate/usssa.com!example.com!1538784000!1538870399.xml samples/aggregate/veeam.com!example.com!1530133200!1530219600.xml + samples/aggregate/rfc9990-sample.xml + samples/aggregate/rfc9990-example.net!example.com!1700000000!1700086399.xml samples/failure/*.eml samples/smtp_tls/*.json samples/smtp_tls/google.com_smtp_tls_report.eml ) - "$PARSEDMARC_BIN" -t 2.0 --dns-retries 1 -c parsedmarc-dev.ini "${SAMPLE_FILES[@]}" || true + # PostgreSQL config is injected via env vars (parsedmarc synthesizes the + # [postgresql] section from PARSEDMARC_POSTGRESQL_*), so the same seed run + # also populates Postgres without touching the gitignored parsedmarc-dev.ini. + # Only wire it in when psycopg is importable: parsedmarc aborts the whole + # run (exit 1, nothing written to *any* backend) if a configured output + # backend can't initialize, so a missing optional extra must not be added. + pg_seed_env=() + seed_python="$(dirname "$PARSEDMARC_BIN")/python" + if [ -x "$seed_python" ] && "$seed_python" -c 'import psycopg' >/dev/null 2>&1; then + pg_seed_env=( + PARSEDMARC_POSTGRESQL_HOST=localhost + PARSEDMARC_POSTGRESQL_PORT=5432 + PARSEDMARC_POSTGRESQL_USER="$PG_USER" + PARSEDMARC_POSTGRESQL_PASSWORD="$PG_PASSWORD" + PARSEDMARC_POSTGRESQL_DATABASE="$PG_DB" + ) + else + # Reached only for an explicit PARSEDMARC_BIN whose env lacks psycopg + # (the auto-resolved venv path installs the extra above). + echo " NOTE: 'psycopg' is not available to ${PARSEDMARC_BIN} — skipping the" + echo " PostgreSQL seed. Enable it with: pip install -e '.[postgresql]'" + fi + env "${pg_seed_env[@]}" \ + "$PARSEDMARC_BIN" -t 2.0 --dns-retries 1 -c parsedmarc-dev.ini "${SAMPLE_FILES[@]}" || true fi # --------------------------------------------------------------------------- @@ -253,11 +324,15 @@ log "Importing OpenSearch Dashboards saved objects" # OSD with the security plugin enabled stores saved objects per tenant. Without # a securitytenant header the import lands in the API user's *private* tenant, # which is invisible to anyone else (and to the same user when their browser -# session is on a different tenant). Target global_tenant — the shared +# session is on a different tenant). Target the Global tenant — the shared # workspace every user has access to and where public dashboards conventionally -# live. To send the import elsewhere set OSD_TENANT=admin_tenant (or any other -# tenant name) before running. -OSD_TENANT="${OSD_TENANT:-global_tenant}" +# live. Its securitytenant token is the literal "global"; any *other* string is +# treated as a custom tenant name, so "global_tenant" would silently create a +# separate "global_tenant" tenant rather than hit Global. (An empty/omitted +# header is *not* equivalent — it falls back to the user's configured default +# tenant, not Global.) To send the import elsewhere set OSD_TENANT=admin_tenant +# (or any other tenant name) before running. +OSD_TENANT="${OSD_TENANT:-global}" curl -sS -X POST 'http://localhost:5602/api/saved_objects/_import?overwrite=true' \ -H 'osd-xsrf: true' \ -H "securitytenant: ${OSD_TENANT}" \ @@ -306,6 +381,38 @@ EOF echo " created datasource '${name}'" done +# PostgreSQL datasource for the PostgreSQL DMARC dashboard. Fixed uid dmarc-pg +# so the dashboard import below can resolve its ${DS_POSTGRESQL} input. Skipped +# when already present. +pg_ds_code=$(curl -sS -u "${GRAFANA_USER}:${GRAFANA_PASSWORD}" \ + -o /dev/null -w "%{http_code}" \ + "http://localhost:3000/api/datasources/name/PostgreSQL") +if [ "$pg_ds_code" = "200" ]; then + echo " datasource 'PostgreSQL' already exists — skipping" +else + pg_ds_body=$(cat < str: return os.path.expanduser(os.path.expandvars(p)) +def _expand_file_path_args(paths: list[str]) -> list[str]: + """Expand CLI file-path arguments into a flat list of file paths. + + A path that already exists on disk is taken literally; only a + non-existent path is treated as a glob pattern. This preserves + shell-style wildcard expansion (e.g. a quoted ``samples/*.xml``) while + ensuring that literal filenames containing glob metacharacters + (``[``, ``]``, ``*``, ``?``) are not silently dropped. Emailed DMARC + failure reports are frequently named like + ``[Provider DMARC Failure Report] Subject.eml``; ``glob()`` treats the + brackets as a character class, matches nothing, and drops the file + (see ). + """ + expanded: list[str] = [] + for path in paths: + if os.path.exists(path): + expanded.append(path) + else: + expanded += glob(path) + return expanded + + # All known INI config section names, used for env var resolution. _KNOWN_SECTIONS = frozenset( { @@ -105,6 +127,7 @@ _KNOWN_SECTIONS = frozenset( "kafka", "smtp", "s3", + "postgresql", "syslog", "gmail_api", "maildir", @@ -2112,11 +2135,9 @@ def _main(): logger.error("Output client error: {0}".format(error_)) exit(1) - file_paths = [] + file_paths = _expand_file_path_args(args.file_path) mbox_paths = [] - for file_path in args.file_path: - file_paths += glob(file_path) for file_path in file_paths: if is_mbox(file_path): mbox_paths.append(file_path) diff --git a/tests/test_cli.py b/tests/test_cli.py index 92546236..f0bc3cf7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -349,6 +349,40 @@ hosts = localhost # Just a section name with no key should not match self.assertEqual(_resolve_section_key("IMAP"), (None, None)) + def test_expand_file_path_args_keeps_bracketed_filenames(self): + """Literal report filenames containing glob metacharacters must not + be dropped. + + Regression test: ``_main`` expanded every file argument with + ``glob()``, which treats ``[...]`` as a character class. A real + file named ``[Provider DMARC Failure Report] Subject.eml`` (the + shape Netease and others use) matched nothing and was silently + skipped, so the report never reached the parser. + See https://docs.python.org/3/library/glob.html. + """ + from glob import glob + from parsedmarc.cli import _expand_file_path_args + + with tempfile.TemporaryDirectory() as d: + bracket = os.path.join(d, "[Netease DMARC Failure Report] Rent.eml") + plain = os.path.join(d, "report.eml") + for p in (bracket, plain): + with open(p, "w") as f: + f.write("x") + + # Sanity: raw glob drops the bracketed path (documents the bug). + self.assertEqual(glob(bracket), []) + + # The literal bracketed path is preserved as-is. + self.assertEqual(_expand_file_path_args([bracket]), [bracket]) + + # Wildcards (non-existent as literal paths) still expand. + wildcard = os.path.join(d, "*.eml") + self.assertEqual( + sorted(_expand_file_path_args([wildcard])), + sorted([bracket, plain]), + ) + def test_apply_env_overrides_injects_values(self): """Env vars are injected into an existing ConfigParser.""" from configparser import ConfigParser @@ -382,6 +416,36 @@ hosts = localhost self.assertTrue(config.has_section("elasticsearch")) self.assertEqual(config.get("elasticsearch", "hosts"), "http://localhost:9200") + def test_apply_env_overrides_postgresql_section(self): + """PARSEDMARC_POSTGRESQL_* env vars must resolve to the [postgresql] + section. + + Regression test: ``postgresql`` was missing from ``_KNOWN_SECTIONS``, + so ``_resolve_section_key`` returned ``(None, None)`` for every + ``PARSEDMARC_POSTGRESQL_*`` var and the override was silently dropped. + The PostgreSQL backend is only initialized when ``"postgresql" in + config.sections()`` (cli.py), so the section must exist for env-var / + Docker-secret configuration of the backend to work at all. + """ + from configparser import ConfigParser + from parsedmarc.cli import _apply_env_overrides + + config = ConfigParser() + + env = { + "PARSEDMARC_POSTGRESQL_HOST": "db.example.com", + "PARSEDMARC_POSTGRESQL_PORT": "5432", + "PARSEDMARC_POSTGRESQL_USER": "parsedmarc", + "PARSEDMARC_POSTGRESQL_DATABASE": "parsedmarc", + } + with patch.dict(os.environ, env, clear=False): + _apply_env_overrides(config) + + self.assertIn("postgresql", config.sections()) + self.assertEqual(config.get("postgresql", "host"), "db.example.com") + self.assertEqual(config.get("postgresql", "port"), "5432") + self.assertEqual(config.get("postgresql", "database"), "parsedmarc") + def test_apply_env_overrides_ignores_config_file_var(self): """PARSEDMARC_CONFIG_FILE is not injected as a config key.""" from configparser import ConfigParser