From ff0ca6538c979298b407bef03abdcfcfce72eb66 Mon Sep 17 00:00:00 2001 From: Sean Whalen Date: Wed, 25 Mar 2026 19:25:21 -0400 Subject: [PATCH] 9.5.0 Add environment variable configuration support and update documentation - Introduced support for configuration via environment variables using the `PARSEDMARC_{SECTION}_{KEY}` format. - Added `PARSEDMARC_CONFIG_FILE` variable to specify the config file path. - Enabled env-only mode for file-less Docker deployments. - Implemented explicit read permission checks on config files. - Updated changelog and usage documentation to reflect these changes. --- CHANGELOG.md | 9 ++ docs/source/usage.md | 90 ++++++++++++++++ parsedmarc/cli.py | 129 ++++++++++++++++++++--- parsedmarc/constants.py | 2 +- tests.py | 221 ++++++++++++++++++++++++++++++++++++++-- 5 files changed, 427 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 501c76e..be31827 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 9.5.0 + +### Added + +- Environment variable configuration support: any config option can now be set via `PARSEDMARC_{SECTION}_{KEY}` environment variables (e.g. `PARSEDMARC_IMAP_PASSWORD`, `PARSEDMARC_SPLUNK_HEC_TOKEN`). Environment variables override config file values but are overridden by CLI arguments. +- `PARSEDMARC_CONFIG_FILE` environment variable to specify the config file path without the `-c` flag. +- Env-only mode: parsedmarc can now run without a config file when `PARSEDMARC_*` environment variables are set, enabling fully file-less Docker deployments. +- Explicit read permission check on config file, giving a clear error message when the container UID cannot read the file (e.g. `chmod 600` with a UID mismatch). + ## 9.4.0 ### Added diff --git a/docs/source/usage.md b/docs/source/usage.md index e920c53..72116d8 100644 --- a/docs/source/usage.md +++ b/docs/source/usage.md @@ -531,6 +531,96 @@ PUT _cluster/settings Increasing this value increases resource usage. ::: +## Environment variable configuration + +Any configuration option can be set via environment variables using the +naming convention `PARSEDMARC_{SECTION}_{KEY}` (uppercase). This is +especially useful for Docker deployments where file permissions make it +difficult to use config files for secrets. + +**Priority order:** CLI arguments > environment variables > config file > defaults + +### Examples + +```bash +# Set IMAP credentials via env vars +export PARSEDMARC_IMAP_HOST=imap.example.com +export PARSEDMARC_IMAP_USER=dmarc@example.com +export PARSEDMARC_IMAP_PASSWORD=secret + +# Elasticsearch +export PARSEDMARC_ELASTICSEARCH_HOSTS=http://localhost:9200 +export PARSEDMARC_ELASTICSEARCH_SSL=false + +# Splunk HEC (note: section name splunk_hec becomes SPLUNK_HEC) +export PARSEDMARC_SPLUNK_HEC_URL=https://splunk.example.com +export PARSEDMARC_SPLUNK_HEC_TOKEN=my-hec-token +export PARSEDMARC_SPLUNK_HEC_INDEX=email + +# General settings +export PARSEDMARC_GENERAL_SAVE_AGGREGATE=true +export PARSEDMARC_GENERAL_DEBUG=true +``` + +### Specifying the config file via environment variable + +```bash +export PARSEDMARC_CONFIG_FILE=/etc/parsedmarc.ini +parsedmarc +``` + +### Running without a config file (env-only mode) + +When no config file is given (neither `-c` flag nor `PARSEDMARC_CONFIG_FILE`), +parsedmarc will still pick up any `PARSEDMARC_*` environment variables. This +enables fully file-less deployments: + +```bash +export PARSEDMARC_GENERAL_SAVE_AGGREGATE=true +export PARSEDMARC_GENERAL_OFFLINE=true +export PARSEDMARC_ELASTICSEARCH_HOSTS=http://elasticsearch:9200 +parsedmarc /path/to/reports/* +``` + +### Docker Compose example + +```yaml +services: + parsedmarc: + image: parsedmarc:latest + environment: + PARSEDMARC_IMAP_HOST: imap.example.com + PARSEDMARC_IMAP_USER: dmarc@example.com + PARSEDMARC_IMAP_PASSWORD: ${IMAP_PASSWORD} + PARSEDMARC_MAILBOX_WATCH: "true" + PARSEDMARC_ELASTICSEARCH_HOSTS: http://elasticsearch:9200 + PARSEDMARC_GENERAL_SAVE_AGGREGATE: "true" + PARSEDMARC_GENERAL_SAVE_FORENSIC: "true" +``` + +### Section name mapping + +For sections with underscores in the name, the full section name is used: + +| Section | Env var prefix | +|------------------|-------------------------------| +| `general` | `PARSEDMARC_GENERAL_` | +| `mailbox` | `PARSEDMARC_MAILBOX_` | +| `imap` | `PARSEDMARC_IMAP_` | +| `msgraph` | `PARSEDMARC_MSGRAPH_` | +| `elasticsearch` | `PARSEDMARC_ELASTICSEARCH_` | +| `opensearch` | `PARSEDMARC_OPENSEARCH_` | +| `splunk_hec` | `PARSEDMARC_SPLUNK_HEC_` | +| `kafka` | `PARSEDMARC_KAFKA_` | +| `smtp` | `PARSEDMARC_SMTP_` | +| `s3` | `PARSEDMARC_S3_` | +| `syslog` | `PARSEDMARC_SYSLOG_` | +| `gmail_api` | `PARSEDMARC_GMAIL_API_` | +| `maildir` | `PARSEDMARC_MAILDIR_` | +| `log_analytics` | `PARSEDMARC_LOG_ANALYTICS_` | +| `gelf` | `PARSEDMARC_GELF_` | +| `webhook` | `PARSEDMARC_WEBHOOK_` | + ## Performance tuning For large mailbox imports or backfills, parsedmarc can consume a noticeable amount diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index 8993f6d..809003c 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -75,6 +75,79 @@ def _str_to_list(s): return list(map(lambda i: i.lstrip(), _list)) +# All known INI config section names, used for env var resolution. +_KNOWN_SECTIONS = frozenset( + { + "general", + "mailbox", + "imap", + "msgraph", + "elasticsearch", + "opensearch", + "splunk_hec", + "kafka", + "smtp", + "s3", + "syslog", + "gmail_api", + "maildir", + "log_analytics", + "gelf", + "webhook", + } +) + + +def _resolve_section_key(suffix: str) -> tuple: + """Resolve an env var suffix like ``IMAP_PASSWORD`` to ``('imap', 'password')``. + + Uses longest-prefix matching against known section names so that + multi-word sections like ``splunk_hec`` are handled correctly. + + Returns ``(None, None)`` when no known section matches. + """ + suffix_lower = suffix.lower() + + best_section = None + best_key = None + for section in _KNOWN_SECTIONS: + section_prefix = section + "_" + if suffix_lower.startswith(section_prefix): + key = suffix_lower[len(section_prefix) :] + if key and (best_section is None or len(section) > len(best_section)): + best_section = section + best_key = key + + return best_section, best_key + + +def _apply_env_overrides(config: ConfigParser) -> None: + """Inject ``PARSEDMARC_*`` environment variables into *config*. + + Environment variables matching ``PARSEDMARC_{SECTION}_{KEY}`` override + (or create) the corresponding config-file values. Sections are created + automatically when they do not yet exist. + """ + prefix = "PARSEDMARC_" + + for env_key, env_value in os.environ.items(): + if not env_key.startswith(prefix) or env_key == "PARSEDMARC_CONFIG_FILE": + continue + + suffix = env_key[len(prefix) :] + section, key = _resolve_section_key(suffix) + + if section is None: + logger.debug("Ignoring unrecognized env var: %s", env_key) + continue + + if not config.has_section(section): + config.add_section(section) + + config.set(section, key, env_value) + logger.debug("Config override from env: [%s] %s", section, key) + + def _configure_logging(log_level, log_file=None): """ Configure logging for the current process. @@ -178,12 +251,39 @@ class ConfigurationError(Exception): pass -def _parse_config_file(config_file, opts): - """Parse a config file and update opts in place. +def _load_config(config_file: str | None = None) -> ConfigParser: + """Load configuration from an INI file and/or environment variables. Args: - config_file: Path to the .ini config file - opts: Namespace object to update with parsed values + config_file: Optional path to an .ini config file. + + Returns: + A ``ConfigParser`` populated from the file (if given) and from any + ``PARSEDMARC_*`` environment variables. + + Raises: + ConfigurationError: If *config_file* is given but does not exist. + """ + config = ConfigParser() + if config_file is not None: + abs_path = os.path.abspath(config_file) + if not os.path.exists(abs_path): + raise ConfigurationError("A file does not exist at {0}".format(abs_path)) + if not os.access(abs_path, os.R_OK): + raise ConfigurationError( + "Unable to read {0} — check file permissions".format(abs_path) + ) + config.read(config_file) + _apply_env_overrides(config) + return config + + +def _parse_config(config: ConfigParser, opts): + """Apply a loaded ``ConfigParser`` to *opts* in place. + + Args: + config: A ``ConfigParser`` (from ``_load_config``). + opts: Namespace object to update with parsed values. Returns: index_prefix_domain_map or None @@ -191,13 +291,8 @@ def _parse_config_file(config_file, opts): Raises: ConfigurationError: If required settings are missing or invalid. """ - abs_path = os.path.abspath(config_file) - if not os.path.exists(abs_path): - raise ConfigurationError("A file does not exist at {0}".format(abs_path)) opts.silent = True - config = ConfigParser() index_prefix_domain_map = None - config.read(config_file) if "general" in config.sections(): general_config = config["general"] if "silent" in general_config: @@ -1683,9 +1778,16 @@ def _main(): index_prefix_domain_map = None - if args.config_file: + config_file = args.config_file or os.environ.get("PARSEDMARC_CONFIG_FILE") + has_env_config = any( + k.startswith("PARSEDMARC_") and k != "PARSEDMARC_CONFIG_FILE" + for k in os.environ + ) + + if config_file or has_env_config: try: - index_prefix_domain_map = _parse_config_file(args.config_file, opts) + config = _load_config(config_file) + index_prefix_domain_map = _parse_config(config, opts) except ConfigurationError as e: logger.critical(str(e)) exit(-1) @@ -2102,9 +2204,8 @@ def _main(): # Build a fresh opts starting from CLI-only defaults so that # sections removed from the config file actually take effect. new_opts = Namespace(**vars(opts_from_cli)) - new_index_prefix_domain_map = _parse_config_file( - args.config_file, new_opts - ) + new_config = _load_config(config_file) + new_index_prefix_domain_map = _parse_config(new_config, new_opts) new_clients = _init_output_clients(new_opts) # All steps succeeded — commit the changes atomically. diff --git a/parsedmarc/constants.py b/parsedmarc/constants.py index 3e948ad..36f2a3a 100644 --- a/parsedmarc/constants.py +++ b/parsedmarc/constants.py @@ -1,3 +1,3 @@ -__version__ = "9.4.0" +__version__ = "9.5.0" USER_AGENT = f"parsedmarc/{__version__}" diff --git a/tests.py b/tests.py index 60d9946..3552624 100755 --- a/tests.py +++ b/tests.py @@ -11,6 +11,7 @@ import sys import tempfile import unittest from base64 import urlsafe_b64encode +from configparser import ConfigParser from glob import glob from pathlib import Path from tempfile import NamedTemporaryFile, TemporaryDirectory @@ -1985,7 +1986,8 @@ watch = true "SIGHUP not available on this platform", ) @patch("parsedmarc.cli._init_output_clients") - @patch("parsedmarc.cli._parse_config_file") + @patch("parsedmarc.cli._parse_config") + @patch("parsedmarc.cli._load_config") @patch("parsedmarc.cli.get_dmarc_reports_from_mailbox") @patch("parsedmarc.cli.watch_inbox") @patch("parsedmarc.cli.IMAPConnection") @@ -1994,6 +1996,7 @@ watch = true mock_imap, mock_watch, mock_get_reports, + mock_load_config, mock_parse_config, mock_init_clients, ): @@ -2007,7 +2010,9 @@ watch = true "smtp_tls_reports": [], } - def parse_side_effect(config_file, opts): + mock_load_config.return_value = ConfigParser() + + def parse_side_effect(config, opts): opts.imap_host = "imap.example.com" opts.imap_user = "user" opts.imap_password = "pass" @@ -2046,7 +2051,7 @@ watch = true self.assertEqual(cm.exception.code, 1) # watch_inbox was called twice: initial run + after reload self.assertEqual(mock_watch.call_count, 2) - # _parse_config_file called for initial load + reload + # _parse_config called for initial load + reload self.assertGreaterEqual(mock_parse_config.call_count, 2) @unittest.skipUnless( @@ -2054,7 +2059,8 @@ watch = true "SIGHUP not available on this platform", ) @patch("parsedmarc.cli._init_output_clients") - @patch("parsedmarc.cli._parse_config_file") + @patch("parsedmarc.cli._parse_config") + @patch("parsedmarc.cli._load_config") @patch("parsedmarc.cli.get_dmarc_reports_from_mailbox") @patch("parsedmarc.cli.watch_inbox") @patch("parsedmarc.cli.IMAPConnection") @@ -2063,6 +2069,7 @@ watch = true mock_imap, mock_watch, mock_get_reports, + mock_load_config, mock_parse_config, mock_init_clients, ): @@ -2076,11 +2083,13 @@ watch = true "smtp_tls_reports": [], } + mock_load_config.return_value = ConfigParser() + # Initial parse sets required opts; reload parse raises initial_map = {"prefix_": ["example.com"]} call_count = [0] - def parse_side_effect(config_file, opts): + def parse_side_effect(config, opts): call_count[0] += 1 opts.imap_host = "imap.example.com" opts.imap_user = "user" @@ -2130,7 +2139,8 @@ watch = true "SIGHUP not available on this platform", ) @patch("parsedmarc.cli._init_output_clients") - @patch("parsedmarc.cli._parse_config_file") + @patch("parsedmarc.cli._parse_config") + @patch("parsedmarc.cli._load_config") @patch("parsedmarc.cli.get_dmarc_reports_from_mailbox") @patch("parsedmarc.cli.watch_inbox") @patch("parsedmarc.cli.IMAPConnection") @@ -2139,6 +2149,7 @@ watch = true mock_imap, mock_watch, mock_get_reports, + mock_load_config, mock_parse_config, mock_init_clients, ): @@ -2152,7 +2163,9 @@ watch = true "smtp_tls_reports": [], } - def parse_side_effect(config_file, opts): + mock_load_config.return_value = ConfigParser() + + def parse_side_effect(config, opts): opts.imap_host = "imap.example.com" opts.imap_user = "user" opts.imap_password = "pass" @@ -2284,7 +2297,8 @@ watch = true "SIGHUP not available on this platform", ) @patch("parsedmarc.cli._init_output_clients") - @patch("parsedmarc.cli._parse_config_file") + @patch("parsedmarc.cli._parse_config") + @patch("parsedmarc.cli._load_config") @patch("parsedmarc.cli.get_dmarc_reports_from_mailbox") @patch("parsedmarc.cli.watch_inbox") @patch("parsedmarc.cli.IMAPConnection") @@ -2293,6 +2307,7 @@ watch = true mock_imap, mock_watch, mock_get_reports, + mock_load_config, mock_parse_config, mock_init_clients, ): @@ -2308,7 +2323,9 @@ watch = true "smtp_tls_reports": [], } - def parse_side_effect(config_file, opts): + mock_load_config.return_value = ConfigParser() + + def parse_side_effect(config, opts): opts.imap_host = "imap.example.com" opts.imap_user = "user" opts.imap_password = "pass" @@ -2474,5 +2491,191 @@ password = test-password self.assertNotIn("unmapped-1", report_ids) +class TestEnvVarConfig(unittest.TestCase): + """Tests for environment variable configuration support.""" + + def test_resolve_section_key_simple(self): + """Simple section names resolve correctly.""" + from parsedmarc.cli import _resolve_section_key + + self.assertEqual(_resolve_section_key("IMAP_PASSWORD"), ("imap", "password")) + self.assertEqual(_resolve_section_key("GENERAL_DEBUG"), ("general", "debug")) + self.assertEqual(_resolve_section_key("S3_BUCKET"), ("s3", "bucket")) + self.assertEqual(_resolve_section_key("GELF_HOST"), ("gelf", "host")) + + def test_resolve_section_key_underscore_sections(self): + """Multi-word section names (splunk_hec, gmail_api, etc.) resolve correctly.""" + from parsedmarc.cli import _resolve_section_key + + self.assertEqual( + _resolve_section_key("SPLUNK_HEC_TOKEN"), ("splunk_hec", "token") + ) + self.assertEqual( + _resolve_section_key("GMAIL_API_CREDENTIALS_FILE"), + ("gmail_api", "credentials_file"), + ) + self.assertEqual( + _resolve_section_key("LOG_ANALYTICS_CLIENT_ID"), + ("log_analytics", "client_id"), + ) + + def test_resolve_section_key_unknown(self): + """Unknown prefixes return (None, None).""" + from parsedmarc.cli import _resolve_section_key + + self.assertEqual(_resolve_section_key("UNKNOWN_FOO"), (None, None)) + # Just a section name with no key should not match + self.assertEqual(_resolve_section_key("IMAP"), (None, None)) + + def test_apply_env_overrides_injects_values(self): + """Env vars are injected into an existing ConfigParser.""" + from configparser import ConfigParser + from parsedmarc.cli import _apply_env_overrides + + config = ConfigParser() + config.add_section("imap") + config.set("imap", "host", "original.example.com") + + env = { + "PARSEDMARC_IMAP_HOST": "new.example.com", + "PARSEDMARC_IMAP_PASSWORD": "secret123", + } + with patch.dict(os.environ, env, clear=False): + _apply_env_overrides(config) + + self.assertEqual(config.get("imap", "host"), "new.example.com") + self.assertEqual(config.get("imap", "password"), "secret123") + + def test_apply_env_overrides_creates_sections(self): + """Env vars create new sections when they don't exist.""" + from configparser import ConfigParser + from parsedmarc.cli import _apply_env_overrides + + config = ConfigParser() + + env = {"PARSEDMARC_ELASTICSEARCH_HOSTS": "http://localhost:9200"} + with patch.dict(os.environ, env, clear=False): + _apply_env_overrides(config) + + self.assertTrue(config.has_section("elasticsearch")) + self.assertEqual(config.get("elasticsearch", "hosts"), "http://localhost:9200") + + def test_apply_env_overrides_ignores_config_file_var(self): + """PARSEDMARC_CONFIG_FILE is not injected as a config key.""" + from configparser import ConfigParser + from parsedmarc.cli import _apply_env_overrides + + config = ConfigParser() + + env = {"PARSEDMARC_CONFIG_FILE": "/some/path.ini"} + with patch.dict(os.environ, env, clear=False): + _apply_env_overrides(config) + + self.assertEqual(config.sections(), []) + + def test_load_config_with_file_and_env_override(self): + """Env vars override values from an INI file.""" + from parsedmarc.cli import _load_config + + with NamedTemporaryFile(mode="w", suffix=".ini", delete=False) as f: + f.write( + "[imap]\nhost = file.example.com\nuser = alice\npassword = fromfile\n" + ) + f.flush() + config_path = f.name + + try: + env = {"PARSEDMARC_IMAP_PASSWORD": "fromenv"} + with patch.dict(os.environ, env, clear=False): + config = _load_config(config_path) + + self.assertEqual(config.get("imap", "host"), "file.example.com") + self.assertEqual(config.get("imap", "user"), "alice") + self.assertEqual(config.get("imap", "password"), "fromenv") + finally: + os.unlink(config_path) + + def test_load_config_env_only(self): + """Config can be loaded purely from env vars with no file.""" + from parsedmarc.cli import _load_config + + env = { + "PARSEDMARC_GENERAL_DEBUG": "true", + "PARSEDMARC_ELASTICSEARCH_HOSTS": "http://localhost:9200", + } + with patch.dict(os.environ, env, clear=False): + config = _load_config(None) + + self.assertEqual(config.get("general", "debug"), "true") + self.assertEqual(config.get("elasticsearch", "hosts"), "http://localhost:9200") + + def test_parse_config_from_env(self): + """Full round-trip: env vars -> ConfigParser -> opts.""" + from argparse import Namespace + from parsedmarc.cli import _load_config, _parse_config + + env = { + "PARSEDMARC_GENERAL_DEBUG": "true", + "PARSEDMARC_GENERAL_SAVE_AGGREGATE": "true", + "PARSEDMARC_GENERAL_OFFLINE": "true", + } + with patch.dict(os.environ, env, clear=False): + config = _load_config(None) + + opts = Namespace() + _parse_config(config, opts) + + self.assertTrue(opts.debug) + self.assertTrue(opts.save_aggregate) + self.assertTrue(opts.offline) + + def test_config_file_env_var(self): + """PARSEDMARC_CONFIG_FILE env var specifies the config file path.""" + from argparse import Namespace + from parsedmarc.cli import _load_config, _parse_config + + with NamedTemporaryFile(mode="w", suffix=".ini", delete=False) as f: + f.write("[general]\ndebug = true\noffline = true\n") + f.flush() + config_path = f.name + + try: + env = {"PARSEDMARC_CONFIG_FILE": config_path} + with patch.dict(os.environ, env, clear=False): + config = _load_config(os.environ.get("PARSEDMARC_CONFIG_FILE")) + + opts = Namespace() + _parse_config(config, opts) + self.assertTrue(opts.debug) + self.assertTrue(opts.offline) + finally: + os.unlink(config_path) + + def test_boolean_values_from_env(self): + """Various boolean string representations work through ConfigParser.""" + from configparser import ConfigParser + from parsedmarc.cli import _apply_env_overrides + + for true_val in ("true", "yes", "1", "on", "True", "YES"): + config = ConfigParser() + env = {"PARSEDMARC_GENERAL_DEBUG": true_val} + with patch.dict(os.environ, env, clear=False): + _apply_env_overrides(config) + self.assertTrue( + config.getboolean("general", "debug"), + f"Expected truthy for {true_val!r}", + ) + + for false_val in ("false", "no", "0", "off", "False", "NO"): + config = ConfigParser() + env = {"PARSEDMARC_GENERAL_DEBUG": false_val} + with patch.dict(os.environ, env, clear=False): + _apply_env_overrides(config) + self.assertFalse( + config.getboolean("general", "debug"), + f"Expected falsy for {false_val!r}", + ) + + if __name__ == "__main__": unittest.main(verbosity=2)