9.5.1

- Correct ISO format for MSGraphConnection timestamps (PR #706)

CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog
 
+## 9.5.1
+
+### Changes
+
+- Correct ISO format for MSGraphConnection timestamps (PR #706)
+
+## 9.5.0
+
+### Added
+
+- Environment variable configuration support: any config option can now be set via `PARSEDMARC_{SECTION}_{KEY}` environment variables (e.g. `PARSEDMARC_IMAP_PASSWORD`, `PARSEDMARC_SPLUNK_HEC_TOKEN`). Environment variables override config file values but are overridden by CLI arguments.
+- `PARSEDMARC_CONFIG_FILE` environment variable to specify the config file path without the `-c` flag.
+- Env-only mode: parsedmarc can now run without a config file when `PARSEDMARC_*` environment variables are set, enabling fully file-less Docker deployments.
+- Explicit read permission check on config file, giving a clear error message when the container UID cannot read the file (e.g. `chmod 600` with a UID mismatch).
+
 ## 9.4.0
 
 ### Added

docs/source/usage.md

@@ -531,6 +531,96 @@ PUT _cluster/settings
 Increasing this value increases resource usage.
 :::
 
+## Environment variable configuration
+
+Any configuration option can be set via environment variables using the
+naming convention `PARSEDMARC_{SECTION}_{KEY}` (uppercase). This is
+especially useful for Docker deployments where file permissions make it
+difficult to use config files for secrets.
+
+**Priority order:** CLI arguments > environment variables > config file > defaults
+
+### Examples
+
+```bash
+# Set IMAP credentials via env vars
+export PARSEDMARC_IMAP_HOST=imap.example.com
+export PARSEDMARC_IMAP_USER=dmarc@example.com
+export PARSEDMARC_IMAP_PASSWORD=secret
+
+# Elasticsearch
+export PARSEDMARC_ELASTICSEARCH_HOSTS=http://localhost:9200
+export PARSEDMARC_ELASTICSEARCH_SSL=false
+
+# Splunk HEC (note: section name splunk_hec becomes SPLUNK_HEC)
+export PARSEDMARC_SPLUNK_HEC_URL=https://splunk.example.com
+export PARSEDMARC_SPLUNK_HEC_TOKEN=my-hec-token
+export PARSEDMARC_SPLUNK_HEC_INDEX=email
+
+# General settings
+export PARSEDMARC_GENERAL_SAVE_AGGREGATE=true
+export PARSEDMARC_GENERAL_DEBUG=true
+```
+
+### Specifying the config file via environment variable
+
+```bash
+export PARSEDMARC_CONFIG_FILE=/etc/parsedmarc.ini
+parsedmarc
+```
+
+### Running without a config file (env-only mode)
+
+When no config file is given (neither `-c` flag nor `PARSEDMARC_CONFIG_FILE`),
+parsedmarc will still pick up any `PARSEDMARC_*` environment variables. This
+enables fully file-less deployments:
+
+```bash
+export PARSEDMARC_GENERAL_SAVE_AGGREGATE=true
+export PARSEDMARC_GENERAL_OFFLINE=true
+export PARSEDMARC_ELASTICSEARCH_HOSTS=http://elasticsearch:9200
+parsedmarc /path/to/reports/*
+```
+
+### Docker Compose example
+
+```yaml
+services:
+  parsedmarc:
+    image: parsedmarc:latest
+    environment:
+      PARSEDMARC_IMAP_HOST: imap.example.com
+      PARSEDMARC_IMAP_USER: dmarc@example.com
+      PARSEDMARC_IMAP_PASSWORD: ${IMAP_PASSWORD}
+      PARSEDMARC_MAILBOX_WATCH: "true"
+      PARSEDMARC_ELASTICSEARCH_HOSTS: http://elasticsearch:9200
+      PARSEDMARC_GENERAL_SAVE_AGGREGATE: "true"
+      PARSEDMARC_GENERAL_SAVE_FORENSIC: "true"
+```
+
+### Section name mapping
+
+For sections with underscores in the name, the full section name is used:
+
+| Section          | Env var prefix                |
+|------------------|-------------------------------|
+| `general`        | `PARSEDMARC_GENERAL_`         |
+| `mailbox`        | `PARSEDMARC_MAILBOX_`         |
+| `imap`           | `PARSEDMARC_IMAP_`            |
+| `msgraph`        | `PARSEDMARC_MSGRAPH_`         |
+| `elasticsearch`  | `PARSEDMARC_ELASTICSEARCH_`   |
+| `opensearch`     | `PARSEDMARC_OPENSEARCH_`      |
+| `splunk_hec`     | `PARSEDMARC_SPLUNK_HEC_`      |
+| `kafka`          | `PARSEDMARC_KAFKA_`           |
+| `smtp`           | `PARSEDMARC_SMTP_`            |
+| `s3`             | `PARSEDMARC_S3_`              |
+| `syslog`         | `PARSEDMARC_SYSLOG_`          |
+| `gmail_api`      | `PARSEDMARC_GMAIL_API_`       |
+| `maildir`        | `PARSEDMARC_MAILDIR_`         |
+| `log_analytics`  | `PARSEDMARC_LOG_ANALYTICS_`   |
+| `gelf`           | `PARSEDMARC_GELF_`            |
+| `webhook`        | `PARSEDMARC_WEBHOOK_`         |
+
 ## Performance tuning
 
 For large mailbox imports or backfills, parsedmarc can consume a noticeable amount

parsedmarc/__init__.py

@@ -1957,7 +1957,7 @@ def get_dmarc_reports_from_mailbox(
         elif isinstance(connection, MSGraphConnection):
             since = (
                 datetime.now(timezone.utc) - timedelta(minutes=_since)
-            ).isoformat() + "Z"
+            ).isoformat()
             current_time = datetime.now(timezone.utc).isoformat() + "Z"
         elif isinstance(connection, GmailConnection):
             since = (datetime.now(timezone.utc) - timedelta(minutes=_since)).strftime(

parsedmarc/cli.py

@@ -75,6 +75,79 @@ def _str_to_list(s):
     return list(map(lambda i: i.lstrip(), _list))
 
 
+# All known INI config section names, used for env var resolution.
+_KNOWN_SECTIONS = frozenset(
+    {
+        "general",
+        "mailbox",
+        "imap",
+        "msgraph",
+        "elasticsearch",
+        "opensearch",
+        "splunk_hec",
+        "kafka",
+        "smtp",
+        "s3",
+        "syslog",
+        "gmail_api",
+        "maildir",
+        "log_analytics",
+        "gelf",
+        "webhook",
+    }
+)
+
+
+def _resolve_section_key(suffix: str) -> tuple:
+    """Resolve an env var suffix like ``IMAP_PASSWORD`` to ``('imap', 'password')``.
+
+    Uses longest-prefix matching against known section names so that
+    multi-word sections like ``splunk_hec`` are handled correctly.
+
+    Returns ``(None, None)`` when no known section matches.
+    """
+    suffix_lower = suffix.lower()
+
+    best_section = None
+    best_key = None
+    for section in _KNOWN_SECTIONS:
+        section_prefix = section + "_"
+        if suffix_lower.startswith(section_prefix):
+            key = suffix_lower[len(section_prefix) :]
+            if key and (best_section is None or len(section) > len(best_section)):
+                best_section = section
+                best_key = key
+
+    return best_section, best_key
+
+
+def _apply_env_overrides(config: ConfigParser) -> None:
+    """Inject ``PARSEDMARC_*`` environment variables into *config*.
+
+    Environment variables matching ``PARSEDMARC_{SECTION}_{KEY}`` override
+    (or create) the corresponding config-file values.  Sections are created
+    automatically when they do not yet exist.
+    """
+    prefix = "PARSEDMARC_"
+
+    for env_key, env_value in os.environ.items():
+        if not env_key.startswith(prefix) or env_key == "PARSEDMARC_CONFIG_FILE":
+            continue
+
+        suffix = env_key[len(prefix) :]
+        section, key = _resolve_section_key(suffix)
+
+        if section is None:
+            logger.debug("Ignoring unrecognized env var: %s", env_key)
+            continue
+
+        if not config.has_section(section):
+            config.add_section(section)
+
+        config.set(section, key, env_value)
+        logger.debug("Config override from env: [%s] %s", section, key)
+
+
 def _configure_logging(log_level, log_file=None):
     """
     Configure logging for the current process.
@@ -178,12 +251,39 @@ class ConfigurationError(Exception):
     pass
 
 
-def _parse_config_file(config_file, opts):
-    """Parse a config file and update opts in place.
+def _load_config(config_file: str | None = None) -> ConfigParser:
+    """Load configuration from an INI file and/or environment variables.
 
     Args:
-        config_file: Path to the .ini config file
-        opts: Namespace object to update with parsed values
+        config_file: Optional path to an .ini config file.
+
+    Returns:
+        A ``ConfigParser`` populated from the file (if given) and from any
+        ``PARSEDMARC_*`` environment variables.
+
+    Raises:
+        ConfigurationError: If *config_file* is given but does not exist.
+    """
+    config = ConfigParser()
+    if config_file is not None:
+        abs_path = os.path.abspath(config_file)
+        if not os.path.exists(abs_path):
+            raise ConfigurationError("A file does not exist at {0}".format(abs_path))
+        if not os.access(abs_path, os.R_OK):
+            raise ConfigurationError(
+                "Unable to read {0} — check file permissions".format(abs_path)
+            )
+        config.read(config_file)
+    _apply_env_overrides(config)
+    return config
+
+
+def _parse_config(config: ConfigParser, opts):
+    """Apply a loaded ``ConfigParser`` to *opts* in place.
+
+    Args:
+        config: A ``ConfigParser`` (from ``_load_config``).
+        opts: Namespace object to update with parsed values.
 
     Returns:
         index_prefix_domain_map or None
@@ -191,13 +291,8 @@ def _parse_config_file(config_file, opts):
     Raises:
         ConfigurationError: If required settings are missing or invalid.
     """
-    abs_path = os.path.abspath(config_file)
-    if not os.path.exists(abs_path):
-        raise ConfigurationError("A file does not exist at {0}".format(abs_path))
     opts.silent = True
-    config = ConfigParser()
     index_prefix_domain_map = None
-    config.read(config_file)
     if "general" in config.sections():
         general_config = config["general"]
         if "silent" in general_config:
@@ -1683,9 +1778,16 @@ def _main():
 
     index_prefix_domain_map = None
 
-    if args.config_file:
+    config_file = args.config_file or os.environ.get("PARSEDMARC_CONFIG_FILE")
+    has_env_config = any(
+        k.startswith("PARSEDMARC_") and k != "PARSEDMARC_CONFIG_FILE"
+        for k in os.environ
+    )
+
+    if config_file or has_env_config:
         try:
-            index_prefix_domain_map = _parse_config_file(args.config_file, opts)
+            config = _load_config(config_file)
+            index_prefix_domain_map = _parse_config(config, opts)
         except ConfigurationError as e:
             logger.critical(str(e))
             exit(-1)
@@ -2102,9 +2204,8 @@ def _main():
                 # Build a fresh opts starting from CLI-only defaults so that
                 # sections removed from the config file actually take effect.
                 new_opts = Namespace(**vars(opts_from_cli))
-                new_index_prefix_domain_map = _parse_config_file(
-                    args.config_file, new_opts
-                )
+                new_config = _load_config(config_file)
+                new_index_prefix_domain_map = _parse_config(new_config, new_opts)
                 new_clients = _init_output_clients(new_opts)
 
                 # All steps succeeded — commit the changes atomically.

parsedmarc/constants.py

@@ -1,3 +1,3 @@
-__version__ = "9.4.0"
+__version__ = "9.5.1"
 
 USER_AGENT = f"parsedmarc/{__version__}"

tests.py

@@ -11,6 +11,7 @@ import sys
 import tempfile
 import unittest
 from base64 import urlsafe_b64encode
+from configparser import ConfigParser
 from glob import glob
 from pathlib import Path
 from tempfile import NamedTemporaryFile, TemporaryDirectory
@@ -1985,7 +1986,8 @@ watch = true
         "SIGHUP not available on this platform",
     )
     @patch("parsedmarc.cli._init_output_clients")
-    @patch("parsedmarc.cli._parse_config_file")
+    @patch("parsedmarc.cli._parse_config")
+    @patch("parsedmarc.cli._load_config")
     @patch("parsedmarc.cli.get_dmarc_reports_from_mailbox")
     @patch("parsedmarc.cli.watch_inbox")
     @patch("parsedmarc.cli.IMAPConnection")
@@ -1994,6 +1996,7 @@ watch = true
         mock_imap,
         mock_watch,
         mock_get_reports,
+        mock_load_config,
         mock_parse_config,
         mock_init_clients,
     ):
@@ -2007,7 +2010,9 @@ watch = true
             "smtp_tls_reports": [],
         }
 
-        def parse_side_effect(config_file, opts):
+        mock_load_config.return_value = ConfigParser()
+
+        def parse_side_effect(config, opts):
             opts.imap_host = "imap.example.com"
             opts.imap_user = "user"
             opts.imap_password = "pass"
@@ -2046,7 +2051,7 @@ watch = true
         self.assertEqual(cm.exception.code, 1)
         # watch_inbox was called twice: initial run + after reload
         self.assertEqual(mock_watch.call_count, 2)
-        # _parse_config_file called for initial load + reload
+        # _parse_config called for initial load + reload
         self.assertGreaterEqual(mock_parse_config.call_count, 2)
 
     @unittest.skipUnless(
@@ -2054,7 +2059,8 @@ watch = true
         "SIGHUP not available on this platform",
     )
     @patch("parsedmarc.cli._init_output_clients")
-    @patch("parsedmarc.cli._parse_config_file")
+    @patch("parsedmarc.cli._parse_config")
+    @patch("parsedmarc.cli._load_config")
     @patch("parsedmarc.cli.get_dmarc_reports_from_mailbox")
     @patch("parsedmarc.cli.watch_inbox")
     @patch("parsedmarc.cli.IMAPConnection")
@@ -2063,6 +2069,7 @@ watch = true
         mock_imap,
         mock_watch,
         mock_get_reports,
+        mock_load_config,
         mock_parse_config,
         mock_init_clients,
     ):
@@ -2076,11 +2083,13 @@ watch = true
             "smtp_tls_reports": [],
         }
 
+        mock_load_config.return_value = ConfigParser()
+
         # Initial parse sets required opts; reload parse raises
         initial_map = {"prefix_": ["example.com"]}
         call_count = [0]
 
-        def parse_side_effect(config_file, opts):
+        def parse_side_effect(config, opts):
             call_count[0] += 1
             opts.imap_host = "imap.example.com"
             opts.imap_user = "user"
@@ -2130,7 +2139,8 @@ watch = true
         "SIGHUP not available on this platform",
     )
     @patch("parsedmarc.cli._init_output_clients")
-    @patch("parsedmarc.cli._parse_config_file")
+    @patch("parsedmarc.cli._parse_config")
+    @patch("parsedmarc.cli._load_config")
     @patch("parsedmarc.cli.get_dmarc_reports_from_mailbox")
     @patch("parsedmarc.cli.watch_inbox")
     @patch("parsedmarc.cli.IMAPConnection")
@@ -2139,6 +2149,7 @@ watch = true
         mock_imap,
         mock_watch,
         mock_get_reports,
+        mock_load_config,
         mock_parse_config,
         mock_init_clients,
     ):
@@ -2152,7 +2163,9 @@ watch = true
             "smtp_tls_reports": [],
         }
 
-        def parse_side_effect(config_file, opts):
+        mock_load_config.return_value = ConfigParser()
+
+        def parse_side_effect(config, opts):
             opts.imap_host = "imap.example.com"
             opts.imap_user = "user"
             opts.imap_password = "pass"
@@ -2284,7 +2297,8 @@ watch = true
         "SIGHUP not available on this platform",
     )
     @patch("parsedmarc.cli._init_output_clients")
-    @patch("parsedmarc.cli._parse_config_file")
+    @patch("parsedmarc.cli._parse_config")
+    @patch("parsedmarc.cli._load_config")
     @patch("parsedmarc.cli.get_dmarc_reports_from_mailbox")
     @patch("parsedmarc.cli.watch_inbox")
     @patch("parsedmarc.cli.IMAPConnection")
@@ -2293,6 +2307,7 @@ watch = true
         mock_imap,
         mock_watch,
         mock_get_reports,
+        mock_load_config,
         mock_parse_config,
         mock_init_clients,
     ):
@@ -2308,7 +2323,9 @@ watch = true
             "smtp_tls_reports": [],
         }
 
-        def parse_side_effect(config_file, opts):
+        mock_load_config.return_value = ConfigParser()
+
+        def parse_side_effect(config, opts):
             opts.imap_host = "imap.example.com"
             opts.imap_user = "user"
             opts.imap_password = "pass"
@@ -2474,5 +2491,191 @@ password = test-password
         self.assertNotIn("unmapped-1", report_ids)
 
 
+class TestEnvVarConfig(unittest.TestCase):
+    """Tests for environment variable configuration support."""
+
+    def test_resolve_section_key_simple(self):
+        """Simple section names resolve correctly."""
+        from parsedmarc.cli import _resolve_section_key
+
+        self.assertEqual(_resolve_section_key("IMAP_PASSWORD"), ("imap", "password"))
+        self.assertEqual(_resolve_section_key("GENERAL_DEBUG"), ("general", "debug"))
+        self.assertEqual(_resolve_section_key("S3_BUCKET"), ("s3", "bucket"))
+        self.assertEqual(_resolve_section_key("GELF_HOST"), ("gelf", "host"))
+
+    def test_resolve_section_key_underscore_sections(self):
+        """Multi-word section names (splunk_hec, gmail_api, etc.) resolve correctly."""
+        from parsedmarc.cli import _resolve_section_key
+
+        self.assertEqual(
+            _resolve_section_key("SPLUNK_HEC_TOKEN"), ("splunk_hec", "token")
+        )
+        self.assertEqual(
+            _resolve_section_key("GMAIL_API_CREDENTIALS_FILE"),
+            ("gmail_api", "credentials_file"),
+        )
+        self.assertEqual(
+            _resolve_section_key("LOG_ANALYTICS_CLIENT_ID"),
+            ("log_analytics", "client_id"),
+        )
+
+    def test_resolve_section_key_unknown(self):
+        """Unknown prefixes return (None, None)."""
+        from parsedmarc.cli import _resolve_section_key
+
+        self.assertEqual(_resolve_section_key("UNKNOWN_FOO"), (None, None))
+        # Just a section name with no key should not match
+        self.assertEqual(_resolve_section_key("IMAP"), (None, None))
+
+    def test_apply_env_overrides_injects_values(self):
+        """Env vars are injected into an existing ConfigParser."""
+        from configparser import ConfigParser
+        from parsedmarc.cli import _apply_env_overrides
+
+        config = ConfigParser()
+        config.add_section("imap")
+        config.set("imap", "host", "original.example.com")
+
+        env = {
+            "PARSEDMARC_IMAP_HOST": "new.example.com",
+            "PARSEDMARC_IMAP_PASSWORD": "secret123",
+        }
+        with patch.dict(os.environ, env, clear=False):
+            _apply_env_overrides(config)
+
+        self.assertEqual(config.get("imap", "host"), "new.example.com")
+        self.assertEqual(config.get("imap", "password"), "secret123")
+
+    def test_apply_env_overrides_creates_sections(self):
+        """Env vars create new sections when they don't exist."""
+        from configparser import ConfigParser
+        from parsedmarc.cli import _apply_env_overrides
+
+        config = ConfigParser()
+
+        env = {"PARSEDMARC_ELASTICSEARCH_HOSTS": "http://localhost:9200"}
+        with patch.dict(os.environ, env, clear=False):
+            _apply_env_overrides(config)
+
+        self.assertTrue(config.has_section("elasticsearch"))
+        self.assertEqual(config.get("elasticsearch", "hosts"), "http://localhost:9200")
+
+    def test_apply_env_overrides_ignores_config_file_var(self):
+        """PARSEDMARC_CONFIG_FILE is not injected as a config key."""
+        from configparser import ConfigParser
+        from parsedmarc.cli import _apply_env_overrides
+
+        config = ConfigParser()
+
+        env = {"PARSEDMARC_CONFIG_FILE": "/some/path.ini"}
+        with patch.dict(os.environ, env, clear=False):
+            _apply_env_overrides(config)
+
+        self.assertEqual(config.sections(), [])
+
+    def test_load_config_with_file_and_env_override(self):
+        """Env vars override values from an INI file."""
+        from parsedmarc.cli import _load_config
+
+        with NamedTemporaryFile(mode="w", suffix=".ini", delete=False) as f:
+            f.write(
+                "[imap]\nhost = file.example.com\nuser = alice\npassword = fromfile\n"
+            )
+            f.flush()
+            config_path = f.name
+
+        try:
+            env = {"PARSEDMARC_IMAP_PASSWORD": "fromenv"}
+            with patch.dict(os.environ, env, clear=False):
+                config = _load_config(config_path)
+
+            self.assertEqual(config.get("imap", "host"), "file.example.com")
+            self.assertEqual(config.get("imap", "user"), "alice")
+            self.assertEqual(config.get("imap", "password"), "fromenv")
+        finally:
+            os.unlink(config_path)
+
+    def test_load_config_env_only(self):
+        """Config can be loaded purely from env vars with no file."""
+        from parsedmarc.cli import _load_config
+
+        env = {
+            "PARSEDMARC_GENERAL_DEBUG": "true",
+            "PARSEDMARC_ELASTICSEARCH_HOSTS": "http://localhost:9200",
+        }
+        with patch.dict(os.environ, env, clear=False):
+            config = _load_config(None)
+
+        self.assertEqual(config.get("general", "debug"), "true")
+        self.assertEqual(config.get("elasticsearch", "hosts"), "http://localhost:9200")
+
+    def test_parse_config_from_env(self):
+        """Full round-trip: env vars -> ConfigParser -> opts."""
+        from argparse import Namespace
+        from parsedmarc.cli import _load_config, _parse_config
+
+        env = {
+            "PARSEDMARC_GENERAL_DEBUG": "true",
+            "PARSEDMARC_GENERAL_SAVE_AGGREGATE": "true",
+            "PARSEDMARC_GENERAL_OFFLINE": "true",
+        }
+        with patch.dict(os.environ, env, clear=False):
+            config = _load_config(None)
+
+        opts = Namespace()
+        _parse_config(config, opts)
+
+        self.assertTrue(opts.debug)
+        self.assertTrue(opts.save_aggregate)
+        self.assertTrue(opts.offline)
+
+    def test_config_file_env_var(self):
+        """PARSEDMARC_CONFIG_FILE env var specifies the config file path."""
+        from argparse import Namespace
+        from parsedmarc.cli import _load_config, _parse_config
+
+        with NamedTemporaryFile(mode="w", suffix=".ini", delete=False) as f:
+            f.write("[general]\ndebug = true\noffline = true\n")
+            f.flush()
+            config_path = f.name
+
+        try:
+            env = {"PARSEDMARC_CONFIG_FILE": config_path}
+            with patch.dict(os.environ, env, clear=False):
+                config = _load_config(os.environ.get("PARSEDMARC_CONFIG_FILE"))
+
+            opts = Namespace()
+            _parse_config(config, opts)
+            self.assertTrue(opts.debug)
+            self.assertTrue(opts.offline)
+        finally:
+            os.unlink(config_path)
+
+    def test_boolean_values_from_env(self):
+        """Various boolean string representations work through ConfigParser."""
+        from configparser import ConfigParser
+        from parsedmarc.cli import _apply_env_overrides
+
+        for true_val in ("true", "yes", "1", "on", "True", "YES"):
+            config = ConfigParser()
+            env = {"PARSEDMARC_GENERAL_DEBUG": true_val}
+            with patch.dict(os.environ, env, clear=False):
+                _apply_env_overrides(config)
+            self.assertTrue(
+                config.getboolean("general", "debug"),
+                f"Expected truthy for {true_val!r}",
+            )
+
+        for false_val in ("false", "no", "0", "off", "False", "NO"):
+            config = ConfigParser()
+            env = {"PARSEDMARC_GENERAL_DEBUG": false_val}
+            with patch.dict(os.environ, env, clear=False):
+                _apply_env_overrides(config)
+            self.assertFalse(
+                config.getboolean("general", "debug"),
+                f"Expected falsy for {false_val!r}",
+            )
+
+
 if __name__ == "__main__":
     unittest.main(verbosity=2)