diff --git a/AGENTS.md b/AGENTS.md index 39b9560b..c2a79dc6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -42,7 +42,7 @@ To skip DNS lookups during testing, set `GITHUB_ACTIONS=true`. ### Key modules - `parsedmarc/__init__.py` — Core parsing logic. Main functions: `parse_report_file()`, `parse_report_email()`, `parse_aggregate_report_xml()`, `parse_forensic_report()`, `parse_smtp_tls_report_json()`, `get_dmarc_reports_from_mailbox()`, `watch_inbox()` -- `parsedmarc/cli.py` — CLI entry point (`_main`), config file parsing, output orchestration +- `parsedmarc/cli.py` — CLI entry point (`_main`), config file parsing (`_load_config` + `_parse_config`), output orchestration. Supports configuration via INI files, `PARSEDMARC_{SECTION}_{KEY}` environment variables, or both (env vars override file values). - `parsedmarc/types.py` — TypedDict definitions for all report types (`AggregateReport`, `ForensicReport`, `SMTPTLSReport`, `ParsingResults`) - `parsedmarc/utils.py` — IP/DNS/GeoIP enrichment, base64 decoding, compression handling - `parsedmarc/mail/` — Polymorphic mail connections: `IMAPConnection`, `GmailConnection`, `MSGraphConnection`, `MaildirConnection` @@ -52,6 +52,10 @@ To skip DNS lookups during testing, set `GITHUB_ACTIONS=true`. `ReportType = Literal["aggregate", "forensic", "smtp_tls"]`. Exception hierarchy: `ParserError` → `InvalidDMARCReport` → `InvalidAggregateReport`/`InvalidForensicReport`, and `InvalidSMTPTLSReport`. +### Configuration + +Config priority: CLI args > env vars > config file > defaults. Env var naming: `PARSEDMARC_{SECTION}_{KEY}` (e.g. `PARSEDMARC_IMAP_PASSWORD`). Section names with underscores use longest-prefix matching (`PARSEDMARC_SPLUNK_HEC_TOKEN` → `[splunk_hec] token`). Some INI keys have short aliases for env var friendliness (e.g. `[maildir] create` for `maildir_create`). File path values are expanded via `os.path.expanduser`/`os.path.expandvars`. Config can be loaded purely from env vars with no file (`PARSEDMARC_CONFIG_FILE` sets the file path). + ### Caching IP address info cached for 4 hours, seen aggregate report IDs cached for 1 hour (via `ExpiringDict`). @@ -62,3 +66,6 @@ IP address info cached for 4 hours, seen aggregate report IDs cached for 1 hour - TypedDict for structured data, type hints throughout - Python ≥3.10 required - Tests are in a single `tests.py` file using unittest; sample reports live in `samples/` +- File path config values must be wrapped with `_expand_path()` in `cli.py` +- Maildir UID checks are intentionally relaxed (warn, don't crash) for Docker compatibility +- Token file writes must create parent directories before opening for write diff --git a/CHANGELOG.md b/CHANGELOG.md index 512f8a19..d9e628e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## 9.5.4 + +### Fixed + +- Maildir `fetch_messages` now respects the `reports_folder` argument. Previously it always read from the top-level Maildir, ignoring the configured reports folder. `fetch_message`, `delete_message`, and `move_message` now also operate on the correct active folder. +- Config key aliases for env var compatibility: `[maildir] create` and `path` are now accepted as aliases for `maildir_create` and `maildir_path`, and `[msgraph] url` for `graph_url`. This allows natural env var names like `PARSEDMARC_MAILDIR_CREATE` to work without the redundant `PARSEDMARC_MAILDIR_MAILDIR_CREATE`. + ## 9.5.3 ### Fixed diff --git a/parsedmarc/cli.py b/parsedmarc/cli.py index 47915752..ab107549 100644 --- a/parsedmarc/cli.py +++ b/parsedmarc/cli.py @@ -581,6 +581,8 @@ def _parse_config(config: ConfigParser, opts): if "graph_url" in graph_config: opts.graph_url = graph_config["graph_url"] + elif "url" in graph_config: + opts.graph_url = graph_config["url"] if "allow_unencrypted_storage" in graph_config: opts.graph_allow_unencrypted_storage = bool( @@ -884,10 +886,15 @@ def _parse_config(config: ConfigParser, opts): if "maildir" in config.sections(): maildir_api_config = config["maildir"] - maildir_p = maildir_api_config.get("maildir_path") + maildir_p = maildir_api_config.get( + "maildir_path", maildir_api_config.get("path") + ) opts.maildir_path = _expand_path(maildir_p) if maildir_p else maildir_p opts.maildir_create = bool( - maildir_api_config.getboolean("maildir_create", fallback=False) + maildir_api_config.getboolean( + "maildir_create", + fallback=maildir_api_config.getboolean("create", fallback=False), + ) ) if "log_analytics" in config.sections(): diff --git a/parsedmarc/constants.py b/parsedmarc/constants.py index 50c75d88..91a01fe0 100644 --- a/parsedmarc/constants.py +++ b/parsedmarc/constants.py @@ -1,3 +1,3 @@ -__version__ = "9.5.3" +__version__ = "9.5.4" USER_AGENT = f"parsedmarc/{__version__}" diff --git a/parsedmarc/mail/maildir.py b/parsedmarc/mail/maildir.py index 453cc9a9..9f075e0b 100644 --- a/parsedmarc/mail/maildir.py +++ b/parsedmarc/mail/maildir.py @@ -46,16 +46,27 @@ class MaildirConnection(MailboxConnection): for subdir in ("cur", "new", "tmp"): os.makedirs(os.path.join(maildir_path, subdir), exist_ok=True) self._client = mailbox.Maildir(maildir_path, create=maildir_create) + self._active_folder: mailbox.Maildir = self._client self._subfolder_client: Dict[str, mailbox.Maildir] = {} + def _get_folder(self, folder_name: str) -> mailbox.Maildir: + """Return a cached subfolder handle, creating it if needed.""" + if folder_name not in self._subfolder_client: + self._subfolder_client[folder_name] = self._client.add_folder(folder_name) + return self._subfolder_client[folder_name] + def create_folder(self, folder_name: str): - self._subfolder_client[folder_name] = self._client.add_folder(folder_name) + self._get_folder(folder_name) def fetch_messages(self, reports_folder: str, **kwargs): - return self._client.keys() + if reports_folder and reports_folder != "INBOX": + self._active_folder = self._get_folder(reports_folder) + else: + self._active_folder = self._client + return self._active_folder.keys() def fetch_message(self, message_id: str) -> str: - msg = self._client.get(message_id) + msg = self._active_folder.get(message_id) if msg is not None: msg = msg.as_string() if msg is not None: @@ -63,16 +74,15 @@ class MaildirConnection(MailboxConnection): return "" def delete_message(self, message_id: str): - self._client.remove(message_id) + self._active_folder.remove(message_id) def move_message(self, message_id: str, folder_name: str): - message_data = self._client.get(message_id) + message_data = self._active_folder.get(message_id) if message_data is None: return - if folder_name not in self._subfolder_client: - self._subfolder_client[folder_name] = self._client.add_folder(folder_name) - self._subfolder_client[folder_name].add(message_data) - self._client.remove(message_id) + dest = self._get_folder(folder_name) + dest.add(message_data) + self._active_folder.remove(message_id) def keepalive(self): return diff --git a/tests.py b/tests.py index a99dccc6..d8e22d9c 100755 --- a/tests.py +++ b/tests.py @@ -2566,6 +2566,164 @@ class TestMaildirConnection(unittest.TestCase): self.assertEqual(len(conn._subfolder_client["archive"].keys()), 1) +class TestMaildirReportsFolder(unittest.TestCase): + """Tests for Maildir reports_folder support in fetch_messages.""" + + def test_fetch_from_subfolder(self): + """fetch_messages with a subfolder name reads from that subfolder.""" + from parsedmarc.mail.maildir import MaildirConnection + + with TemporaryDirectory() as d: + conn = MaildirConnection(d, maildir_create=True) + + # Add message to a subfolder + subfolder = conn._client.add_folder("reports") + msg_key = subfolder.add("From: test@example.com\n\nSubfolder msg") + + # Root should be empty + self.assertEqual(conn.fetch_messages("INBOX"), []) + + # Subfolder should have the message + keys = conn.fetch_messages("reports") + self.assertIn(msg_key, keys) + + def test_fetch_message_uses_active_folder(self): + """fetch_message reads from the folder set by fetch_messages.""" + from parsedmarc.mail.maildir import MaildirConnection + + with TemporaryDirectory() as d: + conn = MaildirConnection(d, maildir_create=True) + + subfolder = conn._client.add_folder("reports") + msg_key = subfolder.add("From: sub@example.com\n\nIn subfolder") + + conn.fetch_messages("reports") + content = conn.fetch_message(msg_key) + self.assertIn("sub@example.com", content) + + def test_delete_message_uses_active_folder(self): + """delete_message removes from the folder set by fetch_messages.""" + from parsedmarc.mail.maildir import MaildirConnection + + with TemporaryDirectory() as d: + conn = MaildirConnection(d, maildir_create=True) + + subfolder = conn._client.add_folder("reports") + msg_key = subfolder.add("From: del@example.com\n\nDelete me") + + conn.fetch_messages("reports") + conn.delete_message(msg_key) + self.assertEqual(conn.fetch_messages("reports"), []) + + def test_move_message_from_subfolder(self): + """move_message works when active folder is a subfolder.""" + from parsedmarc.mail.maildir import MaildirConnection + + with TemporaryDirectory() as d: + conn = MaildirConnection(d, maildir_create=True) + + subfolder = conn._client.add_folder("reports") + msg_key = subfolder.add("From: move@example.com\n\nMove me") + + conn.fetch_messages("reports") + conn.move_message(msg_key, "archive") + + # Source should be empty + self.assertEqual(conn.fetch_messages("reports"), []) + # Destination should have the message + archive_keys = conn.fetch_messages("archive") + self.assertEqual(len(archive_keys), 1) + + def test_inbox_reads_root(self): + """INBOX reads from the top-level Maildir.""" + from parsedmarc.mail.maildir import MaildirConnection + + with TemporaryDirectory() as d: + conn = MaildirConnection(d, maildir_create=True) + + msg_key = conn._client.add("From: root@example.com\n\nRoot msg") + + keys = conn.fetch_messages("INBOX") + self.assertIn(msg_key, keys) + + def test_empty_folder_reads_root(self): + """Empty string reports_folder reads from the top-level Maildir.""" + from parsedmarc.mail.maildir import MaildirConnection + + with TemporaryDirectory() as d: + conn = MaildirConnection(d, maildir_create=True) + + msg_key = conn._client.add("From: root@example.com\n\nRoot msg") + + keys = conn.fetch_messages("") + self.assertIn(msg_key, keys) + + +class TestConfigAliases(unittest.TestCase): + """Tests for config key aliases (env var friendly short names).""" + + def test_maildir_create_alias(self): + """[maildir] create works as alias for maildir_create.""" + from argparse import Namespace + from parsedmarc.cli import _load_config, _parse_config + + env = { + "PARSEDMARC_MAILDIR_CREATE": "true", + "PARSEDMARC_MAILDIR_PATH": "/tmp/test", + } + with patch.dict(os.environ, env, clear=False): + config = _load_config(None) + opts = Namespace() + _parse_config(config, opts) + self.assertTrue(opts.maildir_create) + + def test_maildir_path_alias(self): + """[maildir] path works as alias for maildir_path.""" + from argparse import Namespace + from parsedmarc.cli import _load_config, _parse_config + + env = {"PARSEDMARC_MAILDIR_PATH": "/var/mail/dmarc"} + with patch.dict(os.environ, env, clear=False): + config = _load_config(None) + opts = Namespace() + _parse_config(config, opts) + self.assertEqual(opts.maildir_path, "/var/mail/dmarc") + + def test_msgraph_url_alias(self): + """[msgraph] url works as alias for graph_url.""" + from parsedmarc.cli import _load_config, _parse_config + from argparse import Namespace + + env = { + "PARSEDMARC_MSGRAPH_AUTH_METHOD": "ClientSecret", + "PARSEDMARC_MSGRAPH_CLIENT_ID": "test-id", + "PARSEDMARC_MSGRAPH_CLIENT_SECRET": "test-secret", + "PARSEDMARC_MSGRAPH_TENANT_ID": "test-tenant", + "PARSEDMARC_MSGRAPH_MAILBOX": "test@example.com", + "PARSEDMARC_MSGRAPH_URL": "https://custom.graph.example.com", + } + with patch.dict(os.environ, env, clear=False): + config = _load_config(None) + opts = Namespace() + _parse_config(config, opts) + self.assertEqual(opts.graph_url, "https://custom.graph.example.com") + + def test_original_keys_still_work(self): + """Original INI key names (maildir_create, maildir_path) still work.""" + from argparse import Namespace + from parsedmarc.cli import _parse_config + + config = ConfigParser(interpolation=None) + config.add_section("maildir") + config.set("maildir", "maildir_path", "/original/path") + config.set("maildir", "maildir_create", "true") + + opts = Namespace() + _parse_config(config, opts) + self.assertEqual(opts.maildir_path, "/original/path") + self.assertTrue(opts.maildir_create) + + class TestMaildirUidHandling(unittest.TestCase): """Tests for Maildir UID mismatch handling in Docker-like environments."""