Enhance mailbox connection watch method to support reload functionality

- Updated the `watch` method in `GmailConnection`, `MSGraphConnection`, `IMAPConnection`, `MaildirConnection`, and the abstract `MailboxConnection` class to accept an optional `should_reload` parameter. This allows the method to check if a reload is necessary and exit the loop if so.
- Modified related tests to accommodate the new method signature.
- Changed logger calls from `critical` to `error` for consistency in logging severity.
- Added a new settings file for Claude with specific permissions for testing and code checks.
This commit is contained in:
Sean Whalen
2026-03-20 15:00:21 -04:00
parent dd1a8fd461
commit dfdffe4947
11 changed files with 1042 additions and 874 deletions

16
.claude/settings.json Normal file
View File

@@ -0,0 +1,16 @@
{
"permissions": {
"allow": [
"Bash(python -c \"import py_compile; py_compile.compile\\(''parsedmarc/cli.py'', doraise=True\\)\")",
"Bash(ruff check:*)",
"Bash(ruff format:*)",
"Bash(GITHUB_ACTIONS=true pytest --cov tests.py)",
"Bash(ls tests*)",
"Bash(GITHUB_ACTIONS=true python -m pytest --cov tests.py -x)",
"Bash(GITHUB_ACTIONS=true python -m pytest tests.py -x -v)"
],
"additionalDirectories": [
"/tmp"
]
}
}

View File

@@ -1,5 +1,18 @@
# Changelog
## 9.3.0
### Added
- SIGHUP-based configuration reload for watch mode — update output
destinations, DNS/GeoIP settings, processing flags, and log level
without restarting the service or interrupting in-progress report
processing. Use `systemctl reload parsedmarc` when running under
systemd.
- Extracted `_parse_config_file()` and `_init_output_clients()` from
`_main()` in `cli.py` to support config reload and reduce code
duplication.
## 9.2.1
### Added

View File

@@ -404,6 +404,7 @@ The full set of configuration options are:
retry_attempts = 3
retry_delay = 5
```
- `gmail_api`
- `credentials_file` - str: Path to file containing the
credentials, None to disable (Default: `None`)
@@ -442,7 +443,7 @@ The full set of configuration options are:
- `dcr_smtp_tls_stream` - str: The stream name for the SMTP TLS reports in the DCR
:::{note}
Information regarding the setup of the Data Collection Rule can be found [here](https://learn.microsoft.com/en-us/azure/azure-monitor/logs/tutorial-logs-ingestion-portal).
Information regarding the setup of the Data Collection Rule can be found [in the Azure documentation](https://learn.microsoft.com/en-us/azure/azure-monitor/logs/tutorial-logs-ingestion-portal).
:::
- `gelf`
- `host` - str: The GELF server name or IP address
@@ -602,6 +603,7 @@ After=network.target network-online.target elasticsearch.service
[Service]
ExecStart=/opt/parsedmarc/venv/bin/parsedmarc -c /etc/parsedmarc.ini
ExecReload=/bin/kill -HUP $MAINPID
User=parsedmarc
Group=parsedmarc
Restart=always
@@ -634,6 +636,44 @@ sudo service parsedmarc restart
:::
### Reloading configuration without restarting
When running in watch mode, `parsedmarc` supports reloading its
configuration file without restarting the service or interrupting
report processing that is already in progress. Send a `SIGHUP` signal
to the process, or use `systemctl reload` if the unit file includes
the `ExecReload` line shown above:
```bash
sudo systemctl reload parsedmarc
```
The reload takes effect after the current batch of reports finishes
processing and all output operations (Elasticsearch, Kafka, S3, etc.)
for that batch have completed. The following settings are reloaded:
- All output destinations (Elasticsearch, OpenSearch, Kafka, S3,
Splunk, syslog, GELF, webhooks, Log Analytics)
- Multi-tenant index prefix domain map (`index_prefix_domain_map` —
the referenced YAML file is re-read on reload)
- DNS and GeoIP settings (`nameservers`, `dns_timeout`, `ip_db_path`,
`offline`, etc.)
- Processing flags (`strip_attachment_payloads`, `batch_size`,
`check_timeout`, etc.)
- Log level (`debug`, `verbose`, `warnings`, `silent`)
Mailbox connection settings (IMAP host/credentials, Microsoft Graph,
Gmail API, Maildir path) are **not** reloaded — changing those still
requires a full restart.
If the new configuration file contains errors, the reload is aborted
and the previous configuration remains active. Check the logs for
details:
```bash
journalctl -u parsedmarc.service -r
```
To check the status of the service, run:
```bash

View File

@@ -2195,6 +2195,7 @@ def watch_inbox(
batch_size: int = 10,
since: Optional[Union[datetime, date, str]] = None,
normalize_timespan_threshold_hours: float = 24,
should_reload: Optional[Callable] = None,
):
"""
Watches the mailbox for new messages and
@@ -2222,6 +2223,8 @@ def watch_inbox(
batch_size (int): Number of messages to read and process before saving
since: Search for messages since certain time
normalize_timespan_threshold_hours (float): Normalize timespans beyond this
should_reload: Optional callable that returns True when a config
reload has been requested (e.g. via SIGHUP)
"""
def check_callback(connection):
@@ -2246,7 +2249,11 @@ def watch_inbox(
)
callback(res)
mailbox_connection.watch(check_callback=check_callback, check_timeout=check_timeout)
mailbox_connection.watch(
check_callback=check_callback,
check_timeout=check_timeout,
should_reload=should_reload,
)
def append_json(

File diff suppressed because it is too large Load Diff

View File

@@ -175,11 +175,13 @@ class GmailConnection(MailboxConnection):
# Not needed
pass
def watch(self, check_callback, check_timeout):
def watch(self, check_callback, check_timeout, should_reload=None):
"""Checks the mailbox for new messages every n seconds"""
while True:
sleep(check_timeout)
check_callback(self)
if should_reload and should_reload():
return
@lru_cache(maxsize=10)
def _find_label_id_for_label(self, label_name: str) -> str:

View File

@@ -278,11 +278,13 @@ class MSGraphConnection(MailboxConnection):
# Not needed
pass
def watch(self, check_callback, check_timeout):
def watch(self, check_callback, check_timeout, should_reload=None):
"""Checks the mailbox for new messages every n seconds"""
while True:
sleep(check_timeout)
check_callback(self)
if should_reload and should_reload():
return
@lru_cache(maxsize=10)
def _find_folder_id_from_folder_path(self, folder_name: str) -> str:

View File

@@ -81,7 +81,7 @@ class IMAPConnection(MailboxConnection):
def keepalive(self):
self._client.noop()
def watch(self, check_callback, check_timeout):
def watch(self, check_callback, check_timeout, should_reload=None):
"""
Use an IDLE IMAP connection to parse incoming emails,
and pass the results to a callback function
@@ -111,3 +111,5 @@ class IMAPConnection(MailboxConnection):
except Exception as e:
logger.warning("IMAP connection error. {0}. Reconnecting...".format(e))
sleep(check_timeout)
if should_reload and should_reload():
return

View File

@@ -28,5 +28,5 @@ class MailboxConnection(ABC):
def keepalive(self):
raise NotImplementedError
def watch(self, check_callback, check_timeout):
def watch(self, check_callback, check_timeout, should_reload=None):
raise NotImplementedError

View File

@@ -63,10 +63,12 @@ class MaildirConnection(MailboxConnection):
def keepalive(self):
return
def watch(self, check_callback, check_timeout):
def watch(self, check_callback, check_timeout, should_reload=None):
while True:
try:
check_callback(self)
except Exception as e:
logger.warning("Maildir init error. {0}".format(e))
if should_reload and should_reload():
return
sleep(check_timeout)

View File

@@ -1277,7 +1277,7 @@ class TestMailboxWatchSince(unittest.TestCase):
def testWatchInboxPassesSinceToMailboxFetch(self):
mailbox_connection = SimpleNamespace()
def fake_watch(check_callback, check_timeout):
def fake_watch(check_callback, check_timeout, should_reload=None):
check_callback(mailbox_connection)
raise _BreakLoop()
@@ -1445,7 +1445,7 @@ mailbox = shared@example.com
parsedmarc.cli._main()
self.assertEqual(system_exit.exception.code, -1)
mock_logger.critical.assert_called_once_with(
mock_logger.error.assert_called_once_with(
"certificate_path setting missing from the msgraph config section"
)
mock_graph_connection.assert_not_called()
@@ -1517,7 +1517,7 @@ user = owner@example.com
parsedmarc.cli._main()
self.assertEqual(system_exit.exception.code, -1)
mock_logger.critical.assert_called_once_with(
mock_logger.error.assert_called_once_with(
"password setting missing from the msgraph config section"
)
mock_graph_connection.assert_not_called()
@@ -1674,7 +1674,7 @@ mailbox = shared@example.com
parsedmarc.cli._main()
self.assertEqual(system_exit.exception.code, -1)
mock_logger.critical.assert_called_once_with(
mock_logger.error.assert_called_once_with(
"client_secret setting missing from the msgraph config section"
)
mock_graph_connection.assert_not_called()
@@ -1706,7 +1706,7 @@ mailbox = shared@example.com
parsedmarc.cli._main()
self.assertEqual(system_exit.exception.code, -1)
mock_logger.critical.assert_called_once_with(
mock_logger.error.assert_called_once_with(
"tenant_id setting missing from the msgraph config section"
)
mock_graph_connection.assert_not_called()
@@ -1738,7 +1738,7 @@ tenant_id = tenant-id
parsedmarc.cli._main()
self.assertEqual(system_exit.exception.code, -1)
mock_logger.critical.assert_called_once_with(
mock_logger.error.assert_called_once_with(
"mailbox setting missing from the msgraph config section"
)
mock_graph_connection.assert_not_called()
@@ -1808,7 +1808,7 @@ mailbox = shared@example.com
parsedmarc.cli._main()
self.assertEqual(system_exit.exception.code, -1)
mock_logger.critical.assert_called_once_with(
mock_logger.error.assert_called_once_with(
"tenant_id setting missing from the msgraph config section"
)
mock_graph_connection.assert_not_called()
@@ -1839,7 +1839,7 @@ tenant_id = tenant-id
parsedmarc.cli._main()
self.assertEqual(system_exit.exception.code, -1)
mock_logger.critical.assert_called_once_with(
mock_logger.error.assert_called_once_with(
"mailbox setting missing from the msgraph config section"
)
mock_graph_connection.assert_not_called()
@@ -1871,7 +1871,7 @@ certificate_path = /tmp/msgraph-cert.pem
parsedmarc.cli._main()
self.assertEqual(system_exit.exception.code, -1)
mock_logger.critical.assert_called_once_with(
mock_logger.error.assert_called_once_with(
"tenant_id setting missing from the msgraph config section"
)
mock_graph_connection.assert_not_called()
@@ -1903,7 +1903,7 @@ certificate_path = /tmp/msgraph-cert.pem
parsedmarc.cli._main()
self.assertEqual(system_exit.exception.code, -1)
mock_logger.critical.assert_called_once_with(
mock_logger.error.assert_called_once_with(
"mailbox setting missing from the msgraph config section"
)
mock_graph_connection.assert_not_called()