mirror of
https://github.com/domainaware/parsedmarc.git
synced 2026-08-03 06:02:17 +00:00
* Centralize configuration handling with ParserConfig (#503) Add parsedmarc/config.py with ParserConfig, a frozen dataclass carrying every parsing/enrichment option plus the three shared caches (IP address info, seen aggregate report IDs, reverse DNS map). All eight public parsing/mailbox functions accept a keyword-only config= argument; when provided, the individual option keyword arguments are ignored in favor of the config's values, and every existing per-option keyword argument keeps working unchanged. The three hand-copied parse_kwargs dicts and the dns_timeout<->timeout rename chain are gone; the CLI builds one ParserConfig per run (rebuilt on SIGHUP) and passes it everywhere. Explicitly constructed configs own fresh isolated caches; dataclasses.replace() shares them; pickling drops cache contents and rebinds the unpickling process's module defaults, preserving the per-worker cache behavior of n_procs parallel parsing. The module globals IP_ADDRESS_CACHE / SEEN_AGGREGATE_REPORT_IDS / REVERSE_DNS_MAP remain, identity-preserved, as re-exports of the default caches. Bug fixes that ride along, each with a regression test: - One-shot mailbox runs now honor [general] dns_timeout/dns_retries; the CLI call site never forwarded dns_timeout, so the library's stray 6.0 default silently applied. - Lazily-triggered reverse DNS map loads (get_ip_address_info / get_service_from_reverse_dns_base_domain, including in n_procs workers) now thread psl_overrides_path/psl_overrides_url through to load_reverse_dns_map instead of clobbering operator-configured PSL overrides with the bundled defaults. - get_dmarc_reports_from_mailbox() and watch_inbox() dns_timeout defaults unified to DEFAULT_DNS_TIMEOUT (2.0s) from a stray 6.0, and normalize_timespan_threshold_hours to the float 24.0 used everywhere else. Closes #503 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * Address CI and review feedback on #851 - Wrap the IMAPConnection example in usage.md so ruff format is clean over the docs code blocks (CI runs ruff format --check on the whole repo; the local runs were scoped to parsedmarc/ and tests/ and missed it). - Fix the pre-existing "URL ro a reverse DNS map" docstring typo in get_service_from_reverse_dns_base_domain, caught by Copilot on the adjacent hunk. - Import parsedmarc.config once, as an aliased plain import, in tests/test_config.py instead of mixing import and import-from of the same module (flagged by code quality scanning); the aliased module import also keeps pyright able to resolve the submodule attribute access. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * Address second Copilot review round on #851 - Add parse_aggregate_report_file() to the library entry-point list in usage.md; the following paragraph describes the config= contract for "each of these functions", so the list must name all eight config-accepting entry points. - ParserConfig.__setstate__ now initializes every non-cache field to its class default before applying the pickled state, so a config serialized by an older parsedmarc version (whose state predates fields added later) unpickles with the newer fields at their defaults instead of unset entirely (__init__ never runs during unpickling, so an absent field would raise AttributeError on first access). Covered by a regression test that feeds __setstate__ a partial state dict. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
4f4733003c
commit
4e80047e68
@@ -1236,6 +1236,108 @@ since = 2d
|
||||
self.assertEqual(mock_watch_inbox.call_args.kwargs.get("since"), "2d")
|
||||
|
||||
|
||||
class TestCliParserConfigWiring(unittest.TestCase):
|
||||
"""Tests that _main() builds a single ParserConfig (via
|
||||
_build_parser_config) from parsed opts and passes it as ``config=`` to
|
||||
the library's mailbox-fetching functions, rather than forwarding
|
||||
individual option kwargs (offline, dns_timeout, ip_db_path, etc.) by
|
||||
hand at each call site."""
|
||||
|
||||
def setUp(self):
|
||||
from parsedmarc.log import logger as _logger
|
||||
|
||||
_logger.disabled = True
|
||||
self._stdout_patch = patch("sys.stdout", new_callable=io.StringIO)
|
||||
self._stderr_patch = patch("sys.stderr", new_callable=io.StringIO)
|
||||
self._stdout_patch.start()
|
||||
self._stderr_patch.start()
|
||||
# SEEN_AGGREGATE_REPORT_IDS is a module-level ExpiringDict shared
|
||||
# across tests in this process; clear it both ways so state from an
|
||||
# earlier test class doesn't leak in, and so this class doesn't leak
|
||||
# into a later one. Precedent: TestDirectoryFilePaths.setUp above.
|
||||
parsedmarc.SEEN_AGGREGATE_REPORT_IDS.clear()
|
||||
self.addCleanup(parsedmarc.SEEN_AGGREGATE_REPORT_IDS.clear)
|
||||
|
||||
def tearDown(self):
|
||||
from parsedmarc.log import logger as _logger
|
||||
|
||||
_logger.disabled = False
|
||||
self._stderr_patch.stop()
|
||||
self._stdout_patch.stop()
|
||||
|
||||
def _run_one_shot_mailbox(
|
||||
self, dns_timeout: float | None = None, dns_retries: int | None = None
|
||||
) -> parsedmarc.ParserConfig:
|
||||
"""Runs a real one-shot _main() against a mocked IMAP connection and
|
||||
mocked get_dmarc_reports_from_mailbox, and returns the ParserConfig
|
||||
the CLI passed as ``config=``."""
|
||||
config_lines = ["[general]", "silent = true"]
|
||||
if dns_timeout is not None:
|
||||
config_lines.append(f"dns_timeout = {dns_timeout}")
|
||||
if dns_retries is not None:
|
||||
config_lines.append(f"dns_retries = {dns_retries}")
|
||||
config_lines += [
|
||||
"",
|
||||
"[imap]",
|
||||
"host = imap.example.com",
|
||||
"user = user",
|
||||
"password = pass",
|
||||
]
|
||||
config_text = "\n".join(config_lines) + "\n"
|
||||
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".ini", delete=False) as cfg:
|
||||
cfg.write(config_text)
|
||||
cfg_path = cfg.name
|
||||
self.addCleanup(lambda: os.path.exists(cfg_path) and os.remove(cfg_path))
|
||||
|
||||
with (
|
||||
patch("parsedmarc.cli.get_dmarc_reports_from_mailbox") as mock_get_reports,
|
||||
patch("parsedmarc.cli.IMAPConnection") as mock_imap,
|
||||
):
|
||||
mock_imap.return_value = object()
|
||||
mock_get_reports.return_value = {
|
||||
"aggregate_reports": [],
|
||||
"failure_reports": [],
|
||||
"smtp_tls_reports": [],
|
||||
}
|
||||
with patch.object(sys, "argv", ["parsedmarc", "-c", cfg_path]):
|
||||
parsedmarc.cli._main()
|
||||
return mock_get_reports.call_args.kwargs["config"]
|
||||
|
||||
def test_one_shot_mailbox_run_honors_general_dns_timeout(self):
|
||||
"""Regression test for the one-shot mailbox call site in _main():
|
||||
before this fix, that call to get_dmarc_reports_from_mailbox
|
||||
omitted the dns_timeout/dns_retries kwargs entirely (and, before
|
||||
config= existed, had no way to pass them at all), so a one-shot run
|
||||
silently used the library's own hardcoded default
|
||||
(``dns_timeout: float = 6.0`` on master's
|
||||
get_dmarc_reports_from_mailbox signature) instead of the operator's
|
||||
``[general] dns_timeout`` / ``dns_retries`` config values. Watch-mode
|
||||
runs were unaffected because the watch_inbox call site did pass
|
||||
dns_timeout/dns_retries directly.
|
||||
"""
|
||||
cfg = self._run_one_shot_mailbox(dns_timeout=11.5, dns_retries=3)
|
||||
self.assertEqual(cfg.dns_timeout, 11.5)
|
||||
self.assertEqual(cfg.dns_retries, 3)
|
||||
|
||||
def test_cli_config_binds_module_default_caches(self):
|
||||
"""The ParserConfig built by the CLI must bind the process-wide
|
||||
default caches (parsedmarc.IP_ADDRESS_CACHE,
|
||||
parsedmarc.SEEN_AGGREGATE_REPORT_IDS, parsedmarc.REVERSE_DNS_MAP) by
|
||||
identity, not fresh/isolated caches — otherwise every CLI run would
|
||||
get its own empty caches (defeating the point of the 4-hour IP
|
||||
cache and the 1-hour dedup cache) even though ParserConfig's default
|
||||
factories exist specifically to give library callers isolated
|
||||
caches when they don't pass config=.
|
||||
"""
|
||||
cfg = self._run_one_shot_mailbox()
|
||||
self.assertIs(cfg.ip_address_cache, parsedmarc.IP_ADDRESS_CACHE)
|
||||
self.assertIs(
|
||||
cfg.seen_aggregate_report_ids, parsedmarc.SEEN_AGGREGATE_REPORT_IDS
|
||||
)
|
||||
self.assertIs(cfg.reverse_dns_map, parsedmarc.REVERSE_DNS_MAP)
|
||||
|
||||
|
||||
class TestMailboxPerformance(unittest.TestCase):
|
||||
def setUp(self):
|
||||
from parsedmarc.log import logger as _logger
|
||||
@@ -3035,6 +3137,91 @@ watch = true
|
||||
"Stale entry should have been cleared by reload",
|
||||
)
|
||||
|
||||
@unittest.skipUnless(
|
||||
hasattr(signal, "SIGHUP"),
|
||||
"SIGHUP not available on this platform",
|
||||
)
|
||||
@patch("parsedmarc.cli._init_output_clients")
|
||||
@patch("parsedmarc.cli._parse_config")
|
||||
@patch("parsedmarc.cli._load_config")
|
||||
@patch("parsedmarc.cli.get_dmarc_reports_from_mailbox")
|
||||
@patch("parsedmarc.cli.watch_inbox")
|
||||
@patch("parsedmarc.cli.IMAPConnection")
|
||||
def test_sighup_reload_rebuilds_parser_config(
|
||||
self,
|
||||
mock_imap,
|
||||
mock_watch,
|
||||
mock_get_reports,
|
||||
mock_load_config,
|
||||
mock_parse_config,
|
||||
mock_init_clients,
|
||||
):
|
||||
"""After a SIGHUP reload, the ParserConfig passed to watch_inbox as
|
||||
``config=`` must reflect the reloaded ``[general] dns_timeout``, not
|
||||
the value from the initial config load.
|
||||
|
||||
Guards against _build_parser_config(opts) being called only once at
|
||||
startup: opts itself is correctly refreshed in place by the existing
|
||||
``for k, v in vars(new_opts).items(): setattr(opts, k, v)`` loop, but
|
||||
parser_config is a separate ParserConfig snapshot built from opts —
|
||||
if the reload path forgot to rebuild it, watch_inbox would keep
|
||||
receiving the stale pre-reload config object forever.
|
||||
"""
|
||||
import signal as signal_module
|
||||
|
||||
mock_imap.return_value = object()
|
||||
mock_get_reports.return_value = {
|
||||
"aggregate_reports": [],
|
||||
"failure_reports": [],
|
||||
"smtp_tls_reports": [],
|
||||
}
|
||||
|
||||
mock_load_config.return_value = ConfigParser()
|
||||
|
||||
parse_calls = [0]
|
||||
|
||||
def parse_side_effect(config, opts):
|
||||
parse_calls[0] += 1
|
||||
opts.imap_host = "imap.example.com"
|
||||
opts.imap_user = "user"
|
||||
opts.imap_password = "pass"
|
||||
opts.mailbox_watch = True
|
||||
opts.dns_timeout = 5.0 if parse_calls[0] == 1 else 42.0
|
||||
return None
|
||||
|
||||
mock_parse_config.side_effect = parse_side_effect
|
||||
mock_init_clients.return_value = {}
|
||||
|
||||
watch_calls = [0]
|
||||
|
||||
def watch_side_effect(*args, **kwargs):
|
||||
watch_calls[0] += 1
|
||||
if watch_calls[0] == 1:
|
||||
if hasattr(signal_module, "SIGHUP"):
|
||||
import os
|
||||
|
||||
os.kill(os.getpid(), signal_module.SIGHUP)
|
||||
return
|
||||
else:
|
||||
raise FileExistsError("stop-watch-loop")
|
||||
|
||||
mock_watch.side_effect = watch_side_effect
|
||||
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".ini", delete=False) as cfg:
|
||||
cfg.write(self._BASE_CONFIG)
|
||||
cfg_path = cfg.name
|
||||
self.addCleanup(lambda: os.path.exists(cfg_path) and os.remove(cfg_path))
|
||||
|
||||
with patch.object(sys, "argv", ["parsedmarc", "-c", cfg_path]):
|
||||
with self.assertRaises(SystemExit):
|
||||
parsedmarc.cli._main()
|
||||
|
||||
self.assertEqual(mock_watch.call_count, 2)
|
||||
first_config = mock_watch.call_args_list[0].kwargs["config"]
|
||||
second_config = mock_watch.call_args_list[1].kwargs["config"]
|
||||
self.assertEqual(first_config.dns_timeout, 5.0)
|
||||
self.assertEqual(second_config.dns_timeout, 42.0)
|
||||
|
||||
|
||||
class TestSigtermShutdown(unittest.TestCase):
|
||||
"""Tests for graceful SIGTERM/SIGINT shutdown."""
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
"""Tests for parsedmarc.config"""
|
||||
|
||||
import dataclasses
|
||||
import pickle
|
||||
import unittest
|
||||
|
||||
import parsedmarc
|
||||
import parsedmarc.config as parsedmarc_config
|
||||
|
||||
|
||||
class TestParserConfigCaches(unittest.TestCase):
|
||||
"""Covers per-instance cache isolation for `ParserConfig`."""
|
||||
|
||||
def test_each_instance_gets_isolated_caches(self):
|
||||
"""Two independently constructed ParserConfig instances must never
|
||||
share cache objects with each other or with the module defaults, and
|
||||
mutating one instance's caches must not leak into the others."""
|
||||
cfg_a = parsedmarc_config.ParserConfig()
|
||||
cfg_b = parsedmarc_config.ParserConfig()
|
||||
|
||||
# All distinct objects.
|
||||
self.assertIsNot(cfg_a.ip_address_cache, cfg_b.ip_address_cache)
|
||||
self.assertIsNot(
|
||||
cfg_a.seen_aggregate_report_ids, cfg_b.seen_aggregate_report_ids
|
||||
)
|
||||
self.assertIsNot(cfg_a.reverse_dns_map, cfg_b.reverse_dns_map)
|
||||
|
||||
self.assertIsNot(cfg_a.ip_address_cache, parsedmarc_config.IP_ADDRESS_CACHE)
|
||||
self.assertIsNot(
|
||||
cfg_a.seen_aggregate_report_ids,
|
||||
parsedmarc_config.SEEN_AGGREGATE_REPORT_IDS,
|
||||
)
|
||||
self.assertIsNot(cfg_a.reverse_dns_map, parsedmarc_config.REVERSE_DNS_MAP)
|
||||
|
||||
self.assertIsNot(cfg_b.ip_address_cache, parsedmarc_config.IP_ADDRESS_CACHE)
|
||||
self.assertIsNot(
|
||||
cfg_b.seen_aggregate_report_ids,
|
||||
parsedmarc_config.SEEN_AGGREGATE_REPORT_IDS,
|
||||
)
|
||||
self.assertIsNot(cfg_b.reverse_dns_map, parsedmarc_config.REVERSE_DNS_MAP)
|
||||
|
||||
# Mutating one instance's caches must not affect the other, nor the
|
||||
# module defaults.
|
||||
cfg_a.ip_address_cache["1.2.3.4"] = {"ip_address": "1.2.3.4"}
|
||||
cfg_a.seen_aggregate_report_ids["report-id-a"] = True
|
||||
cfg_a.reverse_dns_map["example.com"] = {"name": "Example", "type": None}
|
||||
|
||||
self.assertNotIn("1.2.3.4", cfg_b.ip_address_cache)
|
||||
self.assertNotIn("report-id-a", cfg_b.seen_aggregate_report_ids)
|
||||
self.assertNotIn("example.com", cfg_b.reverse_dns_map)
|
||||
|
||||
self.assertNotIn("1.2.3.4", parsedmarc_config.IP_ADDRESS_CACHE)
|
||||
self.assertNotIn("report-id-a", parsedmarc_config.SEEN_AGGREGATE_REPORT_IDS)
|
||||
self.assertNotIn("example.com", parsedmarc_config.REVERSE_DNS_MAP)
|
||||
|
||||
def test_module_default_caches_are_the_public_globals(self):
|
||||
"""The three module-default caches in `parsedmarc.config` must be the
|
||||
very same objects re-exported as `parsedmarc.IP_ADDRESS_CACHE`,
|
||||
`parsedmarc.SEEN_AGGREGATE_REPORT_IDS`, and
|
||||
`parsedmarc.REVERSE_DNS_MAP`, so that pre-refactor callers who
|
||||
imported these names directly from the top-level package keep
|
||||
observing the same cache objects as code that goes through
|
||||
ParserConfig.
|
||||
"""
|
||||
self.assertIs(parsedmarc.IP_ADDRESS_CACHE, parsedmarc_config.IP_ADDRESS_CACHE)
|
||||
self.assertIs(
|
||||
parsedmarc.SEEN_AGGREGATE_REPORT_IDS,
|
||||
parsedmarc_config.SEEN_AGGREGATE_REPORT_IDS,
|
||||
)
|
||||
self.assertIs(parsedmarc.REVERSE_DNS_MAP, parsedmarc_config.REVERSE_DNS_MAP)
|
||||
|
||||
|
||||
class TestParserConfigPickling(unittest.TestCase):
|
||||
"""Covers the pickle strategy documented on `ParserConfig`: option fields
|
||||
round-trip, but cache contents never cross process boundaries and the
|
||||
unpickled object's caches rebind to the unpickling process's module
|
||||
defaults."""
|
||||
|
||||
def test_pickle_drops_cache_contents_and_binds_process_defaults(self):
|
||||
"""Pickling and unpickling a ParserConfig must preserve option
|
||||
fields, but must NOT carry cache contents across the round-trip, and
|
||||
must leave the unpickled instance's caches bound to this module's
|
||||
(the "unpickling process's") default cache objects rather than
|
||||
fresh, empty ones. Fresh-per-unpickle caches would silently defeat
|
||||
per-worker caching, since a functools.partial payload carrying a
|
||||
ParserConfig is re-pickled for every task submitted to a
|
||||
multiprocessing worker."""
|
||||
cfg = parsedmarc_config.ParserConfig(
|
||||
dns_timeout=9.5, nameservers=["192.0.2.53"]
|
||||
)
|
||||
cfg.ip_address_cache["sentinel-ip"] = "sentinel-ip-value"
|
||||
cfg.seen_aggregate_report_ids["sentinel-report-id"] = True
|
||||
cfg.reverse_dns_map["sentinel.example"] = {
|
||||
"name": "Sentinel",
|
||||
"type": None,
|
||||
}
|
||||
|
||||
restored = pickle.loads(pickle.dumps(cfg))
|
||||
|
||||
# Option fields compare equal (cache fields are compare=False).
|
||||
self.assertEqual(cfg, restored)
|
||||
self.assertEqual(restored.dns_timeout, 9.5)
|
||||
self.assertEqual(restored.nameservers, ["192.0.2.53"])
|
||||
|
||||
# Caches rebind to this process's module defaults, not fresh copies.
|
||||
self.assertIs(restored.ip_address_cache, parsedmarc_config.IP_ADDRESS_CACHE)
|
||||
self.assertIs(
|
||||
restored.seen_aggregate_report_ids,
|
||||
parsedmarc_config.SEEN_AGGREGATE_REPORT_IDS,
|
||||
)
|
||||
self.assertIs(restored.reverse_dns_map, parsedmarc_config.REVERSE_DNS_MAP)
|
||||
|
||||
# Sentinel entries placed on the original instance's caches must NOT
|
||||
# have crossed into the module defaults.
|
||||
self.assertNotIn("sentinel-ip", parsedmarc_config.IP_ADDRESS_CACHE)
|
||||
self.assertNotIn(
|
||||
"sentinel-report-id", parsedmarc_config.SEEN_AGGREGATE_REPORT_IDS
|
||||
)
|
||||
self.assertNotIn("sentinel.example", parsedmarc_config.REVERSE_DNS_MAP)
|
||||
|
||||
def test_setstate_defaults_fields_missing_from_older_pickles(self):
|
||||
"""__setstate__ must initialize every non-cache field to its class
|
||||
default before applying the pickled state, so a ParserConfig
|
||||
serialized by an older parsedmarc version (whose state lacks fields
|
||||
added since) unpickles with the newer fields at their defaults
|
||||
instead of unset — __init__ never runs during unpickling, so an
|
||||
absent field would otherwise raise AttributeError on first access.
|
||||
Simulated by calling __setstate__ directly with a partial state
|
||||
dict, exactly what pickle.loads does with an old payload."""
|
||||
restored = object.__new__(parsedmarc_config.ParserConfig)
|
||||
restored.__setstate__({"offline": True, "dns_timeout": 7.5})
|
||||
|
||||
self.assertTrue(restored.offline)
|
||||
self.assertEqual(restored.dns_timeout, 7.5)
|
||||
# Fields absent from the old state get their class defaults.
|
||||
self.assertIsNone(restored.ip_db_path)
|
||||
self.assertIsNone(restored.nameservers)
|
||||
self.assertEqual(restored.normalize_timespan_threshold_hours, 24.0)
|
||||
# Caches still rebind to the module defaults.
|
||||
self.assertIs(restored.ip_address_cache, parsedmarc_config.IP_ADDRESS_CACHE)
|
||||
self.assertIs(
|
||||
restored.seen_aggregate_report_ids,
|
||||
parsedmarc_config.SEEN_AGGREGATE_REPORT_IDS,
|
||||
)
|
||||
self.assertIs(restored.reverse_dns_map, parsedmarc_config.REVERSE_DNS_MAP)
|
||||
|
||||
|
||||
class TestParserConfigFrozenAndReplace(unittest.TestCase):
|
||||
"""Covers frozen-dataclass immutability and the `dataclasses.replace`
|
||||
escape hatch for deriving variants that keep sharing caches."""
|
||||
|
||||
def test_frozen_and_replace_shares_caches(self):
|
||||
"""ParserConfig is frozen, so attribute assignment must raise
|
||||
FrozenInstanceError. `dataclasses.replace(cfg, ...)` must return a
|
||||
new config that shares the SAME cache objects as the source (all
|
||||
fields, including the three cache fields, are init fields), since
|
||||
that is the documented way to derive a variant that keeps warm
|
||||
caches."""
|
||||
cfg = parsedmarc_config.ParserConfig()
|
||||
|
||||
with self.assertRaises(dataclasses.FrozenInstanceError):
|
||||
cfg.offline = True # type: ignore[misc]
|
||||
|
||||
variant = dataclasses.replace(cfg, offline=True)
|
||||
|
||||
self.assertTrue(variant.offline)
|
||||
self.assertIs(variant.ip_address_cache, cfg.ip_address_cache)
|
||||
self.assertIs(variant.seen_aggregate_report_ids, cfg.seen_aggregate_report_ids)
|
||||
self.assertIs(variant.reverse_dns_map, cfg.reverse_dns_map)
|
||||
+249
-13
@@ -7,6 +7,7 @@ extract_report, get_dmarc_reports_from_mbox, and the CSV / JSON renderers.
|
||||
|
||||
import base64
|
||||
import gzip
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
import mailbox
|
||||
@@ -24,6 +25,7 @@ from unittest.mock import MagicMock, patch
|
||||
from lxml import etree # type: ignore[import-untyped]
|
||||
|
||||
import parsedmarc
|
||||
import parsedmarc.constants as constants
|
||||
from parsedmarc.mail import MaildirConnection, MSGraphConnection
|
||||
from parsedmarc.types import (
|
||||
AggregateReport,
|
||||
@@ -749,7 +751,9 @@ class Test(unittest.TestCase):
|
||||
"auth_results": {"dkim": [], "spf": []},
|
||||
}
|
||||
with self.assertRaises(ValueError):
|
||||
parsedmarc._parse_report_record(record, offline=True)
|
||||
parsedmarc._parse_report_record(
|
||||
record, config=parsedmarc.ParserConfig(offline=True)
|
||||
)
|
||||
|
||||
def testParseReportRecordMissingDkimSpf(self):
|
||||
"""Record with missing dkim/spf auth results defaults correctly"""
|
||||
@@ -766,7 +770,9 @@ class Test(unittest.TestCase):
|
||||
"identifiers": {"header_from": "example.com"},
|
||||
"auth_results": {},
|
||||
}
|
||||
result = parsedmarc._parse_report_record(record, offline=True)
|
||||
result = parsedmarc._parse_report_record(
|
||||
record, config=parsedmarc.ParserConfig(offline=True)
|
||||
)
|
||||
self.assertEqual(result["auth_results"]["dkim"], [])
|
||||
self.assertEqual(result["auth_results"]["spf"], [])
|
||||
|
||||
@@ -786,7 +792,9 @@ class Test(unittest.TestCase):
|
||||
"identifiers": {"header_from": "example.com"},
|
||||
"auth_results": {"dkim": [], "spf": []},
|
||||
}
|
||||
result = parsedmarc._parse_report_record(record, offline=True)
|
||||
result = parsedmarc._parse_report_record(
|
||||
record, config=parsedmarc.ParserConfig(offline=True)
|
||||
)
|
||||
reasons = result["policy_evaluated"]["policy_override_reasons"]
|
||||
self.assertEqual(len(reasons), 1)
|
||||
self.assertEqual(reasons[0]["type"], "forwarded")
|
||||
@@ -811,7 +819,9 @@ class Test(unittest.TestCase):
|
||||
"identifiers": {"header_from": "example.com"},
|
||||
"auth_results": {"dkim": [], "spf": []},
|
||||
}
|
||||
result = parsedmarc._parse_report_record(record, offline=True)
|
||||
result = parsedmarc._parse_report_record(
|
||||
record, config=parsedmarc.ParserConfig(offline=True)
|
||||
)
|
||||
reasons = result["policy_evaluated"]["policy_override_reasons"]
|
||||
self.assertEqual(len(reasons), 2)
|
||||
self.assertEqual(reasons[0]["comment"], "relay")
|
||||
@@ -835,7 +845,9 @@ class Test(unittest.TestCase):
|
||||
},
|
||||
"auth_results": {"dkim": [], "spf": []},
|
||||
}
|
||||
result = parsedmarc._parse_report_record(record, offline=True)
|
||||
result = parsedmarc._parse_report_record(
|
||||
record, config=parsedmarc.ParserConfig(offline=True)
|
||||
)
|
||||
self.assertIn("identifiers", result)
|
||||
self.assertEqual(result["identifiers"]["header_from"], "example.com")
|
||||
|
||||
@@ -857,7 +869,9 @@ class Test(unittest.TestCase):
|
||||
"spf": [],
|
||||
},
|
||||
}
|
||||
result = parsedmarc._parse_report_record(record, offline=True)
|
||||
result = parsedmarc._parse_report_record(
|
||||
record, config=parsedmarc.ParserConfig(offline=True)
|
||||
)
|
||||
dkim = result["auth_results"]["dkim"][0]
|
||||
self.assertEqual(dkim["selector"], "none")
|
||||
self.assertEqual(dkim["result"], "none")
|
||||
@@ -881,7 +895,9 @@ class Test(unittest.TestCase):
|
||||
"spf": {"domain": "example.com"},
|
||||
},
|
||||
}
|
||||
result = parsedmarc._parse_report_record(record, offline=True)
|
||||
result = parsedmarc._parse_report_record(
|
||||
record, config=parsedmarc.ParserConfig(offline=True)
|
||||
)
|
||||
spf = result["auth_results"]["spf"][0]
|
||||
self.assertEqual(spf["scope"], "mfrom")
|
||||
self.assertEqual(spf["result"], "none")
|
||||
@@ -919,7 +935,9 @@ class Test(unittest.TestCase):
|
||||
],
|
||||
},
|
||||
}
|
||||
result = parsedmarc._parse_report_record(record, offline=True)
|
||||
result = parsedmarc._parse_report_record(
|
||||
record, config=parsedmarc.ParserConfig(offline=True)
|
||||
)
|
||||
self.assertEqual(result["auth_results"]["dkim"][0]["human_result"], "good key")
|
||||
self.assertEqual(
|
||||
result["auth_results"]["spf"][0]["human_result"], "sender valid"
|
||||
@@ -945,7 +963,9 @@ class Test(unittest.TestCase):
|
||||
],
|
||||
},
|
||||
}
|
||||
result = parsedmarc._parse_report_record(record, offline=True)
|
||||
result = parsedmarc._parse_report_record(
|
||||
record, config=parsedmarc.ParserConfig(offline=True)
|
||||
)
|
||||
self.assertEqual(result["identifiers"]["envelope_from"], "bounce.example.com")
|
||||
|
||||
def testParseReportRecordEnvelopeFromNullFallback(self):
|
||||
@@ -971,7 +991,9 @@ class Test(unittest.TestCase):
|
||||
],
|
||||
},
|
||||
}
|
||||
result = parsedmarc._parse_report_record(record, offline=True)
|
||||
result = parsedmarc._parse_report_record(
|
||||
record, config=parsedmarc.ParserConfig(offline=True)
|
||||
)
|
||||
self.assertEqual(result["identifiers"]["envelope_from"], "spf.example.com")
|
||||
|
||||
def testParseReportRecordEnvelopeFromNullNoSpfDomain(self):
|
||||
@@ -998,7 +1020,9 @@ class Test(unittest.TestCase):
|
||||
"spf": [{"scope": "mfrom", "result": "pass"}],
|
||||
},
|
||||
}
|
||||
result = parsedmarc._parse_report_record(record, offline=True)
|
||||
result = parsedmarc._parse_report_record(
|
||||
record, config=parsedmarc.ParserConfig(offline=True)
|
||||
)
|
||||
self.assertIsNone(result["identifiers"]["envelope_from"])
|
||||
|
||||
def testParseReportRecordEnvelopeTo(self):
|
||||
@@ -1020,7 +1044,9 @@ class Test(unittest.TestCase):
|
||||
},
|
||||
"auth_results": {"dkim": [], "spf": []},
|
||||
}
|
||||
result = parsedmarc._parse_report_record(record, offline=True)
|
||||
result = parsedmarc._parse_report_record(
|
||||
record, config=parsedmarc.ParserConfig(offline=True)
|
||||
)
|
||||
self.assertEqual(result["identifiers"]["envelope_to"], "recipient@example.com")
|
||||
|
||||
def testParseReportRecordAlignment(self):
|
||||
@@ -1038,7 +1064,9 @@ class Test(unittest.TestCase):
|
||||
"identifiers": {"header_from": "example.com"},
|
||||
"auth_results": {"dkim": [], "spf": []},
|
||||
}
|
||||
result = parsedmarc._parse_report_record(record, offline=True)
|
||||
result = parsedmarc._parse_report_record(
|
||||
record, config=parsedmarc.ParserConfig(offline=True)
|
||||
)
|
||||
self.assertTrue(result["alignment"]["dkim"])
|
||||
self.assertFalse(result["alignment"]["spf"])
|
||||
self.assertTrue(result["alignment"]["dmarc"])
|
||||
@@ -2709,6 +2737,214 @@ class TestGetDmarcReportsFromMboxParallel(unittest.TestCase):
|
||||
)
|
||||
|
||||
|
||||
class TestCentralizedConfig(unittest.TestCase):
|
||||
"""Regression coverage for the centralize-config-503 refactor
|
||||
(parsedmarc/config.py's ``ParserConfig``): the kwargs-style public API
|
||||
must keep observing/mutating the same module-default caches it always
|
||||
has (via ``_resolve_config`` injecting ``IP_ADDRESS_CACHE`` /
|
||||
``SEEN_AGGREGATE_REPORT_IDS`` / ``REVERSE_DNS_MAP`` rather than letting
|
||||
a fresh ``ParserConfig()`` default-factory hand back empty ones), an
|
||||
explicit ``config=`` must win over individual option keyword arguments
|
||||
when both are given, and the DNS-timeout/retry/normalize-threshold
|
||||
defaults must stay consistent between each public function's own
|
||||
signature and ``ParserConfig``'s field defaults.
|
||||
"""
|
||||
|
||||
AGGREGATE = "samples/aggregate/twilight.eml"
|
||||
|
||||
def _build_single_aggregate_mbox(self) -> str:
|
||||
"""Builds a temporary mbox containing one copy of AGGREGATE."""
|
||||
tmp = mkdtemp()
|
||||
self.addCleanup(rmtree, tmp, ignore_errors=True)
|
||||
path = os.path.join(tmp, "reports.mbox")
|
||||
box = mailbox.mbox(path)
|
||||
box.lock()
|
||||
try:
|
||||
with open(self.AGGREGATE, "rb") as source_file:
|
||||
box.add(mailbox.mboxMessage(source_file.read()))
|
||||
box.flush()
|
||||
finally:
|
||||
box.unlock()
|
||||
box.close()
|
||||
return path
|
||||
|
||||
def test_kwargs_path_uses_module_default_caches_not_fresh_ones(self):
|
||||
"""Regression guard for _resolve_config: calling
|
||||
get_dmarc_reports_from_mbox with plain kwargs (no config=) twice in
|
||||
a row over the same mbox must dedup the second run's aggregate
|
||||
report against the first run's, because both calls must resolve to
|
||||
the SAME module-default parsedmarc.SEEN_AGGREGATE_REPORT_IDS cache.
|
||||
If _resolve_config ever let the kwargs path fall through to
|
||||
ParserConfig's default_factory instead of explicitly injecting the
|
||||
module-default caches, each call would get a fresh, empty cache and
|
||||
this dedup would silently stop working.
|
||||
"""
|
||||
parsedmarc.SEEN_AGGREGATE_REPORT_IDS.clear()
|
||||
self.addCleanup(parsedmarc.SEEN_AGGREGATE_REPORT_IDS.clear)
|
||||
path = self._build_single_aggregate_mbox()
|
||||
|
||||
first = parsedmarc.get_dmarc_reports_from_mbox(path, offline=True)
|
||||
second = parsedmarc.get_dmarc_reports_from_mbox(path, offline=True)
|
||||
|
||||
self.assertEqual(len(first["aggregate_reports"]), 1)
|
||||
self.assertEqual(len(second["aggregate_reports"]), 0)
|
||||
|
||||
report_metadata = first["aggregate_reports"][0]["report_metadata"]
|
||||
report_key = f"{report_metadata['org_name']}_{report_metadata['report_id']}"
|
||||
self.assertIn(report_key, parsedmarc.SEEN_AGGREGATE_REPORT_IDS)
|
||||
|
||||
def test_kwargs_and_config_equivalence(self):
|
||||
"""parse_report_file must produce identical results whether called
|
||||
with individual option keyword arguments or an equivalent explicit
|
||||
ParserConfig, for one sample of each report type. None of these
|
||||
paths touch dedup (that's only in _classify_parsed_email /
|
||||
get_dmarc_reports_from_mbox / get_dmarc_reports_from_mailbox), so
|
||||
no cache clearing is needed between calls.
|
||||
"""
|
||||
sample_paths = [
|
||||
"samples/aggregate/rfc9990-sample.xml",
|
||||
"samples/failure/dmarc_ruf_report_linkedin.eml",
|
||||
"samples/smtp_tls/google.com_smtp_tls_report.eml",
|
||||
]
|
||||
for sample_path in sample_paths:
|
||||
with self.subTest(sample=sample_path):
|
||||
kwargs_result = parsedmarc.parse_report_file(
|
||||
sample_path, offline=True, always_use_local_files=True
|
||||
)
|
||||
config_result = parsedmarc.parse_report_file(
|
||||
sample_path,
|
||||
config=parsedmarc.ParserConfig(
|
||||
offline=True, always_use_local_files=True
|
||||
),
|
||||
)
|
||||
self.assertEqual(kwargs_result, config_result)
|
||||
|
||||
def test_config_wins_over_kwargs_normalize_threshold(self):
|
||||
"""When both config= and normalize_timespan_threshold_hours= are
|
||||
given, the config's value must win -- per the documented contract,
|
||||
an explicit config makes the individual option kwargs inert.
|
||||
|
||||
samples/aggregate/ikea.com!example.de!1538690400!1538776800.xml
|
||||
spans exactly 86400 seconds (24h), so a 1.0-hour threshold
|
||||
normalizes it and a 1000-hour threshold does not; this makes the
|
||||
config-vs-kwarg outcome observable in normalized_timespan.
|
||||
"""
|
||||
sample_path = "samples/aggregate/ikea.com!example.de!1538690400!1538776800.xml"
|
||||
with open(sample_path, "rb") as f:
|
||||
data = f.read()
|
||||
|
||||
# Config's high threshold must win over the kwarg's low threshold:
|
||||
# the report must NOT be normalized.
|
||||
report = parsedmarc.parse_aggregate_report_file(
|
||||
data,
|
||||
offline=True,
|
||||
config=parsedmarc.ParserConfig(
|
||||
offline=True, normalize_timespan_threshold_hours=1000.0
|
||||
),
|
||||
normalize_timespan_threshold_hours=1.0,
|
||||
)
|
||||
for record in report["records"]:
|
||||
self.assertFalse(record["normalized_timespan"]) # type: ignore[typeddict-item]
|
||||
|
||||
# Converse: config's low threshold must win over the kwarg's high
|
||||
# threshold: the report MUST be normalized.
|
||||
report = parsedmarc.parse_aggregate_report_file(
|
||||
data,
|
||||
offline=True,
|
||||
config=parsedmarc.ParserConfig(
|
||||
offline=True, normalize_timespan_threshold_hours=1.0
|
||||
),
|
||||
normalize_timespan_threshold_hours=1000.0,
|
||||
)
|
||||
for record in report["records"]:
|
||||
self.assertTrue(record["normalized_timespan"]) # type: ignore[typeddict-item]
|
||||
|
||||
def test_separate_configs_isolate_dedup_state(self):
|
||||
"""Two independently constructed ParserConfig(offline=True)
|
||||
instances must NOT share dedup state (each has its own
|
||||
seen_aggregate_report_ids cache), but reusing the SAME instance
|
||||
across two calls must dedup, exactly like the module-default-cache
|
||||
kwargs path does.
|
||||
"""
|
||||
path = self._build_single_aggregate_mbox()
|
||||
|
||||
cfg_a = parsedmarc.ParserConfig(offline=True)
|
||||
cfg_b = parsedmarc.ParserConfig(offline=True)
|
||||
result_a = parsedmarc.get_dmarc_reports_from_mbox(path, config=cfg_a)
|
||||
result_b = parsedmarc.get_dmarc_reports_from_mbox(path, config=cfg_b)
|
||||
self.assertEqual(len(result_a["aggregate_reports"]), 1)
|
||||
self.assertEqual(len(result_b["aggregate_reports"]), 1)
|
||||
|
||||
cfg_c = parsedmarc.ParserConfig(offline=True)
|
||||
first = parsedmarc.get_dmarc_reports_from_mbox(path, config=cfg_c)
|
||||
second = parsedmarc.get_dmarc_reports_from_mbox(path, config=cfg_c)
|
||||
self.assertEqual(len(first["aggregate_reports"]), 1)
|
||||
self.assertEqual(len(second["aggregate_reports"]), 0)
|
||||
|
||||
def test_dns_and_normalize_defaults_match_constants_and_parser_config(self):
|
||||
"""DNS timeout/retries and normalize-timespan-threshold defaults
|
||||
must match parsedmarc.constants (the authoritative source --
|
||||
DEFAULT_DNS_TIMEOUT, DEFAULT_DNS_MAX_RETRIES) and
|
||||
parsedmarc.config.ParserConfig's own field defaults, across every
|
||||
public function that accepts them.
|
||||
|
||||
Regression guard: before this refactor, get_dmarc_reports_from_mailbox
|
||||
and watch_inbox each had a stray literal ``dns_timeout=6.0`` (instead
|
||||
of ``DEFAULT_DNS_TIMEOUT == 2.0``) and
|
||||
``normalize_timespan_threshold_hours=24`` (an int, instead of the
|
||||
float ``24.0`` used everywhere else) -- exactly the kind of drift
|
||||
_resolve_config's shared construction path is meant to prevent from
|
||||
silently reappearing.
|
||||
"""
|
||||
functions_and_dns_params = [
|
||||
(parsedmarc.parse_aggregate_report_xml, "timeout", "retries"),
|
||||
(parsedmarc.parse_aggregate_report_file, "dns_timeout", "dns_retries"),
|
||||
(parsedmarc.parse_failure_report, "dns_timeout", "dns_retries"),
|
||||
(parsedmarc.parse_report_email, "dns_timeout", "dns_retries"),
|
||||
(parsedmarc.parse_report_file, "dns_timeout", "dns_retries"),
|
||||
(parsedmarc.get_dmarc_reports_from_mbox, "dns_timeout", "dns_retries"),
|
||||
(
|
||||
parsedmarc.get_dmarc_reports_from_mailbox,
|
||||
"dns_timeout",
|
||||
"dns_retries",
|
||||
),
|
||||
(parsedmarc.watch_inbox, "dns_timeout", "dns_retries"),
|
||||
]
|
||||
|
||||
default_config = parsedmarc.ParserConfig()
|
||||
|
||||
for func, timeout_param, retries_param in functions_and_dns_params:
|
||||
with self.subTest(func=func.__name__, param="dns"):
|
||||
sig = inspect.signature(func)
|
||||
timeout_default = sig.parameters[timeout_param].default
|
||||
retries_default = sig.parameters[retries_param].default
|
||||
self.assertEqual(timeout_default, constants.DEFAULT_DNS_TIMEOUT)
|
||||
self.assertEqual(retries_default, constants.DEFAULT_DNS_MAX_RETRIES)
|
||||
self.assertEqual(timeout_default, default_config.dns_timeout)
|
||||
self.assertEqual(retries_default, default_config.dns_retries)
|
||||
|
||||
# parse_failure_report has no normalize_timespan_threshold_hours
|
||||
# parameter -- normalization is an aggregate-report-only concept.
|
||||
normalize_funcs = [
|
||||
parsedmarc.parse_aggregate_report_xml,
|
||||
parsedmarc.parse_aggregate_report_file,
|
||||
parsedmarc.parse_report_email,
|
||||
parsedmarc.parse_report_file,
|
||||
parsedmarc.get_dmarc_reports_from_mbox,
|
||||
parsedmarc.get_dmarc_reports_from_mailbox,
|
||||
parsedmarc.watch_inbox,
|
||||
]
|
||||
for func in normalize_funcs:
|
||||
with self.subTest(func=func.__name__, param="normalize"):
|
||||
sig = inspect.signature(func)
|
||||
default = sig.parameters["normalize_timespan_threshold_hours"].default
|
||||
self.assertEqual(default, 24.0)
|
||||
self.assertIsInstance(default, float)
|
||||
self.assertEqual(
|
||||
default, default_config.normalize_timespan_threshold_hours
|
||||
)
|
||||
|
||||
|
||||
class TestGetDmarcReportsFromMailboxValidation(unittest.TestCase):
|
||||
"""Input validation on get_dmarc_reports_from_mailbox.
|
||||
|
||||
|
||||
+10
-4
@@ -68,7 +68,9 @@ class TestParallelMapParseReportFile(_ParallelTestCase):
|
||||
for path in SAMPLE_PATHS
|
||||
]
|
||||
|
||||
job = functools.partial(_parse_report_file_job, kwargs=dict(offline=True))
|
||||
job = functools.partial(
|
||||
_parse_report_file_job, config=parsedmarc.ParserConfig(offline=True)
|
||||
)
|
||||
results = list(parallel_map(job, SAMPLE_PATHS, n_procs=2))
|
||||
|
||||
self.assertEqual(len(results), len(SAMPLE_PATHS))
|
||||
@@ -94,7 +96,9 @@ class TestParallelMapJunkFile(_ParallelTestCase):
|
||||
junk_path = tf.name
|
||||
self.addCleanup(os.remove, junk_path)
|
||||
|
||||
job = functools.partial(_parse_report_file_job, kwargs=dict(offline=True))
|
||||
job = functools.partial(
|
||||
_parse_report_file_job, config=parsedmarc.ParserConfig(offline=True)
|
||||
)
|
||||
results = list(parallel_map(job, [junk_path, junk_path], n_procs=2))
|
||||
|
||||
self.assertEqual(len(results), 2)
|
||||
@@ -208,7 +212,9 @@ class TestWorkerLogging(_ParallelTestCase):
|
||||
configure_logging(logging.DEBUG, log_path)
|
||||
|
||||
sample = SAMPLE_PATHS[0]
|
||||
job = functools.partial(_parse_report_file_job, kwargs=dict(offline=True))
|
||||
job = functools.partial(
|
||||
_parse_report_file_job, config=parsedmarc.ParserConfig(offline=True)
|
||||
)
|
||||
results = list(parallel_map(job, [sample], n_procs=2))
|
||||
|
||||
self.assertEqual(len(results), 1)
|
||||
@@ -275,7 +281,7 @@ class TestParseReportEmailJob(_ParallelTestCase):
|
||||
|
||||
def test_invalid_email_returns_parser_error_value(self):
|
||||
result = _parse_report_email_job(
|
||||
b"not a valid email", kwargs=dict(offline=True)
|
||||
b"not a valid email", config=parsedmarc.ParserConfig(offline=True)
|
||||
)
|
||||
self.assertIsInstance(result, parsedmarc.ParserError)
|
||||
|
||||
|
||||
@@ -490,6 +490,67 @@ class TestLoadReverseDnsMapReloadsPSLOverrides(unittest.TestCase):
|
||||
offline=True,
|
||||
)
|
||||
|
||||
def _write_psl_overrides_file(self):
|
||||
"""Write a temp PSL overrides file with a unique suffix and register
|
||||
cleanup, returning its path."""
|
||||
tf = tempfile.NamedTemporaryFile(
|
||||
"w", suffix=".txt", delete=False, encoding="utf-8"
|
||||
)
|
||||
tf.write(".internal.example-503.net\n")
|
||||
tf.close()
|
||||
self.addCleanup(os.unlink, tf.name)
|
||||
return tf.name
|
||||
|
||||
def test_lazy_reverse_dns_map_load_applies_psl_overrides(self):
|
||||
"""Regression test for GitHub issue #503: the lazy
|
||||
``load_reverse_dns_map()`` call inside
|
||||
``get_service_from_reverse_dns_base_domain`` previously omitted
|
||||
``psl_overrides_path``/``psl_overrides_url``, so
|
||||
``load_reverse_dns_map``'s unconditional ``load_psl_overrides()``
|
||||
call (utils.py) silently reloaded ``psl_overrides`` with the bundled
|
||||
defaults, discarding an operator-configured overrides file. This
|
||||
proves the lazy load now threads the caller's
|
||||
``psl_overrides_path`` through: after calling with an empty
|
||||
``reverse_dns_map`` (which forces the lazy load) and a custom
|
||||
``psl_overrides_path``, the custom override must be in effect.
|
||||
"""
|
||||
path = self._write_psl_overrides_file()
|
||||
parsedmarc.utils.get_service_from_reverse_dns_base_domain(
|
||||
"something.example",
|
||||
reverse_dns_map={},
|
||||
always_use_local_file=True,
|
||||
offline=True,
|
||||
psl_overrides_path=path,
|
||||
)
|
||||
self.assertEqual(
|
||||
parsedmarc.utils.get_base_domain("deep.sub.internal.example-503.net"),
|
||||
"internal.example-503.net",
|
||||
)
|
||||
|
||||
def test_get_ip_address_info_lazy_load_applies_psl_overrides(self):
|
||||
"""Regression test for GitHub issue #503, exercising the
|
||||
ASN-fallback lazy ``load_reverse_dns_map()`` call inside
|
||||
``get_ip_address_info`` (used when no PTR record resolves). Before
|
||||
the fix, this lazy load also omitted ``psl_overrides_path``/
|
||||
``psl_overrides_url``, so it clobbered operator-configured PSL
|
||||
overrides with the bundled defaults the same way. ``offline=True``
|
||||
forces the no-PTR path so this call reaches the ASN-fallback lazy
|
||||
load rather than the PTR-driven
|
||||
``get_service_from_reverse_dns_base_domain`` call.
|
||||
"""
|
||||
path = self._write_psl_overrides_file()
|
||||
parsedmarc.utils.get_ip_address_info(
|
||||
"192.0.2.1",
|
||||
offline=True,
|
||||
reverse_dns_map={},
|
||||
always_use_local_files=True,
|
||||
psl_overrides_path=path,
|
||||
)
|
||||
self.assertEqual(
|
||||
parsedmarc.utils.get_base_domain("deep.sub.internal.example-503.net"),
|
||||
"internal.example-503.net",
|
||||
)
|
||||
|
||||
|
||||
class TestGetBaseDomainWithOverrides(unittest.TestCase):
|
||||
"""`get_base_domain` must honour the current psl_overrides list."""
|
||||
|
||||
Reference in New Issue
Block a user