Centralize configuration handling with ParserConfig (#503) (#851)

* Centralize configuration handling with ParserConfig (#503)

Add parsedmarc/config.py with ParserConfig, a frozen dataclass carrying
every parsing/enrichment option plus the three shared caches (IP address
info, seen aggregate report IDs, reverse DNS map). All eight public
parsing/mailbox functions accept a keyword-only config= argument; when
provided, the individual option keyword arguments are ignored in favor
of the config's values, and every existing per-option keyword argument
keeps working unchanged. The three hand-copied parse_kwargs dicts and
the dns_timeout<->timeout rename chain are gone; the CLI builds one
ParserConfig per run (rebuilt on SIGHUP) and passes it everywhere.

Explicitly constructed configs own fresh isolated caches;
dataclasses.replace() shares them; pickling drops cache contents and
rebinds the unpickling process's module defaults, preserving the
per-worker cache behavior of n_procs parallel parsing. The module
globals IP_ADDRESS_CACHE / SEEN_AGGREGATE_REPORT_IDS / REVERSE_DNS_MAP
remain, identity-preserved, as re-exports of the default caches.

Bug fixes that ride along, each with a regression test:

- One-shot mailbox runs now honor [general] dns_timeout/dns_retries;
  the CLI call site never forwarded dns_timeout, so the library's
  stray 6.0 default silently applied.
- Lazily-triggered reverse DNS map loads (get_ip_address_info /
  get_service_from_reverse_dns_base_domain, including in n_procs
  workers) now thread psl_overrides_path/psl_overrides_url through to
  load_reverse_dns_map instead of clobbering operator-configured PSL
  overrides with the bundled defaults.
- get_dmarc_reports_from_mailbox() and watch_inbox() dns_timeout
  defaults unified to DEFAULT_DNS_TIMEOUT (2.0s) from a stray 6.0, and
  normalize_timespan_threshold_hours to the float 24.0 used everywhere
  else.

Closes #503

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* Address CI and review feedback on #851

- Wrap the IMAPConnection example in usage.md so ruff format is clean
  over the docs code blocks (CI runs ruff format --check on the whole
  repo; the local runs were scoped to parsedmarc/ and tests/ and
  missed it).
- Fix the pre-existing "URL ro a reverse DNS map" docstring typo in
  get_service_from_reverse_dns_base_domain, caught by Copilot on the
  adjacent hunk.
- Import parsedmarc.config once, as an aliased plain import, in
  tests/test_config.py instead of mixing import and import-from of the
  same module (flagged by code quality scanning); the aliased module
  import also keeps pyright able to resolve the submodule attribute
  access.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* Address second Copilot review round on #851

- Add parse_aggregate_report_file() to the library entry-point list in
  usage.md; the following paragraph describes the config= contract for
  "each of these functions", so the list must name all eight
  config-accepting entry points.
- ParserConfig.__setstate__ now initializes every non-cache field to
  its class default before applying the pickled state, so a config
  serialized by an older parsedmarc version (whose state predates
  fields added later) unpickles with the newer fields at their
  defaults instead of unset entirely (__init__ never runs during
  unpickling, so an absent field would raise AttributeError on first
  access). Covered by a regression test that feeds __setstate__ a
  partial state dict.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Sean Whalen
2026-07-25 19:27:09 -04:00
committed by GitHub
co-authored by Claude Fable 5
parent 4f4733003c
commit 4e80047e68
13 changed files with 1301 additions and 293 deletions
+187
View File
@@ -1236,6 +1236,108 @@ since = 2d
self.assertEqual(mock_watch_inbox.call_args.kwargs.get("since"), "2d")
class TestCliParserConfigWiring(unittest.TestCase):
"""Tests that _main() builds a single ParserConfig (via
_build_parser_config) from parsed opts and passes it as ``config=`` to
the library's mailbox-fetching functions, rather than forwarding
individual option kwargs (offline, dns_timeout, ip_db_path, etc.) by
hand at each call site."""
def setUp(self):
from parsedmarc.log import logger as _logger
_logger.disabled = True
self._stdout_patch = patch("sys.stdout", new_callable=io.StringIO)
self._stderr_patch = patch("sys.stderr", new_callable=io.StringIO)
self._stdout_patch.start()
self._stderr_patch.start()
# SEEN_AGGREGATE_REPORT_IDS is a module-level ExpiringDict shared
# across tests in this process; clear it both ways so state from an
# earlier test class doesn't leak in, and so this class doesn't leak
# into a later one. Precedent: TestDirectoryFilePaths.setUp above.
parsedmarc.SEEN_AGGREGATE_REPORT_IDS.clear()
self.addCleanup(parsedmarc.SEEN_AGGREGATE_REPORT_IDS.clear)
def tearDown(self):
from parsedmarc.log import logger as _logger
_logger.disabled = False
self._stderr_patch.stop()
self._stdout_patch.stop()
def _run_one_shot_mailbox(
self, dns_timeout: float | None = None, dns_retries: int | None = None
) -> parsedmarc.ParserConfig:
"""Runs a real one-shot _main() against a mocked IMAP connection and
mocked get_dmarc_reports_from_mailbox, and returns the ParserConfig
the CLI passed as ``config=``."""
config_lines = ["[general]", "silent = true"]
if dns_timeout is not None:
config_lines.append(f"dns_timeout = {dns_timeout}")
if dns_retries is not None:
config_lines.append(f"dns_retries = {dns_retries}")
config_lines += [
"",
"[imap]",
"host = imap.example.com",
"user = user",
"password = pass",
]
config_text = "\n".join(config_lines) + "\n"
with tempfile.NamedTemporaryFile("w", suffix=".ini", delete=False) as cfg:
cfg.write(config_text)
cfg_path = cfg.name
self.addCleanup(lambda: os.path.exists(cfg_path) and os.remove(cfg_path))
with (
patch("parsedmarc.cli.get_dmarc_reports_from_mailbox") as mock_get_reports,
patch("parsedmarc.cli.IMAPConnection") as mock_imap,
):
mock_imap.return_value = object()
mock_get_reports.return_value = {
"aggregate_reports": [],
"failure_reports": [],
"smtp_tls_reports": [],
}
with patch.object(sys, "argv", ["parsedmarc", "-c", cfg_path]):
parsedmarc.cli._main()
return mock_get_reports.call_args.kwargs["config"]
def test_one_shot_mailbox_run_honors_general_dns_timeout(self):
"""Regression test for the one-shot mailbox call site in _main():
before this fix, that call to get_dmarc_reports_from_mailbox
omitted the dns_timeout/dns_retries kwargs entirely (and, before
config= existed, had no way to pass them at all), so a one-shot run
silently used the library's own hardcoded default
(``dns_timeout: float = 6.0`` on master's
get_dmarc_reports_from_mailbox signature) instead of the operator's
``[general] dns_timeout`` / ``dns_retries`` config values. Watch-mode
runs were unaffected because the watch_inbox call site did pass
dns_timeout/dns_retries directly.
"""
cfg = self._run_one_shot_mailbox(dns_timeout=11.5, dns_retries=3)
self.assertEqual(cfg.dns_timeout, 11.5)
self.assertEqual(cfg.dns_retries, 3)
def test_cli_config_binds_module_default_caches(self):
"""The ParserConfig built by the CLI must bind the process-wide
default caches (parsedmarc.IP_ADDRESS_CACHE,
parsedmarc.SEEN_AGGREGATE_REPORT_IDS, parsedmarc.REVERSE_DNS_MAP) by
identity, not fresh/isolated caches — otherwise every CLI run would
get its own empty caches (defeating the point of the 4-hour IP
cache and the 1-hour dedup cache) even though ParserConfig's default
factories exist specifically to give library callers isolated
caches when they don't pass config=.
"""
cfg = self._run_one_shot_mailbox()
self.assertIs(cfg.ip_address_cache, parsedmarc.IP_ADDRESS_CACHE)
self.assertIs(
cfg.seen_aggregate_report_ids, parsedmarc.SEEN_AGGREGATE_REPORT_IDS
)
self.assertIs(cfg.reverse_dns_map, parsedmarc.REVERSE_DNS_MAP)
class TestMailboxPerformance(unittest.TestCase):
def setUp(self):
from parsedmarc.log import logger as _logger
@@ -3035,6 +3137,91 @@ watch = true
"Stale entry should have been cleared by reload",
)
@unittest.skipUnless(
hasattr(signal, "SIGHUP"),
"SIGHUP not available on this platform",
)
@patch("parsedmarc.cli._init_output_clients")
@patch("parsedmarc.cli._parse_config")
@patch("parsedmarc.cli._load_config")
@patch("parsedmarc.cli.get_dmarc_reports_from_mailbox")
@patch("parsedmarc.cli.watch_inbox")
@patch("parsedmarc.cli.IMAPConnection")
def test_sighup_reload_rebuilds_parser_config(
self,
mock_imap,
mock_watch,
mock_get_reports,
mock_load_config,
mock_parse_config,
mock_init_clients,
):
"""After a SIGHUP reload, the ParserConfig passed to watch_inbox as
``config=`` must reflect the reloaded ``[general] dns_timeout``, not
the value from the initial config load.
Guards against _build_parser_config(opts) being called only once at
startup: opts itself is correctly refreshed in place by the existing
``for k, v in vars(new_opts).items(): setattr(opts, k, v)`` loop, but
parser_config is a separate ParserConfig snapshot built from opts —
if the reload path forgot to rebuild it, watch_inbox would keep
receiving the stale pre-reload config object forever.
"""
import signal as signal_module
mock_imap.return_value = object()
mock_get_reports.return_value = {
"aggregate_reports": [],
"failure_reports": [],
"smtp_tls_reports": [],
}
mock_load_config.return_value = ConfigParser()
parse_calls = [0]
def parse_side_effect(config, opts):
parse_calls[0] += 1
opts.imap_host = "imap.example.com"
opts.imap_user = "user"
opts.imap_password = "pass"
opts.mailbox_watch = True
opts.dns_timeout = 5.0 if parse_calls[0] == 1 else 42.0
return None
mock_parse_config.side_effect = parse_side_effect
mock_init_clients.return_value = {}
watch_calls = [0]
def watch_side_effect(*args, **kwargs):
watch_calls[0] += 1
if watch_calls[0] == 1:
if hasattr(signal_module, "SIGHUP"):
import os
os.kill(os.getpid(), signal_module.SIGHUP)
return
else:
raise FileExistsError("stop-watch-loop")
mock_watch.side_effect = watch_side_effect
with tempfile.NamedTemporaryFile("w", suffix=".ini", delete=False) as cfg:
cfg.write(self._BASE_CONFIG)
cfg_path = cfg.name
self.addCleanup(lambda: os.path.exists(cfg_path) and os.remove(cfg_path))
with patch.object(sys, "argv", ["parsedmarc", "-c", cfg_path]):
with self.assertRaises(SystemExit):
parsedmarc.cli._main()
self.assertEqual(mock_watch.call_count, 2)
first_config = mock_watch.call_args_list[0].kwargs["config"]
second_config = mock_watch.call_args_list[1].kwargs["config"]
self.assertEqual(first_config.dns_timeout, 5.0)
self.assertEqual(second_config.dns_timeout, 42.0)
class TestSigtermShutdown(unittest.TestCase):
"""Tests for graceful SIGTERM/SIGINT shutdown."""
+169
View File
@@ -0,0 +1,169 @@
"""Tests for parsedmarc.config"""
import dataclasses
import pickle
import unittest
import parsedmarc
import parsedmarc.config as parsedmarc_config
class TestParserConfigCaches(unittest.TestCase):
"""Covers per-instance cache isolation for `ParserConfig`."""
def test_each_instance_gets_isolated_caches(self):
"""Two independently constructed ParserConfig instances must never
share cache objects with each other or with the module defaults, and
mutating one instance's caches must not leak into the others."""
cfg_a = parsedmarc_config.ParserConfig()
cfg_b = parsedmarc_config.ParserConfig()
# All distinct objects.
self.assertIsNot(cfg_a.ip_address_cache, cfg_b.ip_address_cache)
self.assertIsNot(
cfg_a.seen_aggregate_report_ids, cfg_b.seen_aggregate_report_ids
)
self.assertIsNot(cfg_a.reverse_dns_map, cfg_b.reverse_dns_map)
self.assertIsNot(cfg_a.ip_address_cache, parsedmarc_config.IP_ADDRESS_CACHE)
self.assertIsNot(
cfg_a.seen_aggregate_report_ids,
parsedmarc_config.SEEN_AGGREGATE_REPORT_IDS,
)
self.assertIsNot(cfg_a.reverse_dns_map, parsedmarc_config.REVERSE_DNS_MAP)
self.assertIsNot(cfg_b.ip_address_cache, parsedmarc_config.IP_ADDRESS_CACHE)
self.assertIsNot(
cfg_b.seen_aggregate_report_ids,
parsedmarc_config.SEEN_AGGREGATE_REPORT_IDS,
)
self.assertIsNot(cfg_b.reverse_dns_map, parsedmarc_config.REVERSE_DNS_MAP)
# Mutating one instance's caches must not affect the other, nor the
# module defaults.
cfg_a.ip_address_cache["1.2.3.4"] = {"ip_address": "1.2.3.4"}
cfg_a.seen_aggregate_report_ids["report-id-a"] = True
cfg_a.reverse_dns_map["example.com"] = {"name": "Example", "type": None}
self.assertNotIn("1.2.3.4", cfg_b.ip_address_cache)
self.assertNotIn("report-id-a", cfg_b.seen_aggregate_report_ids)
self.assertNotIn("example.com", cfg_b.reverse_dns_map)
self.assertNotIn("1.2.3.4", parsedmarc_config.IP_ADDRESS_CACHE)
self.assertNotIn("report-id-a", parsedmarc_config.SEEN_AGGREGATE_REPORT_IDS)
self.assertNotIn("example.com", parsedmarc_config.REVERSE_DNS_MAP)
def test_module_default_caches_are_the_public_globals(self):
"""The three module-default caches in `parsedmarc.config` must be the
very same objects re-exported as `parsedmarc.IP_ADDRESS_CACHE`,
`parsedmarc.SEEN_AGGREGATE_REPORT_IDS`, and
`parsedmarc.REVERSE_DNS_MAP`, so that pre-refactor callers who
imported these names directly from the top-level package keep
observing the same cache objects as code that goes through
ParserConfig.
"""
self.assertIs(parsedmarc.IP_ADDRESS_CACHE, parsedmarc_config.IP_ADDRESS_CACHE)
self.assertIs(
parsedmarc.SEEN_AGGREGATE_REPORT_IDS,
parsedmarc_config.SEEN_AGGREGATE_REPORT_IDS,
)
self.assertIs(parsedmarc.REVERSE_DNS_MAP, parsedmarc_config.REVERSE_DNS_MAP)
class TestParserConfigPickling(unittest.TestCase):
"""Covers the pickle strategy documented on `ParserConfig`: option fields
round-trip, but cache contents never cross process boundaries and the
unpickled object's caches rebind to the unpickling process's module
defaults."""
def test_pickle_drops_cache_contents_and_binds_process_defaults(self):
"""Pickling and unpickling a ParserConfig must preserve option
fields, but must NOT carry cache contents across the round-trip, and
must leave the unpickled instance's caches bound to this module's
(the "unpickling process's") default cache objects rather than
fresh, empty ones. Fresh-per-unpickle caches would silently defeat
per-worker caching, since a functools.partial payload carrying a
ParserConfig is re-pickled for every task submitted to a
multiprocessing worker."""
cfg = parsedmarc_config.ParserConfig(
dns_timeout=9.5, nameservers=["192.0.2.53"]
)
cfg.ip_address_cache["sentinel-ip"] = "sentinel-ip-value"
cfg.seen_aggregate_report_ids["sentinel-report-id"] = True
cfg.reverse_dns_map["sentinel.example"] = {
"name": "Sentinel",
"type": None,
}
restored = pickle.loads(pickle.dumps(cfg))
# Option fields compare equal (cache fields are compare=False).
self.assertEqual(cfg, restored)
self.assertEqual(restored.dns_timeout, 9.5)
self.assertEqual(restored.nameservers, ["192.0.2.53"])
# Caches rebind to this process's module defaults, not fresh copies.
self.assertIs(restored.ip_address_cache, parsedmarc_config.IP_ADDRESS_CACHE)
self.assertIs(
restored.seen_aggregate_report_ids,
parsedmarc_config.SEEN_AGGREGATE_REPORT_IDS,
)
self.assertIs(restored.reverse_dns_map, parsedmarc_config.REVERSE_DNS_MAP)
# Sentinel entries placed on the original instance's caches must NOT
# have crossed into the module defaults.
self.assertNotIn("sentinel-ip", parsedmarc_config.IP_ADDRESS_CACHE)
self.assertNotIn(
"sentinel-report-id", parsedmarc_config.SEEN_AGGREGATE_REPORT_IDS
)
self.assertNotIn("sentinel.example", parsedmarc_config.REVERSE_DNS_MAP)
def test_setstate_defaults_fields_missing_from_older_pickles(self):
"""__setstate__ must initialize every non-cache field to its class
default before applying the pickled state, so a ParserConfig
serialized by an older parsedmarc version (whose state lacks fields
added since) unpickles with the newer fields at their defaults
instead of unset — __init__ never runs during unpickling, so an
absent field would otherwise raise AttributeError on first access.
Simulated by calling __setstate__ directly with a partial state
dict, exactly what pickle.loads does with an old payload."""
restored = object.__new__(parsedmarc_config.ParserConfig)
restored.__setstate__({"offline": True, "dns_timeout": 7.5})
self.assertTrue(restored.offline)
self.assertEqual(restored.dns_timeout, 7.5)
# Fields absent from the old state get their class defaults.
self.assertIsNone(restored.ip_db_path)
self.assertIsNone(restored.nameservers)
self.assertEqual(restored.normalize_timespan_threshold_hours, 24.0)
# Caches still rebind to the module defaults.
self.assertIs(restored.ip_address_cache, parsedmarc_config.IP_ADDRESS_CACHE)
self.assertIs(
restored.seen_aggregate_report_ids,
parsedmarc_config.SEEN_AGGREGATE_REPORT_IDS,
)
self.assertIs(restored.reverse_dns_map, parsedmarc_config.REVERSE_DNS_MAP)
class TestParserConfigFrozenAndReplace(unittest.TestCase):
"""Covers frozen-dataclass immutability and the `dataclasses.replace`
escape hatch for deriving variants that keep sharing caches."""
def test_frozen_and_replace_shares_caches(self):
"""ParserConfig is frozen, so attribute assignment must raise
FrozenInstanceError. `dataclasses.replace(cfg, ...)` must return a
new config that shares the SAME cache objects as the source (all
fields, including the three cache fields, are init fields), since
that is the documented way to derive a variant that keeps warm
caches."""
cfg = parsedmarc_config.ParserConfig()
with self.assertRaises(dataclasses.FrozenInstanceError):
cfg.offline = True # type: ignore[misc]
variant = dataclasses.replace(cfg, offline=True)
self.assertTrue(variant.offline)
self.assertIs(variant.ip_address_cache, cfg.ip_address_cache)
self.assertIs(variant.seen_aggregate_report_ids, cfg.seen_aggregate_report_ids)
self.assertIs(variant.reverse_dns_map, cfg.reverse_dns_map)
+249 -13
View File
@@ -7,6 +7,7 @@ extract_report, get_dmarc_reports_from_mbox, and the CSV / JSON renderers.
import base64
import gzip
import inspect
import json
import logging
import mailbox
@@ -24,6 +25,7 @@ from unittest.mock import MagicMock, patch
from lxml import etree # type: ignore[import-untyped]
import parsedmarc
import parsedmarc.constants as constants
from parsedmarc.mail import MaildirConnection, MSGraphConnection
from parsedmarc.types import (
AggregateReport,
@@ -749,7 +751,9 @@ class Test(unittest.TestCase):
"auth_results": {"dkim": [], "spf": []},
}
with self.assertRaises(ValueError):
parsedmarc._parse_report_record(record, offline=True)
parsedmarc._parse_report_record(
record, config=parsedmarc.ParserConfig(offline=True)
)
def testParseReportRecordMissingDkimSpf(self):
"""Record with missing dkim/spf auth results defaults correctly"""
@@ -766,7 +770,9 @@ class Test(unittest.TestCase):
"identifiers": {"header_from": "example.com"},
"auth_results": {},
}
result = parsedmarc._parse_report_record(record, offline=True)
result = parsedmarc._parse_report_record(
record, config=parsedmarc.ParserConfig(offline=True)
)
self.assertEqual(result["auth_results"]["dkim"], [])
self.assertEqual(result["auth_results"]["spf"], [])
@@ -786,7 +792,9 @@ class Test(unittest.TestCase):
"identifiers": {"header_from": "example.com"},
"auth_results": {"dkim": [], "spf": []},
}
result = parsedmarc._parse_report_record(record, offline=True)
result = parsedmarc._parse_report_record(
record, config=parsedmarc.ParserConfig(offline=True)
)
reasons = result["policy_evaluated"]["policy_override_reasons"]
self.assertEqual(len(reasons), 1)
self.assertEqual(reasons[0]["type"], "forwarded")
@@ -811,7 +819,9 @@ class Test(unittest.TestCase):
"identifiers": {"header_from": "example.com"},
"auth_results": {"dkim": [], "spf": []},
}
result = parsedmarc._parse_report_record(record, offline=True)
result = parsedmarc._parse_report_record(
record, config=parsedmarc.ParserConfig(offline=True)
)
reasons = result["policy_evaluated"]["policy_override_reasons"]
self.assertEqual(len(reasons), 2)
self.assertEqual(reasons[0]["comment"], "relay")
@@ -835,7 +845,9 @@ class Test(unittest.TestCase):
},
"auth_results": {"dkim": [], "spf": []},
}
result = parsedmarc._parse_report_record(record, offline=True)
result = parsedmarc._parse_report_record(
record, config=parsedmarc.ParserConfig(offline=True)
)
self.assertIn("identifiers", result)
self.assertEqual(result["identifiers"]["header_from"], "example.com")
@@ -857,7 +869,9 @@ class Test(unittest.TestCase):
"spf": [],
},
}
result = parsedmarc._parse_report_record(record, offline=True)
result = parsedmarc._parse_report_record(
record, config=parsedmarc.ParserConfig(offline=True)
)
dkim = result["auth_results"]["dkim"][0]
self.assertEqual(dkim["selector"], "none")
self.assertEqual(dkim["result"], "none")
@@ -881,7 +895,9 @@ class Test(unittest.TestCase):
"spf": {"domain": "example.com"},
},
}
result = parsedmarc._parse_report_record(record, offline=True)
result = parsedmarc._parse_report_record(
record, config=parsedmarc.ParserConfig(offline=True)
)
spf = result["auth_results"]["spf"][0]
self.assertEqual(spf["scope"], "mfrom")
self.assertEqual(spf["result"], "none")
@@ -919,7 +935,9 @@ class Test(unittest.TestCase):
],
},
}
result = parsedmarc._parse_report_record(record, offline=True)
result = parsedmarc._parse_report_record(
record, config=parsedmarc.ParserConfig(offline=True)
)
self.assertEqual(result["auth_results"]["dkim"][0]["human_result"], "good key")
self.assertEqual(
result["auth_results"]["spf"][0]["human_result"], "sender valid"
@@ -945,7 +963,9 @@ class Test(unittest.TestCase):
],
},
}
result = parsedmarc._parse_report_record(record, offline=True)
result = parsedmarc._parse_report_record(
record, config=parsedmarc.ParserConfig(offline=True)
)
self.assertEqual(result["identifiers"]["envelope_from"], "bounce.example.com")
def testParseReportRecordEnvelopeFromNullFallback(self):
@@ -971,7 +991,9 @@ class Test(unittest.TestCase):
],
},
}
result = parsedmarc._parse_report_record(record, offline=True)
result = parsedmarc._parse_report_record(
record, config=parsedmarc.ParserConfig(offline=True)
)
self.assertEqual(result["identifiers"]["envelope_from"], "spf.example.com")
def testParseReportRecordEnvelopeFromNullNoSpfDomain(self):
@@ -998,7 +1020,9 @@ class Test(unittest.TestCase):
"spf": [{"scope": "mfrom", "result": "pass"}],
},
}
result = parsedmarc._parse_report_record(record, offline=True)
result = parsedmarc._parse_report_record(
record, config=parsedmarc.ParserConfig(offline=True)
)
self.assertIsNone(result["identifiers"]["envelope_from"])
def testParseReportRecordEnvelopeTo(self):
@@ -1020,7 +1044,9 @@ class Test(unittest.TestCase):
},
"auth_results": {"dkim": [], "spf": []},
}
result = parsedmarc._parse_report_record(record, offline=True)
result = parsedmarc._parse_report_record(
record, config=parsedmarc.ParserConfig(offline=True)
)
self.assertEqual(result["identifiers"]["envelope_to"], "recipient@example.com")
def testParseReportRecordAlignment(self):
@@ -1038,7 +1064,9 @@ class Test(unittest.TestCase):
"identifiers": {"header_from": "example.com"},
"auth_results": {"dkim": [], "spf": []},
}
result = parsedmarc._parse_report_record(record, offline=True)
result = parsedmarc._parse_report_record(
record, config=parsedmarc.ParserConfig(offline=True)
)
self.assertTrue(result["alignment"]["dkim"])
self.assertFalse(result["alignment"]["spf"])
self.assertTrue(result["alignment"]["dmarc"])
@@ -2709,6 +2737,214 @@ class TestGetDmarcReportsFromMboxParallel(unittest.TestCase):
)
class TestCentralizedConfig(unittest.TestCase):
"""Regression coverage for the centralize-config-503 refactor
(parsedmarc/config.py's ``ParserConfig``): the kwargs-style public API
must keep observing/mutating the same module-default caches it always
has (via ``_resolve_config`` injecting ``IP_ADDRESS_CACHE`` /
``SEEN_AGGREGATE_REPORT_IDS`` / ``REVERSE_DNS_MAP`` rather than letting
a fresh ``ParserConfig()`` default-factory hand back empty ones), an
explicit ``config=`` must win over individual option keyword arguments
when both are given, and the DNS-timeout/retry/normalize-threshold
defaults must stay consistent between each public function's own
signature and ``ParserConfig``'s field defaults.
"""
AGGREGATE = "samples/aggregate/twilight.eml"
def _build_single_aggregate_mbox(self) -> str:
"""Builds a temporary mbox containing one copy of AGGREGATE."""
tmp = mkdtemp()
self.addCleanup(rmtree, tmp, ignore_errors=True)
path = os.path.join(tmp, "reports.mbox")
box = mailbox.mbox(path)
box.lock()
try:
with open(self.AGGREGATE, "rb") as source_file:
box.add(mailbox.mboxMessage(source_file.read()))
box.flush()
finally:
box.unlock()
box.close()
return path
def test_kwargs_path_uses_module_default_caches_not_fresh_ones(self):
"""Regression guard for _resolve_config: calling
get_dmarc_reports_from_mbox with plain kwargs (no config=) twice in
a row over the same mbox must dedup the second run's aggregate
report against the first run's, because both calls must resolve to
the SAME module-default parsedmarc.SEEN_AGGREGATE_REPORT_IDS cache.
If _resolve_config ever let the kwargs path fall through to
ParserConfig's default_factory instead of explicitly injecting the
module-default caches, each call would get a fresh, empty cache and
this dedup would silently stop working.
"""
parsedmarc.SEEN_AGGREGATE_REPORT_IDS.clear()
self.addCleanup(parsedmarc.SEEN_AGGREGATE_REPORT_IDS.clear)
path = self._build_single_aggregate_mbox()
first = parsedmarc.get_dmarc_reports_from_mbox(path, offline=True)
second = parsedmarc.get_dmarc_reports_from_mbox(path, offline=True)
self.assertEqual(len(first["aggregate_reports"]), 1)
self.assertEqual(len(second["aggregate_reports"]), 0)
report_metadata = first["aggregate_reports"][0]["report_metadata"]
report_key = f"{report_metadata['org_name']}_{report_metadata['report_id']}"
self.assertIn(report_key, parsedmarc.SEEN_AGGREGATE_REPORT_IDS)
def test_kwargs_and_config_equivalence(self):
"""parse_report_file must produce identical results whether called
with individual option keyword arguments or an equivalent explicit
ParserConfig, for one sample of each report type. None of these
paths touch dedup (that's only in _classify_parsed_email /
get_dmarc_reports_from_mbox / get_dmarc_reports_from_mailbox), so
no cache clearing is needed between calls.
"""
sample_paths = [
"samples/aggregate/rfc9990-sample.xml",
"samples/failure/dmarc_ruf_report_linkedin.eml",
"samples/smtp_tls/google.com_smtp_tls_report.eml",
]
for sample_path in sample_paths:
with self.subTest(sample=sample_path):
kwargs_result = parsedmarc.parse_report_file(
sample_path, offline=True, always_use_local_files=True
)
config_result = parsedmarc.parse_report_file(
sample_path,
config=parsedmarc.ParserConfig(
offline=True, always_use_local_files=True
),
)
self.assertEqual(kwargs_result, config_result)
def test_config_wins_over_kwargs_normalize_threshold(self):
"""When both config= and normalize_timespan_threshold_hours= are
given, the config's value must win -- per the documented contract,
an explicit config makes the individual option kwargs inert.
samples/aggregate/ikea.com!example.de!1538690400!1538776800.xml
spans exactly 86400 seconds (24h), so a 1.0-hour threshold
normalizes it and a 1000-hour threshold does not; this makes the
config-vs-kwarg outcome observable in normalized_timespan.
"""
sample_path = "samples/aggregate/ikea.com!example.de!1538690400!1538776800.xml"
with open(sample_path, "rb") as f:
data = f.read()
# Config's high threshold must win over the kwarg's low threshold:
# the report must NOT be normalized.
report = parsedmarc.parse_aggregate_report_file(
data,
offline=True,
config=parsedmarc.ParserConfig(
offline=True, normalize_timespan_threshold_hours=1000.0
),
normalize_timespan_threshold_hours=1.0,
)
for record in report["records"]:
self.assertFalse(record["normalized_timespan"]) # type: ignore[typeddict-item]
# Converse: config's low threshold must win over the kwarg's high
# threshold: the report MUST be normalized.
report = parsedmarc.parse_aggregate_report_file(
data,
offline=True,
config=parsedmarc.ParserConfig(
offline=True, normalize_timespan_threshold_hours=1.0
),
normalize_timespan_threshold_hours=1000.0,
)
for record in report["records"]:
self.assertTrue(record["normalized_timespan"]) # type: ignore[typeddict-item]
def test_separate_configs_isolate_dedup_state(self):
"""Two independently constructed ParserConfig(offline=True)
instances must NOT share dedup state (each has its own
seen_aggregate_report_ids cache), but reusing the SAME instance
across two calls must dedup, exactly like the module-default-cache
kwargs path does.
"""
path = self._build_single_aggregate_mbox()
cfg_a = parsedmarc.ParserConfig(offline=True)
cfg_b = parsedmarc.ParserConfig(offline=True)
result_a = parsedmarc.get_dmarc_reports_from_mbox(path, config=cfg_a)
result_b = parsedmarc.get_dmarc_reports_from_mbox(path, config=cfg_b)
self.assertEqual(len(result_a["aggregate_reports"]), 1)
self.assertEqual(len(result_b["aggregate_reports"]), 1)
cfg_c = parsedmarc.ParserConfig(offline=True)
first = parsedmarc.get_dmarc_reports_from_mbox(path, config=cfg_c)
second = parsedmarc.get_dmarc_reports_from_mbox(path, config=cfg_c)
self.assertEqual(len(first["aggregate_reports"]), 1)
self.assertEqual(len(second["aggregate_reports"]), 0)
def test_dns_and_normalize_defaults_match_constants_and_parser_config(self):
"""DNS timeout/retries and normalize-timespan-threshold defaults
must match parsedmarc.constants (the authoritative source --
DEFAULT_DNS_TIMEOUT, DEFAULT_DNS_MAX_RETRIES) and
parsedmarc.config.ParserConfig's own field defaults, across every
public function that accepts them.
Regression guard: before this refactor, get_dmarc_reports_from_mailbox
and watch_inbox each had a stray literal ``dns_timeout=6.0`` (instead
of ``DEFAULT_DNS_TIMEOUT == 2.0``) and
``normalize_timespan_threshold_hours=24`` (an int, instead of the
float ``24.0`` used everywhere else) -- exactly the kind of drift
_resolve_config's shared construction path is meant to prevent from
silently reappearing.
"""
functions_and_dns_params = [
(parsedmarc.parse_aggregate_report_xml, "timeout", "retries"),
(parsedmarc.parse_aggregate_report_file, "dns_timeout", "dns_retries"),
(parsedmarc.parse_failure_report, "dns_timeout", "dns_retries"),
(parsedmarc.parse_report_email, "dns_timeout", "dns_retries"),
(parsedmarc.parse_report_file, "dns_timeout", "dns_retries"),
(parsedmarc.get_dmarc_reports_from_mbox, "dns_timeout", "dns_retries"),
(
parsedmarc.get_dmarc_reports_from_mailbox,
"dns_timeout",
"dns_retries",
),
(parsedmarc.watch_inbox, "dns_timeout", "dns_retries"),
]
default_config = parsedmarc.ParserConfig()
for func, timeout_param, retries_param in functions_and_dns_params:
with self.subTest(func=func.__name__, param="dns"):
sig = inspect.signature(func)
timeout_default = sig.parameters[timeout_param].default
retries_default = sig.parameters[retries_param].default
self.assertEqual(timeout_default, constants.DEFAULT_DNS_TIMEOUT)
self.assertEqual(retries_default, constants.DEFAULT_DNS_MAX_RETRIES)
self.assertEqual(timeout_default, default_config.dns_timeout)
self.assertEqual(retries_default, default_config.dns_retries)
# parse_failure_report has no normalize_timespan_threshold_hours
# parameter -- normalization is an aggregate-report-only concept.
normalize_funcs = [
parsedmarc.parse_aggregate_report_xml,
parsedmarc.parse_aggregate_report_file,
parsedmarc.parse_report_email,
parsedmarc.parse_report_file,
parsedmarc.get_dmarc_reports_from_mbox,
parsedmarc.get_dmarc_reports_from_mailbox,
parsedmarc.watch_inbox,
]
for func in normalize_funcs:
with self.subTest(func=func.__name__, param="normalize"):
sig = inspect.signature(func)
default = sig.parameters["normalize_timespan_threshold_hours"].default
self.assertEqual(default, 24.0)
self.assertIsInstance(default, float)
self.assertEqual(
default, default_config.normalize_timespan_threshold_hours
)
class TestGetDmarcReportsFromMailboxValidation(unittest.TestCase):
"""Input validation on get_dmarc_reports_from_mailbox.
+10 -4
View File
@@ -68,7 +68,9 @@ class TestParallelMapParseReportFile(_ParallelTestCase):
for path in SAMPLE_PATHS
]
job = functools.partial(_parse_report_file_job, kwargs=dict(offline=True))
job = functools.partial(
_parse_report_file_job, config=parsedmarc.ParserConfig(offline=True)
)
results = list(parallel_map(job, SAMPLE_PATHS, n_procs=2))
self.assertEqual(len(results), len(SAMPLE_PATHS))
@@ -94,7 +96,9 @@ class TestParallelMapJunkFile(_ParallelTestCase):
junk_path = tf.name
self.addCleanup(os.remove, junk_path)
job = functools.partial(_parse_report_file_job, kwargs=dict(offline=True))
job = functools.partial(
_parse_report_file_job, config=parsedmarc.ParserConfig(offline=True)
)
results = list(parallel_map(job, [junk_path, junk_path], n_procs=2))
self.assertEqual(len(results), 2)
@@ -208,7 +212,9 @@ class TestWorkerLogging(_ParallelTestCase):
configure_logging(logging.DEBUG, log_path)
sample = SAMPLE_PATHS[0]
job = functools.partial(_parse_report_file_job, kwargs=dict(offline=True))
job = functools.partial(
_parse_report_file_job, config=parsedmarc.ParserConfig(offline=True)
)
results = list(parallel_map(job, [sample], n_procs=2))
self.assertEqual(len(results), 1)
@@ -275,7 +281,7 @@ class TestParseReportEmailJob(_ParallelTestCase):
def test_invalid_email_returns_parser_error_value(self):
result = _parse_report_email_job(
b"not a valid email", kwargs=dict(offline=True)
b"not a valid email", config=parsedmarc.ParserConfig(offline=True)
)
self.assertIsInstance(result, parsedmarc.ParserError)
+61
View File
@@ -490,6 +490,67 @@ class TestLoadReverseDnsMapReloadsPSLOverrides(unittest.TestCase):
offline=True,
)
def _write_psl_overrides_file(self):
"""Write a temp PSL overrides file with a unique suffix and register
cleanup, returning its path."""
tf = tempfile.NamedTemporaryFile(
"w", suffix=".txt", delete=False, encoding="utf-8"
)
tf.write(".internal.example-503.net\n")
tf.close()
self.addCleanup(os.unlink, tf.name)
return tf.name
def test_lazy_reverse_dns_map_load_applies_psl_overrides(self):
"""Regression test for GitHub issue #503: the lazy
``load_reverse_dns_map()`` call inside
``get_service_from_reverse_dns_base_domain`` previously omitted
``psl_overrides_path``/``psl_overrides_url``, so
``load_reverse_dns_map``'s unconditional ``load_psl_overrides()``
call (utils.py) silently reloaded ``psl_overrides`` with the bundled
defaults, discarding an operator-configured overrides file. This
proves the lazy load now threads the caller's
``psl_overrides_path`` through: after calling with an empty
``reverse_dns_map`` (which forces the lazy load) and a custom
``psl_overrides_path``, the custom override must be in effect.
"""
path = self._write_psl_overrides_file()
parsedmarc.utils.get_service_from_reverse_dns_base_domain(
"something.example",
reverse_dns_map={},
always_use_local_file=True,
offline=True,
psl_overrides_path=path,
)
self.assertEqual(
parsedmarc.utils.get_base_domain("deep.sub.internal.example-503.net"),
"internal.example-503.net",
)
def test_get_ip_address_info_lazy_load_applies_psl_overrides(self):
"""Regression test for GitHub issue #503, exercising the
ASN-fallback lazy ``load_reverse_dns_map()`` call inside
``get_ip_address_info`` (used when no PTR record resolves). Before
the fix, this lazy load also omitted ``psl_overrides_path``/
``psl_overrides_url``, so it clobbered operator-configured PSL
overrides with the bundled defaults the same way. ``offline=True``
forces the no-PTR path so this call reaches the ASN-fallback lazy
load rather than the PTR-driven
``get_service_from_reverse_dns_base_domain`` call.
"""
path = self._write_psl_overrides_file()
parsedmarc.utils.get_ip_address_info(
"192.0.2.1",
offline=True,
reverse_dns_map={},
always_use_local_files=True,
psl_overrides_path=path,
)
self.assertEqual(
parsedmarc.utils.get_base_domain("deep.sub.internal.example-503.net"),
"internal.example-503.net",
)
class TestGetBaseDomainWithOverrides(unittest.TestCase):
"""`get_base_domain` must honour the current psl_overrides list."""