Files
parsedmarc/tests/test_config.py
T
4e80047e68 Centralize configuration handling with ParserConfig (#503) (#851)
* Centralize configuration handling with ParserConfig (#503)

Add parsedmarc/config.py with ParserConfig, a frozen dataclass carrying
every parsing/enrichment option plus the three shared caches (IP address
info, seen aggregate report IDs, reverse DNS map). All eight public
parsing/mailbox functions accept a keyword-only config= argument; when
provided, the individual option keyword arguments are ignored in favor
of the config's values, and every existing per-option keyword argument
keeps working unchanged. The three hand-copied parse_kwargs dicts and
the dns_timeout<->timeout rename chain are gone; the CLI builds one
ParserConfig per run (rebuilt on SIGHUP) and passes it everywhere.

Explicitly constructed configs own fresh isolated caches;
dataclasses.replace() shares them; pickling drops cache contents and
rebinds the unpickling process's module defaults, preserving the
per-worker cache behavior of n_procs parallel parsing. The module
globals IP_ADDRESS_CACHE / SEEN_AGGREGATE_REPORT_IDS / REVERSE_DNS_MAP
remain, identity-preserved, as re-exports of the default caches.

Bug fixes that ride along, each with a regression test:

- One-shot mailbox runs now honor [general] dns_timeout/dns_retries;
  the CLI call site never forwarded dns_timeout, so the library's
  stray 6.0 default silently applied.
- Lazily-triggered reverse DNS map loads (get_ip_address_info /
  get_service_from_reverse_dns_base_domain, including in n_procs
  workers) now thread psl_overrides_path/psl_overrides_url through to
  load_reverse_dns_map instead of clobbering operator-configured PSL
  overrides with the bundled defaults.
- get_dmarc_reports_from_mailbox() and watch_inbox() dns_timeout
  defaults unified to DEFAULT_DNS_TIMEOUT (2.0s) from a stray 6.0, and
  normalize_timespan_threshold_hours to the float 24.0 used everywhere
  else.

Closes #503

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* Address CI and review feedback on #851

- Wrap the IMAPConnection example in usage.md so ruff format is clean
  over the docs code blocks (CI runs ruff format --check on the whole
  repo; the local runs were scoped to parsedmarc/ and tests/ and
  missed it).
- Fix the pre-existing "URL ro a reverse DNS map" docstring typo in
  get_service_from_reverse_dns_base_domain, caught by Copilot on the
  adjacent hunk.
- Import parsedmarc.config once, as an aliased plain import, in
  tests/test_config.py instead of mixing import and import-from of the
  same module (flagged by code quality scanning); the aliased module
  import also keeps pyright able to resolve the submodule attribute
  access.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* Address second Copilot review round on #851

- Add parse_aggregate_report_file() to the library entry-point list in
  usage.md; the following paragraph describes the config= contract for
  "each of these functions", so the list must name all eight
  config-accepting entry points.
- ParserConfig.__setstate__ now initializes every non-cache field to
  its class default before applying the pickled state, so a config
  serialized by an older parsedmarc version (whose state predates
  fields added later) unpickles with the newer fields at their
  defaults instead of unset entirely (__init__ never runs during
  unpickling, so an absent field would raise AttributeError on first
  access). Covered by a regression test that feeds __setstate__ a
  partial state dict.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
2026-07-25 19:27:09 -04:00

170 lines
7.9 KiB
Python

"""Tests for parsedmarc.config"""
import dataclasses
import pickle
import unittest
import parsedmarc
import parsedmarc.config as parsedmarc_config
class TestParserConfigCaches(unittest.TestCase):
"""Covers per-instance cache isolation for `ParserConfig`."""
def test_each_instance_gets_isolated_caches(self):
"""Two independently constructed ParserConfig instances must never
share cache objects with each other or with the module defaults, and
mutating one instance's caches must not leak into the others."""
cfg_a = parsedmarc_config.ParserConfig()
cfg_b = parsedmarc_config.ParserConfig()
# All distinct objects.
self.assertIsNot(cfg_a.ip_address_cache, cfg_b.ip_address_cache)
self.assertIsNot(
cfg_a.seen_aggregate_report_ids, cfg_b.seen_aggregate_report_ids
)
self.assertIsNot(cfg_a.reverse_dns_map, cfg_b.reverse_dns_map)
self.assertIsNot(cfg_a.ip_address_cache, parsedmarc_config.IP_ADDRESS_CACHE)
self.assertIsNot(
cfg_a.seen_aggregate_report_ids,
parsedmarc_config.SEEN_AGGREGATE_REPORT_IDS,
)
self.assertIsNot(cfg_a.reverse_dns_map, parsedmarc_config.REVERSE_DNS_MAP)
self.assertIsNot(cfg_b.ip_address_cache, parsedmarc_config.IP_ADDRESS_CACHE)
self.assertIsNot(
cfg_b.seen_aggregate_report_ids,
parsedmarc_config.SEEN_AGGREGATE_REPORT_IDS,
)
self.assertIsNot(cfg_b.reverse_dns_map, parsedmarc_config.REVERSE_DNS_MAP)
# Mutating one instance's caches must not affect the other, nor the
# module defaults.
cfg_a.ip_address_cache["1.2.3.4"] = {"ip_address": "1.2.3.4"}
cfg_a.seen_aggregate_report_ids["report-id-a"] = True
cfg_a.reverse_dns_map["example.com"] = {"name": "Example", "type": None}
self.assertNotIn("1.2.3.4", cfg_b.ip_address_cache)
self.assertNotIn("report-id-a", cfg_b.seen_aggregate_report_ids)
self.assertNotIn("example.com", cfg_b.reverse_dns_map)
self.assertNotIn("1.2.3.4", parsedmarc_config.IP_ADDRESS_CACHE)
self.assertNotIn("report-id-a", parsedmarc_config.SEEN_AGGREGATE_REPORT_IDS)
self.assertNotIn("example.com", parsedmarc_config.REVERSE_DNS_MAP)
def test_module_default_caches_are_the_public_globals(self):
"""The three module-default caches in `parsedmarc.config` must be the
very same objects re-exported as `parsedmarc.IP_ADDRESS_CACHE`,
`parsedmarc.SEEN_AGGREGATE_REPORT_IDS`, and
`parsedmarc.REVERSE_DNS_MAP`, so that pre-refactor callers who
imported these names directly from the top-level package keep
observing the same cache objects as code that goes through
ParserConfig.
"""
self.assertIs(parsedmarc.IP_ADDRESS_CACHE, parsedmarc_config.IP_ADDRESS_CACHE)
self.assertIs(
parsedmarc.SEEN_AGGREGATE_REPORT_IDS,
parsedmarc_config.SEEN_AGGREGATE_REPORT_IDS,
)
self.assertIs(parsedmarc.REVERSE_DNS_MAP, parsedmarc_config.REVERSE_DNS_MAP)
class TestParserConfigPickling(unittest.TestCase):
"""Covers the pickle strategy documented on `ParserConfig`: option fields
round-trip, but cache contents never cross process boundaries and the
unpickled object's caches rebind to the unpickling process's module
defaults."""
def test_pickle_drops_cache_contents_and_binds_process_defaults(self):
"""Pickling and unpickling a ParserConfig must preserve option
fields, but must NOT carry cache contents across the round-trip, and
must leave the unpickled instance's caches bound to this module's
(the "unpickling process's") default cache objects rather than
fresh, empty ones. Fresh-per-unpickle caches would silently defeat
per-worker caching, since a functools.partial payload carrying a
ParserConfig is re-pickled for every task submitted to a
multiprocessing worker."""
cfg = parsedmarc_config.ParserConfig(
dns_timeout=9.5, nameservers=["192.0.2.53"]
)
cfg.ip_address_cache["sentinel-ip"] = "sentinel-ip-value"
cfg.seen_aggregate_report_ids["sentinel-report-id"] = True
cfg.reverse_dns_map["sentinel.example"] = {
"name": "Sentinel",
"type": None,
}
restored = pickle.loads(pickle.dumps(cfg))
# Option fields compare equal (cache fields are compare=False).
self.assertEqual(cfg, restored)
self.assertEqual(restored.dns_timeout, 9.5)
self.assertEqual(restored.nameservers, ["192.0.2.53"])
# Caches rebind to this process's module defaults, not fresh copies.
self.assertIs(restored.ip_address_cache, parsedmarc_config.IP_ADDRESS_CACHE)
self.assertIs(
restored.seen_aggregate_report_ids,
parsedmarc_config.SEEN_AGGREGATE_REPORT_IDS,
)
self.assertIs(restored.reverse_dns_map, parsedmarc_config.REVERSE_DNS_MAP)
# Sentinel entries placed on the original instance's caches must NOT
# have crossed into the module defaults.
self.assertNotIn("sentinel-ip", parsedmarc_config.IP_ADDRESS_CACHE)
self.assertNotIn(
"sentinel-report-id", parsedmarc_config.SEEN_AGGREGATE_REPORT_IDS
)
self.assertNotIn("sentinel.example", parsedmarc_config.REVERSE_DNS_MAP)
def test_setstate_defaults_fields_missing_from_older_pickles(self):
"""__setstate__ must initialize every non-cache field to its class
default before applying the pickled state, so a ParserConfig
serialized by an older parsedmarc version (whose state lacks fields
added since) unpickles with the newer fields at their defaults
instead of unset — __init__ never runs during unpickling, so an
absent field would otherwise raise AttributeError on first access.
Simulated by calling __setstate__ directly with a partial state
dict, exactly what pickle.loads does with an old payload."""
restored = object.__new__(parsedmarc_config.ParserConfig)
restored.__setstate__({"offline": True, "dns_timeout": 7.5})
self.assertTrue(restored.offline)
self.assertEqual(restored.dns_timeout, 7.5)
# Fields absent from the old state get their class defaults.
self.assertIsNone(restored.ip_db_path)
self.assertIsNone(restored.nameservers)
self.assertEqual(restored.normalize_timespan_threshold_hours, 24.0)
# Caches still rebind to the module defaults.
self.assertIs(restored.ip_address_cache, parsedmarc_config.IP_ADDRESS_CACHE)
self.assertIs(
restored.seen_aggregate_report_ids,
parsedmarc_config.SEEN_AGGREGATE_REPORT_IDS,
)
self.assertIs(restored.reverse_dns_map, parsedmarc_config.REVERSE_DNS_MAP)
class TestParserConfigFrozenAndReplace(unittest.TestCase):
"""Covers frozen-dataclass immutability and the `dataclasses.replace`
escape hatch for deriving variants that keep sharing caches."""
def test_frozen_and_replace_shares_caches(self):
"""ParserConfig is frozen, so attribute assignment must raise
FrozenInstanceError. `dataclasses.replace(cfg, ...)` must return a
new config that shares the SAME cache objects as the source (all
fields, including the three cache fields, are init fields), since
that is the documented way to derive a variant that keeps warm
caches."""
cfg = parsedmarc_config.ParserConfig()
with self.assertRaises(dataclasses.FrozenInstanceError):
cfg.offline = True # type: ignore[misc]
variant = dataclasses.replace(cfg, offline=True)
self.assertTrue(variant.offline)
self.assertIs(variant.ip_address_cache, cfg.ip_address_cache)
self.assertIs(variant.seen_aggregate_report_ids, cfg.seen_aggregate_report_ids)
self.assertIs(variant.reverse_dns_map, cfg.reverse_dns_map)