amorfo77/parsedmarc
1
0
Fork 0
mirror of https://github.com/domainaware/parsedmarc.git synced 2026-06-18 00:04:18 +00:00
Code Issues Projects Releases Wiki Activity

Make the whole codebase pass pyright cleanly and enforce it in CI (#798)

Browse Source 
* Make the whole codebase pass pyright cleanly and enforce it in CI

Fix all 102 pyright (1.1.410, standard mode) errors across the library,
tests, and maps scripts, then pin and enforce the zero-errors bar:

- postgres.py: make the optional psycopg import TYPE_CHECKING-aware so
  the module is properly typed while keeping the runtime install-hint
  fallback; import psycopg.types.json explicitly as psycopg_json (the
  old psycopg_types.json attribute access only worked because psycopg
  imports the submodule eagerly); have _connect()/_ensure_connected()
  return the live connection so save methods use a non-Optional local;
  type the DDL list as list[LiteralString] to match psycopg's execute()
  overloads.
- kafkaclient.py: resolve the kafka-python 2.x/3.x bootstrap-error
  fallback statically via TYPE_CHECKING (kafka-python 3.0 removed
  NoBrokersAvailable), which also fixes _BootstrapError's import
  resolution in tests.
- syslog.py: go through getattr/setattr for SysLogHandler.socket
  (absent from typeshed); type the save_* methods with the report
  TypedDicts (single or list, matching cli.py call sites — gelf.py gets
  the same signatures); raise ValueError when retry_attempts < 1
  instead of falling through and registering a None handler (bug fix,
  with a regression test and a CHANGELOG entry).
- elastic.py / opensearch.py: human_result params are Optional[str].
- maps scripts: sort_csv declared a return type but never returned
  (now -> None); seen_sort_field_values was possibly unbound;
  convert_to_utf8's src_encoding is Optional[str].
- tests: cast sample-report dict helpers to their TypedDicts; mark
  deliberate wrong-type calls with targeted pyright ignores; add
  narrowing asserts for Optional results; access the mocked
  KafkaProducer through a cast helper; match the mailsuite
  fetch_message base signature (**kwargs); patch the renamed
  parsedmarc.postgres.psycopg_json in test_postgres's setUpModule.

Enforcement: [tool.pyright] in pyproject.toml (include parsedmarc,
tests, docs; standard mode), pyright==1.1.410 pinned in the [build]
extra (pinned exactly so a new pyright release can't break CI without a
code change), and a "Check types" step in the lint CI job — which now
also runs ruff format --check and installs the [postgresql] extra so
the optional psycopg import resolves. Documented in AGENTS.md.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* Set session headers via update() instead of replacing the dict

requests 2.34 ships inline type annotations, and Session.headers is a
CaseInsensitiveDict[str] — assigning a plain dict fails pyright there
(the CI runner resolved 2.34.2; the local venv's untyped 2.32.4 hid
it). headers.update() is correctly typed against both versions, and is
the documented requests idiom: it overrides User-Agent and the
client-specific headers while keeping the session's defaults
(Accept-Encoding, Connection) instead of wiping them.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Sean Whalen
2026-06-12 21:33:01 -04:00
committed by GitHub
parent 0c456d44ed
commit eaeea4f53d
20 changed files with 236 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_postgres.py
+15 -8
View File
@@ -10,6 +10,7 @@ real-sample round trip, so the tests fail if the dict-key mapping regresses.
import os
import unittest
from glob import glob
from typing import cast
from unittest.mock import MagicMock, patch
import parsedmarc
@@ -27,7 +28,7 @@ OFFLINE_MODE = os.environ.get("GITHUB_ACTIONS", "false").lower() == "true"
# psycopg is an optional dependency and is not installed in CI (which installs
# only the [build] extra). The save methods mock the connection, but the
# failure path also references ``psycopg_types.json.Jsonb`` at module scope, so
# failure path also references ``psycopg_json.Jsonb`` at module scope, so
# mock that SDK boundary for the whole module when psycopg is absent.
_types_patcher = None
@@ -36,8 +37,8 @@ def setUpModule():
global _types_patcher
import parsedmarc.postgres as pg
if pg.psycopg_types is None:
_types_patcher = patch("parsedmarc.postgres.psycopg_types", MagicMock())
if pg.psycopg_json is None:
_types_patcher = patch("parsedmarc.postgres.psycopg_json", MagicMock())
_types_patcher.start()
@@ -99,6 +100,7 @@ class TestPostgreSQLHelpers(unittest.TestCase):
def test_naive_local_to_timestamptz_valid(self):
"""A valid naive string is returned with a timezone offset."""
result = _naive_local_to_timestamptz("2024-01-15 10:30:00")
assert result is not None
self.assertIsInstance(result, str)
self.assertTrue(
"+" in result or "-" in result[10:],
@@ -125,11 +127,13 @@ class TestPostgreSQLHelpers(unittest.TestCase):
def test_normalize_arrival_date_iso_naive_utc(self):
"""A naive ISO string (known UTC) is returned with +00 suffix."""
result = _normalize_arrival_date("2024-01-15 10:30:00")
assert result is not None
self.assertTrue(result.endswith("+00"), f"Expected +00 suffix: {result}")
def test_normalize_arrival_date_rfc2822(self):
"""An RFC 2822 date is converted to UTC with +00 suffix."""
result = _normalize_arrival_date("Fri, 28 Oct 2022 00:34:24 +0800")
assert result is not None
self.assertTrue(result.endswith("+00"), f"Expected +00 suffix: {result}")
# 00:34:24 +0800 is 16:34:24 UTC on 27 Oct 2022.
self.assertIn("2022-10-27", result)
@@ -138,6 +142,7 @@ class TestPostgreSQLHelpers(unittest.TestCase):
def test_normalize_arrival_date_already_utc(self):
"""A string already ending with +00 still works."""
result = _normalize_arrival_date("2024-01-15 10:30:00+00")
assert result is not None
self.assertTrue(result.endswith("+00"), f"Expected +00 suffix: {result}")
def test_normalize_arrival_date_unparseable(self):
@@ -167,7 +172,8 @@ class TestPostgreSQLHelpers(unittest.TestCase):
def test_contact_info_to_text_numeric(self):
"""Non-string scalars are converted via str()."""
self.assertEqual(_contact_info_to_text(123), "123")
# Deliberately outside the annotated parameter types
self.assertEqual(_contact_info_to_text(123), "123") # pyright: ignore[reportArgumentType]
def _make_client():
@@ -180,8 +186,8 @@ def _make_client():
client = PostgreSQLClient(
host="localhost", database="test", user="test", password="test"
)
mock_conn.closed = False
client._conn = mock_conn
client._conn.closed = False
return client, mock_conn
@@ -211,6 +217,7 @@ def _named_params(call):
sql = call.args[0]
m = re.search(r"\(([^)]*?)\)\s*VALUES", sql, re.S)
assert m is not None
cols = [c.strip() for c in m.group(1).split(",") if c.strip()]
return dict(zip(cols, call.args[1]))
@@ -758,7 +765,7 @@ class TestPostgreSQLWithSamples(unittest.TestCase):
num_records = len(report.get("records", []))
_mock_cursor(mock_conn, [(rid,) for rid in range(1, 2 + num_records)])
try:
client.save_aggregate_report_to_postgresql(report)
client.save_aggregate_report_to_postgresql(cast(dict, report))
saved += 1
except Exception as exc:
self.fail(f"aggregate save failed for {sample_path}: {exc}")
@@ -783,7 +790,7 @@ class TestPostgreSQLWithSamples(unittest.TestCase):
# Dedup SELECT returns None (not a dup), then the INSERT id.
_mock_cursor(mock_conn, [None, (1,)])
try:
client.save_failure_report_to_postgresql(report)
client.save_failure_report_to_postgresql(cast(dict, report))
saved += 1
except Exception as exc:
self.fail(f"failure save failed for {sample_path}: {exc}")
@@ -807,7 +814,7 @@ class TestPostgreSQLWithSamples(unittest.TestCase):
num_policies = len(report.get("policies", []))
_mock_cursor(mock_conn, [(rid,) for rid in range(1, 2 + num_policies)])
try:
client.save_smtp_tls_report_to_postgresql(report)
client.save_smtp_tls_report_to_postgresql(cast(dict, report))
saved += 1
except Exception as exc:
self.fail(f"smtp_tls save failed for {sample_path}: {exc}")