Fix blank SMTP TLS failure-detail fields crashing saves; read RFC 8460 additional-information (11.0.3) (#916)

* Treat blank SMTP TLS failure-detail fields as absent; read RFC 8460 additional-information

Some reporters send optional failure-details fields as empty strings
(e.g. "sending-mta-ip": "" on sts-policy-fetch-error). The parser copied
them verbatim, so the PostgreSQL save failed on the INET columns
("invalid input syntax for type inet") and the Elasticsearch/OpenSearch
saves failed Ip() validation, discarding the whole report. Blank or
whitespace-only optional values are now omitted.

RFC 8460 section 4.4 names the URI key "additional-information"; the
parser only read "additional-info-uri" (the schema's value placeholder),
so conformant reports lost the URI. The RFC key is now read, with the old
key kept as a fallback.

Bump version to 11.0.3.

Closes #915

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

* Import parse_report_file directly in the ES/OpenSearch tests

Addresses github-code-quality's "Module is imported with 'import' and
'import from'" findings on tests/test_elastic.py and
tests/test_opensearch.py.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

tests/test_elastic.py

@@ -8,10 +8,11 @@ queries, error wrapping — without needing a running Elasticsearch cluster.
 
 import time
 import unittest
+from typing import cast
 from unittest.mock import MagicMock, call, patch
 
 import parsedmarc.elastic as elastic_module
-from parsedmarc import InvalidFailureReport
+from parsedmarc import InvalidFailureReport, parse_report_file
 from parsedmarc.elastic import (
     AlreadySaved,
     ElasticsearchError,
@@ -1486,6 +1487,46 @@ class TestSaveSmtpTlsReport(unittest.TestCase):
             "https://reports.example.com/tls-help",
         )
 
+    def test_save_blank_failure_detail_fields_omitted_from_ip_fields(self):
+        """Blank optional failure-detail fields must not reach the Ip()
+        fields (issue #915).
+
+        samples/smtp_tls/empty_failure_detail_fields.json reproduces a
+        real reporter's shape: sending-mta-ip/receiving-ip sent as "" on
+        an sts-policy-fetch-error. Verified directly against
+        elasticsearch.dsl: Document.save() validates by calling
+        full_clean() before any network call, and an Ip() field's
+        validation calls Python's ipaddress.ip_address(""), which raises a
+        bare ValueError ("'' does not appear to be an IPv4 or IPv6
+        address") rather than a caught ValidationException -- so an empty
+        string here previously discarded the whole report instead of
+        indexing it with a blank IP.
+
+        save() is mocked (per this module's convention, so no real
+        cluster is needed), which also means it never runs full_clean()
+        itself; call it explicitly on the captured document so this test
+        exercises the same validation a real save() would, and fails with
+        that ValueError on unfixed parsedmarc/__init__.py.
+        """
+        result = parse_report_file(
+            "samples/smtp_tls/empty_failure_detail_fields.json", offline=True
+        )
+        report = cast(dict, result["report"])
+        with (
+            patch("parsedmarc.elastic.Search", return_value=_empty_search()),
+            patch("parsedmarc.elastic.Index"),
+            patch.object(
+                elastic_module._SMTPTLSReportDoc, "save", autospec=True
+            ) as mock_save,
+        ):
+            save_smtp_tls_report_to_elasticsearch(report)
+        doc = mock_save.call_args[0][0]
+        doc.full_clean()
+        failure_detail = doc.policies[0].failure_details[0]
+        self.assertIsNone(failure_detail.sending_mta_ip)
+        self.assertIsNone(failure_detail.receiving_ip)
+        self.assertEqual(failure_detail.failure_reason_code, "no-policy-served")
+
 
 class TestBackwardCompatAlias(unittest.TestCase):
     def test_save_forensic_alias_points_to_save_failure(self):

tests/test_init.py

@@ -963,6 +963,26 @@ class Test(unittest.TestCase):
                 )
             print("Passed!")
 
+    def testSmtpTlsRfc8460SampleReadsAdditionalInformation(self):
+        """samples/smtp_tls/rfc8460.json is RFC 8460 §4.4's own worked
+        example, whose second failure-details entry sets
+        "additional-information" (not the non-RFC "additional-info-uri").
+        Before the fix, the parser only read "additional-info-uri", so this
+        URI was silently dropped even from the RFC's own example."""
+        result = parsedmarc.parse_report_file(
+            "samples/smtp_tls/rfc8460.json", offline=True
+        )
+        report = cast(SMTPTLSReport, result["report"])
+        failure_details = report["policies"][0].get("failure_details", [])
+        starttls_detail = next(
+            d for d in failure_details if d["result_type"] == "starttls-not-supported"
+        )
+        self.assertEqual(
+            starttls_detail.get("additional_info_uri"),
+            "https://reports.company-x.example/report_info?"
+            "id=5065427c-23d3#StarttlsNotSupported",
+        )
+
     def testSmtpTlsCsvStripsNulFromFields(self):
         """A NUL character in an SMTP TLS report text field is stripped
         from CSV output instead of reaching the ``csv`` writer.
@@ -1487,6 +1507,81 @@ class Test(unittest.TestCase):
         self.assertEqual(result["additional_info_uri"], "https://example.com/info")
         self.assertEqual(result["failure_reason_code"], "TLS_ERROR")
 
+    def testParseSmtpTlsFailureDetailsBlankOptionalFieldsOmitted(self):
+        """Empty-string and whitespace-only optional fields are treated as
+        absent, not passed through.
+
+        Real-world reporters send "" for sending-mta-ip/receiving-ip on
+        results like sts-policy-fetch-error (see samples/smtp_tls/
+        empty_failure_detail_fields.json). An empty string is not a valid
+        IP address, so it previously reached PostgreSQL's INET columns
+        verbatim and crashed the PostgreSQL save, and would also have
+        crashed the Elasticsearch/OpenSearch saves' Ip() field validation
+        with a bare ValueError (issue #915).
+        """
+        details = {
+            "result-type": "sts-policy-fetch-error",
+            "failed-session-count": 1,
+            "sending-mta-ip": "",
+            "receiving-ip": "   ",
+            "receiving-mx-hostname": "",
+            "receiving-mx-helo": "\t",
+            "additional-information": "",
+            "additional-info-uri": "",
+            "failure-reason-code": "",
+        }
+        result = parsedmarc._parse_smtp_tls_failure_details(details)
+        self.assertEqual(result["result_type"], "sts-policy-fetch-error")
+        self.assertEqual(result["failed_session_count"], 1)
+        self.assertNotIn("sending_mta_ip", result)
+        self.assertNotIn("receiving_ip", result)
+        self.assertNotIn("receiving_mx_hostname", result)
+        self.assertNotIn("receiving_mx_helo", result)
+        self.assertNotIn("additional_info_uri", result)
+        self.assertNotIn("failure_reason_code", result)
+
+    def testParseSmtpTlsFailureDetailsAdditionalInformationKey(self):
+        """RFC 8460 §4.4's JSON key is "additional-information" (its value
+        is described as an "additional-info-uri"); it is read into the
+        parser's additional_info_uri field.
+        """
+        details = {
+            "result-type": "starttls-not-supported",
+            "failed-session-count": 1,
+            "additional-information": "https://example.com/rfc-key-info",
+        }
+        result = parsedmarc._parse_smtp_tls_failure_details(details)
+        self.assertEqual(
+            result["additional_info_uri"], "https://example.com/rfc-key-info"
+        )
+
+    def testParseSmtpTlsFailureDetailsAdditionalInformationWinsOverLegacy(self):
+        """When both the RFC key and the legacy key are present and
+        non-blank, the RFC key's value wins."""
+        details = {
+            "result-type": "starttls-not-supported",
+            "failed-session-count": 1,
+            "additional-information": "https://example.com/rfc-key-info",
+            "additional-info-uri": "https://example.com/legacy-key-info",
+        }
+        result = parsedmarc._parse_smtp_tls_failure_details(details)
+        self.assertEqual(
+            result["additional_info_uri"], "https://example.com/rfc-key-info"
+        )
+
+    def testParseSmtpTlsFailureDetailsAdditionalInformationBlankFallsBack(self):
+        """A blank RFC-key value falls back to a non-blank legacy value."""
+        details = {
+            "result-type": "starttls-not-supported",
+            "failed-session-count": 1,
+            "additional-information": "",
+            "additional-info-uri": "https://example.com/legacy-key-info",
+        }
+        result = parsedmarc._parse_smtp_tls_failure_details(details)
+        self.assertEqual(
+            result["additional_info_uri"], "https://example.com/legacy-key-info"
+        )
+
     def testParseSmtpTlsFailureDetailsMissingRequired(self):
         """Missing required field raises InvalidSMTPTLSReport"""
         with self.assertRaises(parsedmarc.InvalidSMTPTLSReport):

tests/test_opensearch.py

@@ -8,10 +8,11 @@ queries, error wrapping — without needing a running OpenSearch cluster.
 
 import time
 import unittest
+from typing import cast
 from unittest.mock import MagicMock, call, patch
 
 import parsedmarc.opensearch as opensearch_module
-from parsedmarc import InvalidFailureReport
+from parsedmarc import InvalidFailureReport, parse_report_file
 from parsedmarc.opensearch import (
     AlreadySaved,
     OpenSearchError,
@@ -1477,6 +1478,46 @@ class TestSaveSmtpTlsReport(unittest.TestCase):
             "https://reports.example.com/tls-help",
         )
 
+    def test_save_blank_failure_detail_fields_omitted_from_ip_fields(self):
+        """Blank optional failure-detail fields must not reach the Ip()
+        fields (issue #915).
+
+        samples/smtp_tls/empty_failure_detail_fields.json reproduces a
+        real reporter's shape: sending-mta-ip/receiving-ip sent as "" on
+        an sts-policy-fetch-error. Verified directly against opensearchpy:
+        Document.save() validates by calling full_clean() before any
+        network call, and an Ip() field's
+        validation calls Python's ipaddress.ip_address(""), which raises
+        a bare ValueError ("'' does not appear to be an IPv4 or IPv6
+        address") rather than a caught ValidationException -- so an empty
+        string here previously discarded the whole report instead of
+        indexing it with a blank IP.
+
+        save() is mocked (per this module's convention, so no real
+        cluster is needed), which also means it never runs full_clean()
+        itself; call it explicitly on the captured document so this test
+        exercises the same validation a real save() would, and fails with
+        that ValueError on unfixed parsedmarc/__init__.py.
+        """
+        result = parse_report_file(
+            "samples/smtp_tls/empty_failure_detail_fields.json", offline=True
+        )
+        report = cast(dict, result["report"])
+        with (
+            patch("parsedmarc.opensearch.Search", return_value=_empty_search()),
+            patch("parsedmarc.opensearch.Index"),
+            patch.object(
+                opensearch_module._SMTPTLSReportDoc, "save", autospec=True
+            ) as mock_save,
+        ):
+            save_smtp_tls_report_to_opensearch(report)
+        doc = mock_save.call_args[0][0]
+        doc.full_clean()
+        failure_detail = doc.policies[0].failure_details[0]
+        self.assertIsNone(failure_detail.sending_mta_ip)
+        self.assertIsNone(failure_detail.receiving_ip)
+        self.assertEqual(failure_detail.failure_reason_code, "no-policy-served")
+
 
 class TestBackwardCompatAlias(unittest.TestCase):
     def test_save_forensic_alias_points_to_save_failure(self):

tests/test_postgres.py

@@ -658,6 +658,46 @@ class TestPostgreSQLClientSave(unittest.TestCase):
         self.assertIn(100, policy_params)
         self.assertIn(2, policy_params)
 
+    def test_save_smtp_tls_report_blank_failure_detail_fields_become_null(self):
+        """Blank optional failure-detail fields reach PostgreSQL as None,
+        not "" (issue #915).
+
+        samples/smtp_tls/empty_failure_detail_fields.json reproduces a
+        real reporter's shape: sending-mta-ip/receiving-ip/
+        receiving-mx-hostname/additional-information sent as "" on an
+        sts-policy-fetch-error. "" is not a valid PostgreSQL INET value, so
+        before the parser fix in parsedmarc/__init__.py this reached
+        cur.execute() as an empty string and crashed the save with
+        "invalid input syntax for type inet: \"\"" against a real server.
+        This test goes through the real parser (parsedmarc.parse_report_file)
+        rather than a hand-built dict, so it fails on unfixed
+        parsedmarc/__init__.py even though this test file only mocks
+        psycopg.
+        """
+        client, mock_conn = _make_client()
+        cur = _mock_cursor(mock_conn, [(1,), (10,)])
+
+        result = parsedmarc.parse_report_file(
+            "samples/smtp_tls/empty_failure_detail_fields.json", offline=True
+        )
+        report = result["report"]
+
+        client.save_smtp_tls_report_to_postgresql(cast(dict, report))
+
+        sqls = _executed_sql(cur)
+        self.assertIn("smtp_tls_failure_detail", sqls[2])
+
+        detail_params = _named_params(cur.execute.call_args_list[2])
+        self.assertIsNone(detail_params["sending_mta_ip"])
+        self.assertIsNone(detail_params["receiving_ip"])
+        self.assertIsNone(detail_params["receiving_mx_hostname"])
+        self.assertIsNone(detail_params["additional_info_uri"])
+        # A non-blank optional field is preserved.
+        self.assertEqual(detail_params["failure_reason_code"], "no-policy-served")
+        # Required fields are preserved.
+        self.assertEqual(detail_params["result_type"], "sts-policy-fetch-error")
+        self.assertEqual(detail_params["failed_session_count"], 1)
+
     def test_save_smtp_tls_report_already_saved(self):
         """AlreadySaved is raised when ON CONFLICT returns no row."""
         client, mock_conn = _make_client()