From cb35fb296dfa579e9f786686d8742881bdacfaf1 Mon Sep 17 00:00:00 2001
From: Tobias Kaupat <tobias.kaupat@lobaro.de>
Date: Tue, 20 Oct 2026 10:06:04 +0200
Subject: [PATCH] fix: sabotage check treats failed scenario steps as INVALID

StepFailed is no AssertionError any more; sabotage.py counts a JUnit failure
named StepFailed as error, so an expected-red test only counts when its own
assertion fails.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
---
 tests/helpers.py  |  8 ++++++--
 tests/sabotage.py | 21 ++++++++++++++++++---
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/tests/helpers.py b/tests/helpers.py
index a2c1095..1656261 100644
--- a/tests/helpers.py
+++ b/tests/helpers.py
@@ -41,8 +41,12 @@ def raise_interrupt(signum: int, _frame: object) -> None:
     raise KeyboardInterrupt(f"received {signal.Signals(signum).name}")
 
 
-class StepFailed(AssertionError):
-    pass
+class StepFailed(Exception):
+    """A scenario step could not run; not an assertion about the behaviour under test.
+
+    Deliberately no AssertionError: sabotage.py tells it apart from a real assertion
+    failure by the exception name in the JUnit report.
+    """
 
 
 class Scenario:
diff --git a/tests/sabotage.py b/tests/sabotage.py
index 22725da..c80187f 100644
--- a/tests/sabotage.py
+++ b/tests/sabotage.py
@@ -35,6 +35,9 @@ EXPECTED_RED = (
 )
 
 
+STEP_FAILED = "StepFailed"  # name of helpers.StepFailed in the JUnit report
+
+
 class Invalid(Exception):
     """The check itself is broken; the suite result means nothing."""
 
@@ -58,13 +61,25 @@ def sabotage_tree(tmp: Path) -> None:
 
 
 def red_tests(report: Path) -> tuple[set[str], set[str]]:
-    """Names of tests with an assertion failure, and of tests with an error (setup etc.)."""
+    """Names of tests with an assertion failure, and of tests with an error.
+
+    Errors are setup errors and failed scenario steps (StepFailed): neither is an assertion
+    about the behaviour under test.
+    """
     try:
         cases = list(ET.parse(report).getroot().iter("testcase"))
     except ET.ParseError as exc:
         raise Invalid(f"pytest report is malformed or truncated: {exc}") from exc
-    failures = {c.get("name", "") for c in cases if c.find("failure") is not None}
     errors = {c.get("name", "") for c in cases if c.find("error") is not None}
+    failures = set()
+    for case in cases:
+        failure = case.find("failure")
+        if failure is None:
+            continue
+        if STEP_FAILED in failure.get("message") or "":
+            errors.add(case.get("name", ""))
+        else:
+            failures.add(case.get("name", ""))
     return failures, errors
 
 
@@ -108,7 +123,7 @@ def main() -> int:
         failures, errors = red_tests(report)
         broken = [name for name in EXPECTED_RED if name in errors]
         if broken:
-            raise Invalid(f"expected tests errored instead of failing an assertion: {broken}")
+            raise Invalid(f"expected tests errored or hit a failed scenario step instead of failing an assertion: {broken}")
         missing = [name for name in EXPECTED_RED if name not in failures]
         for name in missing:
             print(