test: sabotage check rejects errors and malformed reports

Only <failure> of the expected tests counts as red; errors in them and a
malformed JUnit report exit 2 (invalid) instead of passing or exiting 1.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Tobias KaupatandClaude Sonnet 5.5 committed 2026-10-20 09:17:03 +02:00
1 parent a6c091b854
commit 651f9900c5
1 file changed
+17 -14
+17 -14
View File
@@ -24,6 +24,8 @@ import uuid
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
from pathlib import Path from pathlib import Path
from helpers import raise_interrupt
ROOT = Path(__file__).resolve().parent.parent ROOT = Path(__file__).resolve().parent.parent
COPIED = ("Dockerfile", "backup.sh", "check.sh", "entry.sh") COPIED = ("Dockerfile", "backup.sh", "check.sh", "entry.sh")
# Tests about the backup itself; each must fail on the sabotaged image. # Tests about the backup itself; each must fail on the sabotaged image.
@@ -37,10 +39,6 @@ class Invalid(Exception):
"""The check itself is broken; the suite result means nothing.""" """The check itself is broken; the suite result means nothing."""
def _raise_interrupt(signum: int, _frame: object) -> None:
raise KeyboardInterrupt(f"received {signal.Signals(signum).name}")
def sabotage_tree(tmp: Path) -> None: def sabotage_tree(tmp: Path) -> None:
for name in COPIED: for name in COPIED:
shutil.copy2(ROOT / name, tmp / name) shutil.copy2(ROOT / name, tmp / name)
@@ -59,13 +57,15 @@ def sabotage_tree(tmp: Path) -> None:
raise Invalid("sabotaged backup.sh is not executable") raise Invalid("sabotaged backup.sh is not executable")
def failed_tests(report: Path) -> set[str]: def red_tests(report: Path) -> tuple[set[str], set[str]]:
cases = ET.parse(report).getroot().iter("testcase") """Names of tests with an assertion failure, and of tests with an error (setup etc.)."""
return { try:
c.get("name", "") cases = list(ET.parse(report).getroot().iter("testcase"))
for c in cases except ET.ParseError as exc:
if c.find("failure") is not None or c.find("error") is not None raise Invalid(f"pytest report is malformed or truncated: {exc}") from exc
} failures = {c.get("name", "") for c in cases if c.find("failure") is not None}
errors = {c.get("name", "") for c in cases if c.find("error") is not None}
return failures, errors
def main() -> int: def main() -> int:
@@ -105,8 +105,11 @@ def main() -> int:
raise Invalid("log contains 'permission denied', suite failed for the wrong reason") raise Invalid("log contains 'permission denied', suite failed for the wrong reason")
if not report.is_file(): if not report.is_file():
raise Invalid(f"pytest produced no report (exit code {proc.returncode})") raise Invalid(f"pytest produced no report (exit code {proc.returncode})")
red = failed_tests(report) failures, errors = red_tests(report)
missing = [name for name in EXPECTED_RED if name not in red] broken = [name for name in EXPECTED_RED if name in errors]
if broken:
raise Invalid(f"expected tests errored instead of failing an assertion: {broken}")
missing = [name for name in EXPECTED_RED if name not in failures]
for name in missing: for name in missing:
print( print(
f"SABOTAGE CHECK FAILED: expected assertion did not fail: {name}", file=sys.stderr f"SABOTAGE CHECK FAILED: expected assertion did not fail: {name}", file=sys.stderr
@@ -124,5 +127,5 @@ def main() -> int:
if __name__ == "__main__": if __name__ == "__main__":
signal.signal(signal.SIGTERM, _raise_interrupt) signal.signal(signal.SIGTERM, raise_interrupt)
sys.exit(main()) sys.exit(main())