mirror of
https://github.com/lobaro/restic-backup-docker.git
synced 2026-11-20 16:56:52 +00:00
test: sabotage check rejects errors and malformed reports
Only <failure> of the expected tests counts as red; errors in them and a malformed JUnit report exit 2 (invalid) instead of passing or exiting 1. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a6c091b854
commit
651f9900c5
1 file changed
+17
-14
+17
-14
@@ -24,6 +24,8 @@ import uuid
|
|||||||
import xml.etree.ElementTree as ET
|
import xml.etree.ElementTree as ET
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from helpers import raise_interrupt
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parent.parent
|
ROOT = Path(__file__).resolve().parent.parent
|
||||||
COPIED = ("Dockerfile", "backup.sh", "check.sh", "entry.sh")
|
COPIED = ("Dockerfile", "backup.sh", "check.sh", "entry.sh")
|
||||||
# Tests about the backup itself; each must fail on the sabotaged image.
|
# Tests about the backup itself; each must fail on the sabotaged image.
|
||||||
@@ -37,10 +39,6 @@ class Invalid(Exception):
|
|||||||
"""The check itself is broken; the suite result means nothing."""
|
"""The check itself is broken; the suite result means nothing."""
|
||||||
|
|
||||||
|
|
||||||
def _raise_interrupt(signum: int, _frame: object) -> None:
|
|
||||||
raise KeyboardInterrupt(f"received {signal.Signals(signum).name}")
|
|
||||||
|
|
||||||
|
|
||||||
def sabotage_tree(tmp: Path) -> None:
|
def sabotage_tree(tmp: Path) -> None:
|
||||||
for name in COPIED:
|
for name in COPIED:
|
||||||
shutil.copy2(ROOT / name, tmp / name)
|
shutil.copy2(ROOT / name, tmp / name)
|
||||||
@@ -59,13 +57,15 @@ def sabotage_tree(tmp: Path) -> None:
|
|||||||
raise Invalid("sabotaged backup.sh is not executable")
|
raise Invalid("sabotaged backup.sh is not executable")
|
||||||
|
|
||||||
|
|
||||||
def failed_tests(report: Path) -> set[str]:
|
def red_tests(report: Path) -> tuple[set[str], set[str]]:
|
||||||
cases = ET.parse(report).getroot().iter("testcase")
|
"""Names of tests with an assertion failure, and of tests with an error (setup etc.)."""
|
||||||
return {
|
try:
|
||||||
c.get("name", "")
|
cases = list(ET.parse(report).getroot().iter("testcase"))
|
||||||
for c in cases
|
except ET.ParseError as exc:
|
||||||
if c.find("failure") is not None or c.find("error") is not None
|
raise Invalid(f"pytest report is malformed or truncated: {exc}") from exc
|
||||||
}
|
failures = {c.get("name", "") for c in cases if c.find("failure") is not None}
|
||||||
|
errors = {c.get("name", "") for c in cases if c.find("error") is not None}
|
||||||
|
return failures, errors
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
@@ -105,8 +105,11 @@ def main() -> int:
|
|||||||
raise Invalid("log contains 'permission denied', suite failed for the wrong reason")
|
raise Invalid("log contains 'permission denied', suite failed for the wrong reason")
|
||||||
if not report.is_file():
|
if not report.is_file():
|
||||||
raise Invalid(f"pytest produced no report (exit code {proc.returncode})")
|
raise Invalid(f"pytest produced no report (exit code {proc.returncode})")
|
||||||
red = failed_tests(report)
|
failures, errors = red_tests(report)
|
||||||
missing = [name for name in EXPECTED_RED if name not in red]
|
broken = [name for name in EXPECTED_RED if name in errors]
|
||||||
|
if broken:
|
||||||
|
raise Invalid(f"expected tests errored instead of failing an assertion: {broken}")
|
||||||
|
missing = [name for name in EXPECTED_RED if name not in failures]
|
||||||
for name in missing:
|
for name in missing:
|
||||||
print(
|
print(
|
||||||
f"SABOTAGE CHECK FAILED: expected assertion did not fail: {name}", file=sys.stderr
|
f"SABOTAGE CHECK FAILED: expected assertion did not fail: {name}", file=sys.stderr
|
||||||
@@ -124,5 +127,5 @@ def main() -> int:
|
|||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
signal.signal(signal.SIGTERM, _raise_interrupt)
|
signal.signal(signal.SIGTERM, raise_interrupt)
|
||||||
sys.exit(main())
|
sys.exit(main())
|
||||||
Reference in new issue
Block a user