From 12e1109a42cfb342d3c853098b8e17fde220aea4 Mon Sep 17 00:00:00 2001 From: Tobias Kaupat Date: Mon, 19 Oct 2026 16:37:41 +0200 Subject: [PATCH] test: port sabotage self-test to task test:sabotage Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/test.yml | 4 ++ Taskfile.yml | 5 ++ tests/sabotage.py | 128 +++++++++++++++++++++++++++++++++++++ 3 files changed, 137 insertions(+) create mode 100644 tests/sabotage.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4fd981a..19413e4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -77,3 +77,7 @@ jobs: - name: Run Python suite run: task test + + # Guards the guard, as in the Bash job: a broken backup must fail the tests. + - name: Verify the Python suite detects a broken backup + run: task test:sabotage diff --git a/Taskfile.yml b/Taskfile.yml index 39bd6d9..e2af618 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -45,3 +45,8 @@ tasks: desc: Run the feature-gated tests cmds: - uv run pytest -m feature {{.CLI_ARGS}} + + test:sabotage: + desc: Prove the compat tests go red on an image with a broken backup + cmds: + - uv run python tests/sabotage.py diff --git a/tests/sabotage.py b/tests/sabotage.py new file mode 100644 index 0000000..51e9d48 --- /dev/null +++ b/tests/sabotage.py @@ -0,0 +1,128 @@ +"""Self test of the suite: prove that it goes red when the backup is broken. + +Ported from `test/run.sh --sabotage`. Run with `task test:sabotage`. + +A throwaway copy of the tree gets a backup.sh whose `restic backup` call is a no-op, the +nastiest kind of regression (the script still reports success). The compat tests run +against an image built from that copy. The check passes only if the assertions about the +backup itself fail by name. Red for another reason (an image that cannot start, a +non-executable /bin/backup) proves nothing and is reported as INVALID. + +Exit codes: 0 sabotage detected, 1 suite did not detect it, 2 check invalid or broken. +Tracked files are never modified. +""" + +from __future__ import annotations + +import os +import shutil +import signal +import subprocess +import sys +import tempfile +import uuid +import xml.etree.ElementTree as ET +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +COPIED = ("Dockerfile", "backup.sh", "check.sh", "entry.sh") +# Tests about the backup itself; each must fail on the sabotaged image. +EXPECTED_RED = ( + "test_one_snapshot_after_one_backup", + "test_second_backup_adds_snapshot", +) + + +class Invalid(Exception): + """The check itself is broken; the suite result means nothing.""" + + +def _raise_interrupt(signum: int, _frame: object) -> None: + raise KeyboardInterrupt(f"received {signal.Signals(signum).name}") + + +def sabotage_tree(tmp: Path) -> None: + for name in COPIED: + shutil.copy2(ROOT / name, tmp / name) + script = tmp / "backup.sh" + lines = script.read_text().splitlines(keepends=True) + patched = [ + ("true " + ln[len("restic backup ") :]) if ln.startswith("restic backup ") else ln + for ln in lines + ] + if patched == lines: + raise Invalid("sabotage patch did not apply, backup.sh changed shape") + script.write_text("".join(patched)) + # Dockerfile COPY has no chmod: the image only works when the file is executable. + script.chmod(script.stat().st_mode | 0o111) + if not os.access(script, os.X_OK): + raise Invalid("sabotaged backup.sh is not executable") + + +def failed_tests(report: Path) -> set[str]: + cases = ET.parse(report).getroot().iter("testcase") + return { + c.get("name", "") + for c in cases + if c.find("failure") is not None or c.find("error") is not None + } + + +def main() -> int: + tag = f"rbd-sabotage:{uuid.uuid4().hex[:8]}" + tmp = Path(tempfile.mkdtemp(prefix="rbd-sabotage-")) + try: + sabotage_tree(tmp) + print(f">>> building {tag} from a sabotaged copy") + build = subprocess.run(["docker", "build", "--quiet", "-t", tag, str(tmp)]) + if build.returncode != 0: + raise Invalid("image build failed") + print(">>> running the compat tests against the sabotaged image (failure expected)") + report = tmp / "report.xml" + proc = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "tests/test_compat.py", + "-p", + "no:cacheprovider", + f"--junitxml={report}", + ], + cwd=ROOT, + env={**os.environ, "IMAGE": tag}, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + print(proc.stdout) + if proc.returncode == 0: + print( + "SABOTAGE CHECK FAILED: the compat tests passed on a broken image", file=sys.stderr + ) + return 1 + if "permission denied" in proc.stdout.lower(): + raise Invalid("log contains 'permission denied', suite failed for the wrong reason") + if not report.is_file(): + raise Invalid(f"pytest produced no report (exit code {proc.returncode})") + red = failed_tests(report) + missing = [name for name in EXPECTED_RED if name not in red] + for name in missing: + print( + f"SABOTAGE CHECK FAILED: expected assertion did not fail: {name}", file=sys.stderr + ) + if missing: + return 1 + print("\nsabotage check ok: the compat tests detect a broken backup") + return 0 + except Invalid as exc: + print(f"SABOTAGE CHECK INVALID: {exc}", file=sys.stderr) + return 2 + finally: + shutil.rmtree(tmp, ignore_errors=True) + subprocess.run(["docker", "rmi", "-f", tag], capture_output=True) + + +if __name__ == "__main__": + signal.signal(signal.SIGTERM, _raise_interrupt) + sys.exit(main())