From 884d988079cf7c257e8bcb598656d012bdcfc78c Mon Sep 17 00:00:00 2001 From: Tobias Kaupat Date: Mon, 19 Oct 2026 17:49:47 +0200 Subject: [PATCH] test: remove the Bash suite Co-Authored-By: Claude Sonnet 5.5 --- test/compat.sh | 275 --------------------------------------- test/features.sh | 172 ------------------------ test/lib.sh | 287 ----------------------------------------- test/run.sh | 183 -------------------------- tests/sabotage.py | 2 +- tests/test_compat.py | 2 +- tests/test_features.py | 2 +- 7 files changed, 3 insertions(+), 920 deletions(-) delete mode 100644 test/compat.sh delete mode 100644 test/features.sh delete mode 100644 test/lib.sh delete mode 100644 test/run.sh diff --git a/test/compat.sh b/test/compat.sh deleted file mode 100644 index 5de7076..0000000 --- a/test/compat.sh +++ /dev/null @@ -1,275 +0,0 @@ -#!/usr/bin/env bash -# Backwards-compatibility suite. -# -# Everything in here describes behaviour the image has today and that users -# depend on. These tests always run, for every pull request. A change that -# turns one of them red is a breaking change, whether or not that was intended. - -set -u - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=test/lib.sh -. "${SCRIPT_DIR}/lib.sh" - -trap stop_container EXIT - -echo "=== compat: repository bootstrap and default backup ===" - -start_container default -wait_for_start || { fail "container starts with default configuration"; summary "compat"; exit 1; } -pass "container starts with default configuration" - -startup_log="$(docker logs "$CONTAINER" 2>&1)" -assert_contains "$startup_log" "Running restic init" \ - "empty repository is initialised on first start" -assert_contains "$startup_log" "Repo init status 0" \ - "repository initialisation succeeds" - -crontab="$(in_container cat /var/spool/cron/crontabs/root)" -assert_contains "$crontab" "/bin/backup" \ - "BACKUP_CRON is written to the crontab" -assert_contains "$crontab" "flock -n /var/run/backup.lock" \ - "backup cron job is guarded by flock" -assert_not_contains "$crontab" "/bin/check" \ - "no check cron job is installed while CHECK_CRON is unset" - -write_file /data/hello.txt "backup me" - -backup_log="$(in_container /bin/backup 2>&1)" -assert_contains "$backup_log" "Backup Successful" \ - "backup of /data succeeds" -assert_contains "$backup_log" "Pre-backup script not found" \ - "missing pre-backup hook is reported, not fatal" -assert_contains "$backup_log" "Post-backup script not found" \ - "missing post-backup hook is reported, not fatal" -assert_not_contains "$backup_log" "notification" \ - "no mail or Teams notification is attempted without configuration" -assert_eq "1" "$(snapshot_count)" \ - "exactly one snapshot exists after one backup" - -snapshots="$(in_container restic snapshots --json 2>/dev/null)" -assert_contains "$snapshots" '"testtag"' \ - "snapshot carries the configured RESTIC_TAG" - -in_container restic restore latest --target /tmp/restore >/dev/null 2>&1 -restore_rc=$? -assert_eq "0" "$restore_rc" \ - "restic restore of the latest snapshot exits 0" -restored="$(in_container cat /tmp/restore/data/hello.txt 2>&1)" -assert_eq "backup me" "$restored" \ - "restored file content matches the original" - -logfile="$(in_container cat /var/log/backup-last.log 2>&1)" -assert_contains "$logfile" "RESTIC_REPOSITORY: /mnt/restic" \ - "backup-last.log records the run parameters" - -in_container /bin/backup >/dev/null 2>&1 -assert_eq "2" "$(snapshot_count)" \ - "a second backup adds a second snapshot" - -echo -echo "=== compat: data integrity check ===" - -check_log="$(in_container /bin/check 2>&1)" -assert_contains "$check_log" "Check Successful" \ - "restic check passes on a healthy repository" -assert_contains "$check_log" "Pre-check script not found" \ - "missing pre-check hook is reported, not fatal" - -echo -echo "=== compat: pre- and post-backup hooks ===" - -write_file /hooks/pre-backup.sh '#!/bin/sh -echo "PRE-HOOK-RAN" -' -write_file /hooks/post-backup.sh '#!/bin/sh -echo "POST-HOOK-RAN with rc=$1" -' -in_container chmod +x /hooks/pre-backup.sh /hooks/post-backup.sh - -hook_log="$(in_container /bin/backup 2>&1)" -assert_contains "$hook_log" "PRE-HOOK-RAN" \ - "pre-backup hook is executed" -assert_contains "$hook_log" "POST-HOOK-RAN with rc=0" \ - "post-backup hook receives the backup return code" - -stop_container - -echo -echo "=== compat: CHECK_CRON and check hooks ===" - -start_container checkcron -e "CHECK_CRON=0 0 31 2 *" -wait_for_start || { fail "container starts with CHECK_CRON"; summary "compat"; exit 1; } - -check_crontab="$(in_container cat /var/spool/cron/crontabs/root)" -check_cron_line="$(echo "$check_crontab" | grep '/bin/check' || true)" -assert_contains "$check_cron_line" "flock -n /var/run/backup.lock" \ - "check cron job is guarded by the shared backup lock" -assert_contains "$check_cron_line" "/bin/check" \ - "CHECK_CRON adds a check cron job to the crontab" -assert_eq "2" "$(echo "$check_crontab" | grep -c .)" \ - "crontab has a backup line and a check line" -assert_contains "$(echo "$check_crontab" | head -n1)" "/bin/backup" \ - "CHECK_CRON does not replace the backup cron job" - -write_file /data/hello.txt "backup me" -in_container /bin/backup >/dev/null 2>&1 -write_file /hooks/pre-check.sh '#!/bin/sh -echo "PRE-CHECK-HOOK-RAN" -' -write_file /hooks/post-check.sh '#!/bin/sh -echo "POST-CHECK-HOOK-RAN with rc=$1" -' -in_container chmod +x /hooks/pre-check.sh /hooks/post-check.sh - -check_hook_log="$(in_container /bin/check 2>&1)" -assert_contains "$check_hook_log" "PRE-CHECK-HOOK-RAN" \ - "pre-check hook is executed" -assert_contains "$check_hook_log" "POST-CHECK-HOOK-RAN with rc=0" \ - "post-check hook receives the check return code" -stop_container - -echo -echo "=== compat: a corrupted repository is reported as failed ===" -# The success path above is not enough: check.sh prints "Check Successful" -# whenever restic exits 0, so a `restic check` that does nothing at all would -# keep this suite green. Damage the repository and require the failure path. -# Runs in its own throwaway container, the repository stays broken afterwards. - -start_container corrupt -wait_for_start || { fail "container starts for the corruption test"; summary "compat"; exit 1; } -write_file /data/hello.txt "backup me" -in_container /bin/backup >/dev/null 2>&1 -assert_eq "1" "$(snapshot_count)" \ - "the repository to be corrupted has a snapshot" - -in_container sh -c 'f=$(find /mnt/restic/data -type f | head -n1); chmod u+w "$f"; printf garbage > "$f"' -broken_check_log="$(in_container /bin/check 2>&1)" -assert_contains "$broken_check_log" "Check Failed with Status" \ - "restic check detects a corrupted repository" -assert_not_contains "$broken_check_log" "Check Successful" \ - "a corrupted repository is not reported as successful" -if in_container test -f /var/log/check-error-last.log; then - pass "failing check writes check-error-last.log" -else - fail "failing check writes check-error-last.log" "file is missing" -fi -stop_container - -echo -echo "=== compat: RESTIC_FORGET_ARGS ===" - -start_container forget -e "RESTIC_FORGET_ARGS=--keep-last 1" -wait_for_start || { fail "container starts with RESTIC_FORGET_ARGS"; summary "compat"; exit 1; } -write_file /data/hello.txt "backup me" -in_container /bin/backup >/dev/null 2>&1 -forget_log="$(in_container /bin/backup 2>&1)" -assert_contains "$forget_log" "Forget Successful" \ - "forget runs after a successful backup when configured" -assert_eq "1" "$(snapshot_count)" \ - "--keep-last 1 leaves exactly one snapshot" -stop_container - -echo -echo "=== compat: RESTIC_JOB_ARGS ===" -# backup-last.log only records the variable, it does not prove restic ever saw -# it. The arguments have to have a visible effect, otherwise a backup that -# silently drops RESTIC_JOB_ARGS still looks healthy. - -start_container jobargs -e "RESTIC_JOB_ARGS=--exclude=/data/skipme.txt" -wait_for_start || { fail "container starts with RESTIC_JOB_ARGS"; summary "compat"; exit 1; } -write_file /data/keep.txt "keep" -write_file /data/skipme.txt "skip" -jobargs_log="$(in_container /bin/backup 2>&1)" -assert_contains "$jobargs_log" "Backup Successful" \ - "backup with RESTIC_JOB_ARGS succeeds" -in_container restic restore latest --target /tmp/restore >/dev/null 2>&1 -assert_eq "keep" "$(in_container cat /tmp/restore/data/keep.txt 2>&1)" \ - "RESTIC_JOB_ARGS does not break a normal backup" -if in_container test -f /tmp/restore/data/skipme.txt; then - fail "RESTIC_JOB_ARGS is passed to restic backup" \ - "the excluded file was backed up anyway" -else - pass "RESTIC_JOB_ARGS is passed to restic backup" -fi -stop_container - -echo -echo "=== compat: BACKUP_SOURCES ===" - -start_container sources -e "BACKUP_SOURCES=/opt/custom" -wait_for_start || { fail "container starts with BACKUP_SOURCES"; summary "compat"; exit 1; } -write_file /opt/custom/custom.txt "custom source" -write_file /data/ignored.txt "should not be backed up" -in_container /bin/backup >/dev/null 2>&1 -in_container restic restore latest --target /tmp/restore >/dev/null 2>&1 -assert_eq "custom source" "$(in_container cat /tmp/restore/opt/custom/custom.txt 2>&1)" \ - "BACKUP_SOURCES overrides the default /data source" -if in_container test -f /tmp/restore/data/ignored.txt; then - fail "BACKUP_SOURCES excludes /data" "/data was backed up although BACKUP_SOURCES was set" -else - pass "BACKUP_SOURCES excludes /data" -fi -stop_container - -echo -echo "=== compat: failure handling ===" - -start_container failure -wait_for_start || { fail "container starts for failure test"; summary "compat"; exit 1; } -write_file /data/hello.txt "backup me" -# Back up a source that does not exist. The repository stays the initialised -# default one, so the snapshot count below looks at the repo the run really used. -fail_log="$(in_container_env BACKUP_SOURCES=/does-not-exist /bin/backup 2>&1)" -assert_contains "$fail_log" "Backup Failed with Status" \ - "a failing backup is reported as failed" -if in_container test -f /var/log/backup-error-last.log; then - pass "failing backup writes backup-error-last.log" -else - fail "failing backup writes backup-error-last.log" "file is missing" -fi -assert_eq "0" "$(snapshot_count)" \ - "a failed backup leaves no snapshot behind" -stop_container - -echo -echo "=== compat: timezone ===" -# The base image already ships tzdata, so TZ works today. Pinned here so that -# neither a base image change nor a slimming effort silently breaks cron times -# for users who already rely on it. - -tz_default="$(docker run --rm --entrypoint date "$IMAGE" +%Z 2>&1 | tr -d '\r')" -assert_eq "UTC" "$tz_default" \ - "the default timezone is UTC" - -tz_berlin="$(docker run --rm -e TZ=Europe/Berlin --entrypoint date "$IMAGE" +%Z 2>&1 | tr -d '\r')" -if [ "$tz_berlin" = "CET" ] || [ "$tz_berlin" = "CEST" ]; then - pass "TZ=Europe/Berlin changes the container timezone" -else - fail "TZ=Europe/Berlin changes the container timezone" "got: ${tz_berlin}" -fi - -echo -echo "=== compat: RESTIC_TAG handling ===" - -start_container tag -wait_for_start || { fail "container starts for RESTIC_TAG test"; summary "compat"; exit 1; } -write_file /data/hello.txt "backup me" - -# The Dockerfile declares RESTIC_TAG="", so in a normal container the variable -# is always set and an empty value means "no tag". Documented here because the -# `${RESTIC_TAG?...}` guard in backup.sh only triggers when it is truly unset. -emptytag_log="$(in_container_env RESTIC_TAG= /bin/backup 2>&1)" -assert_contains "$emptytag_log" "Backup Successful" \ - "an empty RESTIC_TAG creates an untagged backup" -assert_eq "1" "$(snapshot_count)" \ - "an empty RESTIC_TAG still produces a snapshot" - -notag_log="$(in_container env -u RESTIC_TAG /bin/backup 2>&1)" -assert_contains "$notag_log" "Missing environment variable RESTIC_TAG" \ - "backup refuses to run when RESTIC_TAG is unset entirely" -assert_eq "1" "$(snapshot_count)" \ - "no extra snapshot is created when RESTIC_TAG is unset" -stop_container - -summary "compat" diff --git a/test/features.sh b/test/features.sh deleted file mode 100644 index c3f77e4..0000000 --- a/test/features.sh +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env bash -# Feature suite. -# -# Each block is guarded by a detection check against the built image. On master -# every block reports SKIP. As soon as the corresponding pull request is rebased -# onto master, the feature is detected and its tests start running - the PR -# author does not have to write test code for the basics. -# -# After merging a feature, add its gate name to test/required-features.txt. -# From then on a missing feature is a failure instead of a skip. - -set -u - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=test/lib.sh -. "${SCRIPT_DIR}/lib.sh" - -trap stop_container EXIT - -echo "=== feature: stream backup (RESTIC_STREAM) ===" -# Pull request #109 - back up the output of a command via `restic backup --stdin`. - -if require_feature stream "PR #109"; then - start_container stream \ - -e RESTIC_STREAM=1 \ - -e "RESTIC_STREAM_CMD=echo hello-stream" \ - -e RESTIC_STREAM_FILENAME=stream.txt - if wait_for_start; then - stream_log="$(in_container /bin/backup 2>&1)" - assert_contains "$stream_log" "Backup Successful" \ - "stream backup succeeds" - assert_eq "1" "$(snapshot_count)" \ - "stream backup creates exactly one snapshot" - in_container restic restore latest --target /tmp/restore >/dev/null 2>&1 - restored="$(in_container sh -c 'cat /tmp/restore/stream.txt 2>/dev/null' 2>&1)" - assert_eq "hello-stream" "$(echo "$restored" | tr -d '\r')" \ - "stream content is stored under RESTIC_STREAM_FILENAME" - else - fail "stream backup container starts" - fi - stop_container - - # Blocker 1 of PR #109: the stream branch must honour RESTIC_JOB_ARGS, the - # same way the file backup does. --host is used because it is visible in - # the snapshot metadata of a stdin backup. - start_container stream-jobargs \ - -e RESTIC_STREAM=1 \ - -e "RESTIC_STREAM_CMD=echo hello-stream" \ - -e RESTIC_STREAM_FILENAME=stream.txt \ - -e "RESTIC_JOB_ARGS=--host=jobargshost" - if wait_for_start; then - in_container /bin/backup >/dev/null 2>&1 - # Match on the hostname field itself: restic's own --host filter only - # returns snapshots whose hostname equals the value, so the string - # cannot be satisfied by "jobargshost" appearing in some other field. - # Both directions are checked so the filter is known to discriminate. - assert_eq "1" "$(in_container sh -c \ - 'restic snapshots --host jobargshost --json 2>/dev/null | grep -o "\"short_id\"" | wc -l' | tr -d ' \r')" \ - "RESTIC_JOB_ARGS sets the host of the stream backup snapshot" - assert_eq "0" "$(in_container sh -c \ - 'restic snapshots --host some-other-host --json 2>/dev/null | grep -o "\"short_id\"" | wc -l' | tr -d ' \r')" \ - "the host filter does not match an unrelated hostname" - else - fail "stream RESTIC_JOB_ARGS container starts" - fi - stop_container - - # A half-configured stream must not look like a successful backup. - start_container stream-misconfigured -e RESTIC_STREAM=1 - if wait_for_start; then - misconfig_log="$(in_container /bin/backup 2>&1)" - assert_not_contains "$misconfig_log" "Backup Successful" \ - "incomplete stream configuration does not report success" - assert_eq "0" "$(snapshot_count)" \ - "incomplete stream configuration creates no snapshot" - else - fail "stream misconfiguration container starts" - fi - stop_container - - # The classic path must keep working while the feature is available. - start_container stream-disabled - if wait_for_start; then - write_file /data/hello.txt "backup me" - in_container /bin/backup >/dev/null 2>&1 - assert_eq "1" "$(snapshot_count)" \ - "file backup still works while the stream feature is present" - else - fail "stream-disabled container starts" - fi - stop_container -fi - -echo -echo "=== feature: docker cli ===" -# Pull request #99 - talk to the host docker daemon from hooks. - -if require_feature docker-cli "PR #99"; then - docker_version="$(docker run --rm --entrypoint docker "$IMAGE" --version 2>&1)" - assert_contains "$docker_version" "Docker version" \ - "docker cli is usable inside the image" -fi - -echo -echo "=== feature: bash ===" -# Pull request #99 - allow bash syntax in pre/post hooks. - -if require_feature bash "PR #99"; then - start_container bash-hooks - if wait_for_start; then - write_file /hooks/pre-backup.sh '#!/bin/bash -arr=(one two) -if [[ "${arr[1]}" == "two" ]]; then echo "BASH-HOOK-RAN"; fi -' - in_container chmod +x /hooks/pre-backup.sh - write_file /data/hello.txt "backup me" - bash_log="$(in_container /bin/backup 2>&1)" - assert_contains "$bash_log" "BASH-HOOK-RAN" \ - "a hook with a bash shebang and bash syntax runs" - else - fail "bash hook container starts" - fi - stop_container -fi - -echo -echo "=== feature: msmtp mail relay ===" -# Pull request #84 - send notification mails through msmtp. - -if require_feature msmtp "PR #84"; then - sendmail_target="$(docker run --rm --entrypoint sh "$IMAGE" -c \ - 'readlink -f /usr/sbin/sendmail' 2>&1 | tr -d '\r')" - assert_contains "$sendmail_target" "msmtp" \ - "sendmail is provided by msmtp" - - # The config has to be in place when the container starts, entry.sh copies - # it. Re-running entry.sh in a live container would start a second crond. - start_container_with_file msmtp-config /config/msmtprc 'defaults -account default -host localhost -' - if wait_for_start; then - if in_container test -f /etc/msmtprc; then - pass "msmtp config from /config is copied to /etc/msmtprc" - mode="$(in_container stat -c '%a' /etc/msmtprc 2>&1 | tr -d '\r')" - assert_eq "600" "$mode" \ - "copied msmtp config is not world readable" - else - fail "msmtp config from /config is copied to /etc/msmtprc" "file is missing" - fi - else - fail "msmtp config container starts" - fi - stop_container - - # Users who configured mail before msmtp existed must not lose their backups. - start_container msmtp-missing-config \ - -e "MAILX_ARGS=-r from@example.com -s subject to@example.com" - if wait_for_start; then - write_file /data/hello.txt "backup me" - mail_log="$(in_container /bin/backup 2>&1)" - assert_contains "$mail_log" "Backup Successful" \ - "backup succeeds with MAILX_ARGS set but no msmtp config" - assert_eq "1" "$(snapshot_count)" \ - "snapshot is created even when the mail relay is unconfigured" - else - fail "msmtp missing-config container starts" - fi - stop_container -fi - -summary "features" diff --git a/test/lib.sh b/test/lib.sh deleted file mode 100644 index 1dc496f..0000000 --- a/test/lib.sh +++ /dev/null @@ -1,287 +0,0 @@ -#!/usr/bin/env bash -# Shared helpers for the restic-backup-docker test suites. -# -# Deliberately dependency-free: bash + docker only. No test framework, no -# host-side bind mounts (files are moved with `docker cp`) so the suites behave -# identically on Linux CI runners, macOS and Windows/Git-Bash. -# -# Windows/Git-Bash needs one extra thing: MSYS rewrites every argument that -# looks like a POSIX path before the process sees it, so -# `docker exec /usr/bin/backup` reaches the daemon as -# `C:/Program Files/Git/usr/bin/backup` and the container command is not found. -# Both variables are required (one covers the command, one covers its -# arguments) and both are ignored on Linux and macOS. -export MSYS_NO_PATHCONV=1 -export MSYS2_ARG_CONV_EXCL='*' - -IMAGE="${IMAGE:-restic-backup-docker:test}" - -PASS_COUNT=0 -FAIL_COUNT=0 -SKIP_COUNT=0 -FAILED_TESTS=() -SKIPPED_TESTS=() - -_c_red='' -_c_green='' -_c_yellow='' -_c_reset='' -if [ -t 1 ]; then - _c_red=$'\033[31m' - _c_green=$'\033[32m' - _c_yellow=$'\033[33m' - _c_reset=$'\033[0m' -fi - -log() { echo " $*"; } - -pass() { - PASS_COUNT=$((PASS_COUNT + 1)) - echo "${_c_green}PASS${_c_reset} $1" -} - -fail() { - FAIL_COUNT=$((FAIL_COUNT + 1)) - FAILED_TESTS+=("$1") - echo "${_c_red}FAIL${_c_reset} $1" - if [ -n "${2:-}" ]; then - echo "$2" | sed 's/^/ /' - fi -} - -skip() { - SKIP_COUNT=$((SKIP_COUNT + 1)) - SKIPPED_TESTS+=("$1") - echo "${_c_yellow}SKIP${_c_reset} $1 (${2:-not applicable})" -} - -# summary -> exit code 0 when nothing failed -summary() { - echo - echo "--- $1: ${PASS_COUNT} passed, ${FAIL_COUNT} failed, ${SKIP_COUNT} skipped ---" - if [ ${#FAILED_TESTS[@]} -gt 0 ]; then - echo "Failed:" - printf ' - %s\n' "${FAILED_TESTS[@]}" - fi - if [ ${#SKIPPED_TESTS[@]} -gt 0 ]; then - echo "Skipped:" - printf ' - %s\n' "${SKIPPED_TESTS[@]}" - fi - [ "${FAIL_COUNT}" -eq 0 ] -} - -# --------------------------------------------------------------------------- -# assertions -# --------------------------------------------------------------------------- - -assert_eq() { - local expected="$1" actual="$2" name="$3" - if [ "$expected" = "$actual" ]; then - pass "$name" - else - fail "$name" "expected: ${expected} -actual: ${actual}" - fi -} - -assert_contains() { - local haystack="$1" needle="$2" name="$3" - case "$haystack" in - *"$needle"*) pass "$name" ;; - *) fail "$name" "expected to contain: ${needle} -actual: -$(echo "$haystack" | tail -n 25)" ;; - esac -} - -assert_not_contains() { - local haystack="$1" needle="$2" name="$3" - case "$haystack" in - *"$needle"*) fail "$name" "unexpectedly contains: ${needle}" ;; - *) pass "$name" ;; - esac -} - -# --------------------------------------------------------------------------- -# container helpers -# -# Every test gets its own throwaway container. The restic repository lives -# inside the container at the image default (/mnt/restic), so no host paths are -# involved and tests cannot pollute each other. -# --------------------------------------------------------------------------- - -CONTAINER="" - -# start_container [extra docker run args...] -# Starts the image with a cron expression far in the future so that the only -# backup that ever runs is the one a test triggers explicitly. -start_container() { - local suffix="$1" - shift - CONTAINER="rbd-test-${suffix}-$$" - docker rm -f "$CONTAINER" >/dev/null 2>&1 || true - docker run -d --name "$CONTAINER" \ - -e RESTIC_PASSWORD=testpassword \ - -e RESTIC_TAG=testtag \ - -e "BACKUP_CRON=0 0 31 2 *" \ - "$@" \ - "$IMAGE" >/dev/null -} - -# start_container_with_file [extra docker run args...] -# Like start_container, but the file exists before entry.sh runs, so tests can -# cover what the entrypoint does with it at real container startup. The file is -# placed with `docker create` + `docker cp`, no bind mount. -start_container_with_file() { - local suffix="$1" path="$2" content="$3" - shift 3 - local seed seed_src - seed="$(mktemp -d)" - printf '%s' "$content" > "${seed}/$(basename "$path")" - CONTAINER="rbd-test-${suffix}-$$" - docker rm -f "$CONTAINER" >/dev/null 2>&1 || true - docker create --name "$CONTAINER" \ - -e RESTIC_PASSWORD=testpassword \ - -e RESTIC_TAG=testtag \ - -e "BACKUP_CRON=0 0 31 2 *" \ - "$@" \ - "$IMAGE" >/dev/null - if command -v cygpath >/dev/null 2>&1; then - seed_src="$(cygpath -w "$seed")" - else - seed_src="$seed" - fi - docker cp "${seed_src}/." "${CONTAINER}:$(dirname "$path")" - rm -rf "$seed" - docker start "$CONTAINER" >/dev/null -} - -stop_container() { - [ -n "$CONTAINER" ] || return 0 - docker rm -f "$CONTAINER" >/dev/null 2>&1 || true - CONTAINER="" -} - -# wait_for_start -> fails if entry.sh did not finish within the timeout -wait_for_start() { - local waited=0 - while [ "$waited" -lt 60 ]; do - waited=$((waited + 1)) - if docker logs "$CONTAINER" 2>&1 | grep -q "Container started."; then - return 0 - fi - if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null)" != "true" ]; then - echo "container exited early:" - docker logs "$CONTAINER" 2>&1 | sed 's/^/ /' - return 1 - fi - sleep 1 - done - echo "timeout waiting for container start:" - docker logs "$CONTAINER" 2>&1 | sed 's/^/ /' - return 1 -} - -# in_container - run a command inside the running test container -in_container() { - docker exec "$CONTAINER" "$@" -} - -# in_container_env - run with one overridden variable -in_container_env() { - local env_pair="$1" - shift - docker exec -e "$env_pair" "$CONTAINER" "$@" -} - -# write_file -write_file() { - local path="$1" content="$2" - local dir - dir="$(dirname "$path")" - docker exec "$CONTAINER" mkdir -p "$dir" - printf '%s' "$content" | docker exec -i "$CONTAINER" sh -c "cat > '$path'" -} - -# snapshot_count -> number of snapshots in the repository -# A restic error (wrong password, missing repository) is not "zero snapshots": -# the function prints the error text and returns 1, so assert_eq on the result -# fails loudly instead of matching an expected "0". -snapshot_count() { - local out - if ! out="$(docker exec "$CONTAINER" restic snapshots --json 2>&1)"; then - echo "restic snapshots failed: $(echo "$out" | tail -n 3 | tr '\n\r' ' ')" - return 1 - fi - printf '%s' "$out" | grep -o '"short_id"' | wc -l | tr -d ' \r' -} - -# --------------------------------------------------------------------------- -# feature gating -# -# A feature test only runs when the image actually ships the feature. That way -# this suite already covers the currently open pull requests: once such a PR is -# rebased onto master its feature is detected and its tests start running, -# without the PR author having to write any test code. -# -# test/required-features.txt lists features that MUST be present. A feature -# listed there that is not detected turns the skip into a hard failure, so a -# merged feature cannot silently disappear from the image again. -# --------------------------------------------------------------------------- - -REQUIRED_FEATURES_FILE="${REQUIRED_FEATURES_FILE:-$(dirname "${BASH_SOURCE[0]}")/required-features.txt}" - -feature_is_required() { - local feature="$1" - [ -f "$REQUIRED_FEATURES_FILE" ] || return 1 - tr -d '\r' < "$REQUIRED_FEATURES_FILE" \ - | grep -v '^[[:space:]]*#' \ - | grep -qx "[[:space:]]*${feature}[[:space:]]*" -} - -# One throwaway container probes the image for all features at once; the -# result is cached so every has_feature call after the first is free. -FEATURE_PROBE="" -FEATURE_PROBED=0 - -probe_features() { - [ "$FEATURE_PROBED" -eq 0 ] || return 0 - FEATURE_PROBE="$(docker run --rm --entrypoint sh "$IMAGE" -c ' - grep -q RESTIC_STREAM /bin/backup && echo stream - command -v docker >/dev/null && echo docker-cli - command -v bash >/dev/null && echo bash - command -v msmtp >/dev/null && echo msmtp - true' 2>/dev/null | tr -d '\r')" - FEATURE_PROBED=1 -} - -# has_feature - detection runs against the built image, not the source -has_feature() { - case "$1" in - stream|docker-cli|bash|msmtp) - probe_features - printf '%s\n' "$FEATURE_PROBE" | grep -qx "$1" - ;; - *) - echo "unknown feature: $1" >&2 - return 1 - ;; - esac -} - -# require_feature -# Returns 0 when the feature tests should run. Otherwise records a skip, or a -# failure when the feature is listed in required-features.txt. -require_feature() { - local feature="$1" ref="$2" - if has_feature "$feature"; then - return 0 - fi - if feature_is_required "$feature"; then - fail "feature '${feature}'" \ - "listed in $(basename "$REQUIRED_FEATURES_FILE") but not present in the image" - else - skip "feature '${feature}'" "not in image yet, see ${ref}" - fi - return 1 -} diff --git a/test/run.sh b/test/run.sh deleted file mode 100644 index c3b69d9..0000000 --- a/test/run.sh +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env bash -# Entry point for the test suites. -# -# bash test/run.sh build the image and run every suite -# bash test/run.sh --no-build reuse an already built $IMAGE -# bash test/run.sh --suite compat run a single suite (compat|features|structure) -# bash test/run.sh --sabotage self test: break backup.sh on a throwaway -# copy of the tree and prove the suite fails -# -# Requires bash and docker. container-structure-test is optional: without a -# local install the structure suite runs through the official tool image. - -set -u - -# Windows/Git-Bash: MSYS mangles POSIX-looking arguments on their way to the -# docker daemon, which breaks `docker exec` and `--config /test/structure.yml`. -# lib.sh exports the same two variables for the suites that source it; run.sh -# calls docker itself and starts the suites as separate processes, so it needs -# them as well. Both are ignored on Linux and macOS. -export MSYS_NO_PATHCONV=1 -export MSYS2_ARG_CONV_EXCL='*' - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -IMAGE="${IMAGE:-restic-backup-docker:test}" -export IMAGE -# Pinned by digest-free tag on purpose: the structure test tool is only used to -# read the image under test, a floating tag cannot affect the product. -STRUCTURE_TEST_IMAGE="${STRUCTURE_TEST_IMAGE:-gcr.io/gcp-runtimes/container-structure-test:latest}" - -BUILD=1 -SABOTAGE=0 -ONLY_SUITE="" - -while [ $# -gt 0 ]; do - case "$1" in - --no-build) BUILD=0 ;; - --sabotage) SABOTAGE=1 ;; - --suite) - # A missing value, or a following flag, must not be taken as one. - case "${2:-}" in - ""|-*) echo "--suite requires a value (compat|features|structure)" >&2; exit 2 ;; - esac - ONLY_SUITE="$2"; shift ;; - --suite=*) - ONLY_SUITE="${1#--suite=}" - [ -n "$ONLY_SUITE" ] || { echo "--suite requires a value (compat|features|structure)" >&2; exit 2; } ;; - -h|--help) sed -n '2,/^$/{/^#/p;}' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; - *) echo "unknown argument: $1" >&2; exit 2 ;; - esac - shift -done - -if ! docker info >/dev/null 2>&1; then - echo "docker is not available, cannot run the test suite" >&2 - exit 2 -fi - -# Docker on Windows does not understand the /c/... paths Git-Bash produces, -# so build contexts are translated when cygpath is around. No-op elsewhere. -host_path() { - if command -v cygpath >/dev/null 2>&1; then - cygpath -w "$1" - else - printf '%s' "$1" - fi -} - -build_image() { - local context tag - context="$(host_path "$1")" - tag="$2" - echo ">>> building ${tag}" - if ! docker build --quiet -t "$tag" "$context" >/dev/null; then - echo "image build failed" >&2 - return 1 - fi -} - -# --sabotage proves the suite has teeth: it breaks the backup call on a copy of -# the tree and expects the compat suite to go red. If this mode ever passes, -# the tests are not actually testing anything. -if [ "$SABOTAGE" -eq 1 ]; then - tmp="$(mktemp -d)" - trap 'rm -rf "$tmp"; docker rmi -f restic-backup-docker:sabotage >/dev/null 2>&1 || true' EXIT - tar -c -C "$ROOT_DIR" --exclude=.git --exclude=.intent . | tar -x -C "$tmp" - # Silently turn the backup into a no-op, the nastiest kind of regression. - # No `sed -i`: its argument differs between GNU and BSD (macOS) sed. - sed 's|^restic backup |true |' "$tmp/backup.sh" > "$tmp/backup.sh.new" \ - && mv "$tmp/backup.sh.new" "$tmp/backup.sh" - # mv keeps the mode of the new file (644); Dockerfile COPY has no chmod. - chmod +x "$tmp/backup.sh" - if ! grep -q '^true ' "$tmp/backup.sh"; then - echo "sabotage patch did not apply, backup.sh changed shape" >&2 - exit 2 - fi - if [ ! -x "$tmp/backup.sh" ]; then - echo "SABOTAGE CHECK INVALID: sabotaged backup.sh is not executable" >&2 - exit 2 - fi - build_image "$tmp" restic-backup-docker:sabotage || exit 2 - echo ">>> running compat suite against the sabotaged image (failure expected)" - sabotage_log="${tmp}/compat.log" - if IMAGE=restic-backup-docker:sabotage bash "${SCRIPT_DIR}/compat.sh" > "$sabotage_log" 2>&1; then - cat "$sabotage_log" - echo - echo "SABOTAGE CHECK FAILED: the compat suite passed on a broken image" >&2 - exit 1 - fi - cat "$sabotage_log" - # Red for the wrong reason (e.g. a non-executable /bin/backup) proves nothing. - if grep -qi 'permission denied' "$sabotage_log"; then - echo "SABOTAGE CHECK INVALID: log contains 'permission denied', suite failed for the wrong reason" >&2 - exit 2 - fi - # Any red test would do for "it failed"; a broken test environment is red too. - # Demand the assertions that are about the backup itself. - missing=0 - for name in \ - "exactly one snapshot exists after one backup" \ - "a second backup adds a second snapshot"; do - if ! grep -qF "FAIL ${name}" "$sabotage_log"; then - echo "SABOTAGE CHECK FAILED: expected assertion did not fail: ${name}" >&2 - missing=1 - fi - done - [ "$missing" -eq 0 ] || exit 1 - echo - echo "sabotage check ok: the compat suite detects a broken backup" - exit 0 -fi - -if [ "$BUILD" -eq 1 ]; then - build_image "$ROOT_DIR" "$IMAGE" || exit 2 -fi - -rc=0 - -run_structure() { - echo - echo ">>> structure suite" - if command -v container-structure-test >/dev/null 2>&1; then - container-structure-test test --image "$IMAGE" --config "${SCRIPT_DIR}/structure.yml" || rc=1 - else - # Not installed locally: fall back to the official image. It needs the - # docker socket because it inspects and runs the image under test. - echo "container-structure-test not installed, using the official image" - # The config file is copied in with `docker cp` instead of bind mounted, - # so this also works when DOCKER_HOST points at a remote daemon. - local tool_cid tool_rc - tool_cid="$(docker create \ - -v //var/run/docker.sock:/var/run/docker.sock \ - "$STRUCTURE_TEST_IMAGE" \ - test --image "$IMAGE" --config /structure.yml)" || { rc=1; return; } - docker cp "$(host_path "${SCRIPT_DIR}")/structure.yml" "${tool_cid}:/structure.yml" \ - && docker start -a "$tool_cid" - tool_rc=$? - docker rm -f "$tool_cid" >/dev/null 2>&1 || true - [ "$tool_rc" -eq 0 ] || rc=1 - fi -} - -run_suite() { - echo - echo ">>> $1 suite" - bash "${SCRIPT_DIR}/$1.sh" || rc=1 -} - -case "$ONLY_SUITE" in - "") run_structure; run_suite compat; run_suite features ;; - structure) run_structure ;; - compat) run_suite compat ;; - features) run_suite features ;; - *) echo "unknown suite: ${ONLY_SUITE}" >&2; exit 2 ;; -esac - -echo -if [ "$rc" -eq 0 ]; then - echo "all suites passed" -else - echo "at least one suite failed" -fi -exit "$rc" diff --git a/tests/sabotage.py b/tests/sabotage.py index 51e9d48..e8b7ef1 100644 --- a/tests/sabotage.py +++ b/tests/sabotage.py @@ -1,6 +1,6 @@ """Self test of the suite: prove that it goes red when the backup is broken. -Ported from `test/run.sh --sabotage`. Run with `task test:sabotage`. +Run with `task test:sabotage`. A throwaway copy of the tree gets a backup.sh whose `restic backup` call is a no-op, the nastiest kind of regression (the script still reports success). The compat tests run diff --git a/tests/test_compat.py b/tests/test_compat.py index ae1e403..0910bd5 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -1,4 +1,4 @@ -"""Backwards-compatibility suite, ported from test/compat.sh. +"""Backwards-compatibility suite. Everything here describes behaviour the image has today and that users depend on. A change that turns one of these red is a breaking change, whether or not intended. diff --git a/tests/test_features.py b/tests/test_features.py index 12b0bd8..a29096c 100644 --- a/tests/test_features.py +++ b/tests/test_features.py @@ -1,4 +1,4 @@ -"""Feature-gated tests, ported from test/features.sh. +"""Feature-gated tests. Each class is guarded by `@pytest.mark.feature()`: it runs as soon as the built image ships the feature (detected in conftest.py) and is skipped before that. A gate