From 10b197f1f5acf320b8a0e6b3d4785bf1bac2d3c9 Mon Sep 17 00:00:00 2001 From: Tobias Kaupat Date: Sat, 17 Oct 2026 22:47:26 +0200 Subject: [PATCH] test: add container test suites and run them in CI The repository had no tests: CI only built the image, and the old container-structure-test setup was parked as config-broken.yml / docker-compose.test.broken.yml. This adds suites that actually run the image. - test/compat.sh: backup, restore, forget, check, hooks, failure handling and timezone behaviour against the default configuration. These describe what users depend on today, so a change that turns them red is a breaking change. - test/features.sh: optional features, each guarded by a detection check against the built image. Absent features are reported as SKIP, so open pull requests do not turn master red, and their feature tests start running by themselves once the pull request is rebased. Covers #109 (RESTIC_STREAM), #99 (docker cli, bash) and #84 (msmtp). - test/required-features.txt: once a feature is merged its gate is listed here and a missing feature becomes a failure instead of a skip. - test/structure.yml: revived image layout checks, falls back to the official container-structure-test image when the tool is not installed. - test/run.sh --sabotage: breaks the backup call on a throwaway copy of the tree and expects the suite to fail, so the tests cannot be watered down into always passing. CI runs this as its own step. - .github/workflows/test.yml: shellcheck plus all suites on every pull request. Also adds .gitattributes pinning shell scripts, the Dockerfile and YAML to LF. With a Windows checkout (core.autocrlf=true) the scripts were checked out with CRLF, and the resulting image failed to start with "exec /entry.sh: no such file or directory". No runtime file is touched, the image itself is unchanged. Co-Authored-By: Claude Opus 5 (1M context) Agent-Id: agent-7034f75b-1ddc-4aa1-a4ea-3a54b340e8d9 --- .gitattributes | 7 + .github/workflows/test.yml | 54 ++++++++ CHANGELOG.md | 9 ++ README.md | 33 +++++ config-broken.yml | 21 --- docker-compose.test.broken.yml | 9 -- test/compat.sh | 187 ++++++++++++++++++++++++++ test/features.sh | 147 ++++++++++++++++++++ test/lib.sh | 238 +++++++++++++++++++++++++++++++++ test/required-features.txt | 18 +++ test/run.sh | 133 ++++++++++++++++++ test/structure.yml | 64 +++++++++ 12 files changed, 890 insertions(+), 30 deletions(-) create mode 100644 .gitattributes create mode 100644 .github/workflows/test.yml delete mode 100644 config-broken.yml delete mode 100644 docker-compose.test.broken.yml create mode 100644 test/compat.sh create mode 100644 test/features.sh create mode 100644 test/lib.sh create mode 100644 test/required-features.txt create mode 100644 test/run.sh create mode 100644 test/structure.yml diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..f593848 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,7 @@ +# Shell scripts and the Dockerfile are copied into a Linux image. With CRLF +# line endings the shebang resolves to "/bin/sh\r" and the container fails to +# start, which happens on Windows checkouts with core.autocrlf=true. Pin them +# to LF so the image builds the same way on every platform. +*.sh text eol=lf +Dockerfile text eol=lf +*.yml text eol=lf diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..dc7f042 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,54 @@ +name: Test + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +jobs: + shellcheck: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install shellcheck + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq shellcheck + + # The test scripts are new, so they are held to a clean bill of health. + - name: Run shellcheck on the test scripts + run: shellcheck --severity=warning test/*.sh + + # The image scripts declare #!/bin/sh but use a few bash constructs that + # busybox ash happens to accept. Cleaning that up is a separate change, + # so this step reports but does not block. + - name: Run shellcheck on the image scripts + run: | + shellcheck --severity=warning backup.sh check.sh entry.sh build.sh run.sh \ + || echo "::warning::shellcheck reported findings in the image scripts, see the log above" + + test: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Build test image + run: docker build -t restic-backup-docker:test . + + - name: Install container-structure-test + run: | + curl -fsSL -o /usr/local/bin/container-structure-test \ + https://storage.googleapis.com/container-structure-test/v1.19.3/container-structure-test-linux-amd64 + chmod +x /usr/local/bin/container-structure-test + + - name: Run test suites + run: bash test/run.sh --no-build + + # Guards the guard: if someone waters the suites down until a broken + # backup no longer fails them, this step turns red. + - name: Verify the suite detects a broken backup + run: bash test/run.sh --sabotage diff --git a/CHANGELOG.md b/CHANGELOG.md index 7974157..d2b902f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## Unreleased + +### Added +* Test suites in `test/` (compat, features, structure) that run the image as a real container, plus a `Test` GitHub Actions workflow running them on every pull request +* `.gitattributes` pinning shell scripts, the Dockerfile and YAML files to LF line endings. With a Windows checkout (`core.autocrlf=true`) the scripts were checked out with CRLF, which made the built image fail to start with `exec /entry.sh: no such file or directory` + +### Removed +* `config-broken.yml` and `docker-compose.test.broken.yml`, replaced by `test/structure.yml` and the test workflow + ## v1.4.2 ### Removed * Execute a repository check after prune was passed as a parameter in the forget call - use RESTIC_CHECK_CRON Var instead diff --git a/README.md b/README.md index abb3fd7..960edbd 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,39 @@ Pull Requests to improve the image are always wellcome. Please create an issue a When behaviour of the image changes (Features, Bugfixes, Changes in the API) please update the "Unreleased" section of the [CHANGELOG.md](https://github.com/lobaro/restic-backup-docker/blob/master/CHANGELOG.md) +## Tests + +The test suites build the image and exercise it as a real container. They need +`bash` and `docker`, nothing else: + +```sh +bash test/run.sh # build the image and run every suite +bash test/run.sh --no-build # reuse an already built image +bash test/run.sh --suite compat # run a single suite +``` + +There are three suites: + +* **compat** (`test/compat.sh`) - backup, restore, forget, check, hooks and + failure handling against the default configuration. These tests describe the + behaviour users rely on. A change that turns one of them red is a breaking + change. +* **features** (`test/features.sh`) - tests for optional features. Each block + is guarded by a detection check against the built image, so a feature that is + not part of the image is reported as `SKIP` instead of failing. Pull requests + that add such a feature are therefore covered as soon as they are rebased, + without the author having to write the basic tests. After merging a feature, + add its gate name to `test/required-features.txt` so that a missing feature + turns into a failure. +* **structure** (`test/structure.yml`) - image layout checks via + [container-structure-test](https://github.com/GoogleContainerTools/container-structure-test). + When the tool is not installed, its official container image is used instead. + +`bash test/run.sh --sabotage` is a self test: it breaks the backup call on a +throwaway copy of the tree and expects the compat suite to fail. CI runs it so +that the suites cannot be watered down into always passing. + +When you add a feature, please add a block to `test/features.sh` for it. ## Hooks diff --git a/config-broken.yml b/config-broken.yml deleted file mode 100644 index 11d8ce1..0000000 --- a/config-broken.yml +++ /dev/null @@ -1,21 +0,0 @@ -schemaVersion: '2.0.0' -commandTests: - - name: "restic package installation" - setup: [["/entry.sh"]] - command: "which" - args: ["restic"] - expectedOutput: ["/bin/restic"] -fileExistenceTests: -- name: 'log directory exists' - path: '/var/log' - shouldExist: true -- name: 'cron log file exists' - path: '/var/log/cron.log' - shouldExist: true -- name: 'backup script exists' - path: '/bin/backup' - shouldExist: true -metadataTest: - volumes: ["/data"] - entrypoint: ["/entry.sh"] - cmd: ["tail","-fn0", "/var/log/cron.log"] diff --git a/docker-compose.test.broken.yml b/docker-compose.test.broken.yml deleted file mode 100644 index c875700..0000000 --- a/docker-compose.test.broken.yml +++ /dev/null @@ -1,9 +0,0 @@ -version: '2' - -services: - sut: - image: gcr.io/gcp-runtimes/container-structure-test - command: ["test", "--image", "restic-backup", "--config", "config.yml"] - volumes: - - ./config.yml:/config.yml - - /var/run/docker.sock:/var/run/docker.sock diff --git a/test/compat.sh b/test/compat.sh new file mode 100644 index 0000000..fa2f2d8 --- /dev/null +++ b/test/compat.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +# Backwards-compatibility suite. +# +# Everything in here describes behaviour the image has today and that users +# depend on. These tests always run, for every pull request. A change that +# turns one of them red is a breaking change, whether or not that was intended. + +set -u + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=test/lib.sh +. "${SCRIPT_DIR}/lib.sh" + +trap stop_container EXIT + +echo "=== compat: repository bootstrap and default backup ===" + +start_container default +wait_for_start || { fail "container starts with default configuration"; summary "compat"; exit 1; } +pass "container starts with default configuration" + +startup_log="$(docker logs "$CONTAINER" 2>&1)" +assert_contains "$startup_log" "Running restic init" \ + "empty repository is initialised on first start" +assert_contains "$startup_log" "Repo init status 0" \ + "repository initialisation succeeds" + +crontab="$(in_container cat /var/spool/cron/crontabs/root)" +assert_contains "$crontab" "/bin/backup" \ + "BACKUP_CRON is written to the crontab" +assert_contains "$crontab" "flock -n /var/run/backup.lock" \ + "backup cron job is guarded by flock" +assert_not_contains "$crontab" "/bin/check" \ + "no check cron job is installed while CHECK_CRON is unset" + +write_file /data/hello.txt "backup me" + +backup_log="$(in_container /bin/backup 2>&1)" +assert_contains "$backup_log" "Backup Successful" \ + "backup of /data succeeds" +assert_contains "$backup_log" "Pre-backup script not found" \ + "missing pre-backup hook is reported, not fatal" +assert_contains "$backup_log" "Post-backup script not found" \ + "missing post-backup hook is reported, not fatal" +assert_not_contains "$backup_log" "notification" \ + "no mail or Teams notification is attempted without configuration" +assert_eq "1" "$(snapshot_count)" \ + "exactly one snapshot exists after one backup" + +snapshots="$(in_container restic snapshots --json 2>/dev/null)" +assert_contains "$snapshots" '"testtag"' \ + "snapshot carries the configured RESTIC_TAG" + +in_container restic restore latest --target /tmp/restore >/dev/null 2>&1 +restored="$(in_container cat /tmp/restore/data/hello.txt 2>&1)" +assert_eq "backup me" "$restored" \ + "restored file content matches the original" + +logfile="$(in_container cat /var/log/backup-last.log 2>&1)" +assert_contains "$logfile" "RESTIC_REPOSITORY: /mnt/restic" \ + "backup-last.log records the run parameters" + +in_container /bin/backup >/dev/null 2>&1 +assert_eq "2" "$(snapshot_count)" \ + "a second backup adds a second snapshot" + +echo +echo "=== compat: data integrity check ===" + +check_log="$(in_container /bin/check 2>&1)" +assert_contains "$check_log" "Check Successful" \ + "restic check passes on a healthy repository" +assert_contains "$check_log" "Pre-check script not found" \ + "missing pre-check hook is reported, not fatal" + +echo +echo "=== compat: pre- and post-backup hooks ===" + +write_file /hooks/pre-backup.sh '#!/bin/sh +echo "PRE-HOOK-RAN" +' +write_file /hooks/post-backup.sh '#!/bin/sh +echo "POST-HOOK-RAN with rc=$1" +' +in_container chmod +x /hooks/pre-backup.sh /hooks/post-backup.sh + +hook_log="$(in_container /bin/backup 2>&1)" +assert_contains "$hook_log" "PRE-HOOK-RAN" \ + "pre-backup hook is executed" +assert_contains "$hook_log" "POST-HOOK-RAN with rc=0" \ + "post-backup hook receives the backup return code" + +stop_container + +echo +echo "=== compat: RESTIC_FORGET_ARGS ===" + +start_container forget -e "RESTIC_FORGET_ARGS=--keep-last 1" +wait_for_start || { fail "container starts with RESTIC_FORGET_ARGS"; summary "compat"; exit 1; } +write_file /data/hello.txt "backup me" +in_container /bin/backup >/dev/null 2>&1 +forget_log="$(in_container /bin/backup 2>&1)" +assert_contains "$forget_log" "Forget Successful" \ + "forget runs after a successful backup when configured" +assert_eq "1" "$(snapshot_count)" \ + "--keep-last 1 leaves exactly one snapshot" +stop_container + +echo +echo "=== compat: BACKUP_SOURCES ===" + +start_container sources -e "BACKUP_SOURCES=/opt/custom" +wait_for_start || { fail "container starts with BACKUP_SOURCES"; summary "compat"; exit 1; } +write_file /opt/custom/custom.txt "custom source" +write_file /data/ignored.txt "should not be backed up" +in_container /bin/backup >/dev/null 2>&1 +in_container restic restore latest --target /tmp/restore >/dev/null 2>&1 +assert_eq "custom source" "$(in_container cat /tmp/restore/opt/custom/custom.txt 2>&1)" \ + "BACKUP_SOURCES overrides the default /data source" +if in_container test -f /tmp/restore/data/ignored.txt; then + fail "BACKUP_SOURCES excludes /data" "/data was backed up although BACKUP_SOURCES was set" +else + pass "BACKUP_SOURCES excludes /data" +fi +stop_container + +echo +echo "=== compat: failure handling ===" + +start_container failure +wait_for_start || { fail "container starts for failure test"; summary "compat"; exit 1; } +write_file /data/hello.txt "backup me" +# Point the run at a repository that does not exist. Deleting files inside the +# repository is not enough, restic happily recreates its directory structure. +fail_log="$(in_container_env RESTIC_REPOSITORY=/mnt/does-not-exist /bin/backup 2>&1)" +assert_contains "$fail_log" "Backup Failed with Status" \ + "a failing backup is reported as failed" +if in_container test -f /var/log/backup-error-last.log; then + pass "failing backup writes backup-error-last.log" +else + fail "failing backup writes backup-error-last.log" "file is missing" +fi +assert_eq "0" "$(snapshot_count)" \ + "a failed backup leaves no snapshot behind" +stop_container + +echo +echo "=== compat: timezone ===" +# The base image already ships tzdata, so TZ works today. Pinned here so that +# neither a base image change nor a slimming effort silently breaks cron times +# for users who already rely on it. + +tz_default="$(docker run --rm --entrypoint date "$IMAGE" +%Z 2>&1 | tr -d '\r')" +assert_eq "UTC" "$tz_default" \ + "the default timezone is UTC" + +tz_berlin="$(docker run --rm -e TZ=Europe/Berlin --entrypoint date "$IMAGE" +%Z 2>&1 | tr -d '\r')" +if [ "$tz_berlin" = "CET" ] || [ "$tz_berlin" = "CEST" ]; then + pass "TZ=Europe/Berlin changes the container timezone" +else + fail "TZ=Europe/Berlin changes the container timezone" "got: ${tz_berlin}" +fi + +echo +echo "=== compat: RESTIC_TAG handling ===" + +start_container tag +wait_for_start || { fail "container starts for RESTIC_TAG test"; summary "compat"; exit 1; } +write_file /data/hello.txt "backup me" + +# The Dockerfile declares RESTIC_TAG="", so in a normal container the variable +# is always set and an empty value means "no tag". Documented here because the +# `${RESTIC_TAG?...}` guard in backup.sh only triggers when it is truly unset. +emptytag_log="$(in_container_env RESTIC_TAG= /bin/backup 2>&1)" +assert_contains "$emptytag_log" "Backup Successful" \ + "an empty RESTIC_TAG creates an untagged backup" +assert_eq "1" "$(snapshot_count)" \ + "an empty RESTIC_TAG still produces a snapshot" + +notag_log="$(in_container env -u RESTIC_TAG /bin/backup 2>&1)" +assert_contains "$notag_log" "Missing environment variable RESTIC_TAG" \ + "backup refuses to run when RESTIC_TAG is unset entirely" +assert_eq "1" "$(snapshot_count)" \ + "no extra snapshot is created when RESTIC_TAG is unset" +stop_container + +summary "compat" diff --git a/test/features.sh b/test/features.sh new file mode 100644 index 0000000..7a98232 --- /dev/null +++ b/test/features.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +# Feature suite. +# +# Each block is guarded by a detection check against the built image. On master +# every block reports SKIP. As soon as the corresponding pull request is rebased +# onto master, the feature is detected and its tests start running - the PR +# author does not have to write test code for the basics. +# +# After merging a feature, add its gate name to test/required-features.txt. +# From then on a missing feature is a failure instead of a skip. + +set -u + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=test/lib.sh +. "${SCRIPT_DIR}/lib.sh" + +trap stop_container EXIT + +echo "=== feature: stream backup (RESTIC_STREAM) ===" +# Pull request #109 - back up the output of a command via `restic backup --stdin`. + +if require_feature stream "PR #109"; then + start_container stream \ + -e RESTIC_STREAM=1 \ + -e "RESTIC_STREAM_CMD=echo hello-stream" \ + -e RESTIC_STREAM_FILENAME=stream.txt + if wait_for_start; then + stream_log="$(in_container /bin/backup 2>&1)" + assert_contains "$stream_log" "Backup Successful" \ + "stream backup succeeds" + assert_eq "1" "$(snapshot_count)" \ + "stream backup creates exactly one snapshot" + in_container restic restore latest --target /tmp/restore >/dev/null 2>&1 + restored="$(in_container sh -c 'cat /tmp/restore/stream.txt 2>/dev/null' 2>&1)" + assert_eq "hello-stream" "$(echo "$restored" | tr -d '\r')" \ + "stream content is stored under RESTIC_STREAM_FILENAME" + else + fail "stream backup container starts" + fi + stop_container + + # A half-configured stream must not look like a successful backup. + start_container stream-misconfigured -e RESTIC_STREAM=1 + if wait_for_start; then + misconfig_log="$(in_container /bin/backup 2>&1)" + assert_not_contains "$misconfig_log" "Backup Successful" \ + "incomplete stream configuration does not report success" + assert_eq "0" "$(snapshot_count)" \ + "incomplete stream configuration creates no snapshot" + else + fail "stream misconfiguration container starts" + fi + stop_container + + # The classic path must keep working while the feature is available. + start_container stream-disabled + if wait_for_start; then + write_file /data/hello.txt "backup me" + in_container /bin/backup >/dev/null 2>&1 + assert_eq "1" "$(snapshot_count)" \ + "file backup still works while the stream feature is present" + else + fail "stream-disabled container starts" + fi + stop_container +fi + +echo +echo "=== feature: docker cli ===" +# Pull request #99 - talk to the host docker daemon from hooks. + +if require_feature docker-cli "PR #99"; then + docker_version="$(docker run --rm --entrypoint docker "$IMAGE" --version 2>&1)" + assert_contains "$docker_version" "Docker version" \ + "docker cli is usable inside the image" +fi + +echo +echo "=== feature: bash ===" +# Pull request #99 - allow bash syntax in pre/post hooks. + +if require_feature bash "PR #99"; then + start_container bash-hooks + if wait_for_start; then + write_file /hooks/pre-backup.sh '#!/bin/bash +arr=(one two) +if [[ "${arr[1]}" == "two" ]]; then echo "BASH-HOOK-RAN"; fi +' + in_container chmod +x /hooks/pre-backup.sh + write_file /data/hello.txt "backup me" + bash_log="$(in_container /bin/backup 2>&1)" + assert_contains "$bash_log" "BASH-HOOK-RAN" \ + "a hook with a bash shebang and bash syntax runs" + else + fail "bash hook container starts" + fi + stop_container +fi + +echo +echo "=== feature: msmtp mail relay ===" +# Pull request #84 - send notification mails through msmtp. + +if require_feature msmtp "PR #84"; then + sendmail_target="$(docker run --rm --entrypoint sh "$IMAGE" -c \ + 'readlink -f /usr/sbin/sendmail' 2>&1 | tr -d '\r')" + assert_contains "$sendmail_target" "msmtp" \ + "sendmail is provided by msmtp" + + start_container msmtp-config + if wait_for_start; then + write_file /config/msmtprc 'defaults +account default +host localhost +' + in_container /entry.sh true >/dev/null 2>&1 + if in_container test -f /etc/msmtprc; then + pass "msmtp config from /config is copied to /etc/msmtprc" + mode="$(in_container stat -c '%a' /etc/msmtprc 2>&1 | tr -d '\r')" + assert_eq "600" "$mode" \ + "copied msmtp config is not world readable" + else + fail "msmtp config from /config is copied to /etc/msmtprc" "file is missing" + fi + else + fail "msmtp config container starts" + fi + stop_container + + # Users who configured mail before msmtp existed must not lose their backups. + start_container msmtp-missing-config \ + -e "MAILX_ARGS=-r from@example.com -s subject to@example.com" + if wait_for_start; then + write_file /data/hello.txt "backup me" + mail_log="$(in_container /bin/backup 2>&1)" + assert_contains "$mail_log" "Backup Successful" \ + "backup succeeds with MAILX_ARGS set but no msmtp config" + assert_eq "1" "$(snapshot_count)" \ + "snapshot is created even when the mail relay is unconfigured" + else + fail "msmtp missing-config container starts" + fi + stop_container +fi + +summary "features" diff --git a/test/lib.sh b/test/lib.sh new file mode 100644 index 0000000..1541362 --- /dev/null +++ b/test/lib.sh @@ -0,0 +1,238 @@ +#!/usr/bin/env bash +# Shared helpers for the restic-backup-docker test suites. +# +# Deliberately dependency-free: bash + docker only. No test framework, no +# host-side bind mounts (files are moved with `docker cp`) so the suites behave +# identically on Linux CI runners, macOS and Windows/Git-Bash. + +IMAGE="${IMAGE:-restic-backup-docker:test}" + +PASS_COUNT=0 +FAIL_COUNT=0 +SKIP_COUNT=0 +FAILED_TESTS=() +SKIPPED_TESTS=() + +_c_red='' +_c_green='' +_c_yellow='' +_c_reset='' +if [ -t 1 ]; then + _c_red=$'\033[31m' + _c_green=$'\033[32m' + _c_yellow=$'\033[33m' + _c_reset=$'\033[0m' +fi + +log() { echo " $*"; } + +pass() { + PASS_COUNT=$((PASS_COUNT + 1)) + echo "${_c_green}PASS${_c_reset} $1" +} + +fail() { + FAIL_COUNT=$((FAIL_COUNT + 1)) + FAILED_TESTS+=("$1") + echo "${_c_red}FAIL${_c_reset} $1" + if [ -n "${2:-}" ]; then + echo "$2" | sed 's/^/ /' + fi +} + +skip() { + SKIP_COUNT=$((SKIP_COUNT + 1)) + SKIPPED_TESTS+=("$1") + echo "${_c_yellow}SKIP${_c_reset} $1 (${2:-not applicable})" +} + +# summary -> exit code 0 when nothing failed +summary() { + echo + echo "--- $1: ${PASS_COUNT} passed, ${FAIL_COUNT} failed, ${SKIP_COUNT} skipped ---" + if [ ${#FAILED_TESTS[@]} -gt 0 ]; then + echo "Failed:" + printf ' - %s\n' "${FAILED_TESTS[@]}" + fi + if [ ${#SKIPPED_TESTS[@]} -gt 0 ]; then + echo "Skipped:" + printf ' - %s\n' "${SKIPPED_TESTS[@]}" + fi + [ "${FAIL_COUNT}" -eq 0 ] +} + +# --------------------------------------------------------------------------- +# assertions +# --------------------------------------------------------------------------- + +assert_eq() { + local expected="$1" actual="$2" name="$3" + if [ "$expected" = "$actual" ]; then + pass "$name" + else + fail "$name" "expected: ${expected} +actual: ${actual}" + fi +} + +assert_contains() { + local haystack="$1" needle="$2" name="$3" + case "$haystack" in + *"$needle"*) pass "$name" ;; + *) fail "$name" "expected to contain: ${needle} +actual: +$(echo "$haystack" | tail -n 25)" ;; + esac +} + +assert_not_contains() { + local haystack="$1" needle="$2" name="$3" + case "$haystack" in + *"$needle"*) fail "$name" "unexpectedly contains: ${needle}" ;; + *) pass "$name" ;; + esac +} + +# --------------------------------------------------------------------------- +# container helpers +# +# Every test gets its own throwaway container. The restic repository lives +# inside the container at the image default (/mnt/restic), so no host paths are +# involved and tests cannot pollute each other. +# --------------------------------------------------------------------------- + +CONTAINER="" + +# start_container [extra docker run args...] +# Starts the image with a cron expression far in the future so that the only +# backup that ever runs is the one a test triggers explicitly. +start_container() { + local suffix="$1" + shift + CONTAINER="rbd-test-${suffix}-$$" + docker rm -f "$CONTAINER" >/dev/null 2>&1 || true + docker run -d --name "$CONTAINER" \ + -e RESTIC_PASSWORD=testpassword \ + -e RESTIC_TAG=testtag \ + -e "BACKUP_CRON=0 0 31 2 *" \ + "$@" \ + "$IMAGE" >/dev/null +} + +stop_container() { + [ -n "$CONTAINER" ] || return 0 + docker rm -f "$CONTAINER" >/dev/null 2>&1 || true + CONTAINER="" +} + +# wait_for_start -> fails if entry.sh did not finish within the timeout +wait_for_start() { + local waited=0 + while [ "$waited" -lt 60 ]; do + waited=$((waited + 1)) + if docker logs "$CONTAINER" 2>&1 | grep -q "Container started."; then + return 0 + fi + if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null)" != "true" ]; then + echo "container exited early:" + docker logs "$CONTAINER" 2>&1 | sed 's/^/ /' + return 1 + fi + sleep 1 + done + echo "timeout waiting for container start:" + docker logs "$CONTAINER" 2>&1 | sed 's/^/ /' + return 1 +} + +# in_container - run a command inside the running test container +in_container() { + docker exec "$CONTAINER" "$@" +} + +# in_container_env - run with one overridden variable +in_container_env() { + local env_pair="$1" + shift + docker exec -e "$env_pair" "$CONTAINER" "$@" +} + +# write_file +write_file() { + local path="$1" content="$2" + local dir + dir="$(dirname "$path")" + docker exec "$CONTAINER" mkdir -p "$dir" + printf '%s' "$content" | docker exec -i "$CONTAINER" sh -c "cat > '$path'" +} + +# snapshot_count -> number of snapshots in the repository +snapshot_count() { + docker exec "$CONTAINER" sh -c \ + 'restic snapshots --json 2>/dev/null | grep -o "\"short_id\"" | wc -l' | tr -d ' \r' +} + +# --------------------------------------------------------------------------- +# feature gating +# +# A feature test only runs when the image actually ships the feature. That way +# this suite already covers the currently open pull requests: once such a PR is +# rebased onto master its feature is detected and its tests start running, +# without the PR author having to write any test code. +# +# test/required-features.txt lists features that MUST be present. A feature +# listed there that is not detected turns the skip into a hard failure, so a +# merged feature cannot silently disappear from the image again. +# --------------------------------------------------------------------------- + +REQUIRED_FEATURES_FILE="${REQUIRED_FEATURES_FILE:-$(dirname "${BASH_SOURCE[0]}")/required-features.txt}" + +feature_is_required() { + local feature="$1" + [ -f "$REQUIRED_FEATURES_FILE" ] || return 1 + grep -v '^[[:space:]]*#' "$REQUIRED_FEATURES_FILE" 2>/dev/null \ + | grep -qx "[[:space:]]*${feature}[[:space:]]*" +} + +# has_feature - detection runs against the built image, not the source +has_feature() { + case "$1" in + stream) + docker run --rm --entrypoint sh "$IMAGE" -c \ + 'grep -q RESTIC_STREAM /bin/backup' >/dev/null 2>&1 + ;; + docker-cli) + docker run --rm --entrypoint sh "$IMAGE" -c \ + 'command -v docker' >/dev/null 2>&1 + ;; + bash) + docker run --rm --entrypoint sh "$IMAGE" -c \ + 'command -v bash' >/dev/null 2>&1 + ;; + msmtp) + docker run --rm --entrypoint sh "$IMAGE" -c \ + 'command -v msmtp' >/dev/null 2>&1 + ;; + *) + echo "unknown feature: $1" >&2 + return 1 + ;; + esac +} + +# require_feature +# Returns 0 when the feature tests should run. Otherwise records a skip, or a +# failure when the feature is listed in required-features.txt. +require_feature() { + local feature="$1" ref="$2" + if has_feature "$feature"; then + return 0 + fi + if feature_is_required "$feature"; then + fail "feature '${feature}'" \ + "listed in $(basename "$REQUIRED_FEATURES_FILE") but not present in the image" + else + skip "feature '${feature}'" "not in image yet, see ${ref}" + fi + return 1 +} diff --git a/test/required-features.txt b/test/required-features.txt new file mode 100644 index 0000000..5b266d7 --- /dev/null +++ b/test/required-features.txt @@ -0,0 +1,18 @@ +# Optional features that MUST be present in the built image. +# +# One gate name per line. Known gates and the pull request that introduces +# them (see test/features.sh): +# +# stream stream backups via RESTIC_STREAM PR #109 +# docker-cli docker cli inside the container PR #99 +# bash bash for pre/post hooks PR #99 +# msmtp mail relay via msmtp PR #84 +# +# While a gate is NOT listed here its tests are reported as SKIP when the +# feature is absent, so an open pull request does not turn master red. +# +# When you merge a feature, add its gate name below. From then on the feature +# tests are mandatory and a feature that silently disappears from the image +# fails the build. +# +# Empty on purpose: master currently ships none of the optional features. diff --git a/test/run.sh b/test/run.sh new file mode 100644 index 0000000..5295f1b --- /dev/null +++ b/test/run.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# Entry point for the test suites. +# +# bash test/run.sh build the image and run every suite +# bash test/run.sh --no-build reuse an already built $IMAGE +# bash test/run.sh --suite compat run a single suite (compat|features|structure) +# bash test/run.sh --sabotage self test: break backup.sh on a throwaway +# copy of the tree and prove the suite fails +# +# Requires bash and docker. container-structure-test is optional; the structure +# suite is skipped when the tool is not installed. + +set -u + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +IMAGE="${IMAGE:-restic-backup-docker:test}" +export IMAGE +# Pinned by digest-free tag on purpose: the structure test tool is only used to +# read the image under test, a floating tag cannot affect the product. +STRUCTURE_TEST_IMAGE="${STRUCTURE_TEST_IMAGE:-gcr.io/gcp-runtimes/container-structure-test:latest}" + +BUILD=1 +SABOTAGE=0 +ONLY_SUITE="" + +while [ $# -gt 0 ]; do + case "$1" in + --no-build) BUILD=0 ;; + --sabotage) SABOTAGE=1 ;; + --suite) ONLY_SUITE="${2:-}"; shift ;; + --suite=*) ONLY_SUITE="${1#--suite=}" ;; + -h|--help) sed -n '2,14p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "unknown argument: $1" >&2; exit 2 ;; + esac + shift +done + +if ! docker info >/dev/null 2>&1; then + echo "docker is not available, cannot run the test suite" >&2 + exit 2 +fi + +# Docker on Windows does not understand the /c/... paths Git-Bash produces, +# so build contexts are translated when cygpath is around. No-op elsewhere. +host_path() { + if command -v cygpath >/dev/null 2>&1; then + cygpath -w "$1" + else + printf '%s' "$1" + fi +} + +build_image() { + local context tag + context="$(host_path "$1")" + tag="$2" + echo ">>> building ${tag}" + if ! docker build --quiet -t "$tag" "$context" >/dev/null; then + echo "image build failed" >&2 + return 1 + fi +} + +# --sabotage proves the suite has teeth: it breaks the backup call on a copy of +# the tree and expects the compat suite to go red. If this mode ever passes, +# the tests are not actually testing anything. +if [ "$SABOTAGE" -eq 1 ]; then + tmp="$(mktemp -d)" + trap 'rm -rf "$tmp"; docker rmi -f restic-backup-docker:sabotage >/dev/null 2>&1 || true' EXIT + tar -c -C "$ROOT_DIR" --exclude=.git --exclude=.intent . | tar -x -C "$tmp" + # Silently turn the backup into a no-op, the nastiest kind of regression. + sed -i 's|^restic backup |true |' "$tmp/backup.sh" + if ! grep -q '^true ' "$tmp/backup.sh"; then + echo "sabotage patch did not apply, backup.sh changed shape" >&2 + exit 2 + fi + build_image "$tmp" restic-backup-docker:sabotage || exit 2 + echo ">>> running compat suite against the sabotaged image (failure expected)" + if IMAGE=restic-backup-docker:sabotage bash "${SCRIPT_DIR}/compat.sh"; then + echo + echo "SABOTAGE CHECK FAILED: the compat suite passed on a broken image" >&2 + exit 1 + fi + echo + echo "sabotage check ok: the compat suite detects a broken backup" + exit 0 +fi + +if [ "$BUILD" -eq 1 ]; then + build_image "$ROOT_DIR" "$IMAGE" || exit 2 +fi + +rc=0 + +run_structure() { + echo + echo ">>> structure suite" + if command -v container-structure-test >/dev/null 2>&1; then + container-structure-test test --image "$IMAGE" --config "${SCRIPT_DIR}/structure.yml" || rc=1 + else + # Not installed locally: fall back to the official image. It needs the + # docker socket because it inspects and runs the image under test. + echo "container-structure-test not installed, using the official image" + docker run --rm \ + -v //var/run/docker.sock:/var/run/docker.sock \ + -v "$(host_path "${SCRIPT_DIR}")":/test \ + "$STRUCTURE_TEST_IMAGE" \ + test --image "$IMAGE" --config /test/structure.yml || rc=1 + fi +} + +run_suite() { + echo + echo ">>> $1 suite" + bash "${SCRIPT_DIR}/$1.sh" || rc=1 +} + +case "$ONLY_SUITE" in + "") run_structure; run_suite compat; run_suite features ;; + structure) run_structure ;; + compat) run_suite compat ;; + features) run_suite features ;; + *) echo "unknown suite: ${ONLY_SUITE}" >&2; exit 2 ;; +esac + +echo +if [ "$rc" -eq 0 ]; then + echo "all suites passed" +else + echo "at least one suite failed" +fi +exit "$rc" diff --git a/test/structure.yml b/test/structure.yml new file mode 100644 index 0000000..7ebd244 --- /dev/null +++ b/test/structure.yml @@ -0,0 +1,64 @@ +schemaVersion: '2.0.0' + +# Structural contract of the image. Everything asserted here is something a +# user can reach from outside: a binary they call from a hook, a path they +# mount, or the way the container is started. + +commandTests: + - name: 'restic is installed' + command: 'restic' + args: ['version'] + expectedOutput: ['restic \d+\.\d+\.\d+.*'] + - name: 'rclone is installed' + command: 'rclone' + args: ['--version'] + expectedOutput: ['rclone v.*'] + - name: 'curl is installed, used for Teams notifications' + command: 'curl' + args: ['--version'] + expectedOutput: ['curl \d+\..*'] + - name: 'mail is installed, used for mail notifications' + command: 'sh' + args: ['-c', 'command -v mail'] + expectedOutput: ['/usr/bin/mail'] + - name: 'crond is available' + command: 'sh' + args: ['-c', 'command -v crond'] + expectedOutput: ['/usr/sbin/crond'] + - name: 'flock is available, the cron jobs depend on it' + command: 'sh' + args: ['-c', 'test -x /usr/bin/flock && echo ok'] + expectedOutput: ['ok'] + +fileExistenceTests: + - name: 'backup script' + path: '/bin/backup' + shouldExist: true + permissions: '-rwxr-xr-x' + - name: 'check script' + path: '/bin/check' + shouldExist: true + permissions: '-rwxr-xr-x' + - name: 'entrypoint' + path: '/entry.sh' + shouldExist: true + permissions: '-rwxr-xr-x' + - name: 'default repository mount point' + path: '/mnt/restic' + shouldExist: true + - name: 'cron log file' + path: '/var/log/cron.log' + shouldExist: true + - name: 'crontab directory' + path: '/var/spool/cron/crontabs' + shouldExist: true + +metadataTest: + volumes: ['/data'] + entrypoint: ['/entry.sh'] + cmd: ['tail', '-fn0', '/var/log/cron.log'] + envVars: + - key: 'RESTIC_REPOSITORY' + value: '/mnt/restic' + - key: 'BACKUP_CRON' + value: '0 */6 * * *'