diff --git a/Dockerfile b/Dockerfile index 2dfdbf8..c7700d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,9 +19,11 @@ ENV RESTIC_PASSWORD="" ENV RESTIC_TAG="" ENV NFS_TARGET="" ENV BACKUP_CRON="0 */6 * * *" +ENV CHECK_CRON="" ENV RESTIC_INIT_ARGS="" ENV RESTIC_FORGET_ARGS="" ENV RESTIC_JOB_ARGS="" +ENV RESTIC_DATA_SUBSET="" ENV MAILX_ARGS="" ENV OS_AUTH_URL="" ENV OS_PROJECT_ID="" @@ -49,6 +51,7 @@ RUN mkdir /.cache && \ VOLUME /data COPY backup.sh /bin/backup +COPY check.sh /bin/check COPY entry.sh /entry.sh diff --git a/README.md b/README.md index ff391d9..9954dd8 100644 --- a/README.md +++ b/README.md @@ -21,12 +21,12 @@ docker pull lobaro/restic-backup-docker:latest ## Hooks -If you need to execute a script before or after each backup, +If you need to execute a script before or after each backup or check, you need to add your hook script in the container folder `/hooks`: ``` -v ~/home/user/hooks:/hooks ``` -Call your pre backup script `pre-backup.sh` and post backup script `post-backup.sh` +Call your pre backup script `pre-backup.sh` and post backup script `post-backup.sh`. You can also have separate scripts when running data verification checks `pre-check.sh` and `post-check.sh` Please don't hesitate to report any issue you find. **Thanks.** @@ -88,6 +88,12 @@ Backup a single file or directory docker exec -ti restic-backup-var restic backup /data/path/to/dir --tag my-tag +## Data verification check + +To verify backup integrity and consistency manually independent of the CRON run: + + docker exec -ti restic-backup-var /bin/check + ## Restore You might want to mount a separate hostvolume at e.g. `/restore` to not override existing data while restoring. @@ -115,13 +121,15 @@ The container is setup by setting [environment variables](https://docs.docker.co * `RESTIC_TAG` - Optional. To tag the images created by the container. * `NFS_TARGET` - Optional. If set the given NFS is mounted, i.e. `mount -o nolock -v ${NFS_TARGET} /mnt/restic`. `RESTIC_REPOSITORY` must remain it's default value! * `BACKUP_CRON` - A cron expression to run the backup. Note: cron daemon uses UTC time zone. Default: `0 */6 * * *` aka every 6 hours. +* `CHECK_CRON` - Optional. A cron expression to run data integrity check (`restic check`). If left unset data will not be checked. Note: cron daemon uses UTC time zone. Example: `0 23 * * 3` to run 11PM every Tuesday. * `RESTIC_FORGET_ARGS` - Optional. Only if specified `restic forget` is run with the given arguments after each backup. Example value: `-e "RESTIC_FORGET_ARGS=--prune --keep-last 10 --keep-hourly 24 --keep-daily 7 --keep-weekly 52 --keep-monthly 120 --keep-yearly 100"` * `RESTIC_INIT_ARGS` - Optional. Allows to specify extra arguments to `restic init` such as a password file with `--password-file`. * `RESTIC_JOB_ARGS` - Optional. Allows to specify extra arguments to the back up job such as limiting bandwith with `--limit-upload` or excluding file masks with `--exclude`. +* `RESTIC_DATA_SUBSET` - Optional. You can pass value to `--read-data-subset` when repository check is run. If left unset only the structure of the repository is verified. Note: `CHECK_CRON` must be set for check to be run automatically. * `AWS_ACCESS_KEY_ID` - Optional. When using restic with AWS S3 storage. * `AWS_SECRET_ACCESS_KEY` - Optional. When using restic with AWS S3 storage. -* `TEAMS_WEBHOOK_URL` - Optional. If specified, the content of `/var/log/backup-last.log` is sent to your Microsoft Teams channel after each backup. -* `MAILX_ARGS` - Optional. If specified, the content of `/var/log/backup-last.log` is sent via mail after each backup using an *external SMTP*. To have maximum flexibility, you have to specify the mail/smtp parameters by your own. Have a look at the [mailx manpage](https://linux.die.net/man/1/mailx) for further information. Example value: `-e "MAILX_ARGS=-r 'from@example.de' -s 'Result of the last restic backup run' -S smtp='smtp.example.com:587' -S smtp-use-starttls -S smtp-auth=login -S smtp-auth-user='username' -S smtp-auth-password='password' 'to@example.com'"`. +* `TEAMS_WEBHOOK_URL` - Optional. If specified, the content of `/var/log/backup-last.log` and `/var/log/check-last.log` is sent to your Microsoft Teams channel after each backup and data integrity check. +* `MAILX_ARGS` - Optional. If specified, the content of `/var/log/backup-last.log` and `/var/log/check-last.log` is sent via mail after each backup and data integrity check using an *external SMTP*. To have maximum flexibility, you have to specify the mail/smtp parameters by your own. Have a look at the [mailx manpage](https://linux.die.net/man/1/mailx) for further information. Example value: `-e "MAILX_ARGS=-r 'from@example.de' -s 'Result of the last restic run' -S smtp='smtp.example.com:587' -S smtp-use-starttls -S smtp-auth=login -S smtp-auth-user='username' -S smtp-auth-password='password' 'to@example.com'"`. * `OS_AUTH_URL` - Optional. When using restic with OpenStack Swift container. * `OS_PROJECT_ID` - Optional. When using restic with OpenStack Swift container. * `OS_PROJECT_NAME` - Optional. When using restic with OpenStack Swift container. @@ -180,6 +188,34 @@ To use rclone as a backend for restic, simply add the rclone config file as a vo Note that for some backends (Among them Google Drive and Microsoft OneDrive), rclone writes data back to the `rclone.conf` file. In this case it needs to be writable by Docker. If the the container fails to write the new `rclone.conf` file with the error message `Failed to save config after 10 tries: Failed to move previous config to backup location`, add the entire `rclone` directory as volume: `-v /absolute/path/to/rclone-dir:/root/.config/rclone`. + +## Example docker-compose + +This is example `docker-compose.yml`. Container will backup two directories to a SFTP server and check data integrity once a week. + +``` +version: '3' + +services: + restic: + image: lobaro/restic-backup-docker:latest + hostname: nas # This will be visible in restic snapshot list + restart: always + privileged: true + volumes: + - /volume1/Backup:/data/Backup:ro # Backup /volume1/Backup from host + - /home/user:/data/home:ro # Backup /home/user from host + - ./post-backup.sh:/hooks/post-backup.sh:ro # Run script post-backup.sh after every backup + - ./post-check.sh:/hooks/post-check.sh:ro # Run script post-check.sh after every check + - ./ssh:/root/.ssh # SSH keys and config so we can login to "storageserver" without password + environment: + - RESTIC_REPOSITORY=sftp:storageserver:/storage/nas # Backup to server "storageserver" + - RESTIC_PASSWORD=passwordForRestic # Password restic uses for encryption + - BACKUP_CRON=0 22 * * 0 # Start backup every Sunday 22:00 UTC + - CHECK_CRON=0 22 * * 3 # Start check every Wednesday 22:00 UTC + - RESTIC_DATA_SUBSET=50G # Download 50G of data from "storageserver" every Wednesday 22:00 UTC and check the data integrity + - RESTIC_FORGET_ARGS=--prune --keep-last 12 # Only keep the last 12 snapshots +``` # Versioning & Changelog diff --git a/check.sh b/check.sh new file mode 100755 index 0000000..c33c168 --- /dev/null +++ b/check.sh @@ -0,0 +1,76 @@ +#!/bin/sh + +lastLogfile="/var/log/check-last.log" +lastMailLogfile="/var/log/check-mail-last.log" +lastMicrosoftTeamsLogfile="/var/log/check-microsoft-teams-last.log" + +copyErrorLog() { + cp ${lastLogfile} /var/log/check-error-last.log +} + +logLast() { + echo "$1" >> ${lastLogfile} +} + +if [ -f "/hooks/pre-check.sh" ]; then + echo "Starting pre-check script ..." + /hooks/pre-check.sh +else + echo "Pre-check script not found ..." +fi + +start=`date +%s` +rm -f ${lastLogfile} ${lastMailLogfile} +echo "Starting Check at $(date +"%Y-%m-%d %H:%M:%S")" +echo "Starting Check at $(date)" >> ${lastLogfile} +logLast "CHECK_CRON: ${CHECK_CRON}" +logLast "RESTIC_DATA_SUBSET: ${RESTIC_DATA_SUBSET}" +logLast "RESTIC_REPOSITORY: ${RESTIC_REPOSITORY}" +logLast "AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}" + +# Do not save full check log to logfile but to check-last.log +if [ -n "${RESTIC_DATA_SUBSET}" ]; then + restic check --read-data-subset=${RESTIC_DATA_SUBSET} >> ${lastLogfile} 2>&1 +else + restic check >> ${lastLogfile} 2>&1 +fi +checkRC=$? +logLast "Finished check at $(date)" +if [[ $checkRC == 0 ]]; then + echo "Check Successful" +else + echo "Check Failed with Status ${checkRC}" + restic unlock + copyErrorLog +fi + +end=`date +%s` +echo "Finished Check at $(date +"%Y-%m-%d %H:%M:%S") after $((end-start)) seconds" + +if [ -n "${TEAMS_WEBHOOK_URL}" ]; then + teamsTitle="Restic Last Check Log" + teamsMessage=$( cat ${lastLogfile} | sed 's/"/\"/g' | sed "s/'/\'/g" | sed ':a;N;$!ba;s/\n/\n\n/g' ) + teamsReqBody="{\"title\": \"${teamsTitle}\", \"text\": \"${teamsMessage}\" }" + sh -c "curl -H 'Content-Type: application/json' -d '${teamsReqBody}' '${TEAMS_WEBHOOK_URL}' > ${lastMicrosoftTeamsLogfile} 2>&1" + if [ $? == 0 ]; then + echo "Microsoft Teams notification successfully sent." + else + echo "Sending Microsoft Teams notification FAILED. Check ${lastMicrosoftTeamsLogfile} for further information." + fi +fi + +if [ -n "${MAILX_ARGS}" ]; then + sh -c "mailx -v -S sendwait ${MAILX_ARGS} < ${lastLogfile} > ${lastMailLogfile} 2>&1" + if [ $? == 0 ]; then + echo "Mail notification successfully sent." + else + echo "Sending mail notification FAILED. Check ${lastMailLogfile} for further information." + fi +fi + +if [ -f "/hooks/post-check.sh" ]; then + echo "Starting post-check script ..." + /hooks/post-check.sh $checkRC +else + echo "Post-check script not found ..." +fi diff --git a/entry.sh b/entry.sh index 7be1768..e0076c8 100755 --- a/entry.sh +++ b/entry.sh @@ -29,6 +29,12 @@ fi echo "Setup backup cron job with cron expression BACKUP_CRON: ${BACKUP_CRON}" echo "${BACKUP_CRON} /usr/bin/flock -n /var/run/backup.lock /bin/backup >> /var/log/cron.log 2>&1" > /var/spool/cron/crontabs/root +# If CHECK_CRON is set we will enable automatic backup checking +if [ -n "${CHECK_CRON}" ]; then + echo "Setup check cron job with cron expression CHECK_CRON: ${CHECK_CRON}" + echo "${CHECK_CRON} /usr/bin/flock -n /var/run/backup.lock /bin/check >> /var/log/cron.log 2>&1" >> /var/spool/cron/crontabs/root +fi + # Make sure the file exists before we start tail touch /var/log/cron.log @@ -37,4 +43,4 @@ crond echo "Container started." -exec "$@" +exec "$@" \ No newline at end of file