Merge pull request #73 from lojutaan/check

Update restic to version 0.14.0 and add option to check data integrity

Dockerfile

@@ -19,9 +19,11 @@ ENV RESTIC_PASSWORD=""
 ENV RESTIC_TAG=""
 ENV NFS_TARGET=""
 ENV BACKUP_CRON="0 */6 * * *"
+ENV CHECK_CRON=""
 ENV RESTIC_INIT_ARGS=""
 ENV RESTIC_FORGET_ARGS=""
 ENV RESTIC_JOB_ARGS=""
+ENV RESTIC_DATA_SUBSET=""
 ENV MAILX_ARGS=""
 ENV OS_AUTH_URL=""
 ENV OS_PROJECT_ID=""
@@ -49,6 +51,7 @@ RUN mkdir /.cache && \
 VOLUME /data
 
 COPY backup.sh /bin/backup
+COPY check.sh /bin/check
 COPY entry.sh /entry.sh
 
 

README.md

@@ -21,12 +21,12 @@ docker pull lobaro/restic-backup-docker:latest
 
 ## Hooks
 
-If you need to execute a script before or after each backup, 
+If you need to execute a script before or after each backup or check, 
 you need to add your hook script in the container folder `/hooks`:
 ```
 -v ~/home/user/hooks:/hooks
 ```
-Call your pre backup script `pre-backup.sh` and post backup script `post-backup.sh`
+Call your pre backup script `pre-backup.sh` and post backup script `post-backup.sh`. You can also have separate scripts when running data verification checks `pre-check.sh` and `post-check.sh`
 
 Please don't hesitate to report any issue you find. **Thanks.**
 
@@ -88,6 +88,12 @@ Backup a single file or directory
 
     docker exec -ti restic-backup-var restic backup /data/path/to/dir --tag my-tag
 
+## Data verification check
+
+To verify backup integrity and consistency manually independent of the CRON run:
+
+    docker exec -ti restic-backup-var /bin/check
+
 ## Restore
 
 You might want to mount a separate hostvolume at e.g. `/restore` to not override existing data while restoring. 
@@ -115,13 +121,15 @@ The container is setup by setting [environment variables](https://docs.docker.co
 * `RESTIC_TAG` - Optional. To tag the images created by the container.
 * `NFS_TARGET` - Optional. If set the given NFS is mounted, i.e. `mount -o nolock -v ${NFS_TARGET} /mnt/restic`. `RESTIC_REPOSITORY` must remain it's default value!
 * `BACKUP_CRON` - A cron expression to run the backup. Note: cron daemon uses UTC time zone. Default: `0 */6 * * *` aka every 6 hours.
+* `CHECK_CRON` - Optional. A cron expression to run data integrity check (`restic check`). If left unset data will not be checked. Note: cron daemon uses UTC time zone. Example: `0 23 * * 3` to run 11PM every Tuesday.
 * `RESTIC_FORGET_ARGS` - Optional. Only if specified `restic forget` is run with the given arguments after each backup. Example value: `-e "RESTIC_FORGET_ARGS=--prune --keep-last 10 --keep-hourly 24 --keep-daily 7 --keep-weekly 52 --keep-monthly 120 --keep-yearly 100"`
 * `RESTIC_INIT_ARGS` - Optional. Allows to specify extra arguments to `restic init` such as a password file with `--password-file`.
 * `RESTIC_JOB_ARGS` - Optional. Allows to specify extra arguments to the back up job such as limiting bandwith with `--limit-upload` or excluding file masks with `--exclude`.
+* `RESTIC_DATA_SUBSET` - Optional. You can pass value to `--read-data-subset` when repository check is run. If left unset only the structure of the repository is verified. Note: `CHECK_CRON` must be set for check to be run automatically.
 * `AWS_ACCESS_KEY_ID` - Optional. When using restic with AWS S3 storage.
 * `AWS_SECRET_ACCESS_KEY` - Optional. When using restic with AWS S3 storage.
-* `TEAMS_WEBHOOK_URL` - Optional. If specified, the content of `/var/log/backup-last.log` is sent to your Microsoft Teams channel after each backup.
-* `MAILX_ARGS` - Optional. If specified, the content of `/var/log/backup-last.log` is sent via mail after each backup using an *external SMTP*. To have maximum flexibility, you have to specify the mail/smtp parameters by your own. Have a look at the [mailx manpage](https://linux.die.net/man/1/mailx) for further information. Example value: `-e "MAILX_ARGS=-r 'from@example.de' -s 'Result of the last restic backup run' -S smtp='smtp.example.com:587' -S smtp-use-starttls -S smtp-auth=login -S smtp-auth-user='username' -S smtp-auth-password='password' 'to@example.com'"`.
+* `TEAMS_WEBHOOK_URL` - Optional. If specified, the content of `/var/log/backup-last.log` and `/var/log/check-last.log` is sent to your Microsoft Teams channel after each backup and data integrity check.
+* `MAILX_ARGS` - Optional. If specified, the content of `/var/log/backup-last.log` and `/var/log/check-last.log` is sent via mail after each backup and data integrity check using an *external SMTP*. To have maximum flexibility, you have to specify the mail/smtp parameters by your own. Have a look at the [mailx manpage](https://linux.die.net/man/1/mailx) for further information. Example value: `-e "MAILX_ARGS=-r 'from@example.de' -s 'Result of the last restic run' -S smtp='smtp.example.com:587' -S smtp-use-starttls -S smtp-auth=login -S smtp-auth-user='username' -S smtp-auth-password='password' 'to@example.com'"`.
 * `OS_AUTH_URL` - Optional. When using restic with OpenStack Swift container.
 * `OS_PROJECT_ID` - Optional. When using restic with OpenStack Swift container.
 * `OS_PROJECT_NAME` - Optional. When using restic with OpenStack Swift container.
@@ -180,6 +188,34 @@ To use rclone as a backend for restic, simply add the rclone config file as a vo
 Note that for some backends (Among them Google Drive and Microsoft OneDrive), rclone writes data back to the `rclone.conf` file. In this case it needs to be writable by Docker.
 
 If the the container fails to write the new `rclone.conf` file with the error message `Failed to save config after 10 tries: Failed to move previous config to backup location`, add the entire `rclone` directory as volume: `-v /absolute/path/to/rclone-dir:/root/.config/rclone`.
+
+## Example docker-compose
+
+This is example `docker-compose.yml`. Container will backup two directories to a SFTP server and check data integrity once a week. 
+
+```
+version: '3'
+
+services:
+  restic:
+    image: lobaro/restic-backup-docker:latest
+    hostname: nas                                     # This will be visible in restic snapshot list
+    restart: always
+    privileged: true
+    volumes:
+      - /volume1/Backup:/data/Backup:ro               # Backup /volume1/Backup from host
+      - /home/user:/data/home:ro                      # Backup /home/user from host
+      - ./post-backup.sh:/hooks/post-backup.sh:ro     # Run script post-backup.sh after every backup
+      - ./post-check.sh:/hooks/post-check.sh:ro       # Run script post-check.sh after every check
+      - ./ssh:/root/.ssh                              # SSH keys and config so we can login to "storageserver" without password
+    environment:
+      - RESTIC_REPOSITORY=sftp:storageserver:/storage/nas  # Backup to server "storageserver" 
+      - RESTIC_PASSWORD=passwordForRestic                  # Password restic uses for encryption
+      - BACKUP_CRON=0 22 * * 0                             # Start backup every Sunday 22:00 UTC
+      - CHECK_CRON=0 22 * * 3                              # Start check every Wednesday 22:00 UTC
+      - RESTIC_DATA_SUBSET=50G                             # Download 50G of data from "storageserver" every Wednesday 22:00 UTC and check the data integrity
+      - RESTIC_FORGET_ARGS=--prune --keep-last 12          # Only keep the last 12 snapshots
+```
  
 # Versioning & Changelog
 

check.sh

@@ -0,0 +1,76 @@
+#!/bin/sh
+
+lastLogfile="/var/log/check-last.log"
+lastMailLogfile="/var/log/check-mail-last.log"
+lastMicrosoftTeamsLogfile="/var/log/check-microsoft-teams-last.log"
+
+copyErrorLog() {
+  cp ${lastLogfile} /var/log/check-error-last.log
+}
+
+logLast() {
+  echo "$1" >> ${lastLogfile}
+}
+
+if [ -f "/hooks/pre-check.sh" ]; then
+    echo "Starting pre-check script ..."
+    /hooks/pre-check.sh
+else
+    echo "Pre-check script not found ..."
+fi
+
+start=`date +%s`
+rm -f ${lastLogfile} ${lastMailLogfile}
+echo "Starting Check at $(date +"%Y-%m-%d %H:%M:%S")"
+echo "Starting Check at $(date)" >> ${lastLogfile}
+logLast "CHECK_CRON: ${CHECK_CRON}"
+logLast "RESTIC_DATA_SUBSET: ${RESTIC_DATA_SUBSET}"
+logLast "RESTIC_REPOSITORY: ${RESTIC_REPOSITORY}"
+logLast "AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}"
+
+# Do not save full check log to logfile but to check-last.log
+if [ -n "${RESTIC_DATA_SUBSET}" ]; then
+    restic check --read-data-subset=${RESTIC_DATA_SUBSET} >> ${lastLogfile} 2>&1
+else
+    restic check >> ${lastLogfile} 2>&1
+fi
+checkRC=$?
+logLast "Finished check at $(date)"
+if [[ $checkRC == 0 ]]; then
+    echo "Check Successful"
+else
+    echo "Check Failed with Status ${checkRC}"
+    restic unlock
+    copyErrorLog
+fi
+
+end=`date +%s`
+echo "Finished Check at $(date +"%Y-%m-%d %H:%M:%S") after $((end-start)) seconds"
+
+if [ -n "${TEAMS_WEBHOOK_URL}" ]; then
+    teamsTitle="Restic Last Check Log"
+    teamsMessage=$( cat ${lastLogfile} | sed 's/"/\"/g' | sed "s/'/\'/g" | sed ':a;N;$!ba;s/\n/\n\n/g' )
+    teamsReqBody="{\"title\": \"${teamsTitle}\", \"text\": \"${teamsMessage}\" }"
+    sh -c "curl -H 'Content-Type: application/json' -d '${teamsReqBody}' '${TEAMS_WEBHOOK_URL}' > ${lastMicrosoftTeamsLogfile} 2>&1"
+    if [ $? == 0 ]; then
+        echo "Microsoft Teams notification successfully sent."
+    else
+        echo "Sending Microsoft Teams notification FAILED. Check ${lastMicrosoftTeamsLogfile} for further information."
+    fi
+fi
+
+if [ -n "${MAILX_ARGS}" ]; then
+    sh -c "mailx -v -S sendwait ${MAILX_ARGS} < ${lastLogfile} > ${lastMailLogfile} 2>&1"
+    if [ $? == 0 ]; then
+        echo "Mail notification successfully sent."
+    else
+        echo "Sending mail notification FAILED. Check ${lastMailLogfile} for further information."
+    fi
+fi
+
+if [ -f "/hooks/post-check.sh" ]; then
+    echo "Starting post-check script ..."
+    /hooks/post-check.sh $checkRC
+else
+    echo "Post-check script not found ..."
+fi

entry.sh

@@ -29,6 +29,12 @@ fi
 echo "Setup backup cron job with cron expression BACKUP_CRON: ${BACKUP_CRON}"
 echo "${BACKUP_CRON} /usr/bin/flock -n /var/run/backup.lock /bin/backup >> /var/log/cron.log 2>&1" > /var/spool/cron/crontabs/root
 
+# If CHECK_CRON is set we will enable automatic backup checking
+if [ -n "${CHECK_CRON}" ]; then
+    echo "Setup check cron job with cron expression CHECK_CRON: ${CHECK_CRON}"
+    echo "${CHECK_CRON} /usr/bin/flock -n /var/run/backup.lock /bin/check >> /var/log/cron.log 2>&1" >> /var/spool/cron/crontabs/root
+fi
+
 # Make sure the file exists before we start tail
 touch /var/log/cron.log
 
@@ -37,4 +43,4 @@ crond
 
 echo "Container started."
 
-exec "$@"
+exec "$@"