Merge branch 'master' into alpine

#30 Resolve test docker hub

Dockerfile

@@ -1,32 +1,47 @@
-FROM gliderlabs/alpine:3.3
-MAINTAINER info@lobaro.com
+FROM alpine:3.10.1 as certs
+RUN apk add --no-cache ca-certificates
 
-RUN echo http://nl.alpinelinux.org/alpine/v3.4/community >> /etc/apk/repositories
-RUN apk add --no-cache git go nfs-utils openssh fuse
-RUN git clone https://github.com/restic/restic \
-  && cd restic \
-  && go run build.go \
-  && cp restic /usr/local/bin/
-RUN apk del git go
+# Get restic executable
+ENV RESTIC_VERSION=0.9.5
+ADD https://github.com/restic/restic/releases/download/v${RESTIC_VERSION}/restic_${RESTIC_VERSION}_linux_amd64.bz2 /
+RUN bzip2 -d restic_${RESTIC_VERSION}_linux_amd64.bz2 && mv restic_${RESTIC_VERSION}_linux_amd64 /bin/restic && chmod +x /bin/restic
 
-RUN mkdir /mnt/restic
+FROM alpine as rclone
+
+# Get rclone executable
+ADD https://downloads.rclone.org/rclone-current-linux-amd64.zip /
+RUN unzip rclone-current-linux-amd64.zip && mv rclone-*-linux-amd64/rclone /bin/rclone && chmod +x /bin/rclone
+
+FROM busybox:glibc
+
+# install mailx
+RUN apk add --update --no-cache heirloom-mailx
+
+COPY --from=certs /etc/ssl/certs /etc/ssl/certs
+COPY --from=certs /bin/restic /bin/restic
+COPY --from=rclone /bin/rclone /bin/rclone
+
+RUN \
+    mkdir -p /mnt/restic /var/spool/cron/crontabs /var/log; \
+    touch /var/log/cron.log;
 
 ENV RESTIC_REPOSITORY=/mnt/restic
 ENV RESTIC_PASSWORD=""
 ENV RESTIC_TAG=""
 ENV NFS_TARGET=""
-# By default backup every 6 hours
-ENV BACKUP_CRON="* */6 * * *"
+ENV BACKUP_CRON="0 */6 * * *"
 ENV RESTIC_FORGET_ARGS=""
+ENV RESTIC_JOB_ARGS=""
+ENV MAILX_ARGS=""
 
 # /data is the dir where you have to put the data to be backed up
 VOLUME /data
 
 COPY backup.sh /bin/backup
-RUN chmod +x /bin/backup
-
 COPY entry.sh /entry.sh
 
-RUN touch /var/log/cron.log
-ENTRYPOINT ["/entry.sh"]
 
+WORKDIR "/"
+
+ENTRYPOINT ["/entry.sh"]
+CMD ["tail","-fn0","/var/log/cron.log"]

README.md

@@ -4,15 +4,23 @@ A docker container to automate [restic backups](https://restic.github.io/)
 This container runs restic backups in regular intervals. 
 
 * Easy setup and maintanance
-* Support for different targets (currently: Local, NFS, SFTP)
+* Support for different targets (tested with: Local, NFS, SFTP, AWS)
 * Support `restic mount` inside the container to browse the backup files
 
 **Container**: [lobaro/restic-backup-docker](https://hub.docker.com/r/lobaro/restic-backup-docker/)
 
+Stable
 ```
-docker pull lobaro/restic-backup-docker
+docker pull lobaro/restic-backup-docker:1.2-0.9.4
 ```
 
+Latest (experimental)
+```
+docker pull lobaro/restic-backup-docker:latest
+```
+
+Please don't hesitate to report any issue you find. **Thanks.**
+
 # Test the container
 
 Clone this repository
@@ -42,17 +50,17 @@ To enter your container execute
 docker exec -ti backup-test /bin/sh
 ```
 
-Now you can use restic [as documented](https://restic.readthedocs.io/en/stable/Manual/), e.g. try to run `restic snapshots` to list all your snapshots.
+Now you can use restic [as documented](https://restic.readthedocs.io/en/stable/), e.g. try to run `restic snapshots` to list all your snapshots.
 
 ## Logfiles
-Logfiles inside the container:
+Logfiles are inside the container. If needed you can create volumes for them.
 
 ```
 docker logs
 ```
 Shows `/var/log/cron.log`
 
-Additionally you can see the the full log of the last `backup` and `forget` command in `/var/log/backup-last.log` inside the container.
+Additionally you can see the the full log, including restic output, of the last execution in `/var/log/backup-last.log`. When the backup fails the log is copied to `/var/log/restic-error-last.log`. If configured, you can find the full output of the mail notification in `/var/log/mail-last.log`.
 
 # Customize the Container
 
@@ -60,12 +68,16 @@ The container is setup by setting [environment variables](https://docs.docker.co
 
 ## Environment variables
 
-* `RESTIC_REPOSITORY` - the location of the restic repository. Default `/mnt/restic`
+* `RESTIC_REPOSITORY` - the location of the restic repository. Default `/mnt/restic`. For S3: `s3:https://s3.amazonaws.com/BUCKET_NAME`
 * `RESTIC_PASSWORD` - the password for the restic repository. Will also be used for restic init during first start when the repository is not initialized.
 * `RESTIC_TAG` - Optional. To tag the images created by the container.
 * `NFS_TARGET` - Optional. If set the given NFS is mounted, i.e. `mount -o nolock -v ${NFS_TARGET} /mnt/restic`. `RESTIC_REPOSITORY` must remain it's default value!
-* `BACKUP_CRON` - A cron expression to run the backup. Default: `* */6 * * *` aka every 6 hours.
-* `RESTIC_FORGET_ARGS` - Optional. Only if specified `restic forget` is run with the given arguments after each backup. Example value: `-e "RESTIC_FORGET_ARGS=--keep-last 10 --keep-hourly 24 --keep-daily 7 --keep-weekly 52 --keep-monthly 120 --keep-yearly 100"`
+* `BACKUP_CRON` - A cron expression to run the backup. Note: cron daemon uses UTC time zone. Default: `0 */6 * * *` aka every 6 hours.
+* `RESTIC_FORGET_ARGS` - Optional. Only if specified `restic forget` is run with the given arguments after each backup. Example value: `-e "RESTIC_FORGET_ARGS=--prune --keep-last 10 --keep-hourly 24 --keep-daily 7 --keep-weekly 52 --keep-monthly 120 --keep-yearly 100"`
+* `RESTIC_JOB_ARGS` - Optional. Allows to specify extra arguments to the back up job such as limiting bandwith with `--limit-upload` or excluding file masks with `--exclude`.
+* `AWS_ACCESS_KEY_ID` - Optional. When using restic with AWS S3 storage.
+* `AWS_SECRET_ACCESS_KEY` - Optional. When using restic with AWS S3 storage.
+* `MAILX_ARGS` - Optional. If specified, the content of `/var/log/backup-last.log` is sent via mail after each backup using an *external SMTP*. To have maximum flexibility, you have to specify the mail/smtp parameters by your own. Have a look at the [mailx manpage](https://linux.die.net/man/1/mailx) for further information. Example value: `-e "MAILX_ARGS=-r 'from@example.de' -s 'Result of the last restic backup run' -S smtp='smtp.example.com:587' -S smtp-use-starttls -S smtp-auth=login -S smtp-auth-user='username' -S smtp-auth-password='password' 'to@example.com'"`.
 
 ## Volumes
 
@@ -73,11 +85,11 @@ The container is setup by setting [environment variables](https://docs.docker.co
 
 ## Set the hostname
 
-Since restic saves the hostname with each snapshot and the hostname of a docker container is it's id you might want to customize this by setting the hostname of the container to another value.
+Since restic saves the hostname with each snapshot and the hostname of a docker container is derived from it's id you might want to customize this by setting the hostname of the container to another value.
 
 Either by setting the [environment variable](https://docs.docker.com/engine/reference/run/#env-environment-variables) `HOSTNAME` or with `--hostname` in the [network settings](https://docs.docker.com/engine/reference/run/#network-settings)
 
-## Backup to SFTP
+## Backup via SFTP
 
 Since restic needs a **password less login** to the SFTP server make sure you can do `sftp user@host` from inside the container. If you can do so from your host system, the easiest way is to just mount your `.ssh` folder conaining the authorized cert into the container by specifying `-v ~/.ssh:/root/.ssh` as argument for `docker run`.
 
@@ -87,6 +99,16 @@ Now you can simply specify the restic repository to be an [SFTP repository](http
 -e "RESTIC_REPOSITORY=sftp:user@host:/tmp/backup"
 ```
 
-# TODO
+## Backup via rclone
 
-* Add testsetup based on docker-compose
+To use rclone as a backend for restic, simply add the rclone config file as a volume with `-v /absolute/path/to/rclone.conf:/root/.config/rclone/rclone.conf`.
+
+# Versioning & Changelog
+
+Starting from v1.3.0 versioning follows [Semantic versioning](http://semver.org/)
+
+Build metadata is used to declare the Restic version.
+
+**Example:** 1.3.0+0.9.5 (includes Restic 0.9.5)
+
+For changelog see: https://github.com/lobaro/restic-backup-docker/releases

backup.sh

@@ -1,27 +1,62 @@
 #!/bin/sh
 
-echo "Starting Backup"
+lastLogfile="/var/log/backup-last.log"
+lastMailLogfile="/var/log/mail-last.log"
+
+copyErrorLog() {
+  cp ${lastLogfile} /var/log/backup-error-last.log
+}
+
+logLast() {
+  echo "$1" >> ${lastLogfile}
+}
+
+start=`date +%s`
+rm -f ${lastLogfile} ${lastMailLogfile}
+echo "Starting Backup at $(date +"%Y-%m-%d %H:%M:%S")"
+echo "Starting Backup at $(date)" >> ${lastLogfile}
+logLast "BACKUP_CRON: ${BACKUP_CRON}"
+logLast "RESTIC_TAG: ${RESTIC_TAG}"
+logLast "RESTIC_FORGET_ARGS: ${RESTIC_FORGET_ARGS}"
+logLast "RESTIC_JOB_ARGS: ${RESTIC_JOB_ARGS}"
+logLast "RESTIC_REPOSITORY: ${RESTIC_REPOSITORY}"
+logLast "AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}"
 
 # Do not save full backup log to logfile but to backup-last.log
-restic backup /data --tag=${RESTIC_TAG?"Missing environment variable RESTIC_TAG"} > /var/log/backup-last.log 2>&1
+restic backup /data ${RESTIC_JOB_ARGS} --tag=${RESTIC_TAG?"Missing environment variable RESTIC_TAG"} >> ${lastLogfile} 2>&1
 rc=$?
-echo "Finished backup at $(date)" >> /var/log/backup-last.log
+logLast "Finished backup at $(date)"
 if [[ $rc == 0 ]]; then
     echo "Backup Successfull" 
 else
     echo "Backup Failed with Status ${rc}"
     restic unlock
+    copyErrorLog
+    kill 1
 fi
 
 if [ -n "${RESTIC_FORGET_ARGS}" ]; then
     echo "Forget about old snapshots based on RESTIC_FORGET_ARGS = ${RESTIC_FORGET_ARGS}"
-    restic forget ${RESTIC_FORGET_ARGS} >> /var/log/backup-last.log 2>&1
+    restic forget ${RESTIC_FORGET_ARGS} >> ${lastLogfile} 2>&1
     rc=$?
-    echo "Finished forget at $(date)" >> /var/log/backup-last.log
+    logLast "Finished forget at $(date)"
     if [[ $rc == 0 ]]; then
         echo "Forget Successfull"
     else
         echo "Forget Failed with Status ${rc}"
         restic unlock
+        copyErrorLog
+    fi
+fi
+
+end=`date +%s`
+echo "Finished Backup at $(date +"%Y-%m-%d %H:%M:%S") after $((end-start)) seconds"
+
+if [ -n "${MAILX_ARGS}" ]; then
+    sh -c "mailx -v -S sendwait ${MAILX_ARGS} < ${lastLogfile} > ${lastMailLogfile} 2>&1"
+    if [ $? == 0 ]; then
+        echo "Mail notification successfully sent."
+    else
+        echo "Sending mail notification FAILED. Check ${lastMailLogfile} for further information."
     fi
 fi

config.yml

@@ -0,0 +1,21 @@
+schemaVersion: '2.0.0'
+commandTests:  
+  - name: "restic package installation"
+    setup: [["/entry.sh"]]
+    command: "which"
+    args: ["restic"]
+    expectedOutput: ["/bin/restic"]
+fileExistenceTests:
+- name: 'log directory exists'
+  path: '/var/log'
+  shouldExist: true
+- name: 'cron log file exists'
+  path: '/var/log/cron.log'
+  shouldExist: true
+- name: 'backup script exists'
+  path: '/bin/backup'
+  shouldExist: true
+metadataTest:
+  volumes: ["/data"]
+  entrypoint: ["/entry.sh"]
+  cmd: ["tail","-fn0", "/var/log/cron.log"]

docker-compose.test.yml

@@ -0,0 +1,9 @@
+version: '2'
+
+services:
+  sut:
+    image: gcr.io/gcp-runtimes/container-structure-test
+    command: ["test", "--image", "restic-backup", "--config", "config.yml"]
+    volumes:
+      - ./config.yml:/config.yml
+      - /var/run/docker.sock:/var/run/docker.sock

entry.sh

@@ -1,18 +1,37 @@
 #!bin/sh
-set -e
 
 echo "Starting container ..."
 
+RESTIC_CMD=restic
+
+if [ -n "${ROOT_CERT}" ]; then
+	RESTIC_CMD="${RESTIC_CMD} --cert ${ROOT_CERT}"
+fi
+
 if [ -n "${NFS_TARGET}" ]; then
     echo "Mounting NFS based on NFS_TARGET: ${NFS_TARGET}"
     mount -o nolock -v ${NFS_TARGET} /mnt/restic
 fi
 
-if [ ! -f "$RESTIC_REPOSITORY/config" ]; then
+restic snapshots &>/dev/null
+status=$?
+echo "Check Repo status $status"
+
+if [ $status != 0 ]; then
     echo "Restic repository '${RESTIC_REPOSITORY}' does not exists. Running restic init."
-    restic init | true
+    restic init
+
+    init_status=$?
+    echo "Repo init status $init_status"
+
+    if [ $init_status != 0 ]; then
+        echo "Failed to init the repository: '${RESTIC_REPOSITORY}'"
+        exit 1
+    fi
 fi
 
+
+
 echo "Setup backup cron job with cron expression BACKUP_CRON: ${BACKUP_CRON}"
 echo "${BACKUP_CRON} /bin/backup >> /var/log/cron.log 2>&1" > /var/spool/cron/crontabs/root
 
@@ -24,4 +43,4 @@ crond
 
 echo "Container started."
 
-tail -fn0 /var/log/cron.log
+exec "$@"

run.sh

@@ -7,7 +7,8 @@ echo "Start backup-test container. Backup of ~/test-data/ to repository ~/test-r
 docker run --privileged --name backup-test \
 -e "RESTIC_PASSWORD=test" \
 -e "RESTIC_TAG=test" \
--e "BACKUP_CRON=* * * * *" \
+-e "BACKUP_CRON=0 0 * * *" \
+-e "RESTIC_FORGET_ARGS=--keep-last 10" \
 -v ~/test-data:/data \
 -v ~/test-repo/:/mnt/restic \
 -t restic-backup