Add version to CHANGELOG

Update dependencies

.envrc

@@ -1 +0,0 @@
-GOPATH=$PWD:$PWD/vendor

.github/ISSUE_TEMPLATE.md

@@ -1,13 +1,22 @@
 <!--
-NOTE: Not filling out the issue template needs a good reason, otherwise the
-issue may be closed instantly! Please take the time to help us debugging the
-problem by collecting information, even if it seems irrelevant to you. Thanks!
+NOTE: Not filling out the issue template needs a good reason, otherwise it may
+take a lot longer to find the problem! Please take the time to help us
+debugging the problem by collecting information, even if it seems irrelevant to
+you. Thanks!
+
+If you have a question, maybe the forum at https://discourse.restic.net is a
+better place.
 -->
 
 ## Output of `restic version`
 
 
-## How did you start restic exactly? (Include the complete command line)
+## How did you run restic exactly?
+
+<!--
+Include the complete command line and any environment variables you used to
+configure restic's backend access. Make sure to replace sensitive values!
+-->
 
 
 ## What backend/server/service did you use?
@@ -23,3 +32,6 @@ problem by collecting information, even if it seems irrelevant to you. Thanks!
 
 
 ## Do you have any idea what may have caused this?
+
+
+## Do you have an idea how to solve the issue?

.gitignore

@@ -1,6 +1,3 @@
-/pkg
-/bin
 /restic
 /.vagrant
-/vendor/pkg
 /doc/_build

.travis.yml

@@ -2,9 +2,8 @@ language: go
 sudo: false
 
 go:
-  - 1.7.5
-  - 1.8.1
-  - tip
+  - 1.8.3
+  - 1.9
 
 os:
   - linux
@@ -17,19 +16,15 @@ env:
 matrix:
   exclude:
     - os: osx
-      go: 1.7.5
-    - os: osx
-      go: tip
+      go: 1.8.3
     - os: linux
-      go: 1.8.1
+      go: 1.9
   include:
     - os: linux
-      go: 1.8.1
+      go: 1.9
       sudo: true
       env:
         RESTIC_TEST_FUSE=1
-  allow_failures:
-    - go: tip
 
 branches:
   only:

CHANGELOG.md

@@ -1,6 +1,234 @@
 This file describes changes relevant to all users that are made in each
 released version of restic from the perspective of the user.
 
+Important Changes in 0.7.2
+==========================
+
+ * We've added an official docker image and a Dockerfile to build this image in
+   `docker/`.
+   https://github.com/restic/restic/pull/1061
+
+ * The git repository layout was changed to resemble the layout typically used
+   in Go projects, we're not using `gb` for building restic any more and
+   vendoring the dependencies is now taken care of by `dep`.
+   https://github.com/restic/restic/pull/1126
+
+ * We now support saving backups on Google Cloud Storage.
+   https://github.com/restic/restic/pull/1134
+   https://github.com/restic/restic/pull/1052
+   https://github.com/restic/restic/issues/211
+
+ * We've added support for Microsoft Azure Blob Storage as a restic backend.
+   https://github.com/restic/restic/pull/1149
+   https://github.com/restic/restic/pull/1059
+   https://github.com/restic/restic/issues/609
+
+ * In the course of supporting Microsoft Azure Blobe Storage Go 1.8 is now a
+   requirement to build restic.
+
+ * The `restore` command has been improved: When dirs are excluded (or not
+   included) in a restore, they are not loaded from the repo any more.
+   https://github.com/restic/restic/pull/1044
+
+ * Name collisions are now resolved by appending a counter.
+   https://github.com/restic/restic/issues/1179
+   https://github.com/restic/restic/pull/1209
+
+
+Small changes
+-------------
+
+ * The `key` command now prompts for a password even if the original password
+   to access a repo has been specified via the `RESTIC_PASSWORD` environment
+   variable or a password file.
+   https://github.com/restic/restic/issues/1132
+   https://github.com/restic/restic/pull/1133
+
+ * Properly report errors when reading files with exclude patterns.
+   https://github.com/restic/restic/pull/1144
+
+ * We now automatically generate man pages for all restic commands, see the
+   subdir `doc/man`.
+   https://github.com/restic/restic/issues/697
+   https://github.com/restic/restic/pull/1147
+
+ * The `key remove` command was corrected and now works as documented.
+   https://github.com/restic/restic/pull/1164
+
+ * When a restic command other than `init` is used with a local repository and
+   the repository directory does not exist, restic creates the directory
+   structure. That's an error, only the `init` command should create the dir.
+   https://github.com/restic/restic/issues/1167
+   https://github.com/restic/restic/pull/1182
+
+ * Restic now prints stats on all BSD systems (not only on darwin) when SIGINFO
+   is received (usually when ctrl+t is pressed).
+   https://github.com/restic/restic/pull/1203
+   https://github.com/restic/restic/pull/1082#issuecomment-326279920
+
+ * Since a few releases restic had the ability to write profiling files for
+   memory and CPU usage when `debug` is enabled. It was discovered that when
+   restic is interrupted (ctrl+c is pressed), the proper shutdown hook is not
+   run. This is now corrected.
+   https://github.com/restic/restic/pull/1191
+
+ * A new option `--exclude-caches` was added that allows excluding cache
+   directories (that are tagged as such). This is a special case of a more
+   generic option `--exclude-if-present` which excludes a directory if a file
+   with a specific name (and contents) is present.
+   https://github.com/restic/restic/issues/317
+   https://github.com/restic/restic/pull/1170
+   https://github.com/restic/restic/pull/1224
+
+ * The `forget` command now has an option `--group-by` that allows flexible
+   grouping policies.
+   https://github.com/restic/restic/pull/1196
+
+ * The date and time restic records for a new backup can now be specified
+   externally by passing `--time` to the `backup` command.
+   https://github.com/restic/restic/pull/1205
+
+ * The option `--compact` was added to the `snapshots` command to get a better
+   overview of the snapshots in a repo. It limits each snapshot to a single
+   line.
+   https://github.com/restic/restic/issues/1218
+   https://github.com/restic/restic/pull/1223
+
+
+Important Changes in 0.7.1
+==========================
+
+ * The `migrate` command for chaning the `s3legacy` layout to the `default`
+   layout for s3 backends has been improved: It can now be restarted with
+   `restic migrate --force s3_layout` and automatically retries operations on
+   error.
+   https://github.com/restic/restic/issues/1073
+   https://github.com/restic/restic/pull/1075
+
+Small changes
+-------------
+
+ * The local and sftp backends now create the subdirs below `data/` on
+   open/init. This way, restic makes sure that they always exist. This is
+   connected to an issue for the sftp server:
+   https://github.com/restic/rest-server/pull/11#issuecomment-309879710
+   https://github.com/restic/restic/issues/1055
+   https://github.com/restic/restic/pull/1077
+   https://github.com/restic/restic/pull/1105
+
+ * When no S3 credentials are specified in the environment variables, restic
+   now tries to load credentials from an IAM instance profile when the s3
+   backend is used.
+   https://github.com/restic/restic/issues/1067
+   https://github.com/restic/restic/pull/1086
+
+ * On Darwin and FreeBSD, restic now prints stats when SIGINFO is received
+   (usually when ctrl+t is pressed).
+   https://github.com/restic/restic/pull/1082
+
+ * The dependencies have been updated.
+   https://github.com/restic/restic/pull/1108
+   https://github.com/restic/restic/pull/1124
+
+ * A bug was found (and corrected) in the index rebuilding after prune, which
+   led to indexes which include blobs that were not present in the repo any
+   more. There were already checks in place which detected this situation and
+   aborted with an error message. A new run of either `prune` or
+   `rebuild-index` corrected the index files. This is now fixed and a test has
+   been added to detect this.
+   https://github.com/restic/restic/pull/1115
+
+ * Errors for chmod() on Unix for filesystems which do not support it (e.g. smb
+   mounted via gvfs) are now ignored.
+   https://github.com/restic/restic/pull/1080
+   https://github.com/restic/restic/pull/1112
+
+ * The semantic for the `--tags` option to `forget` and `snapshots` was
+   clarified:
+   https://github.com/restic/restic/issues/1081
+   https://github.com/restic/restic/pull/1090
+
+Important Changes in 0.7.0
+==========================
+
+ * New "swift" backend: A new backend for the OpenStack Swift cloud storage
+   protocol has been added, https://wiki.openstack.org/wiki/Swift
+   https://github.com/restic/restic/pull/975
+   https://github.com/restic/restic/pull/648
+
+ * New "b2" backend: A new backend for Backblaze B2 cloud storage
+   service has been added, https://www.backblaze.com
+   https://github.com/restic/restic/issues/512
+   https://github.com/restic/restic/pull/978
+
+ * Improved performance for the `find` command: Restic recognizes paths it has
+   already checked for the files in question, so the number of backend requests
+   is reduced a lot.
+   https://github.com/restic/restic/issues/989
+   https://github.com/restic/restic/pull/993
+
+ * Improved performance for the fuse mount: Listing directories which contain
+   large files now is significantly faster.
+   https://github.com/restic/restic/pull/998
+
+ * The default layout for the s3 backend is now `default` (instead of
+   `s3legacy`). Also, there's a new `migrate` command to convert an existing
+   repo, it can be run like this: `restic migrate s3_layout`
+   https://github.com/restic/restic/issues/965
+   https://github.com/restic/restic/pull/1004
+
+ * The fuse mount now has two more directories: `tags` contains a subdir for
+   each tag, which in turn contains only the snapshots that have this tag. The
+   subdir `hosts` contains a subdir for each host that has a snapshot, and the
+   subdir contains the snapshots for that host.
+   https://github.com/restic/restic/issues/636
+   https://github.com/restic/restic/pull/1050
+
+Small changes
+-------------
+
+ * For the s3 backend we're back to using the high-level API the s3 client
+   library for uploading data, a few users reported dropped connections (which
+   the library will automatically retry now).
+   https://github.com/restic/restic/issues/1013
+   https://github.com/restic/restic/issues/1023
+   https://github.com/restic/restic/pull/1025
+
+ * The `prune` command has been improved and will now remove invalid pack
+   files, for example files that have not been uploaded completely because a
+   backup was interrupted.
+   https://github.com/restic/restic/issues/1029
+   https://github.com/restic/restic/pull/1036
+
+ * restic now tries to detect when an invalid/unknown backend is used and
+   returns an error message.
+   https://github.com/restic/restic/issues/1021
+   https://github.com/restic/restic/pull/1070
+
+Important Changes in 0.6.1
+==========================
+
+This is mostly a bugfix release and only contains small changes:
+
+ * We've fixed a bug where `rebuild-index` would corrupt the index when used
+   with the s3 backend together with the `default` layout. This is not the
+   default setting.
+
+ * Backends based on HTTP now allow several idle connections in parallel. This
+   is especially important for the REST backend, which (when used with a local
+   server) may create a lot connections and exhaust available ports quickly.
+   https://github.com/restic/restic/issues/985
+   https://github.com/restic/restic/pull/986
+
+ * Regular status report: We've removed the status report that was printed
+   every 10 seconds when restic is run non-interactively. You can still force
+   reporting the current status by sending a `USR1` signal to the process.
+   https://github.com/restic/restic/pull/974
+
+ * The `build.go` now strips the temporary directory used for compilation from
+   the binary. This is the first step in enabling reproducible builds.
+   https://github.com/restic/restic/pull/981
+
 Important Changes in 0.6.0
 ==========================
 

CONTRIBUTING.md

@@ -60,50 +60,35 @@ uploading it somewhere or post only the parts that are really relevant.
 Development Environment
 =======================
 
-For development you need the build tool [`gb`](https://getgb.io), it can be
-installed by running the following command:
+In order to compile restic with the `go` tool directly, it needs to be checked
+out at the right path within a `GOPATH`. The concept of a `GOPATH` is explained
+in ["How to write Go code"](https://golang.org/doc/code.html).
 
-    $ go get github.com/constabulary/gb/...
-
-The repository contains two directories with code: `src/` contains the code
-written for restic, whereas `vendor/` contains copies of libraries restic
-depends on. The libraries are managed with the `gb vendor` command.
-
-Just clone the repository, `cd` to it and run `gb build` to build the binary:
+If you do not have a directory with Go code yet, executing the following
+instructions in your shell will create one for you and check out the restic
+repo:
 
+    $ export GOPATH="$HOME/go"
+    $ mkdir -p "$GOPATH/src/github.com/restic"
+    $ cd "$GOPATH/src/github.com/restic"
     $ git clone https://github.com/restic/restic
     $ cd restic
-    $ gb build
-    [...]
-    $ bin/restic version
+
+You can then build restic as follows:
+
+    $ go build ./cmd/restic
+    $ ./restic version
     restic compiled manually
-    compiled at unknown time with go1.7
+    compiled with go1.8.3 on linux/amd64
 
 The following commands can be used to run all the tests:
 
-    $ gb test
-    ok          github.com/restic/restic        8.174s
-    [...]
+    $ go test ./cmd/... ./internal/...
 
-If you want to run your tests on Linux, OpenBSD or FreeBSD, you can use
-[vagrant](https://www.vagrantup.com/) with the provided `Vagrantfile` to
-quickly set up VMs and run the tests, e.g.:
-
-    $ vagrant up freebsd
-    [...]
-
-    $ vagrant ssh freebsd -c 'cd restic/restic; go test -v ./...'
-    [...]
-
-The default `go` tool can also be used by setting the environment variable
-`GOPATH` to the following value while being in the top level directory in the
-git repository:
-
-    $ export GOPATH=$PWD:$PWD/vendor
-
-The file `.envrc` allows automatic `GOPATH` configuration with
-[direnv](https://direnv.net/), inspect the file and then allow automatic
-configuration by running `direnv allow`.
+The repository contains two sets of directories with code: `cmd/` and
+`internal/` contain the code written for restic, whereas `vendor/` contains
+copies of libraries restic depends on. The libraries are managed with the
+[`dep`](https://github.com/golang/dep) tool.
 
 Providing Patches
 =================
@@ -122,7 +107,8 @@ down to the following steps:
 
  2. Clone the repository locally and create a new branch. If you are working on
     the code itself, please set up the development environment as described in
-    the previous section.
+    the previous section. Especially take care to place your forked repository
+    at the correct path (`src/github.com/restic/restic`) within your `GOPATH`.
 
  3. Then commit your changes as fine grained as possible, as smaller patches,
     that handle one and only one issue are easier to discuss and merge.
@@ -143,8 +129,14 @@ down to the following steps:
     next stable release. While writing, ask yourself: If I were the user, what
     would I need to be aware of with this change.
 
- 8. Once your code looks good and passes all the tests, we'll merge it. Thanks
-    a low for your contribution!
+ 8. When your contribution adds and/or changes command-line parameters or help
+    texts, the manual pages need to be regenerated and commited to the
+    repository. In order to do this, compile restic and save the generated
+    updated man pages in the subdir `doc/man` with the following command:
+    `./restic manpage --output-dir doc/man`
+
+ 9. Once your code looks good and passes all the tests, we'll merge it. Thanks
+    a lot for your contribution!
 
 Please provide the patches for each bug or feature in a separate branch and
 open up a pull request for each.
@@ -170,7 +162,7 @@ what the tests are there for.
 Git Commits
 -----------
 
-I would be good if you could follow the same general style regarding Git
+It would be good if you could follow the same general style regarding Git
 commits as the rest of the project, this makes reviewing code, browsing the
 history and triaging bugs much easier.
 

Dockerfile

@@ -1,57 +0,0 @@
-# This Dockerfiles configures a container that is similar to the Travis CI
-# environment and can be used to run tests locally.
-#
-# build the image:
-#   docker build -t restic/test .
-#
-# run all tests and cross-compile restic:
-#   docker run --rm -v $PWD:/home/travis/restic restic/test go run run_integration_tests.go -minio minio
-#
-# run interactively:
-#   docker run --interactive --tty --rm -v $PWD:/home/travis/restic restic/test /bin/bash
-#
-# run a subset of tests:
-#   docker run --rm -v $PWD:/home/travis/restic restic/test gb test -v ./backend
-#
-# build the image for an older version of Go:
-#   docker build --build-arg GOVERSION=1.6.4 -t restic/test:go1.6.4 .
-
-FROM ubuntu:14.04
-
-ARG GOVERSION=1.7.5
-ARG GOARCH=amd64
-
-# install dependencies
-RUN apt-get update
-RUN apt-get install -y --no-install-recommends ca-certificates wget git build-essential openssh-server
-
-# add and configure user
-ENV HOME /home/travis
-RUN useradd -m -d $HOME -s /bin/bash travis
-
-# run everything below as user travis
-USER travis
-WORKDIR $HOME
-
-# download and install Go
-RUN wget -q -O /tmp/go.tar.gz https://storage.googleapis.com/golang/go${GOVERSION}.linux-${GOARCH}.tar.gz
-RUN tar xf /tmp/go.tar.gz && rm -f /tmp/go.tar.gz
-ENV GOROOT $HOME/go
-ENV GOPATH $HOME/gopath
-ENV PATH $PATH:$GOROOT/bin:$GOPATH/bin:$HOME/bin
-
-RUN mkdir -p $HOME/restic
-
-# pre-install tools, this speeds up running the tests itself
-RUN go get github.com/constabulary/gb/...
-RUN go get golang.org/x/tools/cmd/cover
-RUN go get github.com/mitchellh/gox
-RUN go get github.com/pierrre/gotestcover
-RUN mkdir $HOME/bin \
-    && wget -q -O $HOME/bin/minio https://dl.minio.io/server/minio/release/linux-${GOARCH}/minio \
-    && chmod +x $HOME/bin/minio
-
-# set TRAVIS_BUILD_DIR for integration script
-ENV TRAVIS_BUILD_DIR $HOME/restic
-
-WORKDIR $HOME/restic

Gopkg.lock

@@ -0,0 +1,207 @@
+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
+
+[[projects]]
+  branch = "master"
+  name = "bazil.org/fuse"
+  packages = [".","fs","fuseutil"]
+  revision = "371fbbdaa8987b715bdd21d6adc4c9b20155f748"
+
+[[projects]]
+  name = "cloud.google.com/go"
+  packages = ["compute/metadata"]
+  revision = "5a9e19d4e1e41a734154e44a2132b358afb49a03"
+  version = "v0.13.0"
+
+[[projects]]
+  name = "github.com/Azure/azure-sdk-for-go"
+  packages = ["storage"]
+  revision = "df4dd90d076ebbf6e87d08d3f00bfac8ff4bde1a"
+  version = "v10.3.1-beta"
+
+[[projects]]
+  name = "github.com/Azure/go-autorest"
+  packages = ["autorest","autorest/adal","autorest/azure","autorest/date"]
+  revision = "5432abe734f8d95c78340cd56712f912906e6514"
+  version = "v8.3.1"
+
+[[projects]]
+  name = "github.com/cpuguy83/go-md2man"
+  packages = ["md2man"]
+  revision = "1d903dcb749992f3741d744c0f8376b4bd7eb3e1"
+  version = "v1.0.7"
+
+[[projects]]
+  name = "github.com/dgrijalva/jwt-go"
+  packages = ["."]
+  revision = "d2709f9f1f31ebcda9651b03077758c1f3a0018c"
+  version = "v3.0.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/dustin/go-humanize"
+  packages = ["."]
+  revision = "79e699ccd02f240a1f1fbbdcee7e64c1c12e41aa"
+
+[[projects]]
+  name = "github.com/elithrar/simple-scrypt"
+  packages = ["."]
+  revision = "2325946f714c95de4a6088202c402fbdfa64163b"
+  version = "v1.2.0"
+
+[[projects]]
+  name = "github.com/go-ini/ini"
+  packages = ["."]
+  revision = "20b96f641a5ea98f2f8619ff4f3e061cff4833bd"
+  version = "v1.28.2"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/golang/protobuf"
+  packages = ["proto"]
+  revision = "17ce1425424ab154092bbb43af630bd647f3bb0d"
+
+[[projects]]
+  name = "github.com/inconshreveable/mousetrap"
+  packages = ["."]
+  revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
+  version = "v1.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/kr/fs"
+  packages = ["."]
+  revision = "2788f0dbd16903de03cb8186e5c7d97b69ad387b"
+
+[[projects]]
+  name = "github.com/kurin/blazer"
+  packages = ["b2","base","internal/b2types","internal/blog"]
+  revision = "1a870c3ee8b83e17d762307c6eae8f390ac3f4a0"
+  version = "v0.1.1"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/minio/go-homedir"
+  packages = ["."]
+  revision = "21304a94172ae3a09dee2cd86a12fb6f842138c7"
+
+[[projects]]
+  name = "github.com/minio/minio-go"
+  packages = [".","pkg/credentials","pkg/encrypt","pkg/policy","pkg/s3signer","pkg/s3utils","pkg/set"]
+  revision = "4e0f567303d4cc90ceb055a451959fb9fc391fb9"
+  version = "3.0.3"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/ncw/swift"
+  packages = ["."]
+  revision = "9d3f812e23d270d1c66a9a01e20af1005061cdc4"
+
+[[projects]]
+  name = "github.com/pkg/errors"
+  packages = ["."]
+  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
+  version = "v0.8.0"
+
+[[projects]]
+  name = "github.com/pkg/profile"
+  packages = ["."]
+  revision = "5b67d428864e92711fcbd2f8629456121a56d91f"
+  version = "v1.2.1"
+
+[[projects]]
+  name = "github.com/pkg/sftp"
+  packages = ["."]
+  revision = "98203f5a8333288eb3163b7c667d4260fe1333e9"
+  version = "1.0.0"
+
+[[projects]]
+  name = "github.com/pkg/xattr"
+  packages = ["."]
+  revision = "23c75e3f6c1d8b13b3dd905b011a7f38a06044b7"
+  version = "v0.2.1"
+
+[[projects]]
+  name = "github.com/restic/chunker"
+  packages = ["."]
+  revision = "bb2ecf9a98e35a0b336ffc23fc515fb6e7961577"
+  version = "v0.1.0"
+
+[[projects]]
+  name = "github.com/russross/blackfriday"
+  packages = ["."]
+  revision = "4048872b16cc0fc2c5fd9eacf0ed2c2fedaa0c8c"
+  version = "v1.5"
+
+[[projects]]
+  name = "github.com/satori/uuid"
+  packages = ["."]
+  revision = "879c5887cd475cd7864858769793b2ceb0d44feb"
+  version = "v1.1.0"
+
+[[projects]]
+  name = "github.com/sirupsen/logrus"
+  packages = ["."]
+  revision = "f006c2ac4710855cf0f916dd6b77acf6b048dc6e"
+  version = "v1.0.3"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/spf13/cobra"
+  packages = [".","doc"]
+  revision = "b78744579491c1ceeaaa3b40205e56b0591b93a3"
+
+[[projects]]
+  name = "github.com/spf13/pflag"
+  packages = ["."]
+  revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66"
+  version = "v1.0.0"
+
+[[projects]]
+  branch = "master"
+  name = "golang.org/x/crypto"
+  packages = ["curve25519","ed25519","ed25519/internal/edwards25519","pbkdf2","poly1305","scrypt","ssh","ssh/terminal"]
+  revision = "faadfbdc035307d901e69eea569f5dda451a3ee3"
+
+[[projects]]
+  branch = "master"
+  name = "golang.org/x/net"
+  packages = ["context","context/ctxhttp"]
+  revision = "b129b8e0fbeb39c8358e51a07ab6c50ad415e72e"
+
+[[projects]]
+  branch = "master"
+  name = "golang.org/x/oauth2"
+  packages = [".","google","internal","jws","jwt"]
+  revision = "13449ad91cb26cb47661c1b080790392170385fd"
+
+[[projects]]
+  branch = "master"
+  name = "golang.org/x/sys"
+  packages = ["unix","windows"]
+  revision = "062cd7e4e68206d8bab9b18396626e855c992658"
+
+[[projects]]
+  branch = "master"
+  name = "google.golang.org/api"
+  packages = ["gensupport","googleapi","googleapi/internal/uritemplates","storage/v1"]
+  revision = "2fe03ca2dc379c00d654a4459d1a50812cac2848"
+
+[[projects]]
+  name = "google.golang.org/appengine"
+  packages = [".","internal","internal/app_identity","internal/base","internal/datastore","internal/log","internal/modules","internal/remote_api","internal/urlfetch","urlfetch"]
+  revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
+  version = "v1.0.0"
+
+[[projects]]
+  branch = "v2"
+  name = "gopkg.in/yaml.v2"
+  packages = ["."]
+  revision = "eb3733d160e74a9c7e442f435eb3bea458e1d19f"
+
+[solve-meta]
+  analyzer-name = "dep"
+  analyzer-version = 1
+  inputs-digest = "53e4779dc4c7de2cd8b195f13c215c24da5efc5e33acf584615b5c43bfefd2db"
+  solver-name = "gps-cdcl"
+  solver-version = 1

Gopkg.toml

@@ -0,0 +1,78 @@
+
+# Gopkg.toml example
+#
+# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
+# for detailed Gopkg.toml documentation.
+#
+# required = ["github.com/user/thing/cmd/thing"]
+# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
+#
+# [[constraint]]
+#   name = "github.com/user/project"
+#   version = "1.0.0"
+#
+# [[constraint]]
+#   name = "github.com/user/project2"
+#   branch = "dev"
+#   source = "github.com/myfork/project2"
+#
+# [[override]]
+#  name = "github.com/x/y"
+#  version = "2.4.0"
+
+
+[[constraint]]
+  branch = "master"
+  name = "bazil.org/fuse"
+
+[[constraint]]
+  name = "github.com/elithrar/simple-scrypt"
+  branch = "master"
+
+[[constraint]]
+  name = "github.com/kurin/blazer"
+  version = "0.1.0"
+
+[[constraint]]
+  name = "github.com/minio/minio-go"
+  version = "3.0.0"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/ncw/swift"
+
+[[constraint]]
+  name = "github.com/pkg/errors"
+  version = "0.8.0"
+
+[[constraint]]
+  name = "github.com/pkg/profile"
+  version = "1.2.1"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/pkg/sftp"
+
+[[constraint]]
+  name = "github.com/pkg/xattr"
+  version = "0.2.1"
+
+[[constraint]]
+  name = "github.com/restic/chunker"
+  version = "0.1.0"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/spf13/cobra"
+
+[[constraint]]
+  branch = "master"
+  name = "golang.org/x/crypto"
+
+[[constraint]]
+  branch = "master"
+  name = "golang.org/x/net"
+
+[[constraint]]
+  branch = "master"
+  name = "golang.org/x/sys"

Makefile

@@ -6,7 +6,8 @@ restic:
 	go run build.go
 
 clean:
-	rm -rf restic
+	rm -f restic
 
 test:
-	go test ./...
+	go test ./cmd/... ./internal/...
+

README.rst

@@ -7,6 +7,8 @@ restic is a backup program that is fast, efficient and secure.
 
 For detailed usage and installation instructions check out the `documentation <https://restic.readthedocs.io/en/latest>`__.
 
+You can ask questions in our `Discourse forum <https://forum.restic.net>`__.
+
 Quick start
 -----------
 
@@ -35,8 +37,27 @@ and add some data:
     duration: 0:29, 54.47MiB/s
     snapshot 40dc1520 saved
 
+Next you can either use ``restic restore`` to restore files or use ``restic
+mount`` to mount the repository via fuse and browse the files from previous
+snapshots.
+
 For more options check out the `manual guide <https://restic.readthedocs.io/en/latest/manual.html>`__.
 
+Backends
+--------
+
+Saving a backup on the same machine is nice but not a real backup strategy.
+Therefore, restic supports the following backends for storing backups natively:
+
+- `Local directory <https://restic.readthedocs.io/en/latest/manual.html#local>`__
+- `sftp server (via SSH) <https://restic.readthedocs.io/en/latest/manual.html#sftp>`__
+- `HTTP REST server <https://restic.readthedocs.io/en/latest/manual.html#rest-server>`__ (`protocol <doc/rest_backend.rst>`__ `rest-server <https://github.com/restic/rest-server>`__)
+- `AWS S3 <https://restic.readthedocs.io/en/latest/manual.html#amazon-s3>`__ (either from Amazon or using the `Minio <https://minio.io>`__ server)
+- `OpenStack Swift <https://restic.readthedocs.io/en/latest/manual.html#openstack-swift>`__
+- `BackBlaze B2 <https://restic.readthedocs.io/en/latest/manual.html#backblaze-b2>`__
+- `Microsoft Azure Blob Storage <https://restic.readthedocs.io/en/latest/manual.html#microsoft-azure-blob-storage>`__
+- `Google Cloud Storage <https://restic.readthedocs.io/en/latest/manual.html#google-cloud-storage>`__
+
 Design Principles
 -----------------
 
@@ -68,6 +89,15 @@ following principles in mind:
    data should be de-duplicated before it is actually written to the
    storage back end to save precious backup space.
 
+Reproducible Builds
+-------------------
+
+The binaries released with each restic version starting at 0.6.1 are
+`reproducible <https://reproducible-builds.org/>`__, which means that you can
+easily reproduce a byte identical version from the source code for that
+release. Instructions on how to do that are contained in the
+`builder repository <https://github.com/restic/builder>`__.
+
 News
 ----
 
@@ -86,7 +116,7 @@ complete text in ``LICENSE``.
    :target: https://travis-ci.org/restic/restic
 .. |Build status| image:: https://ci.appveyor.com/api/projects/status/nuy4lfbgfbytw92q/branch/master?svg=true
    :target: https://ci.appveyor.com/project/fd0/restic/branch/master
-.. |Report Card| image:: http://goreportcard.com/badge/github.com/restic/restic
-   :target: http://goreportcard.com/report/github.com/restic/restic
+.. |Report Card| image:: https://goreportcard.com/badge/github.com/restic/restic
+   :target: https://goreportcard.com/report/github.com/restic/restic
 .. |Say Thanks| image:: https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg
    :target: https://saythanks.io/to/restic

VERSION

@@ -1 +1 @@
-0.6.0-rc.1
+0.7.2

Vagrantfile

@@ -1,124 +0,0 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
-
-GO_VERSION = '1.7'
-
-def packages_freebsd
-  return <<-EOF
-    pkg install -y git
-    pkg install -y curl
-
-    echo 'fuse_load="YES"' >> /boot/loader.conf
-    echo 'vfs.usermount=1' >> /etc/sysctl.conf
-
-    kldload fuse
-    sysctl vfs.usermount=1
-    pw groupmod operator -M vagrant
-  EOF
-end
-
-def packages_openbsd
-  return <<-EOF
-    . ~/.profile
-    pkg_add git curl bash gtar--
-    ln -sf /usr/local/bin/gtar /usr/local/bin/tar
-  EOF
-end
-
-def packages_linux
-  return <<-EOF
-    apt-get update
-    apt-get install -y git curl
-  EOF
-end
-
-def packages_darwin
-  return <<-EOF
-    ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
-    brew cask install osxfuse
-  EOF
-end
-
-def install_gimme
-  return <<-EOF
-    rm -rf /opt/gimme
-    mkdir -p /opt/gimme || true
-    git clone https://github.com/meatballhat/gimme /opt/gimme
-    perl -p -i -e 's,/bin/bash,/usr/bin/env bash,' /opt/gimme/gimme
-    ln -sf /opt/gimme/gimme /usr/bin/gimme
-  EOF
-end
-
-def prepare_user(boxname)
-  return <<-EOF
-    mkdir -p ~/go/src
-    export PATH=/usr/local/bin:$PATH
-
-    gimme #{GO_VERSION} >> ~/.profile
-    echo export 'GOPATH=/vagrant/go' >> ~/.profile
-    echo export 'PATH=$GOPATH/bin:/usr/local/bin:$PATH' >> ~/.profile
-
-    . ~/.profile
-
-    go get golang.org/x/tools/cmd/cover
-    go get github.com/constabulary/gb/...
-
-    echo
-    echo "Run:"
-    echo "  vagrant rsync #{boxname}"
-    echo "  vagrant ssh #{boxname} -c 'cd /vagrant; gb build && gb test'"
-  EOF
-end
-
-def fix_perms
-  return <<-EOF
-    chown -R vagrant /vagrant
-  EOF
-end
-
-# All Vagrant configuration is done below. The "2" in Vagrant.configure
-# configures the configuration version (we support older styles for
-# backwards compatibility). Please don't change it unless you know what
-# you're doing.
-Vagrant.configure(2) do |config|
-  # use rsync to copy content to the folder
-  config.vm.synced_folder ".", "/vagrant", :type => "rsync"
-
-  # fix permissions on synced folder
-  config.vm.provision "fix perms", :type => :shell, :inline => fix_perms
-
-  config.vm.define "linux" do |b|
-    b.vm.box = "ubuntu/trusty64"
-    b.vm.provision "packages linux", :type => :shell, :inline => packages_linux
-    b.vm.provision "install gimme",  :type => :shell, :inline => install_gimme
-    b.vm.provision "prepare user",   :type => :shell, :privileged => false, :inline => prepare_user("linux")
-
-    # fix network card
-    config.vm.provider "virtualbox" do |v|
-      v.customize ["modifyvm", :id, "--nictype1", "virtio"]
-    end
-  end
-
-  config.vm.define "freebsd" do |b|
-    b.vm.box = "geoffgarside/freebsd-10.1"
-    b.vm.provision "packages freebsd", :type => :shell, :inline => packages_freebsd
-    b.vm.provision "install gimme",  :type => :shell, :inline => install_gimme
-    b.vm.provision "prepare user",   :type => :shell, :privileged => false, :inline => prepare_user("freebsd")
-  end
-
-  config.vm.define "openbsd" do |b|
-    b.vm.box = "tmatilai/openbsd-5.6"
-    b.vm.provision "packages openbsd", :type => :shell, :inline => packages_openbsd
-    b.vm.provision "install gimme",  :type => :shell, :inline => install_gimme
-    b.vm.provision "prepare user",   :type => :shell, :privileged => false, :inline => prepare_user("openbsd")
-  end
-
-  config.vm.define "darwin" do |b|
-    #b.vm.box = "jhcook/osx-yosemite-10.10"
-    b.vm.box = "jhcook/yosemite-clitools"
-    b.vm.provision "packages darwin", :type => :shell, :privileged => false, :inline => packages_darwin
-    b.vm.provision "install gimme",  :type => :shell, :inline => install_gimme
-    b.vm.provision "prepare user",   :type => :shell, :privileged => false, :inline => prepare_user("darwin")
-  end
-
-end

appveyor.yml

@@ -17,8 +17,8 @@ init:
 
 install:
   - rmdir c:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.8.1.windows-amd64.msi
-  - msiexec /i go1.8.1.windows-amd64.msi /q
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.9.windows-amd64.msi
+  - msiexec /i go1.9.windows-amd64.msi /q
   - go version
   - go env
   - appveyor DownloadFile http://sourceforge.netcologne.de/project/gnuwin32/tar/1.13-1/tar-1.13-1-bin.zip -FileName tar.zip

build.go

@@ -27,10 +27,12 @@ var config = struct {
 	Main      string
 	Tests     []string
 }{
-	Name:      "restic",                           // name of the program executable and directory
-	Namespace: "",                                 // subdir of GOPATH, e.g. "github.com/foo/bar"
-	Main:      "cmds/restic",                      // package name for the main package
-	Tests:     []string{"restic/...", "cmds/..."}, // tests to run
+	Name:      "restic",                              // name of the program executable and directory
+	Namespace: "github.com/restic/restic",            // subdir of GOPATH, e.g. "github.com/foo/bar"
+	Main:      "github.com/restic/restic/cmd/restic", // package name for the main package
+	Tests: []string{ // tests to run
+		"github.com/restic/restic/internal/...",
+		"github.com/restic/restic/cmd/..."},
 }
 
 // specialDir returns true if the file begins with a special character ('.' or '_').
@@ -77,7 +79,12 @@ func excludePath(name string) bool {
 //               └── restic
 //                   └── foo.go
 func updateGopath(dst, src, prefix string) error {
+	verbosePrintf("copy contents of %v to %v\n", src, filepath.Join(dst, prefix))
 	return filepath.Walk(src, func(name string, fi os.FileInfo, err error) error {
+		if name == src {
+			return err
+		}
+
 		if specialDir(name) {
 			if fi.IsDir() {
 				return filepath.SkipDir
@@ -86,6 +93,10 @@ func updateGopath(dst, src, prefix string) error {
 			return nil
 		}
 
+		if err != nil {
+			return err
+		}
+
 		if fi.IsDir() {
 			return nil
 		}
@@ -195,8 +206,11 @@ func cleanEnv() (env []string) {
 
 // build runs "go build args..." with GOPATH set to gopath.
 func build(cwd, goos, goarch, gopath string, args ...string) error {
-	args = append([]string{"build"}, args...)
-	cmd := exec.Command("go", args...)
+	a := []string{"build"}
+	a = append(a, "-asmflags", fmt.Sprintf("-trimpath=%s", gopath))
+	a = append(a, "-gcflags", fmt.Sprintf("-trimpath=%s", gopath))
+	a = append(a, args...)
+	cmd := exec.Command("go", a...)
 	cmd.Env = append(cleanEnv(), "GOPATH="+gopath, "GOARCH="+goarch, "GOOS="+goos)
 	if !enableCGO {
 		cmd.Env = append(cmd.Env, "CGO_ENABLED=0")
@@ -288,8 +302,8 @@ func (cs Constants) LDFlags() string {
 
 func main() {
 	ver := runtime.Version()
-	if strings.HasPrefix(ver, "go1") && ver < "go1.7" {
-		fmt.Fprintf(os.Stderr, "Go version %s detected, restic requires at least Go 1.7\n", ver)
+	if strings.HasPrefix(ver, "go1") && ver < "go1.8" {
+		fmt.Fprintf(os.Stderr, "Go version %s detected, restic requires at least Go 1.8\n", ver)
 		os.Exit(1)
 	}
 
@@ -365,13 +379,13 @@ func main() {
 	}
 
 	verbosePrintf("create GOPATH at %v\n", gopath)
-	if err = updateGopath(gopath, filepath.Join(root, "src"), config.Namespace); err != nil {
+	if err = updateGopath(gopath, root, config.Namespace); err != nil {
 		die("copying files from %v/src to %v/src failed: %v\n", root, gopath, err)
 	}
 
-	vendor := filepath.Join(root, "vendor", "src")
+	vendor := filepath.Join(root, "vendor")
 	if directoryExists(vendor) {
-		if err = updateGopath(gopath, vendor, ""); err != nil {
+		if err = updateGopath(gopath, vendor, filepath.Join(config.Namespace, "vendor")); err != nil {
 			die("copying files from %v to %v failed: %v\n", root, gopath, err)
 		}
 	}
@@ -398,7 +412,10 @@ func main() {
 	if err != nil {
 		die("Getwd() returned %v\n", err)
 	}
-	output := filepath.Join(cwd, outputFilename)
+	output := outputFilename
+	if !filepath.IsAbs(output) {
+		output = filepath.Join(cwd, output)
+	}
 
 	version := getVersion()
 	constants := Constants{}

build_release_binaries.sh

@@ -1,64 +0,0 @@
-#!/bin/bash
-
-set -e
-
-if [[ -z "$VERSION" ]]; then
-    echo '$VERSION unset'
-    exit 1
-fi
-
-dir=$(mktemp -d --tmpdir restic-release-XXXXXX)
-echo "path is ${dir}"
-
-for R in       \
-    darwin/386     \
-    darwin/amd64   \
-    freebsd/386    \
-    freebsd/amd64  \
-    freebsd/arm    \
-    linux/386      \
-    linux/amd64    \
-    linux/arm      \
-    linux/arm64    \
-    openbsd/386   \
-    openbsd/amd64 \
-    windows/386    \
-    windows/amd64  \
-    ; do \
-
-    OS=$(dirname $R)
-    ARCH=$(basename $R)
-    filename=restic_${VERSION}_${OS}_${ARCH}
-
-    if [[ "$OS" == "windows" ]]; then
-        filename="${filename}.exe"
-    fi
-
-    echo $filename
-
-    go run build.go --goos $OS --goarch $ARCH --output ${filename}
-    if [[ "$OS" == "windows" ]]; then
-        zip ${filename%.exe}.zip ${filename}
-        rm ${filename}
-        mv ${filename%.exe}.zip ${dir}
-    else
-        bzip2 ${filename}
-        mv ${filename}.bz2 ${dir}
-    fi
-done
-
-echo "packing sources"
-git archive --format=tar --prefix=restic-$VERSION/ v$VERSION | gzip -n > restic-$VERSION.tar.gz
-mv restic-$VERSION.tar.gz ${dir}
-
-echo "creating checksums"
-pushd ${dir}
-sha256sum restic_*.{zip,bz2} restic-$VERSION.tar.gz > SHA256SUMS
-gpg --armor --detach-sign SHA256SUMS
-popd
-
-echo "creating source signature file"
-gpg --armor --detach-sign ${dir}/restic-$VERSION.tar.gz
-
-echo
-echo "done, path is ${dir}"

cmd/restic/background_linux.go

@@ -4,7 +4,7 @@ import (
 	"syscall"
 	"unsafe"
 
-	"restic/debug"
+	"github.com/restic/restic/internal/debug"
 )
 
 // IsProcessBackground returns true if it is running in the background or false if not

cmd/restic/cleanup.go

@@ -7,7 +7,7 @@ import (
 	"sync"
 	"syscall"
 
-	"restic/debug"
+	"github.com/restic/restic/internal/debug"
 )
 
 var cleanupHandlers struct {

cmd/restic/cmd_autocomplete.go

@@ -8,7 +8,7 @@ var autocompleteTarget string
 
 var cmdAutocomplete = &cobra.Command{
 	Use:   "autocomplete",
-	Short: "generate shell autocompletion script",
+	Short: "Generate shell autocompletion script",
 	Long: `The "autocomplete" command generates a shell autocompletion script.
 
 NOTE: The current version supports Bash only.
@@ -19,6 +19,7 @@ for convenience, and the command may need superuser rights, e.g.:
 
 $ sudo restic autocomplete`,
 
+	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		if err := cmdRoot.GenBashCompletionFile(autocompleteTarget); err != nil {
 			return err

cmd/restic/cmd_backup.go

@@ -2,30 +2,41 @@ package main
 
 import (
 	"bufio"
+	"context"
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
-	"restic"
 	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
 
-	"restic/archiver"
-	"restic/debug"
-	"restic/errors"
-	"restic/filter"
-	"restic/fs"
+	"github.com/restic/restic/internal/archiver"
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/fs"
+	"github.com/restic/restic/internal/restic"
 )
 
 var cmdBackup = &cobra.Command{
 	Use:   "backup [flags] FILE/DIR [FILE/DIR] ...",
-	Short: "create a new backup of files and/or directories",
+	Short: "Create a new backup of files and/or directories",
 	Long: `
 The "backup" command creates a new snapshot and saves the files and directories
 given as the arguments.
 `,
+	PreRun: func(cmd *cobra.Command, args []string) {
+		if backupOptions.Hostname == "" {
+			hostname, err := os.Hostname()
+			if err != nil {
+				debug.Log("os.Hostname() returned err: %v", err)
+				return
+			}
+			backupOptions.Hostname = hostname
+		}
+	},
+	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		if backupOptions.Stdin && backupOptions.FilesFrom == "-" {
 			return errors.Fatal("cannot use both `--stdin` and `--files-from -`")
@@ -41,16 +52,19 @@ given as the arguments.
 
 // BackupOptions bundles all options for the backup command.
 type BackupOptions struct {
-	Parent         string
-	Force          bool
-	Excludes       []string
-	ExcludeFiles   []string
-	ExcludeOtherFS bool
-	Stdin          bool
-	StdinFilename  string
-	Tags           []string
-	Hostname       string
-	FilesFrom      string
+	Parent           string
+	Force            bool
+	Excludes         []string
+	ExcludeFiles     []string
+	ExcludeOtherFS   bool
+	ExcludeIfPresent []string
+	ExcludeCaches    bool
+	Stdin            bool
+	StdinFilename    string
+	Tags             []string
+	Hostname         string
+	FilesFrom        string
+	TimeStamp        string
 }
 
 var backupOptions BackupOptions
@@ -58,23 +72,20 @@ var backupOptions BackupOptions
 func init() {
 	cmdRoot.AddCommand(cmdBackup)
 
-	hostname, err := os.Hostname()
-	if err != nil {
-		debug.Log("os.Hostname() returned err: %v", err)
-		hostname = ""
-	}
-
 	f := cmdBackup.Flags()
 	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent snapshot (default: last snapshot in the repo that has the same target files/directories)")
 	f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`)
-	f.StringSliceVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
-	f.StringSliceVar(&backupOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
+	f.StringArrayVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
+	f.StringArrayVar(&backupOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
 	f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems")
+	f.StringArrayVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", nil, "takes filename[:header], exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)")
+	f.BoolVar(&backupOptions.ExcludeCaches, "exclude-caches", false, `excludes cache directories that are marked with a CACHEDIR.TAG file`)
 	f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
 	f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "file name to use when reading from stdin")
-	f.StringSliceVar(&backupOptions.Tags, "tag", nil, "add a `tag` for the new snapshot (can be specified multiple times)")
-	f.StringVar(&backupOptions.Hostname, "hostname", hostname, "set the `hostname` for the snapshot manually")
+	f.StringArrayVar(&backupOptions.Tags, "tag", nil, "add a `tag` for the new snapshot (can be specified multiple times)")
+	f.StringVar(&backupOptions.Hostname, "hostname", "", "set the `hostname` for the snapshot manually")
 	f.StringVar(&backupOptions.FilesFrom, "files-from", "", "read the files to backup from file (can be combined with file args)")
+	f.StringVar(&backupOptions.TimeStamp, "time", "", "time of the backup (ex. '2012-11-01 22:08:41') (default: now)")
 }
 
 func newScanProgress(gopts GlobalOptions) *restic.Progress {
@@ -205,6 +216,7 @@ func filterExisting(items []string) (result []string, err error) {
 	for _, item := range items {
 		_, err := fs.Lstat(item)
 		if err != nil && os.IsNotExist(errors.Cause(err)) {
+			Warnf("%v does not exist, skipping\n", item)
 			continue
 		}
 
@@ -218,27 +230,6 @@ func filterExisting(items []string) (result []string, err error) {
 	return
 }
 
-// gatherDevices returns the set of unique device ids of the files and/or
-// directory paths listed in "items".
-func gatherDevices(items []string) (deviceMap map[string]uint64, err error) {
-	deviceMap = make(map[string]uint64)
-	for _, item := range items {
-		fi, err := fs.Lstat(item)
-		if err != nil {
-			return nil, err
-		}
-		id, err := fs.DeviceID(fi)
-		if err != nil {
-			return nil, err
-		}
-		deviceMap[item] = id
-	}
-	if len(deviceMap) == 0 {
-		return nil, errors.New("zero allowed devices")
-	}
-	return deviceMap, nil
-}
-
 func readBackupFromStdin(opts BackupOptions, gopts GlobalOptions, args []string) error {
 	if len(args) != 0 {
 		return errors.Fatal("when reading from stdin, no additional files can be specified")
@@ -248,7 +239,7 @@ func readBackupFromStdin(opts BackupOptions, gopts GlobalOptions, args []string)
 		return errors.Fatal("filename for backup from stdin must not be empty")
 	}
 
-	if gopts.password == "" && gopts.PasswordFile == "" {
+	if gopts.password == "" {
 		return errors.Fatal("unable to read password from stdin when data is to be read from stdin, use --password-file or $RESTIC_PASSWORD")
 	}
 
@@ -263,7 +254,7 @@ func readBackupFromStdin(opts BackupOptions, gopts GlobalOptions, args []string)
 		return err
 	}
 
-	err = repo.LoadIndex()
+	err = repo.LoadIndex(context.TODO())
 	if err != nil {
 		return err
 	}
@@ -274,7 +265,7 @@ func readBackupFromStdin(opts BackupOptions, gopts GlobalOptions, args []string)
 		Hostname:   opts.Hostname,
 	}
 
-	_, id, err := r.Archive(opts.StdinFilename, os.Stdin, newArchiveStdinProgress(gopts))
+	_, id, err := r.Archive(context.TODO(), opts.StdinFilename, os.Stdin, newArchiveStdinProgress(gopts))
 	if err != nil {
 		return err
 	}
@@ -321,8 +312,8 @@ func readLinesFromFile(filename string) ([]string, error) {
 }
 
 func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
-	if opts.FilesFrom == "-" && gopts.password == "" && gopts.PasswordFile == "" {
-		return errors.Fatal("no password; either use `--password-file` option or put the password into the RESTIC_PASSWORD environment variable")
+	if opts.FilesFrom == "-" && gopts.password == "" {
+		return errors.Fatal("unable to read password from stdin when data is to be read from stdin, use --password-file or $RESTIC_PASSWORD")
 	}
 
 	fromfile, err := readLinesFromFile(opts.FilesFrom)
@@ -335,7 +326,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
 	// same time
 	args = append(args, fromfile...)
 	if len(args) == 0 {
-		return errors.Fatal("wrong number of parameters")
+		return errors.Fatal("nothing to backup, please specify target files/dirs")
 	}
 
 	target := make([]string, 0, len(args))
@@ -351,14 +342,38 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
+	// rejectFuncs collect functions that can reject items from the backup
+	var rejectFuncs []RejectFunc
+
 	// allowed devices
-	var allowedDevs map[string]uint64
 	if opts.ExcludeOtherFS {
-		allowedDevs, err = gatherDevices(target)
+		f, err := rejectByDevice(target)
 		if err != nil {
 			return err
 		}
-		debug.Log("allowed devices: %v\n", allowedDevs)
+		rejectFuncs = append(rejectFuncs, f)
+	}
+
+	// add patterns from file
+	if len(opts.ExcludeFiles) > 0 {
+		opts.Excludes = append(opts.Excludes, readExcludePatternsFromFiles(opts.ExcludeFiles)...)
+	}
+
+	if len(opts.Excludes) > 0 {
+		rejectFuncs = append(rejectFuncs, rejectByPattern(opts.Excludes))
+	}
+
+	if opts.ExcludeCaches {
+		opts.ExcludeIfPresent = append(opts.ExcludeIfPresent, "CACHEDIR.TAG:Signature: 8a477f597d28d172789f06886806bc55")
+	}
+
+	for _, spec := range opts.ExcludeIfPresent {
+		f, err := rejectIfPresent(spec)
+		if err != nil {
+			return err
+		}
+
+		rejectFuncs = append(rejectFuncs, f)
 	}
 
 	repo, err := OpenRepository(gopts)
@@ -372,7 +387,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	err = repo.LoadIndex()
+	err = repo.LoadIndex(context.TODO())
 	if err != nil {
 		return err
 	}
@@ -391,7 +406,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
 
 	// Find last snapshot to set it as parent, if not already set
 	if !opts.Force && parentSnapshotID == nil {
-		id, err := restic.FindLatestSnapshot(repo, target, opts.Tags, opts.Hostname)
+		id, err := restic.FindLatestSnapshot(context.TODO(), repo, target, []restic.TagList{opts.Tags}, opts.Hostname)
 		if err == nil {
 			parentSnapshotID = &id
 		} else if err != restic.ErrNoSnapshotFound {
@@ -405,74 +420,13 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
 
 	Verbosef("scan %v\n", target)
 
-	// add patterns from file
-	if len(opts.ExcludeFiles) > 0 {
-		for _, filename := range opts.ExcludeFiles {
-			file, err := fs.Open(filename)
-			if err != nil {
-				Warnf("error reading exclude patterns: %v", err)
-				return nil
-			}
-
-			scanner := bufio.NewScanner(file)
-			for scanner.Scan() {
-				line := strings.TrimSpace(scanner.Text())
-
-				// ignore empty lines
-				if line == "" {
-					continue
-				}
-
-				// strip comments
-				if strings.HasPrefix(line, "#") {
-					continue
-				}
-
-				line = os.ExpandEnv(line)
-				opts.Excludes = append(opts.Excludes, line)
-			}
-		}
-	}
-
 	selectFilter := func(item string, fi os.FileInfo) bool {
-		matched, err := filter.List(opts.Excludes, item)
-		if err != nil {
-			Warnf("error for exclude pattern: %v", err)
-		}
-
-		if matched {
-			debug.Log("path %q excluded by a filter", item)
-			return false
-		}
-
-		if !opts.ExcludeOtherFS || fi == nil {
-			return true
-		}
-
-		id, err := fs.DeviceID(fi)
-		if err != nil {
-			// This should never happen because gatherDevices() would have
-			// errored out earlier. If it still does that's a reason to panic.
-			panic(err)
-		}
-
-		for dir := item; dir != ""; dir = filepath.Dir(dir) {
-			debug.Log("item %v, test dir %v", item, dir)
-
-			allowedID, ok := allowedDevs[dir]
-			if !ok {
-				continue
-			}
-
-			if allowedID != id {
-				debug.Log("path %q on disallowed device %d", item, id)
+		for _, reject := range rejectFuncs {
+			if reject(item, fi) {
 				return false
 			}
-
-			return true
 		}
-
-		panic(fmt.Sprintf("item %v, device id %v not found, allowedDevs: %v", item, id, allowedDevs))
+		return true
 	}
 
 	stat, err := archiver.Scan(target, selectFilter, newScanProgress(gopts))
@@ -489,7 +443,15 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
 		Warnf("%s\rwarning for %s: %v\n", ClearLine(), dir, err)
 	}
 
-	_, id, err := arch.Snapshot(newArchiveProgress(gopts, stat), target, opts.Tags, opts.Hostname, parentSnapshotID)
+	timeStamp := time.Now()
+	if opts.TimeStamp != "" {
+		timeStamp, err = time.Parse(TimeFormat, opts.TimeStamp)
+		if err != nil {
+			return errors.Fatalf("error in time option: %v\n", err)
+		}
+	}
+
+	_, id, err := arch.Snapshot(context.TODO(), newArchiveProgress(gopts, stat), target, opts.Tags, opts.Hostname, parentSnapshotID, timeStamp)
 	if err != nil {
 		return err
 	}
@@ -498,3 +460,45 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
 
 	return nil
 }
+
+func readExcludePatternsFromFiles(excludeFiles []string) []string {
+	var excludes []string
+	for _, filename := range excludeFiles {
+		err := func() (err error) {
+			file, err := fs.Open(filename)
+			if err != nil {
+				return err
+			}
+			defer func() {
+				// return pre-close error if there was one
+				if errClose := file.Close(); err == nil {
+					err = errClose
+				}
+			}()
+
+			scanner := bufio.NewScanner(file)
+			for scanner.Scan() {
+				line := strings.TrimSpace(scanner.Text())
+
+				// ignore empty lines
+				if line == "" {
+					continue
+				}
+
+				// strip comments
+				if strings.HasPrefix(line, "#") {
+					continue
+				}
+
+				line = os.ExpandEnv(line)
+				excludes = append(excludes, line)
+			}
+			return scanner.Err()
+		}()
+		if err != nil {
+			Warnf("error reading exclude patterns: %v:", err)
+			return nil
+		}
+	}
+	return excludes
+}

cmd/restic/cmd_cat.go

@@ -1,24 +1,26 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"os"
 
 	"github.com/spf13/cobra"
 
-	"restic"
-	"restic/backend"
-	"restic/errors"
-	"restic/repository"
+	"github.com/restic/restic/internal/backend"
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
 )
 
 var cmdCat = &cobra.Command{
 	Use:   "cat [flags] [pack|blob|snapshot|index|key|masterkey|config|lock] ID",
-	Short: "print internal objects to stdout",
+	Short: "Print internal objects to stdout",
 	Long: `
 The "cat" command is used to print internal objects to stdout.
 `,
+	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runCat(globalOptions, args)
 	},
@@ -73,7 +75,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 		fmt.Println(string(buf))
 		return nil
 	case "index":
-		buf, err := repo.LoadAndDecrypt(restic.IndexFile, id)
+		buf, err := repo.LoadAndDecrypt(context.TODO(), restic.IndexFile, id)
 		if err != nil {
 			return err
 		}
@@ -83,7 +85,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 
 	case "snapshot":
 		sn := &restic.Snapshot{}
-		err = repo.LoadJSONUnpacked(restic.SnapshotFile, id, sn)
+		err = repo.LoadJSONUnpacked(context.TODO(), restic.SnapshotFile, id, sn)
 		if err != nil {
 			return err
 		}
@@ -98,7 +100,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 		return nil
 	case "key":
 		h := restic.Handle{Type: restic.KeyFile, Name: id.String()}
-		buf, err := backend.LoadAll(repo.Backend(), h)
+		buf, err := backend.LoadAll(context.TODO(), repo.Backend(), h)
 		if err != nil {
 			return err
 		}
@@ -125,7 +127,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 		fmt.Println(string(buf))
 		return nil
 	case "lock":
-		lock, err := restic.LoadLock(repo, id)
+		lock, err := restic.LoadLock(context.TODO(), repo, id)
 		if err != nil {
 			return err
 		}
@@ -141,7 +143,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 	}
 
 	// load index, handle all the other types
-	err = repo.LoadIndex()
+	err = repo.LoadIndex(context.TODO())
 	if err != nil {
 		return err
 	}
@@ -149,7 +151,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 	switch tpe {
 	case "pack":
 		h := restic.Handle{Type: restic.DataFile, Name: id.String()}
-		buf, err := backend.LoadAll(repo.Backend(), h)
+		buf, err := backend.LoadAll(context.TODO(), repo.Backend(), h)
 		if err != nil {
 			return err
 		}
@@ -171,7 +173,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 			blob := list[0]
 
 			buf := make([]byte, blob.Length)
-			n, err := repo.LoadBlob(t, id, buf)
+			n, err := repo.LoadBlob(context.TODO(), t, id, buf)
 			if err != nil {
 				return err
 			}

cmd/restic/cmd_check.go

@@ -1,24 +1,26 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"time"
 
 	"github.com/spf13/cobra"
 
-	"restic"
-	"restic/checker"
-	"restic/errors"
+	"github.com/restic/restic/internal/checker"
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/restic"
 )
 
 var cmdCheck = &cobra.Command{
 	Use:   "check [flags]",
-	Short: "check the repository for errors",
+	Short: "Check the repository for errors",
 	Long: `
 The "check" command tests the repository for errors and reports any errors it
 finds. It can also be used to read all data and therefore simulate a restore.
 `,
+	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runCheck(checkOptions, globalOptions, args)
 	},
@@ -92,7 +94,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 	chkr := checker.New(repo)
 
 	Verbosef("Load indexes\n")
-	hints, errs := chkr.LoadIndex()
+	hints, errs := chkr.LoadIndex(context.TODO())
 
 	dupFound := false
 	for _, hint := range hints {
@@ -113,14 +115,11 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 		return errors.Fatal("LoadIndex returned errors")
 	}
 
-	done := make(chan struct{})
-	defer close(done)
-
 	errorsFound := false
 	errChan := make(chan error)
 
 	Verbosef("Check all packs\n")
-	go chkr.Packs(errChan, done)
+	go chkr.Packs(context.TODO(), errChan)
 
 	for err := range errChan {
 		errorsFound = true
@@ -129,7 +128,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 
 	Verbosef("Check snapshots, trees and blobs\n")
 	errChan = make(chan error)
-	go chkr.Structure(errChan, done)
+	go chkr.Structure(context.TODO(), errChan)
 
 	for err := range errChan {
 		errorsFound = true
@@ -156,7 +155,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
 		p := newReadProgress(gopts, restic.Stat{Blobs: chkr.CountPacks()})
 		errChan := make(chan error)
 
-		go chkr.ReadData(p, errChan, done)
+		go chkr.ReadData(context.TODO(), p, errChan)
 
 		for err := range errChan {
 			errorsFound = true

cmd/restic/cmd_dump.go

@@ -1,8 +1,9 @@
-// +build debug
+// xbuild debug
 
 package main
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -10,20 +11,21 @@ import (
 
 	"github.com/spf13/cobra"
 
-	"restic"
-	"restic/errors"
-	"restic/pack"
-	"restic/repository"
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/pack"
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
 
-	"restic/worker"
+	"github.com/restic/restic/internal/worker"
 )
 
 var cmdDump = &cobra.Command{
 	Use:   "dump [indexes|snapshots|trees|all|packs]",
-	Short: "dump data structures",
+	Short: "Dump data structures",
 	Long: `
 The "dump" command dumps data structures from the repository as JSON objects. It
 is used for debugging purposes only.`,
+	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runDump(globalOptions, args)
 	},
@@ -44,11 +46,8 @@ func prettyPrintJSON(wr io.Writer, item interface{}) error {
 }
 
 func debugPrintSnapshots(repo *repository.Repository, wr io.Writer) error {
-	done := make(chan struct{})
-	defer close(done)
-
-	for id := range repo.List(restic.SnapshotFile, done) {
-		snapshot, err := restic.LoadSnapshot(repo, id)
+	for id := range repo.List(context.TODO(), restic.SnapshotFile) {
+		snapshot, err := restic.LoadSnapshot(context.TODO(), repo, id)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "LoadSnapshot(%v): %v", id.Str(), err)
 			continue
@@ -83,15 +82,12 @@ type Blob struct {
 }
 
 func printPacks(repo *repository.Repository, wr io.Writer) error {
-	done := make(chan struct{})
-	defer close(done)
-
-	f := func(job worker.Job, done <-chan struct{}) (interface{}, error) {
+	f := func(ctx context.Context, job worker.Job) (interface{}, error) {
 		name := job.Data.(string)
 
 		h := restic.Handle{Type: restic.DataFile, Name: name}
 
-		blobInfo, err := repo.Backend().Stat(h)
+		blobInfo, err := repo.Backend().Stat(ctx, h)
 		if err != nil {
 			return nil, err
 		}
@@ -106,10 +102,10 @@ func printPacks(repo *repository.Repository, wr io.Writer) error {
 
 	jobCh := make(chan worker.Job)
 	resCh := make(chan worker.Job)
-	wp := worker.New(dumpPackWorkers, f, jobCh, resCh)
+	wp := worker.New(context.TODO(), dumpPackWorkers, f, jobCh, resCh)
 
 	go func() {
-		for name := range repo.Backend().List(restic.DataFile, done) {
+		for name := range repo.Backend().List(context.TODO(), restic.DataFile) {
 			jobCh <- worker.Job{Data: name}
 		}
 		close(jobCh)
@@ -146,13 +142,10 @@ func printPacks(repo *repository.Repository, wr io.Writer) error {
 }
 
 func dumpIndexes(repo restic.Repository) error {
-	done := make(chan struct{})
-	defer close(done)
-
-	for id := range repo.List(restic.IndexFile, done) {
+	for id := range repo.List(context.TODO(), restic.IndexFile) {
 		fmt.Printf("index_id: %v\n", id)
 
-		idx, err := repository.LoadIndex(repo, id)
+		idx, err := repository.LoadIndex(context.TODO(), repo, id)
 		if err != nil {
 			return err
 		}
@@ -184,7 +177,7 @@ func runDump(gopts GlobalOptions, args []string) error {
 		}
 	}
 
-	err = repo.LoadIndex()
+	err = repo.LoadIndex(context.TODO())
 	if err != nil {
 		return err
 	}

cmd/restic/cmd_find.go

@@ -9,18 +9,18 @@ import (
 
 	"github.com/spf13/cobra"
 
-	"restic"
-	"restic/debug"
-	"restic/errors"
-	"restic/repository"
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/restic"
 )
 
 var cmdFind = &cobra.Command{
 	Use:   "find [flags] PATTERN",
-	Short: "find a file or directory",
+	Short: "Find a file or directory",
 	Long: `
 The "find" command searches for files or directories in snapshots stored in the
 repo. `,
+	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runFind(findOptions, globalOptions, args)
 	},
@@ -35,7 +35,7 @@ type FindOptions struct {
 	ListLong        bool
 	Host            string
 	Paths           []string
-	Tags            []string
+	Tags            restic.TagLists
 }
 
 var findOptions FindOptions
@@ -46,13 +46,13 @@ func init() {
 	f := cmdFind.Flags()
 	f.StringVarP(&findOptions.Oldest, "oldest", "O", "", "oldest modification date/time")
 	f.StringVarP(&findOptions.Newest, "newest", "N", "", "newest modification date/time")
-	f.StringSliceVarP(&findOptions.Snapshots, "snapshot", "s", nil, "snapshot `id` to search in (can be given multiple times)")
+	f.StringArrayVarP(&findOptions.Snapshots, "snapshot", "s", nil, "snapshot `id` to search in (can be given multiple times)")
 	f.BoolVarP(&findOptions.CaseInsensitive, "ignore-case", "i", false, "ignore case for pattern")
 	f.BoolVarP(&findOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
 
 	f.StringVarP(&findOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
-	f.StringSliceVar(&findOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot-ID is given")
-	f.StringSliceVar(&findOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
+	f.Var(&findOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot-ID is given")
+	f.StringArrayVar(&findOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
 }
 
 type findPattern struct {
@@ -172,59 +172,76 @@ func (s *statefulOutput) Finish() {
 	}
 }
 
-func findInTree(repo *repository.Repository, pat *findPattern, id restic.ID, prefix string, state *statefulOutput) error {
-	debug.Log("checking tree %v\n", id)
+// Finder bundles information needed to find a file or directory.
+type Finder struct {
+	repo     restic.Repository
+	pat      findPattern
+	out      statefulOutput
+	notfound restic.IDSet
+}
 
-	tree, err := repo.LoadTree(id)
+func (f *Finder) findInTree(treeID restic.ID, prefix string) error {
+	if f.notfound.Has(treeID) {
+		debug.Log("%v skipping tree %v, has already been checked", prefix, treeID.Str())
+		return nil
+	}
+
+	debug.Log("%v checking tree %v\n", prefix, treeID.Str())
+
+	tree, err := f.repo.LoadTree(context.TODO(), treeID)
 	if err != nil {
 		return err
 	}
 
+	var found bool
 	for _, node := range tree.Nodes {
 		debug.Log("  testing entry %q\n", node.Name)
 
 		name := node.Name
-		if pat.ignoreCase {
+		if f.pat.ignoreCase {
 			name = strings.ToLower(name)
 		}
 
-		m, err := filepath.Match(pat.pattern, name)
+		m, err := filepath.Match(f.pat.pattern, name)
 		if err != nil {
 			return err
 		}
 
 		if m {
-			debug.Log("    pattern matches\n")
-			if !pat.oldest.IsZero() && node.ModTime.Before(pat.oldest) {
-				debug.Log("    ModTime is older than %s\n", pat.oldest)
+			if !f.pat.oldest.IsZero() && node.ModTime.Before(f.pat.oldest) {
+				debug.Log("    ModTime is older than %s\n", f.pat.oldest)
 				continue
 			}
 
-			if !pat.newest.IsZero() && node.ModTime.After(pat.newest) {
-				debug.Log("    ModTime is newer than %s\n", pat.newest)
+			if !f.pat.newest.IsZero() && node.ModTime.After(f.pat.newest) {
+				debug.Log("    ModTime is newer than %s\n", f.pat.newest)
 				continue
 			}
 
-			state.Print(prefix, node)
-		} else {
-			debug.Log("    pattern does not match\n")
+			debug.Log("    found match\n")
+			found = true
+			f.out.Print(prefix, node)
 		}
 
 		if node.Type == "dir" {
-			if err := findInTree(repo, pat, *node.Subtree, filepath.Join(prefix, node.Name), state); err != nil {
+			if err := f.findInTree(*node.Subtree, filepath.Join(prefix, node.Name)); err != nil {
 				return err
 			}
 		}
 	}
 
+	if !found {
+		f.notfound.Insert(treeID)
+	}
+
 	return nil
 }
 
-func findInSnapshot(repo *repository.Repository, sn *restic.Snapshot, pat findPattern, state *statefulOutput) error {
-	debug.Log("searching in snapshot %s\n  for entries within [%s %s]", sn.ID(), pat.oldest, pat.newest)
+func (f *Finder) findInSnapshot(sn *restic.Snapshot) error {
+	debug.Log("searching in snapshot %s\n  for entries within [%s %s]", sn.ID(), f.pat.oldest, f.pat.newest)
 
-	state.newsn = sn
-	if err := findInTree(repo, &pat, *sn.Tree, string(filepath.Separator), state); err != nil {
+	f.out.newsn = sn
+	if err := f.findInTree(*sn.Tree, string(filepath.Separator)); err != nil {
 		return err
 	}
 	return nil
@@ -267,19 +284,25 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
-	if err = repo.LoadIndex(); err != nil {
+	if err = repo.LoadIndex(context.TODO()); err != nil {
 		return err
 	}
 
 	ctx, cancel := context.WithCancel(gopts.ctx)
 	defer cancel()
-	state := statefulOutput{ListLong: opts.ListLong, JSON: globalOptions.JSON}
+
+	f := &Finder{
+		repo:     repo,
+		pat:      pat,
+		out:      statefulOutput{ListLong: opts.ListLong, JSON: globalOptions.JSON},
+		notfound: restic.NewIDSet(),
+	}
 	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, opts.Snapshots) {
-		if err = findInSnapshot(repo, sn, pat, &state); err != nil {
+		if err = f.findInSnapshot(sn); err != nil {
 			return err
 		}
 	}
-	state.Finish()
+	f.out.Finish()
 
 	return nil
 }

cmd/restic/cmd_forget.go

@@ -3,21 +3,23 @@ package main
 import (
 	"context"
 	"encoding/json"
-	"restic"
 	"sort"
 	"strings"
 
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/restic"
 	"github.com/spf13/cobra"
 )
 
 var cmdForget = &cobra.Command{
 	Use:   "forget [flags] [snapshot ID] [...]",
-	Short: "forget removes snapshots from the repository",
+	Short: "Remove snapshots from the repository",
 	Long: `
 The "forget" command removes snapshots according to a policy. Please note that
 this command really only deletes the snapshot object in the repository, which
 is a reference to data stored there. In order to remove this (now unreferenced)
 data after 'forget' was run successfully, see the 'prune' command. `,
+	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runForget(forgetOptions, globalOptions, args)
 	},
@@ -31,15 +33,16 @@ type ForgetOptions struct {
 	Weekly   int
 	Monthly  int
 	Yearly   int
-	KeepTags []string
+	KeepTags restic.TagLists
 
 	Host  string
-	Tags  []string
+	Tags  restic.TagLists
 	Paths []string
 
-	GroupByTags bool
-	DryRun      bool
-	Prune       bool
+	// Grouping
+	GroupBy string
+	DryRun  bool
+	Prune   bool
 }
 
 var forgetOptions ForgetOptions
@@ -55,15 +58,15 @@ func init() {
 	f.IntVarP(&forgetOptions.Monthly, "keep-monthly", "m", 0, "keep the last `n` monthly snapshots")
 	f.IntVarP(&forgetOptions.Yearly, "keep-yearly", "y", 0, "keep the last `n` yearly snapshots")
 
-	f.StringSliceVar(&forgetOptions.KeepTags, "keep-tag", []string{}, "keep snapshots with this `tag` (can be specified multiple times)")
-	f.BoolVarP(&forgetOptions.GroupByTags, "group-by-tags", "G", false, "Group by host,paths,tags instead of just host,paths")
+	f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)")
 	// Sadly the commonly used shortcut `H` is already used.
 	f.StringVar(&forgetOptions.Host, "host", "", "only consider snapshots with the given `host`")
 	// Deprecated since 2017-03-07.
 	f.StringVar(&forgetOptions.Host, "hostname", "", "only consider snapshots with the given `hostname` (deprecated)")
-	f.StringSliceVar(&forgetOptions.Tags, "tag", nil, "only consider snapshots which include this `tag` (can be specified multiple times)")
-	f.StringSliceVar(&forgetOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` (can be specified multiple times)")
+	f.Var(&forgetOptions.Tags, "tag", "only consider snapshots which include this `taglist` in the format `tag[,tag,...]` (can be specified multiple times)")
+	f.StringArrayVar(&forgetOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` (can be specified multiple times)")
 
+	f.StringVarP(&forgetOptions.GroupBy, "group-by", "g", "host,paths", "string for grouping snapshots by host,paths,tags")
 	f.BoolVarP(&forgetOptions.DryRun, "dry-run", "n", false, "do not delete anything, just print what would be done")
 	f.BoolVar(&forgetOptions.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed")
 
@@ -90,6 +93,27 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 	}
 	snapshotGroups := make(map[string]restic.Snapshots)
 
+	var GroupByTag bool
+	var GroupByHost bool
+	var GroupByPath bool
+	var GroupOptionList []string
+
+	GroupOptionList = strings.Split(opts.GroupBy, ",")
+
+	for _, option := range GroupOptionList {
+		switch option {
+		case "host":
+			GroupByHost = true
+		case "paths":
+			GroupByPath = true
+		case "tags":
+			GroupByTag = true
+		case "":
+		default:
+			return errors.Fatal("unknown grouping option: '" + option + "'")
+		}
+	}
+
 	ctx, cancel := context.WithCancel(gopts.ctx)
 	defer cancel()
 	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
@@ -97,7 +121,7 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 			// When explicit snapshots args are given, remove them immediately.
 			if !opts.DryRun {
 				h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
-				if err = repo.Backend().Remove(h); err != nil {
+				if err = repo.Backend().Remove(context.TODO(), h); err != nil {
 					return err
 				}
 				Verbosef("removed snapshot %v\n", sn.ID().Str())
@@ -105,13 +129,28 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 				Verbosef("would have removed snapshot %v\n", sn.ID().Str())
 			}
 		} else {
+			// Determing grouping-keys
 			var tags []string
-			if opts.GroupByTags {
+			var hostname string
+			var paths []string
+
+			if GroupByTag {
 				tags = sn.Tags
 				sort.StringSlice(tags).Sort()
 			}
+			if GroupByHost {
+				hostname = sn.Hostname
+			}
+			if GroupByPath {
+				paths = sn.Paths
+			}
+
 			sort.StringSlice(sn.Paths).Sort()
-			k, err := json.Marshal(key{Hostname: sn.Hostname, Tags: tags, Paths: sn.Paths})
+			var k []byte
+			var err error
+
+			k, err = json.Marshal(key{Tags: tags, Hostname: hostname, Paths: paths})
+
 			if err != nil {
 				return err
 			}
@@ -143,22 +182,35 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 		if json.Unmarshal([]byte(k), &key) != nil {
 			return err
 		}
-		if opts.GroupByTags {
-			Verbosef("snapshots for host %v, tags [%v], paths: [%v]:\n\n", key.Hostname, strings.Join(key.Tags, ", "), strings.Join(key.Paths, ", "))
-		} else {
-			Verbosef("snapshots for host %v, paths: [%v]:\n\n", key.Hostname, strings.Join(key.Paths, ", "))
+
+		// Info
+		Verbosef("snapshots")
+		var infoStrings []string
+		if GroupByTag {
+			infoStrings = append(infoStrings, "tags ["+strings.Join(key.Tags, ", ")+"]")
 		}
+		if GroupByHost {
+			infoStrings = append(infoStrings, "host ["+key.Hostname+"]")
+		}
+		if GroupByPath {
+			infoStrings = append(infoStrings, "paths ["+strings.Join(key.Paths, ", ")+"]")
+		}
+		if infoStrings != nil {
+			Verbosef(" for (" + strings.Join(infoStrings, ", ") + ")")
+		}
+		Verbosef(":\n\n")
+
 		keep, remove := restic.ApplyPolicy(snapshotGroup, policy)
 
 		if len(keep) != 0 && !gopts.Quiet {
 			Printf("keep %d snapshots:\n", len(keep))
-			PrintSnapshots(globalOptions.stdout, keep)
+			PrintSnapshots(globalOptions.stdout, keep, false)
 			Printf("\n")
 		}
 
 		if len(remove) != 0 && !gopts.Quiet {
 			Printf("remove %d snapshots:\n", len(remove))
-			PrintSnapshots(globalOptions.stdout, remove)
+			PrintSnapshots(globalOptions.stdout, remove, false)
 			Printf("\n")
 		}
 
@@ -167,7 +219,7 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 		if !opts.DryRun {
 			for _, sn := range remove {
 				h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
-				err = repo.Backend().Remove(h)
+				err = repo.Backend().Remove(context.TODO(), h)
 				if err != nil {
 					return err
 				}

cmd/restic/cmd_init.go

@@ -1,18 +1,21 @@
 package main
 
 import (
-	"restic/errors"
-	"restic/repository"
+	"context"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/repository"
 
 	"github.com/spf13/cobra"
 )
 
 var cmdInit = &cobra.Command{
 	Use:   "init",
-	Short: "initialize a new repository",
+	Short: "Initialize a new repository",
 	Long: `
 The "init" command initializes a new repository.
 `,
+	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runInit(globalOptions, args)
 	},
@@ -32,18 +35,16 @@ func runInit(gopts GlobalOptions, args []string) error {
 		return errors.Fatalf("create backend at %s failed: %v\n", gopts.Repo, err)
 	}
 
-	if gopts.password == "" {
-		gopts.password, err = ReadPasswordTwice(gopts,
-			"enter password for new backend: ",
-			"enter password again: ")
-		if err != nil {
-			return err
-		}
+	gopts.password, err = ReadPasswordTwice(gopts,
+		"enter password for new backend: ",
+		"enter password again: ")
+	if err != nil {
+		return err
 	}
 
 	s := repository.New(be)
 
-	err = s.Init(gopts.password)
+	err = s.Init(context.TODO(), gopts.password)
 	if err != nil {
 		return errors.Fatalf("create key in backend at %s failed: %v\n", gopts.Repo, err)
 	}

cmd/restic/cmd_key.go

@@ -3,19 +3,21 @@ package main
 import (
 	"context"
 	"fmt"
-	"restic"
-	"restic/errors"
-	"restic/repository"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
 
 	"github.com/spf13/cobra"
 )
 
 var cmdKey = &cobra.Command{
-	Use:   "key [list|add|rm|passwd] [ID]",
-	Short: "manage keys (passwords)",
+	Use:   "key [list|add|remove|passwd] [ID]",
+	Short: "Manage keys (passwords)",
 	Long: `
 The "key" command manages keys (passwords) for accessing the repository.
 `,
+	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runKey(globalOptions, args)
 	},
@@ -30,8 +32,8 @@ func listKeys(ctx context.Context, s *repository.Repository) error {
 	tab.Header = fmt.Sprintf(" %-10s  %-10s  %-10s  %s", "ID", "User", "Host", "Created")
 	tab.RowFormat = "%s%-10s  %-10s  %-10s  %s"
 
-	for id := range s.List(restic.KeyFile, ctx.Done()) {
-		k, err := repository.LoadKey(s, id.String())
+	for id := range s.List(ctx, restic.KeyFile) {
+		k, err := repository.LoadKey(ctx, s, id.String())
 		if err != nil {
 			Warnf("LoadKey() failed: %v\n", err)
 			continue
@@ -58,7 +60,12 @@ func getNewPassword(gopts GlobalOptions) (string, error) {
 		return testKeyNewPassword, nil
 	}
 
-	return ReadPasswordTwice(gopts,
+	// Since we already have an open repository, temporary remove the password
+	// to prompt the user for the passwd.
+	newopts := gopts
+	newopts.password = ""
+
+	return ReadPasswordTwice(newopts,
 		"enter password for new key: ",
 		"enter password again: ")
 }
@@ -69,7 +76,7 @@ func addKey(gopts GlobalOptions, repo *repository.Repository) error {
 		return err
 	}
 
-	id, err := repository.AddKey(repo, pw, repo.Key())
+	id, err := repository.AddKey(context.TODO(), repo, pw, repo.Key())
 	if err != nil {
 		return errors.Fatalf("creating new key failed: %v\n", err)
 	}
@@ -85,7 +92,7 @@ func deleteKey(repo *repository.Repository, name string) error {
 	}
 
 	h := restic.Handle{Type: restic.KeyFile, Name: name}
-	err := repo.Backend().Remove(h)
+	err := repo.Backend().Remove(context.TODO(), h)
 	if err != nil {
 		return err
 	}
@@ -100,13 +107,13 @@ func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
 		return err
 	}
 
-	id, err := repository.AddKey(repo, pw, repo.Key())
+	id, err := repository.AddKey(context.TODO(), repo, pw, repo.Key())
 	if err != nil {
 		return errors.Fatalf("creating new key failed: %v\n", err)
 	}
 
 	h := restic.Handle{Type: restic.KeyFile, Name: repo.KeyName()}
-	err = repo.Backend().Remove(h)
+	err = repo.Backend().Remove(context.TODO(), h)
 	if err != nil {
 		return err
 	}
@@ -117,7 +124,7 @@ func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
 }
 
 func runKey(gopts GlobalOptions, args []string) error {
-	if len(args) < 1 || (args[0] == "rm" && len(args) != 2) || (args[0] != "rm" && len(args) != 1) {
+	if len(args) < 1 || (args[0] == "remove" && len(args) != 2) || (args[0] != "remove" && len(args) != 1) {
 		return errors.Fatal("wrong number of arguments")
 	}
 
@@ -146,7 +153,7 @@ func runKey(gopts GlobalOptions, args []string) error {
 		}
 
 		return addKey(gopts, repo)
-	case "rm":
+	case "remove":
 		lock, err := lockRepoExclusive(repo)
 		defer unlockRepo(lock)
 		if err != nil {

cmd/restic/cmd_list.go

@@ -1,20 +1,23 @@
 package main
 
 import (
+	"context"
 	"fmt"
-	"restic"
-	"restic/errors"
-	"restic/index"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/index"
+	"github.com/restic/restic/internal/restic"
 
 	"github.com/spf13/cobra"
 )
 
 var cmdList = &cobra.Command{
 	Use:   "list [blobs|packs|index|snapshots|keys|locks]",
-	Short: "list objects in the repository",
+	Short: "List objects in the repository",
 	Long: `
 The "list" command allows listing objects in the repository based on type.
 `,
+	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runList(globalOptions, args)
 	},
@@ -55,7 +58,7 @@ func runList(opts GlobalOptions, args []string) error {
 	case "locks":
 		t = restic.LockFile
 	case "blobs":
-		idx, err := index.Load(repo, nil)
+		idx, err := index.Load(context.TODO(), repo, nil)
 		if err != nil {
 			return err
 		}
@@ -71,7 +74,7 @@ func runList(opts GlobalOptions, args []string) error {
 		return errors.Fatal("invalid type")
 	}
 
-	for id := range repo.List(t, nil) {
+	for id := range repo.List(context.TODO(), t) {
 		Printf("%s\n", id)
 	}
 

cmd/restic/cmd_ls.go

@@ -6,19 +6,20 @@ import (
 
 	"github.com/spf13/cobra"
 
-	"restic"
-	"restic/errors"
-	"restic/repository"
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
 )
 
 var cmdLs = &cobra.Command{
 	Use:   "ls [flags] [snapshot-ID ...]",
-	Short: "list files in a snapshot",
+	Short: "List files in a snapshot",
 	Long: `
 The "ls" command allows listing files and directories in a snapshot.
 
 The special snapshot-ID "latest" can be used to list files and directories of the latest snapshot in the repository.
 `,
+	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runLs(lsOptions, globalOptions, args)
 	},
@@ -28,7 +29,7 @@ The special snapshot-ID "latest" can be used to list files and directories of th
 type LsOptions struct {
 	ListLong bool
 	Host     string
-	Tags     []string
+	Tags     restic.TagLists
 	Paths    []string
 }
 
@@ -41,18 +42,18 @@ func init() {
 	flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
 
 	flags.StringVarP(&lsOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
-	flags.StringSliceVar(&lsOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot ID is given")
-	flags.StringSliceVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
+	flags.Var(&lsOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot ID is given")
+	flags.StringArrayVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
 }
 
 func printTree(repo *repository.Repository, id *restic.ID, prefix string) error {
-	tree, err := repo.LoadTree(*id)
+	tree, err := repo.LoadTree(context.TODO(), *id)
 	if err != nil {
 		return err
 	}
 
 	for _, entry := range tree.Nodes {
-		Printf(formatNode(prefix, entry, lsOptions.ListLong) + "\n")
+		Printf("%s\n", formatNode(prefix, entry, lsOptions.ListLong))
 
 		if entry.Type == "dir" && entry.Subtree != nil {
 			if err = printTree(repo, entry.Subtree, filepath.Join(prefix, entry.Name)); err != nil {
@@ -74,7 +75,7 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	if err = repo.LoadIndex(); err != nil {
+	if err = repo.LoadIndex(context.TODO()); err != nil {
 		return err
 	}
 

cmd/restic/cmd_manpage.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"os"
+	"time"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/spf13/cobra"
+	"github.com/spf13/cobra/doc"
+)
+
+var cmdManpage = &cobra.Command{
+	Use:   "manpage [command]",
+	Short: "Generate manual pages",
+	Long: `
+The "manpage" command generates a manual page for a single command. It can also
+be used to write all manual pages to a directory. If the output directory is
+set and no command is specified, all manpages are written to the directory.
+`,
+	DisableAutoGenTag: true,
+	RunE:              runManpage,
+}
+
+var manpageOpts = struct {
+	OutputDir string
+}{}
+
+func init() {
+	cmdRoot.AddCommand(cmdManpage)
+	fs := cmdManpage.Flags()
+	fs.StringVar(&manpageOpts.OutputDir, "output-dir", "", "write man pages to this `directory`")
+}
+
+func runManpage(cmd *cobra.Command, args []string) error {
+	// use a fixed date for the man pages so that generating them is deterministic
+	date, err := time.Parse("Jan 2006", "Jan 2017")
+	if err != nil {
+		return err
+	}
+
+	header := &doc.GenManHeader{
+		Title:   "restic backup",
+		Section: "1",
+		Source:  "generated by `restic manpage`",
+		Date:    &date,
+	}
+
+	dir := manpageOpts.OutputDir
+	if dir != "" {
+		Verbosef("writing man pages to directory %v\n", dir)
+		return doc.GenManTree(cmdRoot, header, dir)
+	}
+
+	switch {
+	case len(args) == 0:
+		return errors.Fatalf("no command given")
+	case len(args) > 1:
+		return errors.Fatalf("more than one command given: %v", args)
+	}
+
+	name := args[0]
+
+	for _, cmd := range cmdRoot.Commands() {
+		if cmd.Name() == name {
+			return doc.GenMan(cmd, header, os.Stdout)
+		}
+	}
+
+	return errors.Fatalf("command %q is not known", args)
+}

cmd/restic/cmd_migrate.go

@@ -0,0 +1,108 @@
+package main
+
+import (
+	"github.com/restic/restic/internal/migrations"
+	"github.com/restic/restic/internal/restic"
+
+	"github.com/spf13/cobra"
+)
+
+var cmdMigrate = &cobra.Command{
+	Use:   "migrate [name]",
+	Short: "Apply migrations",
+	Long: `
+The "migrate" command applies migrations to a repository. When no migration
+name is explicitly given, a list of migrations that can be applied is printed.
+`,
+	DisableAutoGenTag: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runMigrate(migrateOptions, globalOptions, args)
+	},
+}
+
+// MigrateOptions bundles all options for the 'check' command.
+type MigrateOptions struct {
+	Force bool
+}
+
+var migrateOptions MigrateOptions
+
+func init() {
+	cmdRoot.AddCommand(cmdMigrate)
+	f := cmdMigrate.Flags()
+	f.BoolVarP(&migrateOptions.Force, "force", "f", false, `apply a migration a second time`)
+}
+
+func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository) error {
+	ctx := gopts.ctx
+	Printf("available migrations:\n")
+	for _, m := range migrations.All {
+		ok, err := m.Check(ctx, repo)
+		if err != nil {
+			return err
+		}
+
+		if ok {
+			Printf("  %v: %v\n", m.Name(), m.Desc())
+		}
+	}
+
+	return nil
+}
+
+func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository, args []string) error {
+	ctx := gopts.ctx
+
+	var firsterr error
+	for _, name := range args {
+		for _, m := range migrations.All {
+			if m.Name() == name {
+				ok, err := m.Check(ctx, repo)
+				if err != nil {
+					return err
+				}
+
+				if !ok {
+					if !opts.Force {
+						Warnf("migration %v cannot be applied: check failed\nIf you want to apply this migration anyway, re-run with option --force\n", m.Name())
+						continue
+					}
+
+					Warnf("check for migration %v failed, continuing anyway\n", m.Name())
+				}
+
+				Printf("applying migration %v...\n", m.Name())
+				if err = m.Apply(ctx, repo); err != nil {
+					Warnf("migration %v failed: %v\n", m.Name(), err)
+					if firsterr == nil {
+						firsterr = err
+					}
+					continue
+				}
+
+				Printf("migration %v: success\n", m.Name())
+			}
+		}
+	}
+
+	return firsterr
+}
+
+func runMigrate(opts MigrateOptions, gopts GlobalOptions, args []string) error {
+	repo, err := OpenRepository(gopts)
+	if err != nil {
+		return err
+	}
+
+	lock, err := lockRepoExclusive(repo)
+	defer unlockRepo(lock)
+	if err != nil {
+		return err
+	}
+
+	if len(args) == 0 {
+		return checkMigrations(opts, gopts, repo)
+	}
+
+	return applyMigrations(opts, gopts, repo, args)
+}

cmd/restic/cmd_mount.go

@@ -4,15 +4,17 @@
 package main
 
 import (
+	"context"
 	"os"
 
 	"github.com/spf13/cobra"
 
-	"restic/debug"
-	"restic/errors"
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/restic"
 
-	resticfs "restic/fs"
-	"restic/fuse"
+	resticfs "github.com/restic/restic/internal/fs"
+	"github.com/restic/restic/internal/fuse"
 
 	systemFuse "bazil.org/fuse"
 	"bazil.org/fuse/fs"
@@ -20,11 +22,12 @@ import (
 
 var cmdMount = &cobra.Command{
 	Use:   "mount [flags] mountpoint",
-	Short: "mount the repository",
+	Short: "Mount the repository",
 	Long: `
 The "mount" command mounts the repository via fuse to a directory. This is a
 read-only mount.
 `,
+	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runMount(mountOptions, globalOptions, args)
 	},
@@ -36,7 +39,7 @@ type MountOptions struct {
 	AllowRoot  bool
 	AllowOther bool
 	Host       string
-	Tags       []string
+	Tags       restic.TagLists
 	Paths      []string
 }
 
@@ -51,8 +54,8 @@ func init() {
 	mountFlags.BoolVar(&mountOptions.AllowOther, "allow-other", false, "allow other users to access the data in the mounted directory")
 
 	mountFlags.StringVarP(&mountOptions.Host, "host", "H", "", `only consider snapshots for this host`)
-	mountFlags.StringSliceVar(&mountOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`")
-	mountFlags.StringSliceVar(&mountOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`")
+	mountFlags.Var(&mountOptions.Tags, "tag", "only consider snapshots which include this `taglist`")
+	mountFlags.StringArrayVar(&mountOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`")
 }
 
 func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
@@ -64,7 +67,7 @@ func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
 		return err
 	}
 
-	err = repo.LoadIndex()
+	err = repo.LoadIndex(context.TODO())
 	if err != nil {
 		return err
 	}
@@ -95,14 +98,26 @@ func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
 		return err
 	}
 
+	systemFuse.Debug = func(msg interface{}) {
+		debug.Log("fuse: %v", msg)
+	}
+
+	cfg := fuse.Config{
+		OwnerIsRoot: opts.OwnerRoot,
+		Host:        opts.Host,
+		Tags:        opts.Tags,
+		Paths:       opts.Paths,
+	}
+	root, err := fuse.NewRoot(context.TODO(), repo, cfg)
+	if err != nil {
+		return err
+	}
+
 	Printf("Now serving the repository at %s\n", mountpoint)
 	Printf("Don't forget to umount after quitting!\n")
 
-	root := fs.Tree{}
-	root.Add("snapshots", fuse.NewSnapshotsDir(repo, opts.OwnerRoot, opts.Paths, opts.Tags, opts.Host))
-
 	debug.Log("serving mount at %v", mountpoint)
-	err = fs.Serve(c, &root)
+	err = fs.Serve(c, root)
 	if err != nil {
 		return err
 	}

cmd/restic/cmd_options.go

@@ -2,18 +2,20 @@ package main
 
 import (
 	"fmt"
-	"restic/options"
+
+	"github.com/restic/restic/internal/options"
 
 	"github.com/spf13/cobra"
 )
 
 var optionsCmd = &cobra.Command{
 	Use:   "options",
-	Short: "print list of extended options",
+	Short: "Print list of extended options",
 	Long: `
 The "options" command prints a list of extended options.
 `,
-	Hidden: true,
+	Hidden:            true,
+	DisableAutoGenTag: true,
 	Run: func(cmd *cobra.Command, args []string) {
 		fmt.Printf("All Extended Options:\n")
 		for _, opt := range options.List() {

cmd/restic/cmd_prune.go

@@ -1,25 +1,26 @@
 package main
 
 import (
-	"context"
 	"fmt"
-	"restic"
-	"restic/debug"
-	"restic/errors"
-	"restic/index"
-	"restic/repository"
 	"time"
 
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/index"
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
+
 	"github.com/spf13/cobra"
 )
 
 var cmdPrune = &cobra.Command{
 	Use:   "prune [flags]",
-	Short: "remove unneeded data from the repository",
+	Short: "Remove unneeded data from the repository",
 	Long: `
 The "prune" command checks the repository and removes data that is not
 referenced and therefore not needed any more.
 `,
+	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runPrune(globalOptions)
 	},
@@ -29,6 +30,18 @@ func init() {
 	cmdRoot.AddCommand(cmdPrune)
 }
 
+func shortenStatus(maxLength int, s string) string {
+	if len(s) <= maxLength {
+		return s
+	}
+
+	if maxLength < 3 {
+		return s[:maxLength]
+	}
+
+	return s[:maxLength-3] + "..."
+}
+
 // newProgressMax returns a progress that counts blobs.
 func newProgressMax(show bool, max uint64, description string) *restic.Progress {
 	if !show {
@@ -44,10 +57,7 @@ func newProgressMax(show bool, max uint64, description string) *restic.Progress
 			s.Blobs, max, description)
 
 		if w := stdoutTerminalWidth(); w > 0 {
-			if len(status) > w {
-				max := w - len(status) - 4
-				status = status[:max] + "... "
-			}
+			status = shortenStatus(w, status)
 		}
 
 		PrintProgress("%s", status)
@@ -76,14 +86,13 @@ func runPrune(gopts GlobalOptions) error {
 }
 
 func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
-	err := repo.LoadIndex()
+	ctx := gopts.ctx
+
+	err := repo.LoadIndex(ctx)
 	if err != nil {
 		return err
 	}
 
-	ctx, cancel := context.WithCancel(gopts.ctx)
-	defer cancel()
-
 	var stats struct {
 		blobs     int
 		packs     int
@@ -92,18 +101,22 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
 	}
 
 	Verbosef("counting files in repo\n")
-	for range repo.List(restic.DataFile, ctx.Done()) {
+	for range repo.List(ctx, restic.DataFile) {
 		stats.packs++
 	}
 
 	Verbosef("building new index for repo\n")
 
 	bar := newProgressMax(!gopts.Quiet, uint64(stats.packs), "packs")
-	idx, err := index.New(repo, bar)
+	idx, invalidFiles, err := index.New(ctx, repo, restic.NewIDSet(), bar)
 	if err != nil {
 		return err
 	}
 
+	for _, id := range invalidFiles {
+		Warnf("incomplete pack file (will be removed): %v\n", id)
+	}
+
 	blobs := 0
 	for _, pack := range idx.Packs {
 		stats.bytes += pack.Size
@@ -135,7 +148,7 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
 	Verbosef("load all snapshots\n")
 
 	// find referenced blobs
-	snapshots, err := restic.LoadAllSnapshots(repo)
+	snapshots, err := restic.LoadAllSnapshots(ctx, repo)
 	if err != nil {
 		return err
 	}
@@ -152,12 +165,16 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
 	for _, sn := range snapshots {
 		debug.Log("process snapshot %v", sn.ID().Str())
 
-		err = restic.FindUsedBlobs(repo, *sn.Tree, usedBlobs, seenBlobs)
+		err = restic.FindUsedBlobs(ctx, repo, *sn.Tree, usedBlobs, seenBlobs)
 		if err != nil {
+			if repo.Backend().IsNotExist(err) {
+				return errors.Fatal("unable to load a tree from the repo: " + err.Error())
+			}
+
 			return err
 		}
 
-		debug.Log("found %v blobs for snapshot %v", sn.ID().Str())
+		debug.Log("processed snapshot %v", sn.ID().Str())
 		bar.Report(restic.Stat{Blobs: 1})
 	}
 	bar.Done()
@@ -185,6 +202,12 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
 
 	// find packs that are unneeded
 	removePacks := restic.NewIDSet()
+
+	Verbosef("will remove %d invalid files\n", len(invalidFiles))
+	for _, id := range invalidFiles {
+		removePacks.Insert(id)
+	}
+
 	for packID, p := range idx.Packs {
 
 		hasActiveBlob := false
@@ -214,22 +237,29 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
 	Verbosef("will delete %d packs and rewrite %d packs, this frees %s\n",
 		len(removePacks), len(rewritePacks), formatBytes(uint64(removeBytes)))
 
+	var obsoletePacks restic.IDSet
 	if len(rewritePacks) != 0 {
 		bar = newProgressMax(!gopts.Quiet, uint64(len(rewritePacks)), "packs rewritten")
 		bar.Start()
-		err = repository.Repack(repo, rewritePacks, usedBlobs, bar)
+		obsoletePacks, err = repository.Repack(ctx, repo, rewritePacks, usedBlobs, bar)
 		if err != nil {
 			return err
 		}
 		bar.Done()
 	}
 
+	removePacks.Merge(obsoletePacks)
+
+	if err = rebuildIndex(ctx, repo, removePacks); err != nil {
+		return err
+	}
+
 	if len(removePacks) != 0 {
 		bar = newProgressMax(!gopts.Quiet, uint64(len(removePacks)), "packs deleted")
 		bar.Start()
 		for packID := range removePacks {
 			h := restic.Handle{Type: restic.DataFile, Name: packID.String()}
-			err = repo.Backend().Remove(h)
+			err = repo.Backend().Remove(ctx, h)
 			if err != nil {
 				Warnf("unable to remove file %v from the repository\n", packID.Str())
 			}
@@ -238,10 +268,6 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
 		bar.Done()
 	}
 
-	if err = rebuildIndex(ctx, repo); err != nil {
-		return err
-	}
-
 	Verbosef("done\n")
 	return nil
 }

cmd/restic/cmd_rebuild_index.go

@@ -2,19 +2,21 @@ package main
 
 import (
 	"context"
-	"restic"
-	"restic/index"
+
+	"github.com/restic/restic/internal/index"
+	"github.com/restic/restic/internal/restic"
 
 	"github.com/spf13/cobra"
 )
 
 var cmdRebuildIndex = &cobra.Command{
 	Use:   "rebuild-index [flags]",
-	Short: "build a new index file",
+	Short: "Build a new index file",
 	Long: `
 The "rebuild-index" command creates a new index based on the pack files in the
 repository.
 `,
+	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runRebuildIndex(globalOptions)
 	},
@@ -38,19 +40,19 @@ func runRebuildIndex(gopts GlobalOptions) error {
 
 	ctx, cancel := context.WithCancel(gopts.ctx)
 	defer cancel()
-	return rebuildIndex(ctx, repo)
+	return rebuildIndex(ctx, repo, restic.NewIDSet())
 }
 
-func rebuildIndex(ctx context.Context, repo restic.Repository) error {
+func rebuildIndex(ctx context.Context, repo restic.Repository, ignorePacks restic.IDSet) error {
 	Verbosef("counting files in repo\n")
 
 	var packs uint64
-	for range repo.List(restic.DataFile, ctx.Done()) {
+	for range repo.List(ctx, restic.DataFile) {
 		packs++
 	}
 
-	bar := newProgressMax(!globalOptions.Quiet, packs, "packs")
-	idx, err := index.New(repo, bar)
+	bar := newProgressMax(!globalOptions.Quiet, packs-uint64(len(ignorePacks)), "packs")
+	idx, _, err := index.New(ctx, repo, ignorePacks, bar)
 	if err != nil {
 		return err
 	}
@@ -58,11 +60,11 @@ func rebuildIndex(ctx context.Context, repo restic.Repository) error {
 	Verbosef("finding old index files\n")
 
 	var supersedes restic.IDs
-	for id := range repo.List(restic.IndexFile, ctx.Done()) {
+	for id := range repo.List(ctx, restic.IndexFile) {
 		supersedes = append(supersedes, id)
 	}
 
-	id, err := idx.Save(repo, supersedes)
+	id, err := idx.Save(ctx, repo, supersedes)
 	if err != nil {
 		return err
 	}
@@ -72,7 +74,7 @@ func rebuildIndex(ctx context.Context, repo restic.Repository) error {
 	Verbosef("remove %d old index files\n", len(supersedes))
 
 	for _, id := range supersedes {
-		if err := repo.Backend().Remove(restic.Handle{
+		if err := repo.Backend().Remove(ctx, restic.Handle{
 			Type: restic.IndexFile,
 			Name: id.String(),
 		}); err != nil {

cmd/restic/cmd_restore.go

@@ -1,17 +1,17 @@
 package main
 
 import (
-	"restic"
-	"restic/debug"
-	"restic/errors"
-	"restic/filter"
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/filter"
+	"github.com/restic/restic/internal/restic"
 
 	"github.com/spf13/cobra"
 )
 
 var cmdRestore = &cobra.Command{
 	Use:   "restore [flags] snapshotID",
-	Short: "extract the data from a snapshot",
+	Short: "Extract the data from a snapshot",
 	Long: `
 The "restore" command extracts the data from a snapshot from the repository to
 a directory.
@@ -19,6 +19,7 @@ a directory.
 The special snapshot "latest" can be used to restore the latest snapshot in the
 repository.
 `,
+	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runRestore(restoreOptions, globalOptions, args)
 	},
@@ -31,7 +32,7 @@ type RestoreOptions struct {
 	Target  string
 	Host    string
 	Paths   []string
-	Tags    []string
+	Tags    restic.TagLists
 }
 
 var restoreOptions RestoreOptions
@@ -40,16 +41,18 @@ func init() {
 	cmdRoot.AddCommand(cmdRestore)
 
 	flags := cmdRestore.Flags()
-	flags.StringSliceVarP(&restoreOptions.Exclude, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
-	flags.StringSliceVarP(&restoreOptions.Include, "include", "i", nil, "include a `pattern`, exclude everything else (can be specified multiple times)")
+	flags.StringArrayVarP(&restoreOptions.Exclude, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
+	flags.StringArrayVarP(&restoreOptions.Include, "include", "i", nil, "include a `pattern`, exclude everything else (can be specified multiple times)")
 	flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to")
 
 	flags.StringVarP(&restoreOptions.Host, "host", "H", "", `only consider snapshots for this host when the snapshot ID is "latest"`)
-	flags.StringSliceVar(&restoreOptions.Tags, "tag", nil, "only consider snapshots which include this `tag` for snapshot ID \"latest\"")
-	flags.StringSliceVar(&restoreOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
+	flags.Var(&restoreOptions.Tags, "tag", "only consider snapshots which include this `taglist` for snapshot ID \"latest\"")
+	flags.StringArrayVar(&restoreOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
 }
 
 func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
+	ctx := gopts.ctx
+
 	if len(args) != 1 {
 		return errors.Fatal("no snapshot ID specified")
 	}
@@ -79,7 +82,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
-	err = repo.LoadIndex()
+	err = repo.LoadIndex(ctx)
 	if err != nil {
 		return err
 	}
@@ -87,7 +90,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 	var id restic.ID
 
 	if snapshotIDString == "latest" {
-		id, err = restic.FindLatestSnapshot(repo, opts.Paths, opts.Tags, opts.Host)
+		id, err = restic.FindLatestSnapshot(ctx, repo, opts.Paths, opts.Tags, opts.Host)
 		if err != nil {
 			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Host:%v", err, opts.Paths, opts.Host)
 		}
@@ -110,22 +113,32 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 		return nil
 	}
 
-	selectExcludeFilter := func(item string, dstpath string, node *restic.Node) bool {
-		matched, err := filter.List(opts.Exclude, item)
+	selectExcludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
+		matched, _, err := filter.List(opts.Exclude, item)
 		if err != nil {
 			Warnf("error for exclude pattern: %v", err)
 		}
 
-		return !matched
+		// An exclude filter is basically a 'wildcard but foo',
+		// so even if a childMayMatch, other children of a dir may not,
+		// therefore childMayMatch does not matter, but we should not go down
+		// unless the dir is selected for restore
+		selectedForRestore = !matched
+		childMayBeSelected = selectedForRestore && node.Type == "dir"
+
+		return selectedForRestore, childMayBeSelected
 	}
 
-	selectIncludeFilter := func(item string, dstpath string, node *restic.Node) bool {
-		matched, err := filter.List(opts.Include, item)
+	selectIncludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
+		matched, childMayMatch, err := filter.List(opts.Include, item)
 		if err != nil {
 			Warnf("error for include pattern: %v", err)
 		}
 
-		return matched
+		selectedForRestore = matched
+		childMayBeSelected = childMayMatch && node.Type == "dir"
+
+		return selectedForRestore, childMayBeSelected
 	}
 
 	if len(opts.Exclude) > 0 {
@@ -136,7 +149,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 
 	Verbosef("restoring %s to %s\n", res.Snapshot(), opts.Target)
 
-	err = res.RestoreTo(opts.Target)
+	err = res.RestoreTo(ctx, opts.Target)
 	if totalErrors > 0 {
 		Printf("There were %d errors\n", totalErrors)
 	}

cmd/restic/cmd_snapshots.go

@@ -7,17 +7,17 @@ import (
 	"io"
 	"sort"
 
+	"github.com/restic/restic/internal/restic"
 	"github.com/spf13/cobra"
-
-	"restic"
 )
 
 var cmdSnapshots = &cobra.Command{
 	Use:   "snapshots [snapshotID ...]",
-	Short: "list all snapshots",
+	Short: "List all snapshots",
 	Long: `
 The "snapshots" command lists all snapshots stored in the repository.
 `,
+	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runSnapshots(snapshotOptions, globalOptions, args)
 	},
@@ -25,9 +25,10 @@ The "snapshots" command lists all snapshots stored in the repository.
 
 // SnapshotOptions bundles all options for the snapshots command.
 type SnapshotOptions struct {
-	Host  string
-	Tags  []string
-	Paths []string
+	Host    string
+	Tags    restic.TagLists
+	Paths   []string
+	Compact bool
 }
 
 var snapshotOptions SnapshotOptions
@@ -37,8 +38,9 @@ func init() {
 
 	f := cmdSnapshots.Flags()
 	f.StringVarP(&snapshotOptions.Host, "host", "H", "", "only consider snapshots for this `host`")
-	f.StringSliceVar(&snapshotOptions.Tags, "tag", nil, "only consider snapshots which include this `tag` (can be specified multiple times)")
-	f.StringSliceVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)")
+	f.Var(&snapshotOptions.Tags, "tag", "only consider snapshots which include this `taglist` (can be specified multiple times)")
+	f.StringArrayVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)")
+	f.BoolVarP(&snapshotOptions.Compact, "compact", "c", false, "use compact format")
 }
 
 func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error {
@@ -71,13 +73,13 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 		}
 		return nil
 	}
-	PrintSnapshots(gopts.stdout, list)
+	PrintSnapshots(gopts.stdout, list, opts.Compact)
 
 	return nil
 }
 
 // PrintSnapshots prints a text table of the snapshots in list to stdout.
-func PrintSnapshots(stdout io.Writer, list restic.Snapshots) {
+func PrintSnapshots(stdout io.Writer, list restic.Snapshots, compact bool) {
 
 	// Determine the max widths for host and tag.
 	maxHost, maxTag := 10, 6
@@ -93,8 +95,13 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots) {
 	}
 
 	tab := NewTable()
-	tab.Header = fmt.Sprintf("%-8s  %-19s  %-*s  %-*s  %-3s %s", "ID", "Date", -maxHost, "Host", -maxTag, "Tags", "", "Directory")
-	tab.RowFormat = fmt.Sprintf("%%-8s  %%-19s  %%%ds  %%%ds  %%-3s %%s", -maxHost, -maxTag)
+	if !compact {
+		tab.Header = fmt.Sprintf("%-8s  %-19s  %-*s  %-*s  %-3s %s", "ID", "Date", -maxHost, "Host", -maxTag, "Tags", "", "Directory")
+		tab.RowFormat = fmt.Sprintf("%%-8s  %%-19s  %%%ds  %%%ds  %%-3s %%s", -maxHost, -maxTag)
+	} else {
+		tab.Header = fmt.Sprintf("%-8s  %-19s  %-*s  %-*s", "ID", "Date", -maxHost, "Host", -maxTag, "Tags")
+		tab.RowFormat = fmt.Sprintf("%%-8s  %%-19s  %%%ds  %%s", -maxHost)
+	}
 
 	for _, sn := range list {
 		if len(sn.Paths) == 0 {
@@ -116,7 +123,16 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots) {
 			treeElement = "┌──"
 		}
 
-		tab.Rows = append(tab.Rows, []interface{}{sn.ID().Str(), sn.Time.Format(TimeFormat), sn.Hostname, firstTag, treeElement, sn.Paths[0]})
+		if !compact {
+			tab.Rows = append(tab.Rows, []interface{}{sn.ID().Str(), sn.Time.Format(TimeFormat), sn.Hostname, firstTag, treeElement, sn.Paths[0]})
+		} else {
+			allTags := ""
+			for _, tag := range sn.Tags {
+				allTags += tag + " "
+			}
+			tab.Rows = append(tab.Rows, []interface{}{sn.ID().Str(), sn.Time.Format(TimeFormat), sn.Hostname, allTags})
+			continue
+		}
 
 		if len(sn.Tags) > rows {
 			rows = len(sn.Tags)

cmd/restic/cmd_tag.go

@@ -5,15 +5,15 @@ import (
 
 	"github.com/spf13/cobra"
 
-	"restic"
-	"restic/debug"
-	"restic/errors"
-	"restic/repository"
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
 )
 
 var cmdTag = &cobra.Command{
 	Use:   "tag [flags] [snapshot-ID ...]",
-	Short: "modifies tags on snapshots",
+	Short: "Modify tags on snapshots",
 	Long: `
 The "tag" command allows you to modify tags on exiting snapshots.
 
@@ -22,6 +22,7 @@ add tags to/remove tags from the existing set.
 
 When no snapshot-ID is given, all snapshots matching the host, tag and path filter criteria are modified.
 `,
+	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runTag(tagOptions, globalOptions, args)
 	},
@@ -31,7 +32,7 @@ When no snapshot-ID is given, all snapshots matching the host, tag and path filt
 type TagOptions struct {
 	Host       string
 	Paths      []string
-	Tags       []string
+	Tags       restic.TagLists
 	SetTags    []string
 	AddTags    []string
 	RemoveTags []string
@@ -48,8 +49,8 @@ func init() {
 	tagFlags.StringSliceVar(&tagOptions.RemoveTags, "remove", nil, "`tag` which will be removed from the existing tags (can be given multiple times)")
 
 	tagFlags.StringVarP(&tagOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
-	tagFlags.StringSliceVar(&tagOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot-ID is given")
-	tagFlags.StringSliceVar(&tagOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
+	tagFlags.Var(&tagOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot-ID is given")
+	tagFlags.StringArrayVar(&tagOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
 }
 
 func changeTags(repo *repository.Repository, sn *restic.Snapshot, setTags, addTags, removeTags []string) (bool, error) {
@@ -76,7 +77,7 @@ func changeTags(repo *repository.Repository, sn *restic.Snapshot, setTags, addTa
 		}
 
 		// Save the new snapshot.
-		id, err := repo.SaveJSONUnpacked(restic.SnapshotFile, sn)
+		id, err := repo.SaveJSONUnpacked(context.TODO(), restic.SnapshotFile, sn)
 		if err != nil {
 			return false, err
 		}
@@ -89,7 +90,7 @@ func changeTags(repo *repository.Repository, sn *restic.Snapshot, setTags, addTa
 
 		// Remove the old snapshot.
 		h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
-		if err = repo.Backend().Remove(h); err != nil {
+		if err = repo.Backend().Remove(context.TODO(), h); err != nil {
 			return false, err
 		}
 

cmd/restic/cmd_unlock.go

@@ -1,17 +1,19 @@
 package main
 
 import (
-	"restic"
+	"context"
 
+	"github.com/restic/restic/internal/restic"
 	"github.com/spf13/cobra"
 )
 
 var unlockCmd = &cobra.Command{
 	Use:   "unlock",
-	Short: "remove locks other processes created",
+	Short: "Remove locks other processes created",
 	Long: `
 The "unlock" command removes stale locks that have been created by other restic processes.
 `,
+	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runUnlock(unlockOptions, globalOptions)
 	},
@@ -41,7 +43,7 @@ func runUnlock(opts UnlockOptions, gopts GlobalOptions) error {
 		fn = restic.RemoveAllLocks
 	}
 
-	err = fn(repo)
+	err = fn(context.TODO(), repo)
 	if err != nil {
 		return err
 	}

cmd/restic/cmd_version.go

@@ -9,11 +9,12 @@ import (
 
 var versionCmd = &cobra.Command{
 	Use:   "version",
-	Short: "print version information",
+	Short: "Print version information",
 	Long: `
 The "version" command prints detailed information about the build environment
 and the version of this software.
 `,
+	DisableAutoGenTag: true,
 	Run: func(cmd *cobra.Command, args []string) {
 		fmt.Printf("restic %s\ncompiled with %v on %v/%v\n",
 			version, runtime.Version(), runtime.GOOS, runtime.GOARCH)

cmd/restic/exclude.go

@@ -0,0 +1,179 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/filter"
+	"github.com/restic/restic/internal/fs"
+)
+
+// RejectFunc is a function that takes a filename and os.FileInfo of a
+// file that would be included in the backup. The function returns true if it
+// should be excluded (rejected) from the backup.
+type RejectFunc func(path string, fi os.FileInfo) bool
+
+// rejectByPattern returns a RejectFunc which rejects files that match
+// one of the patterns.
+func rejectByPattern(patterns []string) RejectFunc {
+	return func(item string, fi os.FileInfo) bool {
+		matched, _, err := filter.List(patterns, item)
+		if err != nil {
+			Warnf("error for exclude pattern: %v", err)
+		}
+
+		if matched {
+			debug.Log("path %q excluded by an exclude pattern", item)
+			return true
+		}
+
+		return false
+	}
+}
+
+// rejectIfPresent returns a RejectFunc which itself returns whether a path
+// should be excluded. The RejectFunc considers a file to be excluded when
+// it resides in a directory with an exclusion file, that is specified by
+// excludeFileSpec in the form "filename[:content]". The returned error is
+// non-nil if the filename component of excludeFileSpec is empty.
+func rejectIfPresent(excludeFileSpec string) (RejectFunc, error) {
+	if excludeFileSpec == "" {
+		return nil, errors.New("name for exclusion tagfile is empty")
+	}
+	colon := strings.Index(excludeFileSpec, ":")
+	if colon == 0 {
+		return nil, fmt.Errorf("no name for exclusion tagfile provided")
+	}
+	tf, tc := "", ""
+	if colon > 0 {
+		tf = excludeFileSpec[:colon]
+		tc = excludeFileSpec[colon+1:]
+	} else {
+		tf = excludeFileSpec
+	}
+	debug.Log("using %q as exclusion tagfile", tf)
+	fn := func(filename string, _ os.FileInfo) bool {
+		return isExcludedByFile(filename, tf, tc)
+	}
+	return fn, nil
+}
+
+// isExcludedByFile interprets filename as a path and returns true if that file
+// is in a excluded directory. A directory is identified as excluded if it contains a
+// tagfile which bears the name specified in tagFilename and starts with header.
+func isExcludedByFile(filename, tagFilename, header string) bool {
+	if tagFilename == "" {
+		return false
+	}
+	dir, base := filepath.Split(filename)
+	if base == tagFilename {
+		return false // do not exclude the tagfile itself
+	}
+	tf := filepath.Join(dir, tagFilename)
+	_, err := fs.Lstat(tf)
+	if os.IsNotExist(err) {
+		return false
+	}
+	if err != nil {
+		Warnf("could not access exclusion tagfile: %v", err)
+		return false
+	}
+	// when no signature is given, the mere presence of tf is enough reason
+	// to exclude filename
+	if len(header) == 0 {
+		return true
+	}
+	// From this stage, errors mean tagFilename exists but it is malformed.
+	// Warnings will be generated so that the user is informed that the
+	// indented ignore-action is not performed.
+	f, err := os.Open(tf)
+	if err != nil {
+		Warnf("could not open exclusion tagfile: %v", err)
+		return false
+	}
+	defer f.Close()
+	buf := make([]byte, len(header))
+	_, err = io.ReadFull(f, buf)
+	// EOF is handled with a dedicated message, otherwise the warning were too cryptic
+	if err == io.EOF {
+		Warnf("invalid (too short) signature in exclusion tagfile %q\n", tf)
+		return false
+	}
+	if err != nil {
+		Warnf("could not read signature from exclusion tagfile %q: %v\n", tf, err)
+		return false
+	}
+	if bytes.Compare(buf, []byte(header)) != 0 {
+		Warnf("invalid signature in exclusion tagfile %q\n", tf)
+		return false
+	}
+	return true
+}
+
+// gatherDevices returns the set of unique device ids of the files and/or
+// directory paths listed in "items".
+func gatherDevices(items []string) (deviceMap map[string]uint64, err error) {
+	deviceMap = make(map[string]uint64)
+	for _, item := range items {
+		fi, err := fs.Lstat(item)
+		if err != nil {
+			return nil, err
+		}
+		id, err := fs.DeviceID(fi)
+		if err != nil {
+			return nil, err
+		}
+		deviceMap[item] = id
+	}
+	if len(deviceMap) == 0 {
+		return nil, errors.New("zero allowed devices")
+	}
+	return deviceMap, nil
+}
+
+// rejectByDevice returns a RejectFunc that rejects files which are on a
+// different file systems than the files/dirs in samples.
+func rejectByDevice(samples []string) (RejectFunc, error) {
+	allowed, err := gatherDevices(samples)
+	if err != nil {
+		return nil, err
+	}
+	debug.Log("allowed devices: %v\n", allowed)
+
+	return func(item string, fi os.FileInfo) bool {
+		if fi == nil {
+			return false
+		}
+
+		id, err := fs.DeviceID(fi)
+		if err != nil {
+			// This should never happen because gatherDevices() would have
+			// errored out earlier. If it still does that's a reason to panic.
+			panic(err)
+		}
+
+		for dir := item; dir != ""; dir = filepath.Dir(dir) {
+			debug.Log("item %v, test dir %v", item, dir)
+
+			allowedID, ok := allowed[dir]
+			if !ok {
+				continue
+			}
+
+			if allowedID != id {
+				debug.Log("path %q on disallowed device %d", item, id)
+				return true
+			}
+
+			return false
+		}
+
+		panic(fmt.Sprintf("item %v, device id %v not found, allowedDevs: %v", item, id, allowed))
+	}, nil
+}

cmd/restic/exclude_test.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"io/ioutil"
+	"path/filepath"
+	"testing"
+
+	"github.com/restic/restic/internal/test"
+)
+
+func TestRejectByPattern(t *testing.T) {
+	var tests = []struct {
+		filename string
+		reject   bool
+	}{
+		{filename: "/home/user/foo.go", reject: true},
+		{filename: "/home/user/foo.c", reject: false},
+		{filename: "/home/user/foobar", reject: false},
+		{filename: "/home/user/foobar/x", reject: true},
+		{filename: "/home/user/README", reject: false},
+		{filename: "/home/user/README.md", reject: true},
+	}
+
+	patterns := []string{"*.go", "README.md", "/home/user/foobar/*"}
+
+	for _, tc := range tests {
+		t.Run("", func(t *testing.T) {
+			reject := rejectByPattern(patterns)
+			res := reject(tc.filename, nil)
+			if res != tc.reject {
+				t.Fatalf("wrong result for filename %v: want %v, got %v",
+					tc.filename, tc.reject, res)
+			}
+		})
+	}
+}
+
+func TestIsExcludedByFile(t *testing.T) {
+	const (
+		tagFilename = "CACHEDIR.TAG"
+		header      = "Signature: 8a477f597d28d172789f06886806bc55"
+	)
+	tests := []struct {
+		name    string
+		tagFile string
+		content string
+		want    bool
+	}{
+		{"NoTagfile", "", "", false},
+		{"EmptyTagfile", tagFilename, "", true},
+		{"UnnamedTagFile", "", header, false},
+		{"WrongTagFile", "notatagfile", header, false},
+		{"IncorrectSig", tagFilename, header[1:], false},
+		{"ValidSig", tagFilename, header, true},
+		{"ValidPlusStuff", tagFilename, header + "foo", true},
+		{"ValidPlusNewlineAndStuff", tagFilename, header + "\nbar", true},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			tempDir, cleanup := test.TempDir(t)
+			defer cleanup()
+
+			foo := filepath.Join(tempDir, "foo")
+			err := ioutil.WriteFile(foo, []byte("foo"), 0666)
+			if err != nil {
+				t.Fatalf("could not write file: %v", err)
+			}
+			if tc.tagFile != "" {
+				tagFile := filepath.Join(tempDir, tc.tagFile)
+				err = ioutil.WriteFile(tagFile, []byte(tc.content), 0666)
+				if err != nil {
+					t.Fatalf("could not write tagfile: %v", err)
+				}
+			}
+			h := header
+			if tc.content == "" {
+				h = ""
+			}
+			if got := isExcludedByFile(foo, tagFilename, h); tc.want != got {
+				t.Fatalf("expected %v, got %v", tc.want, got)
+			}
+		})
+	}
+}

cmd/restic/find.go

@@ -3,12 +3,12 @@ package main
 import (
 	"context"
 
-	"restic"
-	"restic/repository"
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
 )
 
 // FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots.
-func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, host string, tags []string, paths []string, snapshotIDs []string) <-chan *restic.Snapshot {
+func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, host string, tags []restic.TagList, paths []string, snapshotIDs []string) <-chan *restic.Snapshot {
 	out := make(chan *restic.Snapshot)
 	go func() {
 		defer close(out)
@@ -22,7 +22,7 @@ func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, hos
 			// Process all snapshot IDs given as arguments.
 			for _, s := range snapshotIDs {
 				if s == "latest" {
-					id, err = restic.FindLatestSnapshot(repo, paths, tags, host)
+					id, err = restic.FindLatestSnapshot(ctx, repo, paths, tags, host)
 					if err != nil {
 						Warnf("Ignoring %q, no snapshot matched given filter (Paths:%v Tags:%v Host:%v)\n", s, paths, tags, host)
 						usedFilter = true
@@ -44,7 +44,7 @@ func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, hos
 			}
 
 			for _, id := range ids.Uniq() {
-				sn, err := restic.LoadSnapshot(repo, id)
+				sn, err := restic.LoadSnapshot(ctx, repo, id)
 				if err != nil {
 					Warnf("Ignoring %q, could not load snapshot: %v\n", id, err)
 					continue
@@ -58,15 +58,7 @@ func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, hos
 			return
 		}
 
-		for id := range repo.List(restic.SnapshotFile, ctx.Done()) {
-			sn, err := restic.LoadSnapshot(repo, id)
-			if err != nil {
-				Warnf("Ignoring %q, could not load snapshot: %v\n", id, err)
-				continue
-			}
-			if (host != "" && host != sn.Hostname) || !sn.HasTags(tags) || !sn.HasPaths(paths) {
-				continue
-			}
+		for _, sn := range restic.FindFilteredSnapshots(ctx, repo, host, tags, paths) {
 			select {
 			case <-ctx.Done():
 				return

cmd/restic/flags_test.go

@@ -8,10 +8,6 @@ import (
 // TestFlags checks for double defined flags, the commands will panic on
 // ParseFlags() when a shorthand flag is defined twice.
 func TestFlags(t *testing.T) {
-	type FlagParser interface {
-		ParseFlags([]string) error
-	}
-
 	for _, cmd := range cmdRoot.Commands() {
 		t.Run(cmd.Name(), func(t *testing.T) {
 			cmd.Flags().SetOutput(ioutil.Discard)

cmd/restic/format.go

@@ -6,7 +6,7 @@ import (
 	"path/filepath"
 	"time"
 
-	"restic"
+	"github.com/restic/restic/internal/restic"
 )
 
 func formatBytes(c uint64) string {

cmd/restic/global.go

@@ -6,21 +6,25 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
-	"restic"
 	"runtime"
 	"strings"
 	"syscall"
 
-	"restic/backend/local"
-	"restic/backend/location"
-	"restic/backend/rest"
-	"restic/backend/s3"
-	"restic/backend/sftp"
-	"restic/debug"
-	"restic/options"
-	"restic/repository"
+	"github.com/restic/restic/internal/backend/azure"
+	"github.com/restic/restic/internal/backend/b2"
+	"github.com/restic/restic/internal/backend/gs"
+	"github.com/restic/restic/internal/backend/local"
+	"github.com/restic/restic/internal/backend/location"
+	"github.com/restic/restic/internal/backend/rest"
+	"github.com/restic/restic/internal/backend/s3"
+	"github.com/restic/restic/internal/backend/sftp"
+	"github.com/restic/restic/internal/backend/swift"
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/options"
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
 
-	"restic/errors"
+	"github.com/restic/restic/internal/errors"
 
 	"golang.org/x/crypto/ssh/terminal"
 )
@@ -51,11 +55,6 @@ var globalOptions = GlobalOptions{
 }
 
 func init() {
-	pw := os.Getenv("RESTIC_PASSWORD")
-	if pw != "" {
-		globalOptions.password = pw
-	}
-
 	var cancel context.CancelFunc
 	globalOptions.ctx, cancel = context.WithCancel(context.Background())
 	AddCleanupHandler(func() error {
@@ -65,7 +64,7 @@ func init() {
 
 	f := cmdRoot.PersistentFlags()
 	f.StringVarP(&globalOptions.Repo, "repo", "r", os.Getenv("RESTIC_REPOSITORY"), "repository to backup to or restore from (default: $RESTIC_REPOSITORY)")
-	f.StringVarP(&globalOptions.PasswordFile, "password-file", "p", "", "read the repository password from a file")
+	f.StringVarP(&globalOptions.PasswordFile, "password-file", "p", os.Getenv("RESTIC_PASSWORD_FILE"), "read the repository password from a file (default: $RESTIC_PASSWORD_FILE)")
 	f.BoolVarP(&globalOptions.Quiet, "quiet", "q", false, "do not output comprehensive progress report")
 	f.BoolVar(&globalOptions.NoLock, "no-lock", false, "do not lock the repo, this allows some operations on read-only repos")
 	f.BoolVarP(&globalOptions.JSON, "json", "", false, "set output mode to JSON for commands that support it")
@@ -205,6 +204,23 @@ func Exitf(exitcode int, format string, args ...interface{}) {
 	Exit(exitcode)
 }
 
+// resolvePassword determines the password to be used for opening the repository.
+func resolvePassword(opts GlobalOptions, env string) (string, error) {
+	if opts.PasswordFile != "" {
+		s, err := ioutil.ReadFile(opts.PasswordFile)
+		if os.IsNotExist(err) {
+			return "", errors.Fatalf("%s does not exist", opts.PasswordFile)
+		}
+		return strings.TrimSpace(string(s)), errors.Wrap(err, "Readfile")
+	}
+
+	if pwd := os.Getenv(env); pwd != "" {
+		return pwd, nil
+	}
+
+	return "", nil
+}
+
 // readPassword reads the password from the given reader directly.
 func readPassword(in io.Reader) (password string, err error) {
 	buf := make([]byte, 1000)
@@ -236,13 +252,8 @@ func readPasswordTerminal(in *os.File, out io.Writer, prompt string) (password s
 // ReadPassword reads the password from a password file, the environment
 // variable RESTIC_PASSWORD or prompts the user.
 func ReadPassword(opts GlobalOptions, prompt string) (string, error) {
-	if opts.PasswordFile != "" {
-		s, err := ioutil.ReadFile(opts.PasswordFile)
-		return strings.TrimSpace(string(s)), errors.Wrap(err, "Readfile")
-	}
-
-	if pwd := os.Getenv("RESTIC_PASSWORD"); pwd != "" {
-		return pwd, nil
+	if opts.password != "" {
+		return opts.password, nil
 	}
 
 	var (
@@ -301,16 +312,14 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
 
 	s := repository.New(be)
 
-	if opts.password == "" {
-		opts.password, err = ReadPassword(opts, "enter password for repository: ")
-		if err != nil {
-			return nil, err
-		}
+	opts.password, err = ReadPassword(opts, "enter password for repository: ")
+	if err != nil {
+		return nil, err
 	}
 
-	err = s.SearchKey(opts.password, maxKeys)
+	err = s.SearchKey(context.TODO(), opts.password, maxKeys)
 	if err != nil {
-		return nil, errors.Fatalf("unable to open repo: %v", err)
+		return nil, err
 	}
 
 	return s, nil
@@ -356,6 +365,79 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
 		debug.Log("opening s3 repository at %#v", cfg)
 		return cfg, nil
 
+	case "gs":
+		cfg := loc.Config.(gs.Config)
+		if cfg.ProjectID == "" {
+			cfg.ProjectID = os.Getenv("GOOGLE_PROJECT_ID")
+		}
+
+		if cfg.JSONKeyPath == "" {
+			if path := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS"); path != "" {
+				// Check read access
+				if _, err := ioutil.ReadFile(path); err != nil {
+					return nil, errors.Fatalf("Failed to read google credential from file %v: %v", path, err)
+				}
+				cfg.JSONKeyPath = path
+			} else {
+				return nil, errors.Fatal("No credential file path is set")
+			}
+		}
+
+		if err := opts.Apply(loc.Scheme, &cfg); err != nil {
+			return nil, err
+		}
+
+		debug.Log("opening gs repository at %#v", cfg)
+		return cfg, nil
+
+	case "azure":
+		cfg := loc.Config.(azure.Config)
+		if cfg.AccountName == "" {
+			cfg.AccountName = os.Getenv("AZURE_ACCOUNT_NAME")
+		}
+
+		if cfg.AccountKey == "" {
+			cfg.AccountKey = os.Getenv("AZURE_ACCOUNT_KEY")
+		}
+
+		if err := opts.Apply(loc.Scheme, &cfg); err != nil {
+			return nil, err
+		}
+
+		debug.Log("opening gs repository at %#v", cfg)
+		return cfg, nil
+
+	case "swift":
+		cfg := loc.Config.(swift.Config)
+
+		if err := swift.ApplyEnvironment("", &cfg); err != nil {
+			return nil, err
+		}
+
+		if err := opts.Apply(loc.Scheme, &cfg); err != nil {
+			return nil, err
+		}
+
+		debug.Log("opening swift repository at %#v", cfg)
+		return cfg, nil
+
+	case "b2":
+		cfg := loc.Config.(b2.Config)
+
+		if cfg.AccountID == "" {
+			cfg.AccountID = os.Getenv("B2_ACCOUNT_ID")
+		}
+
+		if cfg.Key == "" {
+			cfg.Key = os.Getenv("B2_ACCOUNT_KEY")
+		}
+
+		if err := opts.Apply(loc.Scheme, &cfg); err != nil {
+			return nil, err
+		}
+
+		debug.Log("opening b2 repository at %#v", cfg)
+		return cfg, nil
 	case "rest":
 		cfg := loc.Config.(rest.Config)
 		if err := opts.Apply(loc.Scheme, &cfg); err != nil {
@@ -391,6 +473,14 @@ func open(s string, opts options.Options) (restic.Backend, error) {
 		be, err = sftp.Open(cfg.(sftp.Config))
 	case "s3":
 		be, err = s3.Open(cfg.(s3.Config))
+	case "gs":
+		be, err = gs.Open(cfg.(gs.Config))
+	case "azure":
+		be, err = azure.Open(cfg.(azure.Config))
+	case "swift":
+		be, err = swift.Open(cfg.(swift.Config))
+	case "b2":
+		be, err = b2.Open(cfg.(b2.Config))
 	case "rest":
 		be, err = rest.Open(cfg.(rest.Config))
 
@@ -403,7 +493,7 @@ func open(s string, opts options.Options) (restic.Backend, error) {
 	}
 
 	// check if config is there
-	fi, err := be.Stat(restic.Handle{Type: restic.ConfigFile})
+	fi, err := be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
 	if err != nil {
 		return nil, errors.Fatalf("unable to open config file: %v\nIs there a repository at the following location?\n%v", err, s)
 	}
@@ -434,7 +524,15 @@ func create(s string, opts options.Options) (restic.Backend, error) {
 	case "sftp":
 		return sftp.Create(cfg.(sftp.Config))
 	case "s3":
-		return s3.Open(cfg.(s3.Config))
+		return s3.Create(cfg.(s3.Config))
+	case "gs":
+		return gs.Create(cfg.(gs.Config))
+	case "azure":
+		return azure.Create(cfg.(azure.Config))
+	case "swift":
+		return swift.Open(cfg.(swift.Config))
+	case "b2":
+		return b2.Create(cfg.(b2.Config))
 	case "rest":
 		return rest.Create(cfg.(rest.Config))
 	}

cmd/restic/global_debug.go

@@ -7,8 +7,9 @@ import (
 	"net/http"
 	_ "net/http/pprof"
 	"os"
-	"restic/errors"
-	"restic/repository"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/repository"
 
 	"github.com/pkg/profile"
 )
@@ -18,10 +19,6 @@ var (
 	memProfilePath      string
 	cpuProfilePath      string
 	insecure            bool
-
-	prof interface {
-		Stop()
-	}
 )
 
 func init() {
@@ -53,10 +50,21 @@ func runDebug() error {
 		return errors.Fatal("only one profile (memory or CPU) may be activated at the same time")
 	}
 
+	var prof interface {
+		Stop()
+	}
+
 	if memProfilePath != "" {
-		prof = profile.Start(profile.Quiet, profile.MemProfile, profile.ProfilePath(memProfilePath))
+		prof = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.MemProfile, profile.ProfilePath(memProfilePath))
 	} else if cpuProfilePath != "" {
-		prof = profile.Start(profile.Quiet, profile.CPUProfile, profile.ProfilePath(cpuProfilePath))
+		prof = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.CPUProfile, profile.ProfilePath(cpuProfilePath))
+	}
+
+	if prof != nil {
+		AddCleanupHandler(func() error {
+			prof.Stop()
+			return nil
+		})
 	}
 
 	if insecure {
@@ -65,9 +73,3 @@ func runDebug() error {
 
 	return nil
 }
-
-func shutdownDebug() {
-	if prof != nil {
-		prof.Stop()
-	}
-}

cmd/restic/global_release.go

@@ -4,6 +4,3 @@ package main
 
 // runDebug is a noop without the debug tag.
 func runDebug() error { return nil }
-
-// shutdownDebug is a noop without the debug tag.
-func shutdownDebug() {}

cmd/restic/integration_fuse_test.go

@@ -1,20 +1,19 @@
-// +build ignore
 // +build !openbsd
 // +build !windows
 
 package main
 
 import (
+	"context"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"testing"
 	"time"
 
-	"restic"
-	"restic/repository"
-	. "restic/test"
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
+	. "github.com/restic/restic/internal/test"
 )
 
 const (
@@ -55,17 +54,15 @@ func waitForMount(t testing.TB, dir string) {
 	t.Errorf("subdir %q of dir %s never appeared", mountTestSubdir, dir)
 }
 
-func mount(t testing.TB, global GlobalOptions, dir string) {
-	cmd := &CmdMount{global: &global}
-	OK(t, cmd.Mount(dir))
+func testRunMount(t testing.TB, gopts GlobalOptions, dir string) {
+	opts := MountOptions{}
+	OK(t, runMount(opts, gopts, []string{dir}))
 }
 
-func umount(t testing.TB, global GlobalOptions, dir string) {
-	cmd := &CmdMount{global: &global}
-
+func testRunUmount(t testing.TB, gopts GlobalOptions, dir string) {
 	var err error
 	for i := 0; i < mountWait; i++ {
-		if err = cmd.Umount(dir); err == nil {
+		if err = umount(dir); err == nil {
 			t.Logf("directory %v umounted", dir)
 			return
 		}
@@ -87,9 +84,10 @@ func listSnapshots(t testing.TB, dir string) []string {
 
 func checkSnapshots(t testing.TB, global GlobalOptions, repo *repository.Repository, mountpoint, repodir string, snapshotIDs restic.IDs) {
 	t.Logf("checking for %d snapshots: %v", len(snapshotIDs), snapshotIDs)
-	go mount(t, global, mountpoint)
+
+	go testRunMount(t, global, mountpoint)
 	waitForMount(t, mountpoint)
-	defer umount(t, global, mountpoint)
+	defer testRunUmount(t, global, mountpoint)
 
 	if !snapshotsDirExists(t, mountpoint) {
 		t.Fatal(`virtual directory "snapshots" doesn't exist`)
@@ -110,7 +108,7 @@ func checkSnapshots(t testing.TB, global GlobalOptions, repo *repository.Reposit
 	}
 
 	for _, id := range snapshotIDs {
-		snapshot, err := restic.LoadSnapshot(repo, id)
+		snapshot, err := restic.LoadSnapshot(context.TODO(), repo, id)
 		OK(t, err)
 
 		ts := snapshot.Time.Format(time.RFC3339)
@@ -144,46 +142,45 @@ func TestMount(t *testing.T) {
 		t.Skip("Skipping fuse tests")
 	}
 
-	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
 
-		cmdInit(t, global)
-		repo, err := global.OpenRepository()
-		OK(t, err)
+	testRunInit(t, env.gopts)
 
-		mountpoint, err := ioutil.TempDir(TestTempDir, "restic-test-mount-")
-		OK(t, err)
+	repo, err := OpenRepository(env.gopts)
+	OK(t, err)
 
-		// We remove the mountpoint now to check that cmdMount creates it
-		RemoveAll(t, mountpoint)
+	// We remove the mountpoint now to check that cmdMount creates it
+	RemoveAll(t, env.mountpoint)
 
-		checkSnapshots(t, global, repo, mountpoint, env.repo, []restic.ID{})
+	checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, []restic.ID{})
 
-		SetupTarTestFixture(t, env.testdata, filepath.Join("testdata", "backup-data.tar.gz"))
+	SetupTarTestFixture(t, env.testdata, filepath.Join("testdata", "backup-data.tar.gz"))
 
-		// first backup
-		cmdBackup(t, global, []string{env.testdata}, nil)
-		snapshotIDs := cmdList(t, global, "snapshots")
-		Assert(t, len(snapshotIDs) == 1,
-			"expected one snapshot, got %v", snapshotIDs)
+	// first backup
+	testRunBackup(t, []string{env.testdata}, BackupOptions{}, env.gopts)
+	snapshotIDs := testRunList(t, "snapshots", env.gopts)
+	Assert(t, len(snapshotIDs) == 1,
+		"expected one snapshot, got %v", snapshotIDs)
 
-		checkSnapshots(t, global, repo, mountpoint, env.repo, snapshotIDs)
+	checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, snapshotIDs)
 
-		// second backup, implicit incremental
-		cmdBackup(t, global, []string{env.testdata}, nil)
-		snapshotIDs = cmdList(t, global, "snapshots")
-		Assert(t, len(snapshotIDs) == 2,
-			"expected two snapshots, got %v", snapshotIDs)
+	// second backup, implicit incremental
+	testRunBackup(t, []string{env.testdata}, BackupOptions{}, env.gopts)
+	snapshotIDs = testRunList(t, "snapshots", env.gopts)
+	Assert(t, len(snapshotIDs) == 2,
+		"expected two snapshots, got %v", snapshotIDs)
 
-		checkSnapshots(t, global, repo, mountpoint, env.repo, snapshotIDs)
+	checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, snapshotIDs)
 
-		// third backup, explicit incremental
-		cmdBackup(t, global, []string{env.testdata}, &snapshotIDs[0])
-		snapshotIDs = cmdList(t, global, "snapshots")
-		Assert(t, len(snapshotIDs) == 3,
-			"expected three snapshots, got %v", snapshotIDs)
+	// third backup, explicit incremental
+	bopts := BackupOptions{Parent: snapshotIDs[0].String()}
+	testRunBackup(t, []string{env.testdata}, bopts, env.gopts)
+	snapshotIDs = testRunList(t, "snapshots", env.gopts)
+	Assert(t, len(snapshotIDs) == 3,
+		"expected three snapshots, got %v", snapshotIDs)
 
-		checkSnapshots(t, global, repo, mountpoint, env.repo, snapshotIDs)
-	})
+	checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, snapshotIDs)
 }
 
 func TestMountSameTimestamps(t *testing.T) {
@@ -191,21 +188,19 @@ func TestMountSameTimestamps(t *testing.T) {
 		t.Skip("Skipping fuse tests")
 	}
 
-	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
-		SetupTarTestFixture(t, env.base, filepath.Join("testdata", "repo-same-timestamps.tar.gz"))
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
 
-		repo, err := global.OpenRepository()
-		OK(t, err)
+	SetupTarTestFixture(t, env.base, filepath.Join("testdata", "repo-same-timestamps.tar.gz"))
 
-		mountpoint, err := ioutil.TempDir(TestTempDir, "restic-test-mount-")
-		OK(t, err)
+	repo, err := OpenRepository(env.gopts)
+	OK(t, err)
 
-		ids := []restic.ID{
-			restic.TestParseID("280303689e5027328889a06d718b729e96a1ce6ae9ef8290bff550459ae611ee"),
-			restic.TestParseID("75ad6cdc0868e082f2596d5ab8705e9f7d87316f5bf5690385eeff8dbe49d9f5"),
-			restic.TestParseID("5fd0d8b2ef0fa5d23e58f1e460188abb0f525c0f0c4af8365a1280c807a80a1b"),
-		}
+	ids := []restic.ID{
+		restic.TestParseID("280303689e5027328889a06d718b729e96a1ce6ae9ef8290bff550459ae611ee"),
+		restic.TestParseID("75ad6cdc0868e082f2596d5ab8705e9f7d87316f5bf5690385eeff8dbe49d9f5"),
+		restic.TestParseID("5fd0d8b2ef0fa5d23e58f1e460188abb0f525c0f0c4af8365a1280c807a80a1b"),
+	}
 
-		checkSnapshots(t, global, repo, mountpoint, env.repo, ids)
-	})
+	checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, ids)
 }

cmd/restic/integration_helpers_test.go

@@ -9,9 +9,9 @@ import (
 	"runtime"
 	"testing"
 
-	"restic/options"
-	"restic/repository"
-	. "restic/test"
+	"github.com/restic/restic/internal/options"
+	"github.com/restic/restic/internal/repository"
+	. "github.com/restic/restic/internal/test"
 )
 
 type dirEntry struct {
@@ -167,12 +167,13 @@ func dirStats(dir string) (stat dirStat) {
 }
 
 type testEnvironment struct {
-	base, cache, repo, testdata string
+	base, cache, repo, mountpoint, testdata string
+	gopts                                   GlobalOptions
 }
 
-// withTestEnvironment creates a test environment and calls f with it. After f has
-// returned, the temporary directory is removed.
-func withTestEnvironment(t testing.TB, f func(*testEnvironment, GlobalOptions)) {
+// withTestEnvironment creates a test environment and returns a cleanup
+// function which removes it.
+func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) {
 	if !RunIntegrationTest {
 		t.Skip("integration tests disabled")
 	}
@@ -182,18 +183,20 @@ func withTestEnvironment(t testing.TB, f func(*testEnvironment, GlobalOptions))
 	tempdir, err := ioutil.TempDir(TestTempDir, "restic-test-")
 	OK(t, err)
 
-	env := testEnvironment{
-		base:     tempdir,
-		cache:    filepath.Join(tempdir, "cache"),
-		repo:     filepath.Join(tempdir, "repo"),
-		testdata: filepath.Join(tempdir, "testdata"),
+	env = &testEnvironment{
+		base:       tempdir,
+		cache:      filepath.Join(tempdir, "cache"),
+		repo:       filepath.Join(tempdir, "repo"),
+		testdata:   filepath.Join(tempdir, "testdata"),
+		mountpoint: filepath.Join(tempdir, "mount"),
 	}
 
+	OK(t, os.MkdirAll(env.mountpoint, 0700))
 	OK(t, os.MkdirAll(env.testdata, 0700))
 	OK(t, os.MkdirAll(env.cache, 0700))
 	OK(t, os.MkdirAll(env.repo, 0700))
 
-	gopts := GlobalOptions{
+	env.gopts = GlobalOptions{
 		Repo:     env.repo,
 		Quiet:    true,
 		ctx:      context.Background(),
@@ -204,14 +207,15 @@ func withTestEnvironment(t testing.TB, f func(*testEnvironment, GlobalOptions))
 	}
 
 	// always overwrite global options
-	globalOptions = gopts
+	globalOptions = env.gopts
 
-	f(&env, gopts)
-
-	if !TestCleanupTempDirs {
-		t.Logf("leaving temporary directory %v used for test", tempdir)
-		return
+	cleanup = func() {
+		if !TestCleanupTempDirs {
+			t.Logf("leaving temporary directory %v used for test", tempdir)
+			return
+		}
+		RemoveAll(t, tempdir)
 	}
 
-	RemoveAll(t, tempdir)
+	return env, cleanup
 }

cmd/restic/integration_helpers_unix_test.go

@@ -52,11 +52,6 @@ func nlink(info os.FileInfo) uint64 {
 	return uint64(stat.Nlink)
 }
 
-func inode(info os.FileInfo) uint64 {
-	stat, _ := info.Sys().(*syscall.Stat_t)
-	return uint64(stat.Ino)
-}
-
 func createFileSetPerHardlink(dir string) map[uint64][]string {
 	var stat syscall.Stat_t
 	linkTests := make(map[uint64][]string)

cmd/restic/local_layout_test.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"path/filepath"
+	"testing"
+
+	. "github.com/restic/restic/internal/test"
+)
+
+func TestRestoreLocalLayout(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	var tests = []struct {
+		filename string
+		layout   string
+	}{
+		{"repo-layout-default.tar.gz", ""},
+		{"repo-layout-s3legacy.tar.gz", ""},
+		{"repo-layout-default.tar.gz", "default"},
+		{"repo-layout-s3legacy.tar.gz", "s3legacy"},
+	}
+
+	for _, test := range tests {
+		datafile := filepath.Join("..", "..", "internal", "backend", "testdata", test.filename)
+
+		SetupTarTestFixture(t, env.base, datafile)
+
+		env.gopts.extended["local.layout"] = test.layout
+
+		// check the repo
+		testRunCheck(t, env.gopts)
+
+		// restore latest snapshot
+		target := filepath.Join(env.base, "restore")
+		testRunRestoreLatest(t, env.gopts, target, nil, "")
+
+		RemoveAll(t, filepath.Join(env.base, "repo"))
+		RemoveAll(t, target)
+	}
+}

cmd/restic/lock.go

@@ -1,14 +1,15 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"sync"
 	"time"
 
-	"restic"
-	"restic/debug"
-	"restic/repository"
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
 )
 
 var globalLocks struct {
@@ -32,7 +33,7 @@ func lockRepository(repo *repository.Repository, exclusive bool) (*restic.Lock,
 		lockFn = restic.NewExclusiveLock
 	}
 
-	lock, err := lockFn(repo)
+	lock, err := lockFn(context.TODO(), repo)
 	if err != nil {
 		return nil, err
 	}
@@ -75,7 +76,7 @@ func refreshLocks(wg *sync.WaitGroup, done <-chan struct{}) {
 			debug.Log("refreshing locks")
 			globalLocks.Lock()
 			for _, lock := range globalLocks.locks {
-				err := lock.Refresh()
+				err := lock.Refresh(context.TODO())
 				if err != nil {
 					fmt.Fprintf(os.Stderr, "unable to refresh lock: %v\n", err)
 				}

cmd/restic/main.go

@@ -6,25 +6,28 @@ import (
 	"fmt"
 	"log"
 	"os"
-	"restic"
-	"restic/debug"
-	"restic/options"
+	"runtime"
+
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/options"
+	"github.com/restic/restic/internal/restic"
 
 	"github.com/spf13/cobra"
 
-	"restic/errors"
+	"github.com/restic/restic/internal/errors"
 )
 
 // cmdRoot is the base command when no other command has been specified.
 var cmdRoot = &cobra.Command{
 	Use:   "restic",
-	Short: "backup and restore files",
+	Short: "Backup and restore files",
 	Long: `
 restic is a backup program which allows saving multiple revisions of files and
 directories in an encrypted repository stored on different backends.
 `,
-	SilenceErrors: true,
-	SilenceUsage:  true,
+	SilenceErrors:     true,
+	SilenceUsage:      true,
+	DisableAutoGenTag: true,
 
 	PersistentPreRunE: func(*cobra.Command, []string) error {
 		// parse extended options
@@ -34,6 +37,13 @@ directories in an encrypted repository stored on different backends.
 		}
 		globalOptions.extended = opts
 
+		pwd, err := resolvePassword(globalOptions, "RESTIC_PASSWORD")
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Resolving password failed: %v\n", err)
+			Exit(1)
+		}
+		globalOptions.password = pwd
+
 		// run the debug functions for all subcommands (if build tag "debug" is
 		// enabled)
 		if err := runDebug(); err != nil {
@@ -42,9 +52,6 @@ directories in an encrypted repository stored on different backends.
 
 		return nil
 	},
-	PersistentPostRun: func(*cobra.Command, []string) {
-		shutdownDebug()
-	},
 }
 
 var logBuffer = bytes.NewBuffer(nil)
@@ -57,6 +64,8 @@ func init() {
 
 func main() {
 	debug.Log("main %#v", os.Args)
+	debug.Log("restic %s, compiled with %v on %v/%v",
+		version, runtime.Version(), runtime.GOOS, runtime.GOARCH)
 	err := cmdRoot.Execute()
 
 	switch {

codecov.yml

@@ -1,2 +0,0 @@
-codecov:
-  disable_default_path_fixes: true

doc/design.rst

@@ -315,13 +315,12 @@ divided into a 16 byte AES key ``k`` followed by 16 bytes of secret key
 ``r``. The key ``r`` is then masked for use with Poly1305 (see the paper
 for details).
 
-Those message authentication keys (``k`` and ``r``) are used to compute
-a MAC over the bytes contained in the JSON field ``data`` (after
-removing the Base64 encoding and not including the last 32 byte). If the
+Those keys are used to authenticate and decrypt the bytes contained in
+the JSON field ``data`` with AES-256 and Poly1305-AES as if they were
+any other blob (after removing the Base64 encoding). If the
 password is incorrect or the key file has been tampered with, the
 computed MAC will not match the last 16 bytes of the data, and restic
-exits with an error. Otherwise, the data is decrypted with the
-encryption key derived from ``scrypt``. This yields a JSON document
+exits with an error. Otherwise, the data yields a JSON document
 which contains the master encryption and message authentication keys for
 this repository (encoded in Base64). The command
 ``restic cat masterkey`` can be used as follows to decrypt and

doc/development.rst

@@ -9,7 +9,7 @@ new feature. This way, duplicate work is prevented and we can discuss
 your ideas and design first.
 
 More information and a description of the development environment can be
-found in `CONTRIBUTING.md <CONTRIBUTING.md>`__.
+found in `CONTRIBUTING.md <https://github.com/restic/restic/blob/master/CONTRIBUTING.md>`__.
 A document describing the design of restic and the data structures stored on the
 back end is contained in `Design <https://restic.readthedocs.io/en/latest/design.html>`__.
 

doc/installation.rst

@@ -8,7 +8,7 @@ Mac OS X
 ~~~~~~~~~
 
 If you are using Mac OS X, you can install restic using the
-`homebrew <http://brew.sh/>`__ packet manager:
+`homebrew <http://brew.sh/>`__ package manager:
 
 .. code-block:: console
 
@@ -35,7 +35,7 @@ From Source
 -----------
 
 restic is written in the Go programming language and you need at least
-Go version 1.7. Building restic may also work with older versions of Go,
+Go version 1.8. Building restic may also work with older versions of Go,
 but that's not supported. See the `Getting
 started <https://golang.org/doc/install>`__ guide of the Go project for
 instructions how to install Go.

doc/man/restic-autocomplete.1

@@ -0,0 +1,70 @@
+.TH "restic backup" "1" "Jan 2017" "generated by `restic manpage`" "" 
+.nh
+.ad l
+
+
+.SH NAME
+.PP
+restic\-autocomplete \- Generate shell autocompletion script
+
+
+.SH SYNOPSIS
+.PP
+\fBrestic autocomplete [flags]\fP
+
+
+.SH DESCRIPTION
+.PP
+The "autocomplete" command generates a shell autocompletion script.
+
+.PP
+NOTE: The current version supports Bash only.
+      This should work for *nix systems with Bash installed.
+
+.PP
+By default, the file is written directly to /etc/bash\_completion.d
+for convenience, and the command may need superuser rights, e.g.:
+
+.PP
+$ sudo restic autocomplete
+
+
+.SH OPTIONS
+.PP
+\fB\-\-completionfile\fP="/etc/bash\_completion.d/restic.sh"
+    autocompletion file
+
+.PP
+\fB\-h\fP, \fB\-\-help\fP[=false]
+    help for autocomplete
+
+
+.SH OPTIONS INHERITED FROM PARENT COMMANDS
+.PP
+\fB\-\-json\fP[=false]
+    set output mode to JSON for commands that support it
+
+.PP
+\fB\-\-no\-lock\fP[=false]
+    do not lock the repo, this allows some operations on read\-only repos
+
+.PP
+\fB\-o\fP, \fB\-\-option\fP=[]
+    set extended option (\fB\fCkey=value\fR, can be specified multiple times)
+
+.PP
+\fB\-p\fP, \fB\-\-password\-file\fP=""
+    read the repository password from a file (default: $RESTIC\_PASSWORD\_FILE)
+
+.PP
+\fB\-q\fP, \fB\-\-quiet\fP[=false]
+    do not output comprehensive progress report
+
+.PP
+\fB\-r\fP, \fB\-\-repo\fP=""
+    repository to backup to or restore from (default: $RESTIC\_REPOSITORY)
+
+
+.SH SEE ALSO
+.PP
+\fBrestic(1)\fP

doc/man/restic-backup.1

@@ -0,0 +1,108 @@
+.TH "restic backup" "1" "Jan 2017" "generated by `restic manpage`" "" 
+.nh
+.ad l
+
+
+.SH NAME
+.PP
+restic\-backup \- Create a new backup of files and/or directories
+
+
+.SH SYNOPSIS
+.PP
+\fBrestic backup [flags] FILE/DIR [FILE/DIR] ...\fP
+
+
+.SH DESCRIPTION
+.PP
+The "backup" command creates a new snapshot and saves the files and directories
+given as the arguments.
+
+
+.SH OPTIONS
+.PP
+\fB\-e\fP, \fB\-\-exclude\fP=[]
+    exclude a \fB\fCpattern\fR (can be specified multiple times)
+
+.PP
+\fB\-\-exclude\-caches\fP[=false]
+    excludes cache directories that are marked with a CACHEDIR.TAG file
+
+.PP
+\fB\-\-exclude\-file\fP=[]
+    read exclude patterns from a \fB\fCfile\fR (can be specified multiple times)
+
+.PP
+\fB\-\-exclude\-if\-present\fP=[]
+    takes filename[:header], exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)
+
+.PP
+\fB\-\-files\-from\fP=""
+    read the files to backup from file (can be combined with file args)
+
+.PP
+\fB\-f\fP, \fB\-\-force\fP[=false]
+    force re\-reading the target files/directories (overrides the "parent" flag)
+
+.PP
+\fB\-h\fP, \fB\-\-help\fP[=false]
+    help for backup
+
+.PP
+\fB\-\-hostname\fP=""
+    set the \fB\fChostname\fR for the snapshot manually
+
+.PP
+\fB\-x\fP, \fB\-\-one\-file\-system\fP[=false]
+    exclude other file systems
+
+.PP
+\fB\-\-parent\fP=""
+    use this parent snapshot (default: last snapshot in the repo that has the same target files/directories)
+
+.PP
+\fB\-\-stdin\fP[=false]
+    read backup from stdin
+
+.PP
+\fB\-\-stdin\-filename\fP="stdin"
+    file name to use when reading from stdin
+
+.PP
+\fB\-\-tag\fP=[]
+    add a \fB\fCtag\fR for the new snapshot (can be specified multiple times)
+
+.PP
+\fB\-\-time\fP=""
+    time of the backup (ex. '2012\-11\-01 22:08:41') (default: now)
+
+
+.SH OPTIONS INHERITED FROM PARENT COMMANDS
+.PP
+\fB\-\-json\fP[=false]
+    set output mode to JSON for commands that support it
+
+.PP
+\fB\-\-no\-lock\fP[=false]
+    do not lock the repo, this allows some operations on read\-only repos
+
+.PP
+\fB\-o\fP, \fB\-\-option\fP=[]
+    set extended option (\fB\fCkey=value\fR, can be specified multiple times)
+
+.PP
+\fB\-p\fP, \fB\-\-password\-file\fP=""
+    read the repository password from a file (default: $RESTIC\_PASSWORD\_FILE)
+
+.PP
+\fB\-q\fP, \fB\-\-quiet\fP[=false]
+    do not output comprehensive progress report
+
+.PP
+\fB\-r\fP, \fB\-\-repo\fP=""
+    repository to backup to or restore from (default: $RESTIC\_REPOSITORY)
+
+
+.SH SEE ALSO
+.PP
+\fBrestic(1)\fP

doc/man/restic-cat.1

@@ -0,0 +1,55 @@
+.TH "restic backup" "1" "Jan 2017" "generated by `restic manpage`" "" 
+.nh
+.ad l
+
+
+.SH NAME
+.PP
+restic\-cat \- Print internal objects to stdout
+
+
+.SH SYNOPSIS
+.PP
+\fBrestic cat [flags] [pack|blob|snapshot|index|key|masterkey|config|lock] ID\fP
+
+
+.SH DESCRIPTION
+.PP
+The "cat" command is used to print internal objects to stdout.
+
+
+.SH OPTIONS
+.PP
+\fB\-h\fP, \fB\-\-help\fP[=false]
+    help for cat
+
+
+.SH OPTIONS INHERITED FROM PARENT COMMANDS
+.PP
+\fB\-\-json\fP[=false]
+    set output mode to JSON for commands that support it
+
+.PP
+\fB\-\-no\-lock\fP[=false]
+    do not lock the repo, this allows some operations on read\-only repos
+
+.PP
+\fB\-o\fP, \fB\-\-option\fP=[]
+    set extended option (\fB\fCkey=value\fR, can be specified multiple times)
+
+.PP
+\fB\-p\fP, \fB\-\-password\-file\fP=""
+    read the repository password from a file (default: $RESTIC\_PASSWORD\_FILE)
+
+.PP
+\fB\-q\fP, \fB\-\-quiet\fP[=false]
+    do not output comprehensive progress report
+
+.PP
+\fB\-r\fP, \fB\-\-repo\fP=""
+    repository to backup to or restore from (default: $RESTIC\_REPOSITORY)
+
+
+.SH SEE ALSO
+.PP
+\fBrestic(1)\fP

doc/man/restic-check.1

@@ -0,0 +1,64 @@
+.TH "restic backup" "1" "Jan 2017" "generated by `restic manpage`" "" 
+.nh
+.ad l
+
+
+.SH NAME
+.PP
+restic\-check \- Check the repository for errors
+
+
+.SH SYNOPSIS
+.PP
+\fBrestic check [flags]\fP
+
+
+.SH DESCRIPTION
+.PP
+The "check" command tests the repository for errors and reports any errors it
+finds. It can also be used to read all data and therefore simulate a restore.
+
+
+.SH OPTIONS
+.PP
+\fB\-\-check\-unused\fP[=false]
+    find unused blobs
+
+.PP
+\fB\-h\fP, \fB\-\-help\fP[=false]
+    help for check
+
+.PP
+\fB\-\-read\-data\fP[=false]
+    read all data blobs
+
+
+.SH OPTIONS INHERITED FROM PARENT COMMANDS
+.PP
+\fB\-\-json\fP[=false]
+    set output mode to JSON for commands that support it
+
+.PP
+\fB\-\-no\-lock\fP[=false]
+    do not lock the repo, this allows some operations on read\-only repos
+
+.PP
+\fB\-o\fP, \fB\-\-option\fP=[]
+    set extended option (\fB\fCkey=value\fR, can be specified multiple times)
+
+.PP
+\fB\-p\fP, \fB\-\-password\-file\fP=""
+    read the repository password from a file (default: $RESTIC\_PASSWORD\_FILE)
+
+.PP
+\fB\-q\fP, \fB\-\-quiet\fP[=false]
+    do not output comprehensive progress report
+
+.PP
+\fB\-r\fP, \fB\-\-repo\fP=""
+    repository to backup to or restore from (default: $RESTIC\_REPOSITORY)
+
+
+.SH SEE ALSO
+.PP
+\fBrestic(1)\fP

doc/man/restic-dump.1

@@ -0,0 +1,56 @@
+.TH "restic backup" "1" "Jan 2017" "generated by `restic manpage`" "" 
+.nh
+.ad l
+
+
+.SH NAME
+.PP
+restic\-dump \- Dump data structures
+
+
+.SH SYNOPSIS
+.PP
+\fBrestic dump [indexes|snapshots|trees|all|packs] [flags]\fP
+
+
+.SH DESCRIPTION
+.PP
+The "dump" command dumps data structures from the repository as JSON objects. It
+is used for debugging purposes only.
+
+
+.SH OPTIONS
+.PP
+\fB\-h\fP, \fB\-\-help\fP[=false]
+    help for dump
+
+
+.SH OPTIONS INHERITED FROM PARENT COMMANDS
+.PP
+\fB\-\-json\fP[=false]
+    set output mode to JSON for commands that support it
+
+.PP
+\fB\-\-no\-lock\fP[=false]
+    do not lock the repo, this allows some operations on read\-only repos
+
+.PP
+\fB\-o\fP, \fB\-\-option\fP=[]
+    set extended option (\fB\fCkey=value\fR, can be specified multiple times)
+
+.PP
+\fB\-p\fP, \fB\-\-password\-file\fP=""
+    read the repository password from a file (default: $RESTIC\_PASSWORD\_FILE)
+
+.PP
+\fB\-q\fP, \fB\-\-quiet\fP[=false]
+    do not output comprehensive progress report
+
+.PP
+\fB\-r\fP, \fB\-\-repo\fP=""
+    repository to backup to or restore from (default: $RESTIC\_REPOSITORY)
+
+
+.SH SEE ALSO
+.PP
+\fBrestic(1)\fP

doc/man/restic-find.1

@@ -0,0 +1,88 @@
+.TH "restic backup" "1" "Jan 2017" "generated by `restic manpage`" "" 
+.nh
+.ad l
+
+
+.SH NAME
+.PP
+restic\-find \- Find a file or directory
+
+
+.SH SYNOPSIS
+.PP
+\fBrestic find [flags] PATTERN\fP
+
+
+.SH DESCRIPTION
+.PP
+The "find" command searches for files or directories in snapshots stored in the
+repo.
+
+
+.SH OPTIONS
+.PP
+\fB\-h\fP, \fB\-\-help\fP[=false]
+    help for find
+
+.PP
+\fB\-H\fP, \fB\-\-host\fP=""
+    only consider snapshots for this \fB\fChost\fR, when no snapshot ID is given
+
+.PP
+\fB\-i\fP, \fB\-\-ignore\-case\fP[=false]
+    ignore case for pattern
+
+.PP
+\fB\-l\fP, \fB\-\-long\fP[=false]
+    use a long listing format showing size and mode
+
+.PP
+\fB\-N\fP, \fB\-\-newest\fP=""
+    newest modification date/time
+
+.PP
+\fB\-O\fP, \fB\-\-oldest\fP=""
+    oldest modification date/time
+
+.PP
+\fB\-\-path\fP=[]
+    only consider snapshots which include this (absolute) \fB\fCpath\fR, when no snapshot\-ID is given
+
+.PP
+\fB\-s\fP, \fB\-\-snapshot\fP=[]
+    snapshot \fB\fCid\fR to search in (can be given multiple times)
+
+.PP
+\fB\-\-tag\fP=[]
+    only consider snapshots which include this \fB\fCtaglist\fR, when no snapshot\-ID is given
+
+
+.SH OPTIONS INHERITED FROM PARENT COMMANDS
+.PP
+\fB\-\-json\fP[=false]
+    set output mode to JSON for commands that support it
+
+.PP
+\fB\-\-no\-lock\fP[=false]
+    do not lock the repo, this allows some operations on read\-only repos
+
+.PP
+\fB\-o\fP, \fB\-\-option\fP=[]
+    set extended option (\fB\fCkey=value\fR, can be specified multiple times)
+
+.PP
+\fB\-p\fP, \fB\-\-password\-file\fP=""
+    read the repository password from a file (default: $RESTIC\_PASSWORD\_FILE)
+
+.PP
+\fB\-q\fP, \fB\-\-quiet\fP[=false]
+    do not output comprehensive progress report
+
+.PP
+\fB\-r\fP, \fB\-\-repo\fP=""
+    repository to backup to or restore from (default: $RESTIC\_REPOSITORY)
+
+
+.SH SEE ALSO
+.PP
+\fBrestic(1)\fP

doc/man/restic-forget.1

@@ -0,0 +1,114 @@
+.TH "restic backup" "1" "Jan 2017" "generated by `restic manpage`" "" 
+.nh
+.ad l
+
+
+.SH NAME
+.PP
+restic\-forget \- Remove snapshots from the repository
+
+
+.SH SYNOPSIS
+.PP
+\fBrestic forget [flags] [snapshot ID] [...]\fP
+
+
+.SH DESCRIPTION
+.PP
+The "forget" command removes snapshots according to a policy. Please note that
+this command really only deletes the snapshot object in the repository, which
+is a reference to data stored there. In order to remove this (now unreferenced)
+data after 'forget' was run successfully, see the 'prune' command.
+
+
+.SH OPTIONS
+.PP
+\fB\-l\fP, \fB\-\-keep\-last\fP=0
+    keep the last \fB\fCn\fR snapshots
+
+.PP
+\fB\-H\fP, \fB\-\-keep\-hourly\fP=0
+    keep the last \fB\fCn\fR hourly snapshots
+
+.PP
+\fB\-d\fP, \fB\-\-keep\-daily\fP=0
+    keep the last \fB\fCn\fR daily snapshots
+
+.PP
+\fB\-w\fP, \fB\-\-keep\-weekly\fP=0
+    keep the last \fB\fCn\fR weekly snapshots
+
+.PP
+\fB\-m\fP, \fB\-\-keep\-monthly\fP=0
+    keep the last \fB\fCn\fR monthly snapshots
+
+.PP
+\fB\-y\fP, \fB\-\-keep\-yearly\fP=0
+    keep the last \fB\fCn\fR yearly snapshots
+
+.PP
+\fB\-\-keep\-tag\fP=[]
+    keep snapshots with this \fB\fCtaglist\fR (can be specified multiple times)
+
+.PP
+\fB\-\-host\fP=""
+    only consider snapshots with the given \fB\fChost\fR
+
+.PP
+\fB\-\-hostname\fP=""
+    only consider snapshots with the given \fB\fChostname\fR (deprecated)
+
+.PP
+\fB\-\-tag\fP=[]
+    only consider snapshots which include this \fB\fCtaglist\fR in the format \fB\fCtag[,tag,...]\fR (can be specified multiple times)
+
+.PP
+\fB\-\-path\fP=[]
+    only consider snapshots which include this (absolute) \fB\fCpath\fR (can be specified multiple times)
+
+.PP
+\fB\-g\fP, \fB\-\-group\-by\fP="host,paths"
+    string for grouping snapshots by host,paths,tags
+
+.PP
+\fB\-n\fP, \fB\-\-dry\-run\fP[=false]
+    do not delete anything, just print what would be done
+
+.PP
+\fB\-\-prune\fP[=false]
+    automatically run the 'prune' command if snapshots have been removed
+
+.PP
+\fB\-h\fP, \fB\-\-help\fP[=false]
+    help for forget
+
+
+.SH OPTIONS INHERITED FROM PARENT COMMANDS
+.PP
+\fB\-\-json\fP[=false]
+    set output mode to JSON for commands that support it
+
+.PP
+\fB\-\-no\-lock\fP[=false]
+    do not lock the repo, this allows some operations on read\-only repos
+
+.PP
+\fB\-o\fP, \fB\-\-option\fP=[]
+    set extended option (\fB\fCkey=value\fR, can be specified multiple times)
+
+.PP
+\fB\-p\fP, \fB\-\-password\-file\fP=""
+    read the repository password from a file (default: $RESTIC\_PASSWORD\_FILE)
+
+.PP
+\fB\-q\fP, \fB\-\-quiet\fP[=false]
+    do not output comprehensive progress report
+
+.PP
+\fB\-r\fP, \fB\-\-repo\fP=""
+    repository to backup to or restore from (default: $RESTIC\_REPOSITORY)
+
+
+.SH SEE ALSO
+.PP
+\fBrestic(1)\fP

doc/man/restic-init.1

@@ -0,0 +1,55 @@
+.TH "restic backup" "1" "Jan 2017" "generated by `restic manpage`" "" 
+.nh
+.ad l
+
+
+.SH NAME
+.PP
+restic\-init \- Initialize a new repository
+
+
+.SH SYNOPSIS
+.PP
+\fBrestic init [flags]\fP
+
+
+.SH DESCRIPTION
+.PP
+The "init" command initializes a new repository.
+
+
+.SH OPTIONS
+.PP
+\fB\-h\fP, \fB\-\-help\fP[=false]
+    help for init
+
+
+.SH OPTIONS INHERITED FROM PARENT COMMANDS
+.PP
+\fB\-\-json\fP[=false]
+    set output mode to JSON for commands that support it
+
+.PP
+\fB\-\-no\-lock\fP[=false]
+    do not lock the repo, this allows some operations on read\-only repos
+
+.PP
+\fB\-o\fP, \fB\-\-option\fP=[]
+    set extended option (\fB\fCkey=value\fR, can be specified multiple times)
+
+.PP
+\fB\-p\fP, \fB\-\-password\-file\fP=""
+    read the repository password from a file (default: $RESTIC\_PASSWORD\_FILE)
+
+.PP
+\fB\-q\fP, \fB\-\-quiet\fP[=false]
+    do not output comprehensive progress report
+
+.PP
+\fB\-r\fP, \fB\-\-repo\fP=""
+    repository to backup to or restore from (default: $RESTIC\_REPOSITORY)
+
+
+.SH SEE ALSO
+.PP
+\fBrestic(1)\fP

doc/man/restic-key.1

@@ -0,0 +1,55 @@
+.TH "restic backup" "1" "Jan 2017" "generated by `restic manpage`" "" 
+.nh
+.ad l
+
+
+.SH NAME
+.PP
+restic\-key \- Manage keys (passwords)
+
+
+.SH SYNOPSIS
+.PP
+\fBrestic key [list|add|remove|passwd] [ID] [flags]\fP
+
+
+.SH DESCRIPTION
+.PP
+The "key" command manages keys (passwords) for accessing the repository.
+
+
+.SH OPTIONS
+.PP
+\fB\-h\fP, \fB\-\-help\fP[=false]
+    help for key
+
+
+.SH OPTIONS INHERITED FROM PARENT COMMANDS
+.PP
+\fB\-\-json\fP[=false]
+    set output mode to JSON for commands that support it
+
+.PP
+\fB\-\-no\-lock\fP[=false]
+    do not lock the repo, this allows some operations on read\-only repos
+
+.PP
+\fB\-o\fP, \fB\-\-option\fP=[]
+    set extended option (\fB\fCkey=value\fR, can be specified multiple times)
+
+.PP
+\fB\-p\fP, \fB\-\-password\-file\fP=""
+    read the repository password from a file (default: $RESTIC\_PASSWORD\_FILE)
+
+.PP
+\fB\-q\fP, \fB\-\-quiet\fP[=false]
+    do not output comprehensive progress report
+
+.PP
+\fB\-r\fP, \fB\-\-repo\fP=""
+    repository to backup to or restore from (default: $RESTIC\_REPOSITORY)
+
+
+.SH SEE ALSO
+.PP
+\fBrestic(1)\fP

doc/man/restic-list.1

@@ -0,0 +1,55 @@
+.TH "restic backup" "1" "Jan 2017" "generated by `restic manpage`" "" 
+.nh
+.ad l
+
+
+.SH NAME
+.PP
+restic\-list \- List objects in the repository
+
+
+.SH SYNOPSIS
+.PP
+\fBrestic list [blobs|packs|index|snapshots|keys|locks] [flags]\fP
+
+
+.SH DESCRIPTION
+.PP
+The "list" command allows listing objects in the repository based on type.
+
+
+.SH OPTIONS
+.PP
+\fB\-h\fP, \fB\-\-help\fP[=false]
+    help for list
+
+
+.SH OPTIONS INHERITED FROM PARENT COMMANDS
+.PP
+\fB\-\-json\fP[=false]
+    set output mode to JSON for commands that support it
+
+.PP
+\fB\-\-no\-lock\fP[=false]
+    do not lock the repo, this allows some operations on read\-only repos
+
+.PP
+\fB\-o\fP, \fB\-\-option\fP=[]
+    set extended option (\fB\fCkey=value\fR, can be specified multiple times)
+
+.PP
+\fB\-p\fP, \fB\-\-password\-file\fP=""
+    read the repository password from a file (default: $RESTIC\_PASSWORD\_FILE)
+
+.PP
+\fB\-q\fP, \fB\-\-quiet\fP[=false]
+    do not output comprehensive progress report
+
+.PP
+\fB\-r\fP, \fB\-\-repo\fP=""
+    repository to backup to or restore from (default: $RESTIC\_REPOSITORY)
+
+
+.SH SEE ALSO
+.PP
+\fBrestic(1)\fP

doc/man/restic-ls.1

@@ -0,0 +1,74 @@
+.TH "restic backup" "1" "Jan 2017" "generated by `restic manpage`" "" 
+.nh
+.ad l
+
+
+.SH NAME
+.PP
+restic\-ls \- List files in a snapshot
+
+
+.SH SYNOPSIS
+.PP
+\fBrestic ls [flags] [snapshot\-ID ...]\fP
+
+
+.SH DESCRIPTION
+.PP
+The "ls" command allows listing files and directories in a snapshot.
+
+.PP
+The special snapshot\-ID "latest" can be used to list files and directories of the latest snapshot in the repository.
+
+
+.SH OPTIONS
+.PP
+\fB\-h\fP, \fB\-\-help\fP[=false]
+    help for ls
+
+.PP
+\fB\-H\fP, \fB\-\-host\fP=""
+    only consider snapshots for this \fB\fChost\fR, when no snapshot ID is given
+
+.PP
+\fB\-l\fP, \fB\-\-long\fP[=false]
+    use a long listing format showing size and mode
+
+.PP
+\fB\-\-path\fP=[]
+    only consider snapshots which include this (absolute) \fB\fCpath\fR, when no snapshot ID is given
+
+.PP
+\fB\-\-tag\fP=[]
+    only consider snapshots which include this \fB\fCtaglist\fR, when no snapshot ID is given
+
+
+.SH OPTIONS INHERITED FROM PARENT COMMANDS
+.PP
+\fB\-\-json\fP[=false]
+    set output mode to JSON for commands that support it
+
+.PP
+\fB\-\-no\-lock\fP[=false]
+    do not lock the repo, this allows some operations on read\-only repos
+
+.PP
+\fB\-o\fP, \fB\-\-option\fP=[]
+    set extended option (\fB\fCkey=value\fR, can be specified multiple times)
+
+.PP
+\fB\-p\fP, \fB\-\-password\-file\fP=""
+    read the repository password from a file (default: $RESTIC\_PASSWORD\_FILE)
+
+.PP
+\fB\-q\fP, \fB\-\-quiet\fP[=false]
+    do not output comprehensive progress report
+
+.PP
+\fB\-r\fP, \fB\-\-repo\fP=""
+    repository to backup to or restore from (default: $RESTIC\_REPOSITORY)
+
+
+.SH SEE ALSO
+.PP
+\fBrestic(1)\fP

doc/man/restic-manpage.1

@@ -0,0 +1,61 @@
+.TH "restic backup" "1" "Jan 2017" "generated by `restic manpage`" "" 
+.nh
+.ad l
+
+
+.SH NAME
+.PP
+restic\-manpage \- Generate manual pages
+
+
+.SH SYNOPSIS
+.PP
+\fBrestic manpage [command] [flags]\fP
+
+
+.SH DESCRIPTION
+.PP
+The "manpage" command generates a manual page for a single command. It can also
+be used to write all manual pages to a directory. If the output directory is
+set and no command is specified, all manpages are written to the directory.
+
+
+.SH OPTIONS
+.PP
+\fB\-h\fP, \fB\-\-help\fP[=false]
+    help for manpage
+
+.PP
+\fB\-\-output\-dir\fP=""
+    write man pages to this \fB\fCdirectory\fR
+
+
+.SH OPTIONS INHERITED FROM PARENT COMMANDS
+.PP
+\fB\-\-json\fP[=false]
+    set output mode to JSON for commands that support it
+
+.PP
+\fB\-\-no\-lock\fP[=false]
+    do not lock the repo, this allows some operations on read\-only repos
+
+.PP
+\fB\-o\fP, \fB\-\-option\fP=[]
+    set extended option (\fB\fCkey=value\fR, can be specified multiple times)
+
+.PP
+\fB\-p\fP, \fB\-\-password\-file\fP=""
+    read the repository password from a file (default: $RESTIC\_PASSWORD\_FILE)
+
+.PP
+\fB\-q\fP, \fB\-\-quiet\fP[=false]
+    do not output comprehensive progress report
+
+.PP
+\fB\-r\fP, \fB\-\-repo\fP=""
+    repository to backup to or restore from (default: $RESTIC\_REPOSITORY)
+
+
+.SH SEE ALSO
+.PP
+\fBrestic(1)\fP

doc/man/restic-migrate.1

@@ -0,0 +1,60 @@
+.TH "restic backup" "1" "Jan 2017" "generated by `restic manpage`" "" 
+.nh
+.ad l
+
+
+.SH NAME
+.PP
+restic\-migrate \- Apply migrations
+
+
+.SH SYNOPSIS
+.PP
+\fBrestic migrate [name] [flags]\fP
+
+
+.SH DESCRIPTION
+.PP
+The "migrate" command applies migrations to a repository. When no migration
+name is explicitly given, a list of migrations that can be applied is printed.
+
+
+.SH OPTIONS
+.PP
+\fB\-f\fP, \fB\-\-force\fP[=false]
+    apply a migration a second time
+
+.PP
+\fB\-h\fP, \fB\-\-help\fP[=false]
+    help for migrate
+
+
+.SH OPTIONS INHERITED FROM PARENT COMMANDS
+.PP
+\fB\-\-json\fP[=false]
+    set output mode to JSON for commands that support it
+
+.PP
+\fB\-\-no\-lock\fP[=false]
+    do not lock the repo, this allows some operations on read\-only repos
+
+.PP
+\fB\-o\fP, \fB\-\-option\fP=[]
+    set extended option (\fB\fCkey=value\fR, can be specified multiple times)
+
+.PP
+\fB\-p\fP, \fB\-\-password\-file\fP=""
+    read the repository password from a file (default: $RESTIC\_PASSWORD\_FILE)
+
+.PP
+\fB\-q\fP, \fB\-\-quiet\fP[=false]
+    do not output comprehensive progress report
+
+.PP
+\fB\-r\fP, \fB\-\-repo\fP=""
+    repository to backup to or restore from (default: $RESTIC\_REPOSITORY)
+
+
+.SH SEE ALSO
+.PP
+\fBrestic(1)\fP

doc/man/restic-mount.1

@@ -0,0 +1,80 @@
+.TH "restic backup" "1" "Jan 2017" "generated by `restic manpage`" "" 
+.nh
+.ad l
+
+
+.SH NAME
+.PP
+restic\-mount \- Mount the repository
+
+
+.SH SYNOPSIS
+.PP
+\fBrestic mount [flags] mountpoint\fP
+
+
+.SH DESCRIPTION
+.PP
+The "mount" command mounts the repository via fuse to a directory. This is a
+read\-only mount.
+
+
+.SH OPTIONS
+.PP
+\fB\-\-allow\-other\fP[=false]
+    allow other users to access the data in the mounted directory
+
+.PP
+\fB\-\-allow\-root\fP[=false]
+    allow root user to access the data in the mounted directory
+
+.PP
+\fB\-h\fP, \fB\-\-help\fP[=false]
+    help for mount
+
+.PP
+\fB\-H\fP, \fB\-\-host\fP=""
+    only consider snapshots for this host
+
+.PP
+\fB\-\-owner\-root\fP[=false]
+    use 'root' as the owner of files and dirs
+
+.PP
+\fB\-\-path\fP=[]
+    only consider snapshots which include this (absolute) \fB\fCpath\fR
+
+.PP
+\fB\-\-tag\fP=[]
+    only consider snapshots which include this \fB\fCtaglist\fR
+
+
+.SH OPTIONS INHERITED FROM PARENT COMMANDS
+.PP
+\fB\-\-json\fP[=false]
+    set output mode to JSON for commands that support it
+
+.PP
+\fB\-\-no\-lock\fP[=false]
+    do not lock the repo, this allows some operations on read\-only repos
+
+.PP
+\fB\-o\fP, \fB\-\-option\fP=[]
+    set extended option (\fB\fCkey=value\fR, can be specified multiple times)
+
+.PP
+\fB\-p\fP, \fB\-\-password\-file\fP=""
+    read the repository password from a file (default: $RESTIC\_PASSWORD\_FILE)
+
+.PP
+\fB\-q\fP, \fB\-\-quiet\fP[=false]
+    do not output comprehensive progress report
+
+.PP
+\fB\-r\fP, \fB\-\-repo\fP=""
+    repository to backup to or restore from (default: $RESTIC\_REPOSITORY)
+
+
+.SH SEE ALSO
+.PP
+\fBrestic(1)\fP

doc/man/restic-prune.1

@@ -0,0 +1,56 @@
+.TH "restic backup" "1" "Jan 2017" "generated by `restic manpage`" "" 
+.nh
+.ad l
+
+
+.SH NAME
+.PP
+restic\-prune \- Remove unneeded data from the repository
+
+
+.SH SYNOPSIS
+.PP
+\fBrestic prune [flags]\fP
+
+
+.SH DESCRIPTION
+.PP
+The "prune" command checks the repository and removes data that is not
+referenced and therefore not needed any more.
+
+
+.SH OPTIONS
+.PP
+\fB\-h\fP, \fB\-\-help\fP[=false]
+    help for prune
+
+
+.SH OPTIONS INHERITED FROM PARENT COMMANDS
+.PP
+\fB\-\-json\fP[=false]
+    set output mode to JSON for commands that support it
+
+.PP
+\fB\-\-no\-lock\fP[=false]
+    do not lock the repo, this allows some operations on read\-only repos
+
+.PP
+\fB\-o\fP, \fB\-\-option\fP=[]
+    set extended option (\fB\fCkey=value\fR, can be specified multiple times)
+
+.PP
+\fB\-p\fP, \fB\-\-password\-file\fP=""
+    read the repository password from a file (default: $RESTIC\_PASSWORD\_FILE)
+
+.PP
+\fB\-q\fP, \fB\-\-quiet\fP[=false]
+    do not output comprehensive progress report
+
+.PP
+\fB\-r\fP, \fB\-\-repo\fP=""
+    repository to backup to or restore from (default: $RESTIC\_REPOSITORY)
+
+
+.SH SEE ALSO
+.PP
+\fBrestic(1)\fP

doc/man/restic-rebuild-index.1

@@ -0,0 +1,56 @@
+.TH "restic backup" "1" "Jan 2017" "generated by `restic manpage`" "" 
+.nh
+.ad l
+
+
+.SH NAME
+.PP
+restic\-rebuild\-index \- Build a new index file
+
+
+.SH SYNOPSIS
+.PP
+\fBrestic rebuild\-index [flags]\fP
+
+
+.SH DESCRIPTION
+.PP
+The "rebuild\-index" command creates a new index based on the pack files in the
+repository.
+
+
+.SH OPTIONS
+.PP
+\fB\-h\fP, \fB\-\-help\fP[=false]
+    help for rebuild\-index
+
+
+.SH OPTIONS INHERITED FROM PARENT COMMANDS
+.PP
+\fB\-\-json\fP[=false]
+    set output mode to JSON for commands that support it
+
+.PP
+\fB\-\-no\-lock\fP[=false]
+    do not lock the repo, this allows some operations on read\-only repos
+
+.PP
+\fB\-o\fP, \fB\-\-option\fP=[]
+    set extended option (\fB\fCkey=value\fR, can be specified multiple times)
+
+.PP
+\fB\-p\fP, \fB\-\-password\-file\fP=""
+    read the repository password from a file (default: $RESTIC\_PASSWORD\_FILE)
+
+.PP
+\fB\-q\fP, \fB\-\-quiet\fP[=false]
+    do not output comprehensive progress report
+
+.PP
+\fB\-r\fP, \fB\-\-repo\fP=""
+    repository to backup to or restore from (default: $RESTIC\_REPOSITORY)
+
+
+.SH SEE ALSO
+.PP
+\fBrestic(1)\fP

doc/man/restic-restore.1

@@ -0,0 +1,84 @@
+.TH "restic backup" "1" "Jan 2017" "generated by `restic manpage`" "" 
+.nh
+.ad l
+
+
+.SH NAME
+.PP
+restic\-restore \- Extract the data from a snapshot
+
+
+.SH SYNOPSIS
+.PP
+\fBrestic restore [flags] snapshotID\fP
+
+
+.SH DESCRIPTION
+.PP
+The "restore" command extracts the data from a snapshot from the repository to
+a directory.
+
+.PP
+The special snapshot "latest" can be used to restore the latest snapshot in the
+repository.
+
+
+.SH OPTIONS
+.PP
+\fB\-e\fP, \fB\-\-exclude\fP=[]
+    exclude a \fB\fCpattern\fR (can be specified multiple times)
+
+.PP
+\fB\-h\fP, \fB\-\-help\fP[=false]
+    help for restore
+
+.PP
+\fB\-H\fP, \fB\-\-host\fP=""
+    only consider snapshots for this host when the snapshot ID is "latest"
+
+.PP
+\fB\-i\fP, \fB\-\-include\fP=[]
+    include a \fB\fCpattern\fR, exclude everything else (can be specified multiple times)
+
+.PP
+\fB\-\-path\fP=[]
+    only consider snapshots which include this (absolute) \fB\fCpath\fR for snapshot ID "latest"
+
+.PP
+\fB\-\-tag\fP=[]
+    only consider snapshots which include this \fB\fCtaglist\fR for snapshot ID "latest"
+
+.PP
+\fB\-t\fP, \fB\-\-target\fP=""
+    directory to extract data to
+
+
+.SH OPTIONS INHERITED FROM PARENT COMMANDS
+.PP
+\fB\-\-json\fP[=false]
+    set output mode to JSON for commands that support it
+
+.PP
+\fB\-\-no\-lock\fP[=false]
+    do not lock the repo, this allows some operations on read\-only repos
+
+.PP
+\fB\-o\fP, \fB\-\-option\fP=[]
+    set extended option (\fB\fCkey=value\fR, can be specified multiple times)
+
+.PP
+\fB\-p\fP, \fB\-\-password\-file\fP=""
+    read the repository password from a file (default: $RESTIC\_PASSWORD\_FILE)
+
+.PP
+\fB\-q\fP, \fB\-\-quiet\fP[=false]
+    do not output comprehensive progress report
+
+.PP
+\fB\-r\fP, \fB\-\-repo\fP=""
+    repository to backup to or restore from (default: $RESTIC\_REPOSITORY)
+
+
+.SH SEE ALSO
+.PP
+\fBrestic(1)\fP

doc/man/restic-snapshots.1

@@ -0,0 +1,71 @@
+.TH "restic backup" "1" "Jan 2017" "generated by `restic manpage`" "" 
+.nh
+.ad l
+
+
+.SH NAME
+.PP
+restic\-snapshots \- List all snapshots
+
+
+.SH SYNOPSIS
+.PP
+\fBrestic snapshots [snapshotID ...] [flags]\fP
+
+
+.SH DESCRIPTION
+.PP
+The "snapshots" command lists all snapshots stored in the repository.
+
+
+.SH OPTIONS
+.PP
+\fB\-c\fP, \fB\-\-compact\fP[=false]
+    use compact format
+
+.PP
+\fB\-h\fP, \fB\-\-help\fP[=false]
+    help for snapshots
+
+.PP
+\fB\-H\fP, \fB\-\-host\fP=""
+    only consider snapshots for this \fB\fChost\fR
+
+.PP
+\fB\-\-path\fP=[]
+    only consider snapshots for this \fB\fCpath\fR (can be specified multiple times)
+
+.PP
+\fB\-\-tag\fP=[]
+    only consider snapshots which include this \fB\fCtaglist\fR (can be specified multiple times)
+
+
+.SH OPTIONS INHERITED FROM PARENT COMMANDS
+.PP
+\fB\-\-json\fP[=false]
+    set output mode to JSON for commands that support it
+
+.PP
+\fB\-\-no\-lock\fP[=false]
+    do not lock the repo, this allows some operations on read\-only repos
+
+.PP
+\fB\-o\fP, \fB\-\-option\fP=[]
+    set extended option (\fB\fCkey=value\fR, can be specified multiple times)
+
+.PP
+\fB\-p\fP, \fB\-\-password\-file\fP=""
+    read the repository password from a file (default: $RESTIC\_PASSWORD\_FILE)
+
+.PP
+\fB\-q\fP, \fB\-\-quiet\fP[=false]
+    do not output comprehensive progress report
+
+.PP
+\fB\-r\fP, \fB\-\-repo\fP=""
+    repository to backup to or restore from (default: $RESTIC\_REPOSITORY)
+
+
+.SH SEE ALSO
+.PP
+\fBrestic(1)\fP

doc/man/restic-tag.1

@@ -0,0 +1,86 @@
+.TH "restic backup" "1" "Jan 2017" "generated by `restic manpage`" "" 
+.nh
+.ad l
+
+
+.SH NAME
+.PP
+restic\-tag \- Modify tags on snapshots
+
+
+.SH SYNOPSIS
+.PP
+\fBrestic tag [flags] [snapshot\-ID ...]\fP
+
+
+.SH DESCRIPTION
+.PP
+The "tag" command allows you to modify tags on exiting snapshots.
+
+.PP
+You can either set/replace the entire set of tags on a snapshot, or
+add tags to/remove tags from the existing set.
+
+.PP
+When no snapshot\-ID is given, all snapshots matching the host, tag and path filter criteria are modified.
+
+
+.SH OPTIONS
+.PP
+\fB\-\-add\fP=[]
+    \fB\fCtag\fR which will be added to the existing tags (can be given multiple times)
+
+.PP
+\fB\-h\fP, \fB\-\-help\fP[=false]
+    help for tag
+
+.PP
+\fB\-H\fP, \fB\-\-host\fP=""
+    only consider snapshots for this \fB\fChost\fR, when no snapshot ID is given
+
+.PP
+\fB\-\-path\fP=[]
+    only consider snapshots which include this (absolute) \fB\fCpath\fR, when no snapshot\-ID is given
+
+.PP
+\fB\-\-remove\fP=[]
+    \fB\fCtag\fR which will be removed from the existing tags (can be given multiple times)
+
+.PP
+\fB\-\-set\fP=[]
+    \fB\fCtag\fR which will replace the existing tags (can be given multiple times)
+
+.PP
+\fB\-\-tag\fP=[]
+    only consider snapshots which include this \fB\fCtaglist\fR, when no snapshot\-ID is given
+
+
+.SH OPTIONS INHERITED FROM PARENT COMMANDS
+.PP
+\fB\-\-json\fP[=false]
+    set output mode to JSON for commands that support it
+
+.PP
+\fB\-\-no\-lock\fP[=false]
+    do not lock the repo, this allows some operations on read\-only repos
+
+.PP
+\fB\-o\fP, \fB\-\-option\fP=[]
+    set extended option (\fB\fCkey=value\fR, can be specified multiple times)
+
+.PP
+\fB\-p\fP, \fB\-\-password\-file\fP=""
+    read the repository password from a file (default: $RESTIC\_PASSWORD\_FILE)
+
+.PP
+\fB\-q\fP, \fB\-\-quiet\fP[=false]
+    do not output comprehensive progress report
+
+.PP
+\fB\-r\fP, \fB\-\-repo\fP=""
+    repository to backup to or restore from (default: $RESTIC\_REPOSITORY)
+
+
+.SH SEE ALSO
+.PP
+\fBrestic(1)\fP

doc/man/restic-unlock.1

@@ -0,0 +1,59 @@
+.TH "restic backup" "1" "Jan 2017" "generated by `restic manpage`" "" 
+.nh
+.ad l
+
+
+.SH NAME
+.PP
+restic\-unlock \- Remove locks other processes created
+
+
+.SH SYNOPSIS
+.PP
+\fBrestic unlock [flags]\fP
+
+
+.SH DESCRIPTION
+.PP
+The "unlock" command removes stale locks that have been created by other restic processes.
+
+
+.SH OPTIONS
+.PP
+\fB\-h\fP, \fB\-\-help\fP[=false]
+    help for unlock
+
+.PP
+\fB\-\-remove\-all\fP[=false]
+    remove all locks, even non\-stale ones
+
+
+.SH OPTIONS INHERITED FROM PARENT COMMANDS
+.PP
+\fB\-\-json\fP[=false]
+    set output mode to JSON for commands that support it
+
+.PP
+\fB\-\-no\-lock\fP[=false]
+    do not lock the repo, this allows some operations on read\-only repos
+
+.PP
+\fB\-o\fP, \fB\-\-option\fP=[]
+    set extended option (\fB\fCkey=value\fR, can be specified multiple times)
+
+.PP
+\fB\-p\fP, \fB\-\-password\-file\fP=""
+    read the repository password from a file (default: $RESTIC\_PASSWORD\_FILE)
+
+.PP
+\fB\-q\fP, \fB\-\-quiet\fP[=false]
+    do not output comprehensive progress report
+
+.PP
+\fB\-r\fP, \fB\-\-repo\fP=""
+    repository to backup to or restore from (default: $RESTIC\_REPOSITORY)
+
+
+.SH SEE ALSO
+.PP
+\fBrestic(1)\fP

doc/man/restic-version.1

@@ -0,0 +1,56 @@
+.TH "restic backup" "1" "Jan 2017" "generated by `restic manpage`" "" 
+.nh
+.ad l
+
+
+.SH NAME
+.PP
+restic\-version \- Print version information
+
+
+.SH SYNOPSIS
+.PP
+\fBrestic version [flags]\fP
+
+
+.SH DESCRIPTION
+.PP
+The "version" command prints detailed information about the build environment
+and the version of this software.
+
+
+.SH OPTIONS
+.PP
+\fB\-h\fP, \fB\-\-help\fP[=false]
+    help for version
+
+
+.SH OPTIONS INHERITED FROM PARENT COMMANDS
+.PP
+\fB\-\-json\fP[=false]
+    set output mode to JSON for commands that support it
+
+.PP
+\fB\-\-no\-lock\fP[=false]
+    do not lock the repo, this allows some operations on read\-only repos
+
+.PP
+\fB\-o\fP, \fB\-\-option\fP=[]
+    set extended option (\fB\fCkey=value\fR, can be specified multiple times)
+
+.PP
+\fB\-p\fP, \fB\-\-password\-file\fP=""
+    read the repository password from a file (default: $RESTIC\_PASSWORD\_FILE)
+
+.PP
+\fB\-q\fP, \fB\-\-quiet\fP[=false]
+    do not output comprehensive progress report
+
+.PP
+\fB\-r\fP, \fB\-\-repo\fP=""
+    repository to backup to or restore from (default: $RESTIC\_REPOSITORY)
+
+
+.SH SEE ALSO
+.PP
+\fBrestic(1)\fP

doc/man/restic.1

@@ -0,0 +1,54 @@
+.TH "restic backup" "1" "Jan 2017" "generated by `restic manpage`" "" 
+.nh
+.ad l
+
+
+.SH NAME
+.PP
+restic \- Backup and restore files
+
+
+.SH SYNOPSIS
+.PP
+\fBrestic [flags]\fP
+
+
+.SH DESCRIPTION
+.PP
+restic is a backup program which allows saving multiple revisions of files and
+directories in an encrypted repository stored on different backends.
+
+
+.SH OPTIONS
+.PP
+\fB\-h\fP, \fB\-\-help\fP[=false]
+    help for restic
+
+.PP
+\fB\-\-json\fP[=false]
+    set output mode to JSON for commands that support it
+
+.PP
+\fB\-\-no\-lock\fP[=false]
+    do not lock the repo, this allows some operations on read\-only repos
+
+.PP
+\fB\-o\fP, \fB\-\-option\fP=[]
+    set extended option (\fB\fCkey=value\fR, can be specified multiple times)
+
+.PP
+\fB\-p\fP, \fB\-\-password\-file\fP=""
+    read the repository password from a file (default: $RESTIC\_PASSWORD\_FILE)
+
+.PP
+\fB\-q\fP, \fB\-\-quiet\fP[=false]
+    do not output comprehensive progress report
+
+.PP
+\fB\-r\fP, \fB\-\-repo\fP=""
+    repository to backup to or restore from (default: $RESTIC\_REPOSITORY)
+
+
+.SH SEE ALSO
+.PP
+\fBrestic\-autocomplete(1)\fP, \fBrestic\-backup(1)\fP, \fBrestic\-cat(1)\fP, \fBrestic\-check(1)\fP, \fBrestic\-dump(1)\fP, \fBrestic\-find(1)\fP, \fBrestic\-forget(1)\fP, \fBrestic\-init(1)\fP, \fBrestic\-key(1)\fP, \fBrestic\-list(1)\fP, \fBrestic\-ls(1)\fP, \fBrestic\-manpage(1)\fP, \fBrestic\-migrate(1)\fP, \fBrestic\-mount(1)\fP, \fBrestic\-prune(1)\fP, \fBrestic\-rebuild\-index(1)\fP, \fBrestic\-restore(1)\fP, \fBrestic\-snapshots(1)\fP, \fBrestic\-tag(1)\fP, \fBrestic\-unlock(1)\fP, \fBrestic\-version(1)\fP

doc/manual.rst

@@ -16,23 +16,25 @@ Usage help is available:
       restic [command]
 
     Available Commands:
-      autocomplete  generate shell autocompletion script
-      backup        create a new backup of files and/or directories
-      cat           print internal objects to stdout
-      check         check the repository for errors
-      find          find a file or directory
-      forget        forget removes snapshots from the repository
-      init          initialize a new repository
-      key           manage keys (passwords)
-      list          list items in the repository
-      ls            list files in a snapshot
-      mount         mount the repository
-      prune         remove unneeded data from the repository
-      rebuild-index build a new index file
-      restore       extract the data from a snapshot
-      snapshots     list all snapshots
-      tag           modifies tags on snapshots
-      unlock        remove locks other processes created
+      autocomplete  Generate shell autocompletion script
+      backup        Create a new backup of files and/or directories
+      cat           Print internal objects to stdout
+      check         Check the repository for errors
+      dump          Dump data structures
+      find          Find a file or directory
+      forget        Remove snapshots from the repository
+      help          Help about any command
+      init          Initialize a new repository
+      key           Manage keys (passwords)
+      list          List items in the repository
+      ls            List files in a snapshot
+      mount         Mount the repository
+      prune         Remove unneeded data from the repository
+      rebuild-index Build a new index file
+      restore       Extract the data from a snapshot
+      snapshots     List all snapshots
+      tag           Modify tags on snapshots
+      unlock        Remove locks other processes created
       version       Print version information
 
     Flags:
@@ -69,6 +71,7 @@ command:
           --stdin                   read backup from stdin
           --stdin-filename string   file name to use when reading from stdin
           --tag tag                 add a tag for the new snapshot (can be specified multiple times)
+          --time string             time of the backup (ex. '2012-11-01 22:08:41') (default: now)
 
     Global Flags:
           --json                   set output mode to JSON for commands that support it
@@ -108,8 +111,8 @@ command and enter the same password twice:
     Please note that knowledge of your password is required to access the repository.
     Losing your password means that your data is irrecoverably lost.
 
-Other backends like sftp and s3 are `described in a later
-section <#create-an-sftp-repository>`__ of this document.
+Other backends like sftp and s3 are described in the following
+sections.
 
 Remembering your password is important! If you lose it, you won't be
 able to access data stored in the repository.
@@ -282,6 +285,154 @@ this command.
     Please note that knowledge of your password is required to access
     the repository. Losing your password means that your data is irrecoverably lost.
 
+OpenStack Swift
+~~~~~~~~~~~~~~~
+
+Restic can backup data to an OpenStack Swift container. Because Swift supports
+various authentication methods, credentials are passed through environment
+variables. In order to help integration with existing OpenStack installations,
+the naming convention of those variables follows official python swift client:
+
+.. code-block:: console
+
+   # For keystone v1 authentication
+   $ export ST_AUTH=<MY_AUTH_URL>
+   $ export ST_USER=<MY_USER_NAME>
+   $ export ST_KEY=<MY_USER_PASSWORD>
+
+   # For keystone v2 authentication (some variables are optional)
+   $ export OS_AUTH_URL=<MY_AUTH_URL>
+   $ export OS_REGION_NAME=<MY_REGION_NAME>
+   $ export OS_USERNAME=<MY_USERNAME>
+   $ export OS_PASSWORD=<MY_PASSWORD>
+   $ export OS_TENANT_ID=<MY_TENANT_ID>
+   $ export OS_TENANT_NAME=<MY_TENANT_NAME>
+
+   # For keystone v3 authentication (some variables are optional)
+   $ export OS_AUTH_URL=<MY_AUTH_URL>
+   $ export OS_REGION_NAME=<MY_REGION_NAME>
+   $ export OS_USERNAME=<MY_USERNAME>
+   $ export OS_PASSWORD=<MY_PASSWORD>
+   $ export OS_USER_DOMAIN_NAME=<MY_DOMAIN_NAME>
+   $ export OS_PROJECT_NAME=<MY_PROJECT_NAME>
+   $ export OS_PROJECT_DOMAIN_NAME=<MY_PROJECT_DOMAIN_NAME>
+
+   # For authentication based on tokens
+   $ export OS_STORAGE_URL=<MY_STORAGE_URL>
+   $ export OS_AUTH_TOKEN=<MY_AUTH_TOKEN>
+
+
+Restic should be compatible with [OpenStack RC
+file](https://docs.openstack.org/user-guide/common/cli-set-environment-variables-using-openstack-rc.html)
+in most cases.
+
+Once environment variables are set up, a new repository can be created. The
+name of swift container and optional path can be specified. If
+the container does not exist, it will be created automatically:
+
+.. code-block:: console
+
+   $ restic -r swift:container_name:/path init   # path is optional
+   enter password for new backend:
+   enter password again:
+   created restic backend eefee03bbd at swift:container_name:/path
+   Please note that knowledge of your password is required to access the repository.
+   Losing your password means that your data is irrecoverably lost.
+
+The policy of new container created by restic can be changed using environment variable:
+
+.. code-block:: console
+
+   $ export SWIFT_DEFAULT_CONTAINER_POLICY=<MY_CONTAINER_POLICY>
+
+
+Backblaze B2
+~~~~~~~~~~~~
+
+Restic can backup data to any Backblaze B2 bucket. You need to first setup the
+following environment variables with the credentials you obtained when signed
+into your B2 account:
+
+.. code-block:: console
+
+    $ export B2_ACCOUNT_ID=<MY_ACCOUNT_ID>
+    $ export B2_ACCOUNT_KEY=<MY_SECRET_ACCOUNT_KEY>
+
+You can then easily initialize a repository stored at Backblaze B2. If the
+bucket does not exist yet, it will be created:
+
+.. code-block:: console
+
+    $ restic -r b2:bucketname:path/to/repo init
+    enter password for new backend:
+    enter password again:
+    created restic backend eefee03bbd at b2:bucketname:path/to/repo
+    Please note that knowledge of your password is required to access the repository.
+    Losing your password means that your data is irrecoverably lost.
+
+The number of concurrent connections to the B2 service can be set with the `-o
+b2.connections=10`. By default, at most five parallel connections are
+established.
+
+Microsoft Azure Blob Storage
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can also store backups on Microsoft Azure Blob Storage. Export the Azure
+account name and key as follows:
+
+.. code-block:: console
+
+    $ export AZURE_ACCOUNT_NAME=<ACCOUNT_NAME>
+    $ export AZURE_ACCOUNT_KEY=<SECRET_KEY>
+
+Afterwards you can initialize a repository in a container called `foo` in the
+root path like this:
+
+.. code-block:: console
+
+    $ restic -r azure:foo:/ init
+    enter password for new backend:
+    enter password again:
+
+    created restic backend a934bac191 at azure:foo:/
+    [...]
+
+The number of concurrent connections to the B2 service can be set with the
+`-o azure.connections=10`. By default, at most five parallel connections are
+established.
+
+
+Google Cloud Storage
+~~~~~~~~~~~~~~~~~~~~
+
+Restic supports Google Cloud Storage as a backend. In order for this to work
+you first need create a "service account" and download the JSON key file for
+it. In addition, you need the Google Project ID that you can see in the Google
+Cloud Platform console at the "Storage/Settings" menu. Export the path to the
+JSON credentials file and the project ID as follows:
+
+.. code-block:: console
+
+    $ export GOOGLE_PROJECT_ID=123123123123
+    $ export GOOGLE_APPLICATION_CREDENTIALS=$HOME/.config/gs-secret-restic-key.json
+
+Then you can use the ``gs:`` backend type to create a new repository in the
+bucket `foo` at the root path:
+
+.. code-block:: console
+
+    $ restic -r gs:foo:/ init
+    enter password for new backend:
+    enter password again:
+
+    created restic backend bde47d6254 at gs:restic-dev-an:foo2
+    [...]
+
+The number of concurrent connections to the GCS service can be set with the
+`-o gs.connections=10`. By default, at most five parallel connections are
+established.
+
+
 Password prompt on Windows
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -329,7 +480,7 @@ work!
 
 .. code-block:: console
 
-    $ restic -r /tmp/backup backup ~/shared/work/web
+    $ restic -r /tmp/backup backup ~/work
     enter password for repository:
     using parent snapshot 40dc1520aa6a07b7b3ae561786770a01951245d2367241e71e9485f18ae8228c
     scan [/home/user/work]
@@ -343,7 +494,7 @@ You can even backup individual files in the same repository.
 .. code-block:: console
 
     $ restic -r /tmp/backup backup ~/work.txt
-    scan [~/work.txt]
+    scan [/home/user/work.txt]
     scanned 0 directories, 1 files in 0:00
     [0:00] 100.00%  0B/s  220B / 220B  1 / 1 items  0 errors  ETA 0:00
     duration: 0:00, 0.03MiB/s
@@ -436,18 +587,20 @@ specified with ``--stdin-filename``, e.g. like this:
 
     $ mysqldump [...] | restic -r /tmp/backup backup --stdin --stdin-filename production.sql
 
-Tags
-~~~~
+Tags for backup
+~~~~~~~~~~~~~~~
 
 Snapshots can have one or more tags, short strings which add identifying
-information. Just specify the tags for a snapshot with ``--tag``:
+information. Just specify the tags for a snapshot one by one with ``--tag``:
 
 .. code-block:: console
 
-    $ restic -r /tmp/backup backup --tag projectX ~/shared/work/web
+    $ restic -r /tmp/backup backup --tag projectX --tag foo --tag bar ~/work
     [...]
 
-The tags can later be used to keep (or forget) snapshots.
+The tags can later be used to keep (or forget) snapshots with the ``forget``
+command. The command ``tag`` can be used to modify tags on an existing
+snapshot.
 
 List all snapshots
 ------------------
@@ -499,7 +652,7 @@ command to restore the contents of the latest snapshot to
 
 .. code-block:: console
 
-    $ restic -r /tmp/backup restore 79766175 --target ~/tmp/restore-work
+    $ restic -r /tmp/backup restore 79766175 --target /tmp/restore-work
     enter password for repository:
     restoring <Snapshot of [/home/user/work] at 2015-05-08 21:40:19.884408621 +0200 CEST> to /tmp/restore-work
 
@@ -509,9 +662,20 @@ backup for a specific host, path or both.
 
 .. code-block:: console
 
-    $ restic -r /tmp/backup restore latest --target ~/tmp/restore-work --path "/home/art" --host luigi
+    $ restic -r /tmp/backup restore latest --target /tmp/restore-art --path "/home/art" --host luigi
     enter password for repository:
-    restoring <Snapshot of [/home/art] at 2015-05-08 21:45:17.884408621 +0200 CEST> to /tmp/restore-work
+    restoring <Snapshot of [/home/art] at 2015-05-08 21:45:17.884408621 +0200 CEST> to /tmp/restore-art
+
+Use ``--exclude`` and ``--include`` to restrict the restore to a subset of
+files in the snapshot. For example, to restore a single file:
+
+.. code-block:: console
+
+    $ restic -r /tmp/backup restore 79766175 --target /tmp/restore-work --include /work/foo
+    enter password for repository:
+    restoring <Snapshot of [/home/user/work] at 2015-05-08 21:40:19.884408621 +0200 CEST> to /tmp/restore-work
+
+This will restore the file ``foo`` to ``/tmp/restore-work/work/foo``.
 
 Manage repository keys
 ----------------------
@@ -554,7 +718,7 @@ command does that:
 
 .. code-block:: console
 
-    $ restic -r /tmp/backup tag --set NL,CH 590c8fc8
+    $ restic -r /tmp/backup tag --set NL --set CH 590c8fc8
     Create exclusive lock for repository
     Modified tags on 1 snapshots
 
@@ -608,7 +772,7 @@ yield the same error:
 
 .. code-block:: console
 
-    $ restic -r /tmp/backup restore 79766175 --target ~/tmp/restore-work
+    $ restic -r /tmp/backup restore 79766175 --target /tmp/restore-work
     Load indexes
     ciphertext verification failed
 
@@ -624,7 +788,7 @@ command to serve the repository with FUSE:
     $ mkdir /mnt/restic
     $ restic -r /tmp/backup mount /mnt/restic
     enter password for repository:
-    Now serving /tmp/backup at /tmp/restic
+    Now serving /tmp/backup at /mnt/restic
     Don't forget to umount after quitting!
 
 Mounting repositories via FUSE is not possible on Windows and OpenBSD.
@@ -759,9 +923,10 @@ instructs restic to not remove anything but print which snapshots would
 be removed.
 
 When ``forget`` is run with a policy, restic loads the list of all
-snapshots, then groups these by host name and list of directories. The
-policy is then applied to each group of snapshots separately. This is a
-safety feature.
+snapshots, then groups these by host name and list of directories. The grouping
+options can be set with ``--group-by``, to only group snapshots by paths and
+tags use ``--group-by paths,tags``. The policy is then applied to each group of
+snapshots separately. This is a safety feature.
 
 The ``forget`` command accepts the following parameters:
 
@@ -782,7 +947,26 @@ The ``forget`` command accepts the following parameters:
 Additionally, you can restrict removing snapshots to those which have a
 particular hostname with the ``--hostname`` parameter, or tags with the
 ``--tag`` option. When multiple tags are specified, only the snapshots
-which have all the tags are considered.
+which have all the tags are considered. For example, the following command
+removes all but the latest snapshot of all snapshots that have the tag ``foo``:
+
+.. code-block:: console
+
+   $ restic forget --tag foo --keep-last 1
+
+This command removes all but the last snapshot of all snapshots that have
+either the ``foo`` or ``bar`` tag set:
+
+.. code-block:: console
+
+   $ restic forget --tag foo --tag bar --keep-last 1
+
+To only keep the last snapshot of all snapshots with both the tag ``foo`` and
+``bar`` set use:
+
+.. code-block:: console
+
+   $ restic forget --tag foo,tag bar --keep-last 1
 
 All the ``--keep-*`` options above only count
 hours/days/weeks/months/years which have a snapshot, so those without a
@@ -820,8 +1004,10 @@ Restic can write out a bash compatible autocompletion script:
     NOTE: The current version supports Bash only.
           This should work for *nix systems with Bash installed.
 
-    By default, the file is written directly to /etc/bash_completion.d
-    for convenience, and the command may need superuser rights, e.g.:
+By default, the file is written directly to ``/etc/bash_completion.d/``
+for convenience, and the command may need superuser rights, e.g.
+
+.. code-block:: console
 
     $ sudo restic autocomplete
 
@@ -894,7 +1080,7 @@ Under the hood: Browse repository objects
 
 Internally, a repository stores data of several different types
 described in the `design
-documentation <https://github.com/restic/restic/blob/master/doc/Design.md>`__.
+documentation <https://github.com/restic/restic/blob/master/doc/Design.rst>`__.
 You can ``list`` objects such as blobs, packs, index, snapshots, keys or
 locks with the following command:
 

doc/tutorial_aws_s3.rst

@@ -64,7 +64,7 @@ changes:
 .. image:: images/aws_s3/05_bucket_create_review.png
    :alt: Review Bucket Creation
 
-The newly created ``restic-demo`` bucket will no appear on the list of S3
+The newly created ``restic-demo`` bucket will now appear on the list of S3
 buckets:
 
 .. image:: images/aws_s3/06_buckets_list_after.png

docker/Dockerfile

@@ -0,0 +1,7 @@
+FROM alpine:3.6
+
+COPY restic /usr/bin
+
+RUN apk add --update --no-cache ca-certificates fuse
+
+ENTRYPOINT ["/usr/bin/restic"]

docker/README.md

@@ -0,0 +1,24 @@
+# Docker image
+
+## Build
+
+From the root of this repository run:
+
+```
+./docker/build.sh
+```
+
+image name will be `restic/restic:latest`
+
+## Run
+
+Set environment variable `RESTIC_REPOSITORY` and map volume to directories and
+files like:
+
+```
+docker run --rm -ti \
+    -v $HOME/.restic/passfile:/pass \
+    -v $HOME/importantdirectory:/data \
+    -e RESTIC_REPOSITORY=rest:https://user:pass@hostname/ \
+    restic/restic -p /pass backup /data
+```

docker/build.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+set -e
+
+echo "Build binary using golang docker image"
+docker run --rm -ti \
+    -v `pwd`:/go/src/github.com/restic/restic \
+    -w /go/src/github.com/restic/restic golang:1.8.3-alpine go run build.go
+
+echo "Build docker image restic/restic:latest"
+docker build --rm -t restic/restic:latest -f docker/Dockerfile .

internal/archiver/archive_reader.go

@@ -1,12 +1,14 @@
 package archiver
 
 import (
+	"context"
 	"io"
-	"restic"
-	"restic/debug"
 	"time"
 
-	"restic/errors"
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/restic"
+
+	"github.com/restic/restic/internal/errors"
 
 	"github.com/restic/chunker"
 )
@@ -20,13 +22,13 @@ type Reader struct {
 }
 
 // Archive reads data from the reader and saves it to the repo.
-func (r *Reader) Archive(name string, rd io.Reader, p *restic.Progress) (*restic.Snapshot, restic.ID, error) {
+func (r *Reader) Archive(ctx context.Context, name string, rd io.Reader, p *restic.Progress) (*restic.Snapshot, restic.ID, error) {
 	if name == "" {
 		return nil, restic.ID{}, errors.New("no filename given")
 	}
 
 	debug.Log("start archiving %s", name)
-	sn, err := restic.NewSnapshot([]string{name}, r.Tags, r.Hostname)
+	sn, err := restic.NewSnapshot([]string{name}, r.Tags, r.Hostname, time.Now())
 	if err != nil {
 		return nil, restic.ID{}, err
 	}
@@ -53,7 +55,7 @@ func (r *Reader) Archive(name string, rd io.Reader, p *restic.Progress) (*restic
 		id := restic.Hash(chunk.Data)
 
 		if !repo.Index().Has(id, restic.DataBlob) {
-			_, err := repo.SaveBlob(restic.DataBlob, chunk.Data, id)
+			_, err := repo.SaveBlob(ctx, restic.DataBlob, chunk.Data, id)
 			if err != nil {
 				return nil, restic.ID{}, err
 			}
@@ -87,14 +89,14 @@ func (r *Reader) Archive(name string, rd io.Reader, p *restic.Progress) (*restic
 		},
 	}
 
-	treeID, err := repo.SaveTree(tree)
+	treeID, err := repo.SaveTree(ctx, tree)
 	if err != nil {
 		return nil, restic.ID{}, err
 	}
 	sn.Tree = &treeID
 	debug.Log("tree saved as %v", treeID.Str())
 
-	id, err := repo.SaveJSONUnpacked(restic.SnapshotFile, sn)
+	id, err := repo.SaveJSONUnpacked(ctx, restic.SnapshotFile, sn)
 	if err != nil {
 		return nil, restic.ID{}, err
 	}
@@ -106,7 +108,7 @@ func (r *Reader) Archive(name string, rd io.Reader, p *restic.Progress) (*restic
 		return nil, restic.ID{}, err
 	}
 
-	err = repo.SaveIndex()
+	err = repo.SaveIndex(ctx)
 	if err != nil {
 		return nil, restic.ID{}, err
 	}

internal/archiver/archive_reader_test.go

@@ -2,17 +2,19 @@ package archiver
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"io"
 	"math/rand"
-	"restic"
-	"restic/checker"
-	"restic/repository"
 	"testing"
+
+	"github.com/restic/restic/internal/checker"
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
 )
 
 func loadBlob(t *testing.T, repo restic.Repository, id restic.ID, buf []byte) int {
-	n, err := repo.LoadBlob(restic.DataBlob, id, buf)
+	n, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf)
 	if err != nil {
 		t.Fatalf("LoadBlob(%v) returned error %v", id, err)
 	}
@@ -21,7 +23,7 @@ func loadBlob(t *testing.T, repo restic.Repository, id restic.ID, buf []byte) in
 }
 
 func checkSavedFile(t *testing.T, repo restic.Repository, treeID restic.ID, name string, rd io.Reader) {
-	tree, err := repo.LoadTree(treeID)
+	tree, err := repo.LoadTree(context.TODO(), treeID)
 	if err != nil {
 		t.Fatalf("LoadTree() returned error %v", err)
 	}
@@ -85,7 +87,7 @@ func TestArchiveReader(t *testing.T) {
 		Tags:       []string{"test"},
 	}
 
-	sn, id, err := r.Archive("fakefile", f, nil)
+	sn, id, err := r.Archive(context.TODO(), "fakefile", f, nil)
 	if err != nil {
 		t.Fatalf("ArchiveReader() returned error %v", err)
 	}
@@ -111,7 +113,7 @@ func TestArchiveReaderNull(t *testing.T) {
 		Tags:       []string{"test"},
 	}
 
-	sn, id, err := r.Archive("fakefile", bytes.NewReader(nil), nil)
+	sn, id, err := r.Archive(context.TODO(), "fakefile", bytes.NewReader(nil), nil)
 	if err != nil {
 		t.Fatalf("ArchiveReader() returned error %v", err)
 	}
@@ -132,11 +134,8 @@ func (e errReader) Read([]byte) (int, error) {
 }
 
 func countSnapshots(t testing.TB, repo restic.Repository) int {
-	done := make(chan struct{})
-	defer close(done)
-
 	snapshots := 0
-	for range repo.List(restic.SnapshotFile, done) {
+	for range repo.List(context.TODO(), restic.SnapshotFile) {
 		snapshots++
 	}
 	return snapshots
@@ -152,7 +151,7 @@ func TestArchiveReaderError(t *testing.T) {
 		Tags:       []string{"test"},
 	}
 
-	sn, id, err := r.Archive("fakefile", errReader("error returned by reading stdin"), nil)
+	sn, id, err := r.Archive(context.TODO(), "fakefile", errReader("error returned by reading stdin"), nil)
 	if err == nil {
 		t.Errorf("expected error not returned")
 	}
@@ -195,7 +194,7 @@ func BenchmarkArchiveReader(t *testing.B) {
 	t.ResetTimer()
 
 	for i := 0; i < t.N; i++ {
-		_, _, err := r.Archive("fakefile", bytes.NewReader(buf), nil)
+		_, _, err := r.Archive(context.TODO(), "fakefile", bytes.NewReader(buf), nil)
 		if err != nil {
 			t.Fatal(err)
 		}

internal/archiver/archiver.go

@@ -1,22 +1,23 @@
 package archiver
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
-	"restic"
 	"sort"
 	"sync"
 	"time"
 
-	"restic/errors"
-	"restic/walk"
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/walk"
 
-	"restic/debug"
-	"restic/fs"
-	"restic/pipe"
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/fs"
+	"github.com/restic/restic/internal/pipe"
 
 	"github.com/restic/chunker"
 )
@@ -92,7 +93,7 @@ func (arch *Archiver) isKnownBlob(id restic.ID, t restic.BlobType) bool {
 }
 
 // Save stores a blob read from rd in the repository.
-func (arch *Archiver) Save(t restic.BlobType, data []byte, id restic.ID) error {
+func (arch *Archiver) Save(ctx context.Context, t restic.BlobType, data []byte, id restic.ID) error {
 	debug.Log("Save(%v, %v)\n", t, id.Str())
 
 	if arch.isKnownBlob(id, restic.DataBlob) {
@@ -100,7 +101,7 @@ func (arch *Archiver) Save(t restic.BlobType, data []byte, id restic.ID) error {
 		return nil
 	}
 
-	_, err := arch.repo.SaveBlob(t, data, id)
+	_, err := arch.repo.SaveBlob(ctx, t, data, id)
 	if err != nil {
 		debug.Log("Save(%v, %v): error %v\n", t, id.Str(), err)
 		return err
@@ -111,7 +112,7 @@ func (arch *Archiver) Save(t restic.BlobType, data []byte, id restic.ID) error {
 }
 
 // SaveTreeJSON stores a tree in the repository.
-func (arch *Archiver) SaveTreeJSON(tree *restic.Tree) (restic.ID, error) {
+func (arch *Archiver) SaveTreeJSON(ctx context.Context, tree *restic.Tree) (restic.ID, error) {
 	data, err := json.Marshal(tree)
 	if err != nil {
 		return restic.ID{}, errors.Wrap(err, "Marshal")
@@ -124,7 +125,7 @@ func (arch *Archiver) SaveTreeJSON(tree *restic.Tree) (restic.ID, error) {
 		return id, nil
 	}
 
-	return arch.repo.SaveBlob(restic.TreeBlob, data, id)
+	return arch.repo.SaveBlob(ctx, restic.TreeBlob, data, id)
 }
 
 func (arch *Archiver) reloadFileIfChanged(node *restic.Node, file fs.File) (*restic.Node, error) {
@@ -153,13 +154,14 @@ type saveResult struct {
 	bytes uint64
 }
 
-func (arch *Archiver) saveChunk(chunk chunker.Chunk, p *restic.Progress, token struct{}, file fs.File, resultChannel chan<- saveResult) {
+func (arch *Archiver) saveChunk(ctx context.Context, chunk chunker.Chunk, p *restic.Progress, token struct{}, file fs.File, resultChannel chan<- saveResult) {
 	defer freeBuf(chunk.Data)
 
 	id := restic.Hash(chunk.Data)
-	err := arch.Save(restic.DataBlob, chunk.Data, id)
+	err := arch.Save(ctx, restic.DataBlob, chunk.Data, id)
 	// TODO handle error
 	if err != nil {
+		debug.Log("Save(%v) failed: %v", id.Str(), err)
 		panic(err)
 	}
 
@@ -206,12 +208,12 @@ func updateNodeContent(node *restic.Node, results []saveResult) error {
 
 // SaveFile stores the content of the file on the backend as a Blob by calling
 // Save for each chunk.
-func (arch *Archiver) SaveFile(p *restic.Progress, node *restic.Node) (*restic.Node, error) {
+func (arch *Archiver) SaveFile(ctx context.Context, p *restic.Progress, node *restic.Node) (*restic.Node, error) {
 	file, err := fs.Open(node.Path)
-	defer file.Close()
 	if err != nil {
 		return node, errors.Wrap(err, "Open")
 	}
+	defer file.Close()
 
 	debug.RunHook("archiver.SaveFile", node.Path)
 
@@ -234,7 +236,7 @@ func (arch *Archiver) SaveFile(p *restic.Progress, node *restic.Node) (*restic.N
 		}
 
 		resCh := make(chan saveResult, 1)
-		go arch.saveChunk(chunk, p, <-arch.blobToken, file, resCh)
+		go arch.saveChunk(ctx, chunk, p, <-arch.blobToken, file, resCh)
 		resultChannels = append(resultChannels, resCh)
 	}
 
@@ -247,7 +249,7 @@ func (arch *Archiver) SaveFile(p *restic.Progress, node *restic.Node) (*restic.N
 	return node, err
 }
 
-func (arch *Archiver) fileWorker(wg *sync.WaitGroup, p *restic.Progress, done <-chan struct{}, entCh <-chan pipe.Entry) {
+func (arch *Archiver) fileWorker(ctx context.Context, wg *sync.WaitGroup, p *restic.Progress, entCh <-chan pipe.Entry) {
 	defer func() {
 		debug.Log("done")
 		wg.Done()
@@ -305,7 +307,7 @@ func (arch *Archiver) fileWorker(wg *sync.WaitGroup, p *restic.Progress, done <-
 			// otherwise read file normally
 			if node.Type == "file" && len(node.Content) == 0 {
 				debug.Log("   read and save %v", e.Path())
-				node, err = arch.SaveFile(p, node)
+				node, err = arch.SaveFile(ctx, p, node)
 				if err != nil {
 					fmt.Fprintf(os.Stderr, "error for %v: %v\n", node.Path, err)
 					arch.Warn(e.Path(), nil, err)
@@ -322,14 +324,14 @@ func (arch *Archiver) fileWorker(wg *sync.WaitGroup, p *restic.Progress, done <-
 			debug.Log("   processed %v, %d blobs", e.Path(), len(node.Content))
 			e.Result() <- node
 			p.Report(restic.Stat{Files: 1})
-		case <-done:
+		case <-ctx.Done():
 			// pipeline was cancelled
 			return
 		}
 	}
 }
 
-func (arch *Archiver) dirWorker(wg *sync.WaitGroup, p *restic.Progress, done <-chan struct{}, dirCh <-chan pipe.Dir) {
+func (arch *Archiver) dirWorker(ctx context.Context, wg *sync.WaitGroup, p *restic.Progress, dirCh <-chan pipe.Dir) {
 	debug.Log("start")
 	defer func() {
 		debug.Log("done")
@@ -381,7 +383,22 @@ func (arch *Archiver) dirWorker(wg *sync.WaitGroup, p *restic.Progress, done <-c
 						panic("invalid null subtree restic.ID")
 					}
 				}
-				tree.Insert(node)
+
+				// insert node into tree, resolve name collisions
+				name := node.Name
+				i := 0
+				for {
+					i++
+					err := tree.Insert(node)
+					if err == nil {
+						break
+					}
+
+					newName := fmt.Sprintf("%v-%d", name, i)
+					fmt.Fprintf(os.Stderr, "%v: name collision for %q, renaming to %q\n", filepath.Dir(node.Path), node.Name, newName)
+					node.Name = newName
+				}
+
 			}
 
 			node := &restic.Node{}
@@ -398,7 +415,7 @@ func (arch *Archiver) dirWorker(wg *sync.WaitGroup, p *restic.Progress, done <-c
 				node.Error = err.Error()
 			}
 
-			id, err := arch.SaveTreeJSON(tree)
+			id, err := arch.SaveTreeJSON(ctx, tree)
 			if err != nil {
 				panic(err)
 			}
@@ -415,7 +432,7 @@ func (arch *Archiver) dirWorker(wg *sync.WaitGroup, p *restic.Progress, done <-c
 			if dir.Path() != "" {
 				p.Report(restic.Stat{Dirs: 1})
 			}
-		case <-done:
+		case <-ctx.Done():
 			// pipeline was cancelled
 			return
 		}
@@ -427,7 +444,7 @@ type archivePipe struct {
 	New <-chan pipe.Job
 }
 
-func copyJobs(done <-chan struct{}, in <-chan pipe.Job, out chan<- pipe.Job) {
+func copyJobs(ctx context.Context, in <-chan pipe.Job, out chan<- pipe.Job) {
 	var (
 		// disable sending on the outCh until we received a job
 		outCh chan<- pipe.Job
@@ -439,7 +456,7 @@ func copyJobs(done <-chan struct{}, in <-chan pipe.Job, out chan<- pipe.Job) {
 
 	for {
 		select {
-		case <-done:
+		case <-ctx.Done():
 			return
 		case job, ok = <-inCh:
 			if !ok {
@@ -462,7 +479,7 @@ type archiveJob struct {
 	new    pipe.Job
 }
 
-func (a *archivePipe) compare(done <-chan struct{}, out chan<- pipe.Job) {
+func (a *archivePipe) compare(ctx context.Context, out chan<- pipe.Job) {
 	defer func() {
 		close(out)
 		debug.Log("done")
@@ -488,7 +505,7 @@ func (a *archivePipe) compare(done <-chan struct{}, out chan<- pipe.Job) {
 					out <- archiveJob{new: newJob}.Copy()
 				}
 
-				copyJobs(done, a.New, out)
+				copyJobs(ctx, a.New, out)
 				return
 			}
 
@@ -585,7 +602,7 @@ func (j archiveJob) Copy() pipe.Job {
 const saveIndexTime = 30 * time.Second
 
 // saveIndexes regularly queries the master index for full indexes and saves them.
-func (arch *Archiver) saveIndexes(wg *sync.WaitGroup, done <-chan struct{}) {
+func (arch *Archiver) saveIndexes(ctx context.Context, wg *sync.WaitGroup) {
 	defer wg.Done()
 
 	ticker := time.NewTicker(saveIndexTime)
@@ -593,11 +610,11 @@ func (arch *Archiver) saveIndexes(wg *sync.WaitGroup, done <-chan struct{}) {
 
 	for {
 		select {
-		case <-done:
+		case <-ctx.Done():
 			return
 		case <-ticker.C:
 			debug.Log("saving full indexes")
-			err := arch.repo.SaveFullIndex()
+			err := arch.repo.SaveFullIndex(ctx)
 			if err != nil {
 				debug.Log("save indexes returned an error: %v", err)
 				fmt.Fprintf(os.Stderr, "error saving preliminary index: %v\n", err)
@@ -634,7 +651,7 @@ func (p baseNameSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
 // Snapshot creates a snapshot of the given paths. If parentrestic.ID is set, this is
 // used to compare the files to the ones archived at the time this snapshot was
 // taken.
-func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, hostname string, parentID *restic.ID) (*restic.Snapshot, restic.ID, error) {
+func (arch *Archiver) Snapshot(ctx context.Context, p *restic.Progress, paths, tags []string, hostname string, parentID *restic.ID, time time.Time) (*restic.Snapshot, restic.ID, error) {
 	paths = unique(paths)
 	sort.Sort(baseNameSlice(paths))
 
@@ -643,14 +660,13 @@ func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, hostnam
 	debug.RunHook("Archiver.Snapshot", nil)
 
 	// signal the whole pipeline to stop
-	done := make(chan struct{})
 	var err error
 
 	p.Start()
 	defer p.Done()
 
 	// create new snapshot
-	sn, err := restic.NewSnapshot(paths, tags, hostname)
+	sn, err := restic.NewSnapshot(paths, tags, hostname, time)
 	if err != nil {
 		return nil, restic.ID{}, err
 	}
@@ -663,14 +679,14 @@ func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, hostnam
 		sn.Parent = parentID
 
 		// load parent snapshot
-		parent, err := restic.LoadSnapshot(arch.repo, *parentID)
+		parent, err := restic.LoadSnapshot(ctx, arch.repo, *parentID)
 		if err != nil {
 			return nil, restic.ID{}, err
 		}
 
 		// start walker on old tree
 		ch := make(chan walk.TreeJob)
-		go walk.Tree(arch.repo, *parent.Tree, done, ch)
+		go walk.Tree(ctx, arch.repo, *parent.Tree, ch)
 		jobs.Old = ch
 	} else {
 		// use closed channel
@@ -683,13 +699,13 @@ func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, hostnam
 	pipeCh := make(chan pipe.Job)
 	resCh := make(chan pipe.Result, 1)
 	go func() {
-		pipe.Walk(paths, arch.SelectFilter, done, pipeCh, resCh)
+		pipe.Walk(ctx, paths, arch.SelectFilter, pipeCh, resCh)
 		debug.Log("pipe.Walk done")
 	}()
 	jobs.New = pipeCh
 
 	ch := make(chan pipe.Job)
-	go jobs.compare(done, ch)
+	go jobs.compare(ctx, ch)
 
 	var wg sync.WaitGroup
 	entCh := make(chan pipe.Entry)
@@ -708,22 +724,22 @@ func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, hostnam
 	// run workers
 	for i := 0; i < maxConcurrency; i++ {
 		wg.Add(2)
-		go arch.fileWorker(&wg, p, done, entCh)
-		go arch.dirWorker(&wg, p, done, dirCh)
+		go arch.fileWorker(ctx, &wg, p, entCh)
+		go arch.dirWorker(ctx, &wg, p, dirCh)
 	}
 
 	// run index saver
 	var wgIndexSaver sync.WaitGroup
-	stopIndexSaver := make(chan struct{})
+	indexCtx, indexCancel := context.WithCancel(ctx)
 	wgIndexSaver.Add(1)
-	go arch.saveIndexes(&wgIndexSaver, stopIndexSaver)
+	go arch.saveIndexes(indexCtx, &wgIndexSaver)
 
 	// wait for all workers to terminate
 	debug.Log("wait for workers")
 	wg.Wait()
 
 	// stop index saver
-	close(stopIndexSaver)
+	indexCancel()
 	wgIndexSaver.Wait()
 
 	debug.Log("workers terminated")
@@ -740,7 +756,7 @@ func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, hostnam
 	sn.Tree = root.Subtree
 
 	// load top-level tree again to see if it is empty
-	toptree, err := arch.repo.LoadTree(*root.Subtree)
+	toptree, err := arch.repo.LoadTree(ctx, *root.Subtree)
 	if err != nil {
 		return nil, restic.ID{}, err
 	}
@@ -750,7 +766,7 @@ func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, hostnam
 	}
 
 	// save index
-	err = arch.repo.SaveIndex()
+	err = arch.repo.SaveIndex(ctx)
 	if err != nil {
 		debug.Log("error saving index: %v", err)
 		return nil, restic.ID{}, err
@@ -759,7 +775,7 @@ func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, hostnam
 	debug.Log("saved indexes")
 
 	// save snapshot
-	id, err := arch.repo.SaveJSONUnpacked(restic.SnapshotFile, sn)
+	id, err := arch.repo.SaveJSONUnpacked(ctx, restic.SnapshotFile, sn)
 	if err != nil {
 		return nil, restic.ID{}, err
 	}