Add VERSION file for 0.5.0-rc.1

Add --tag filtering to every command, where applicable

.travis.yml

@@ -2,8 +2,9 @@ language: go
 sudo: false
 
 go:
-  - 1.6.3
-  - 1.7.1
+  - 1.7.5
+  - 1.8
+  - tip
 
 os:
   - linux
@@ -16,16 +17,23 @@ env:
 matrix:
   exclude:
     - os: osx
-      go: 1.6.3
+      go: 1.7.5
+    - os: osx
+      go: tip
     - os: linux
-      go: 1.7.1
+      go: 1.8
   include:
     - os: linux
-      go: 1.7.1
+      go: 1.8
       sudo: true
       env:
         RESTIC_TEST_FUSE=1
+  allow_failures:
+    - go: tip
 
+branches:
+  only:
+    - master
 
 notifications:
   irc:
@@ -40,11 +48,9 @@ install:
   - export GOBIN="$GOPATH/bin"
   - export PATH="$PATH:$GOBIN"
   - go env
-  - ulimit -n 2048
 
 script:
   - go run run_integration_tests.go
 
 after_success:
-  - GOPATH=$PWD:$PWD/vendor goveralls -coverprofile=all.cov -service=travis-ci -repotoken "$COVERALLS_TOKEN"
   - bash <(curl -s https://codecov.io/bash) -f all.cov

CONTRIBUTING.md

@@ -3,7 +3,10 @@ This document describes the way you can contribute to the restic project.
 Ways to Help Out
 ================
 
-Thank you for your contribution!
+Thank you for your contribution! Please **open an issue first** (or add a
+comment to an existing issue) if you plan to work on any code or add a new
+feature. This way, duplicate work is prevented and we can discuss your ideas
+and design first.
 
 There are several ways you can help us out. First of all code contributions and
 bug fixes are most welcome. However even "minor" details as fixing spelling
@@ -74,7 +77,7 @@ Just clone the repository, `cd` to it and run `gb build` to build the binary:
     [...]
     $ bin/restic version
     restic compiled manually
-    compiled at unknown time with go1.6
+    compiled at unknown time with go1.7
 
 The following commands can be used to run all the tests:
 
@@ -83,7 +86,7 @@ The following commands can be used to run all the tests:
     [...]
 
 If you want to run your tests on Linux, OpenBSD or FreeBSD, you can use
-[vagrant](https://www.vagrantup.com/) with the proveded `Vagrantfile` to
+[vagrant](https://www.vagrantup.com/) with the provided `Vagrantfile` to
 quickly set up VMs and run the tests, e.g.:
 
     $ vagrant up freebsd

Dockerfile

@@ -14,11 +14,11 @@
 #   docker run --rm -v $PWD:/home/travis/restic restic/test gb test -v ./backend
 #
 # build the image for an older version of Go:
-#   docker build --build-arg GOVERSION=1.3.3 -t restic/test:go1.3.3 .
+#   docker build --build-arg GOVERSION=1.6.4 -t restic/test:go1.6.4 .
 
 FROM ubuntu:14.04
 
-ARG GOVERSION=1.7
+ARG GOVERSION=1.7.5
 ARG GOARCH=amd64
 
 # install dependencies

README.md

@@ -11,17 +11,30 @@ restic is a backup program that is fast, efficient and secure. Detailed
 information can be found in [the documentation](doc/index.md) and [the user
 manual](doc/Manual.md). The [design document](doc/Design.md) lists the
 technical background and gives detailed information about the structure of the
-repository and the data saved therein.
+repository and the data saved therein. The file [FAQ.md](doc/FAQ.md) lists the
+most frequently asked questions.
 
 The latest documentation can be viewed online at
 <https://restic.readthedocs.io/en/latest>. On the bottom left corner there is
 a menu that allows switching to the documentation and user manual for the
 latest released version.
 
+News
+====
+
+You can follow the restic project on Twitter
+[@resticbackup](https://twitter.com/resticbackup) or by subscribing to the
+[development blog](https://restic.github.io/blog/).
+
+Install restic
+==============
+
+You can download the latest pre-compiled binary from the [restic release page](https://github.com/restic/restic/releases/latest).
+
 Build restic
 ============
 
-Install Go/Golang (at least version 1.6), then run `go run build.go`,
+Install Go/Golang (at least version 1.7), then run `go run build.go`,
 afterwards you'll find the binary in the current directory:
 
     $ go run build.go
@@ -39,10 +52,15 @@ Building restic with gccgo may work, but is not supported.
 Contribute and Documentation
 ============================
 
-Contributions are welcome! More information and a description of the
-development environment can be found in [`CONTRIBUTING.md`](CONTRIBUTING.md). A
-document describing the design of restic and the data structures stored on the
-back end is contained in [`doc/Design.md`](doc/Design.md).
+Contributions are welcome! Please **open an issue first** (or add a comment to
+an existing issue) if you plan to work on any code or add a new feature. This
+way, duplicate work is prevented and we can discuss your ideas and design
+first.
+
+More information and a description of the development environment can be found
+in [`CONTRIBUTING.md`](CONTRIBUTING.md). A document describing the design of
+restic and the data structures stored on the back end is contained in
+[`doc/Design.md`](doc/Design.md).
 
 If you'd like to start contributing to restic, but don't know exactly what do
 to, have a look at this great article by Dave Cheney:

VERSION

@@ -1 +1 @@
-0.3.1
+0.5.0-rc.1

Vagrantfile

@@ -1,7 +1,7 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 
-GO_VERSION = '1.6'
+GO_VERSION = '1.7'
 
 def packages_freebsd
   return <<-EOF

appveyor.yml

@@ -3,6 +3,10 @@ clone_folder: c:\restic
 environment:
   GOPATH: c:\gopath
 
+branches:
+  only:
+    - master
+
 init:
   - ps: >-
       $app = Get-WmiObject -Class Win32_Product -Filter "Vendor = 'http://golang.org'"
@@ -13,8 +17,8 @@ init:
 
 install:
   - rmdir c:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.7.windows-amd64.msi
-  - msiexec /i go1.7.windows-amd64.msi /q
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.8.windows-amd64.msi
+  - msiexec /i go1.8.windows-amd64.msi /q
   - go version
   - go env
   - appveyor DownloadFile http://sourceforge.netcologne.de/project/gnuwin32/tar/1.13-1/tar-1.13-1-bin.zip -FileName tar.zip

build.go

@@ -12,7 +12,6 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
-	"time"
 )
 
 var (
@@ -33,8 +32,6 @@ var config = struct {
 	Tests:     []string{"restic/...", "cmds/..."}, // tests to run
 }
 
-const timeFormat = "2006-01-02 15:04:05"
-
 // specialDir returns true if the file begins with a special character ('.' or '_').
 func specialDir(name string) bool {
 	if name == "." {
@@ -177,6 +174,7 @@ func showUsage(output io.Writer) {
 	fmt.Fprintf(output, "  -t     --tags          specify additional build tags\n")
 	fmt.Fprintf(output, "  -k     --keep-gopath   do not remove the GOPATH after build\n")
 	fmt.Fprintf(output, "  -T     --test          run tests\n")
+	fmt.Fprintf(output, "  -o     --output        set output file name\n")
 	fmt.Fprintf(output, "         --goos value    set GOOS for cross-compilation\n")
 	fmt.Fprintf(output, "         --goarch value  set GOARCH for cross-compilation\n")
 }
@@ -207,7 +205,7 @@ func cleanEnv() (env []string) {
 func build(cwd, goos, goarch, gopath string, args ...string) error {
 	args = append([]string{"build"}, args...)
 	cmd := exec.Command("go", args...)
-	cmd.Env = append(cleanEnv(), "GOPATH="+gopath, "GOARCH="+goarch, "GOOS="+goos)
+	cmd.Env = append(cleanEnv(), "GOPATH="+gopath, "GOARCH="+goarch, "GOOS="+goos, "CGO_ENABLED=0")
 	cmd.Dir = cwd
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
@@ -301,6 +299,8 @@ func main() {
 	targetGOOS := runtime.GOOS
 	targetGOARCH := runtime.GOARCH
 
+	var outputFilename string
+
 	for i, arg := range params {
 		if skipNext {
 			skipNext = false
@@ -318,6 +318,9 @@ func main() {
 			}
 			skipNext = true
 			buildTags = strings.Split(params[i+1], " ")
+		case "-o", "--output":
+			skipNext = true
+			outputFilename = params[i+1]
 		case "-T", "--test":
 			runTests = true
 		case "--goos":
@@ -380,9 +383,11 @@ func main() {
 		}
 	}()
 
-	outputFilename := config.Name
-	if targetGOOS == "windows" {
-		outputFilename += ".exe"
+	if outputFilename == "" {
+		outputFilename = config.Name
+		if targetGOOS == "windows" {
+			outputFilename += ".exe"
+		}
 	}
 
 	cwd, err := os.Getwd()
@@ -392,8 +397,7 @@ func main() {
 	output := filepath.Join(cwd, outputFilename)
 
 	version := getVersion()
-	compileTime := time.Now().Format(timeFormat)
-	constants := Constants{`main.compiledAt`: compileTime}
+	constants := Constants{}
 	if version != "" {
 		constants["main.version"] = version
 	}

build_release_binaries.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+set -e
+
+if [[ -z "$VERSION" ]]; then
+    echo '$VERSION unset'
+    exit 1
+fi
+
+dir=$(mktemp -d --tmpdir restic-release-XXXXXX)
+echo "path is ${dir}"
+
+for R in       \
+    darwin/386     \
+    darwin/amd64   \
+    freebsd/386    \
+    freebsd/amd64  \
+    freebsd/arm    \
+    linux/386      \
+    linux/amd64    \
+    linux/arm      \
+    linux/arm64    \
+    openbsd/386   \
+    openbsd/amd64 \
+    windows/386    \
+    windows/amd64  \
+    ; do \
+
+    OS=$(dirname $R)
+    ARCH=$(basename $R)
+    filename=restic_${VERSION}_${OS}_${ARCH}
+
+    if [[ "$OS" == "windows" ]]; then
+        filename="${filename}.exe"
+    fi
+
+    echo $filename
+
+    go run build.go --goos $OS --goarch $ARCH --output ${filename}
+    if [[ "$OS" == "windows" ]]; then
+        zip ${filename%.exe}.zip ${filename}
+        rm ${filename}
+        mv ${filename%.exe}.zip ${dir}
+    else
+        bzip2 ${filename}
+        mv ${filename}.bz2 ${dir}
+    fi
+done
+
+echo "packing sources"
+git archive --format=tar --prefix=restic-$VERSION/ v$VERSION | gzip -n > restic-$VERSION.tar.gz
+mv restic-$VERSION.tar.gz ${dir}
+
+echo "creating checksums"
+pushd ${dir}
+sha256sum restic_*.{zip,bz2} restic-$VERSION.tar.gz > SHA256SUMS
+gpg --armor --detach-sign SHA256SUMS
+popd
+
+echo "creating source signature file"
+gpg --armor --detach-sign ${dir}/restic-$VERSION.tar.gz
+
+echo
+echo "done, path is ${dir}"

doc/Design.md

@@ -285,7 +285,7 @@ This way, the password can be changed without having to re-encrypt all data.
 Snapshots
 ---------
 
-A snapshots represents a directory with all files and sub-directories at a
+A snapshot represents a directory with all files and sub-directories at a
 given point in time. For each backup that is made, a new snapshot is created. A
 snapshot is a JSON document that is stored in an encrypted file below the
 directory `snapshots` in the repository. The filename is the storage ID. This
@@ -294,6 +294,31 @@ string is unique and used within restic to uniquely identify a snapshot.
 The command `restic cat snapshot` can be used as follows to decrypt and
 pretty-print the contents of a snapshot file:
 
+```console
+$ restic -r /tmp/restic-repo cat snapshot 251c2e58
+enter password for repository:
+{
+  "time": "2015-01-02T18:10:50.895208559+01:00",
+  "tree": "2da81727b6585232894cfbb8f8bdab8d1eccd3d8f7c92bc934d62e62e618ffdf",
+  "dir": "/tmp/testdata",
+  "hostname": "kasimir",
+  "username": "fd0",
+  "uid": 1000,
+  "gid": 100,
+  "tags": [
+    "NL"
+  ]
+}
+```
+
+Here it can be seen that this snapshot represents the contents of the directory
+`/tmp/testdata`. The most important field is `tree`. When the meta data (e.g.
+the tags) of a snapshot change, the snapshot needs to be re-encrypted and saved.
+This will change the storage ID, so in order to relate these seemingly
+different snapshots, a field `original` is introduced which contains the ID of
+the original snapshot, e.g. after adding the tag `DE` to the snapshot above it
+becomes:
+
 ```console
 $ restic -r /tmp/restic-repo cat snapshot 22a5af1b
 enter password for repository:
@@ -304,12 +329,17 @@ enter password for repository:
   "hostname": "kasimir",
   "username": "fd0",
   "uid": 1000,
-  "gid": 100
+  "gid": 100,
+  "tags": [
+    "NL",
+    "DE"
+  ],
+  "original": "251c2e5841355f743f9d4ffd3260bee765acee40a6229857e32b60446991b837"
 }
 ```
 
-Here it can be seen that this snapshot represents the contents of the directory
-`/tmp/testdata`. The most important field is `tree`.
+Once introduced, the `original` field is not modified when the snapshot's meta
+data is changed again.
 
 All content within a restic repository is referenced according to its SHA-256
 hash. Before saving, each file is split into variable sized Blobs of data. The
@@ -327,10 +357,11 @@ A snapshot references a tree by the SHA-256 hash of the JSON string
 representation of its contents. Trees and data are saved in pack files in a
 subdirectory of the directory `data`.
 
-The command `restic cat tree` can be used to inspect the tree referenced above:
+The command `restic cat blob` can be used to inspect the tree referenced above
+(piping the output of the command to `jq .` so that the JSON is indented):
 
 ```console
-$ restic -r /tmp/restic-repo cat tree b8138ab08a4722596ac89c917827358da4672eac68e3c03a8115b88dbf4bfb59
+$ restic -r /tmp/restic-repo cat blob b8138ab08a4722596ac89c917827358da4672eac68e3c03a8115b88dbf4bfb59 | jq .
 enter password for repository:
 {
   "nodes": [
@@ -356,11 +387,11 @@ A tree contains a list of entries (in the field `nodes`) which contain meta
 data like a name and timestamps. When the entry references a directory, the
 field `subtree` contains the plain text ID of another tree object.
 
-When the command `restic cat tree` is used, the storage hash is needed to print
+When the command `restic cat blob` is used, the plaintext ID is needed to print
 a tree. The tree referenced above can be dumped as follows:
 
 ```console
-$ restic -r /tmp/restic-repo cat tree 8b238c8811cc362693e91a857460c78d3acf7d9edb2f111048691976803cf16e
+$ restic -r /tmp/restic-repo cat blob 8b238c8811cc362693e91a857460c78d3acf7d9edb2f111048691976803cf16e
 enter password for repository:
 {
   "nodes": [
@@ -389,8 +420,8 @@ enter password for repository:
 This tree contains a file entry. This time, the `subtree` field is not present
 and the `content` field contains a list with one plain text SHA-256 hash.
 
-The command `restic cat data` can be used to extract and decrypt data given a
-plaintext ID, e.g. for the data mentioned above:
+The command `restic cat blob` can also be used to extract and decrypt data
+given a plaintext ID, e.g. for the data mentioned above:
 
 ```console
 $ restic -r /tmp/restic-repo cat blob 50f77b3b4291e8411a027b9f9b9e64658181cc676ce6ba9958b95f268cb1109d | sha256sum

doc/FAQ.md

@@ -0,0 +1,20 @@
+FAQ
+===
+
+This is the list of Frequently Asked Questions for restic.
+
+`restic check` reports packs that aren't referenced in any index, is my repository broken?
+------------------------------------------------------------------------------------------
+
+When `restic check` reports that there are pack files in the repository that are not referenced in any index, that's (in contrast to what restic reports at the moment) not a source for concern. The output looks like this:
+
+    $ restic check
+    Create exclusive lock for repository
+    Load indexes
+    Check all packs
+    pack 819a9a52e4f51230afa89aefbf90df37fb70996337ae57e6f7a822959206a85e: not referenced in any index
+    pack de299e69fb075354a3775b6b045d152387201f1cdc229c31d1caa34c3b340141: not referenced in any index
+    Check snapshots, trees and blobs
+    Fatal: repository contains errors
+
+The message means that there is more data stored in the repo than strictly necessary. With high probability this is duplicate data. In order to clean it up, the command `restic prune` can be used. The cause of this bug is not yet known.

doc/Manual.md

@@ -1,7 +1,13 @@
 Thanks for using restic. This document will give you an overview of the basic
 functionality provided by restic.
 
-# Building/installing restic
+# Installing restic
+
+## from pre-compiled binary
+
+You can download the latest pre-compiled binary from the [restic release page](https://github.com/restic/restic/releases/latest).
+
+## Mac OS X
 
 If you are using Mac OS X, you can install restic using the
 [homebrew](http://brew.sh/) packet manager:
@@ -11,25 +17,19 @@ $ brew tap restic/restic
 $ brew install restic
 ```
 
+## archlinux
+
 On archlinux, there is a package called `restic-git` which can be installed from AUR, e.g. with `pacaur`:
 
 ```console
 $ pacaur -S restic-git
 ```
 
-At debian stable you can install 'go' directly from the repositories (as root):
+# Building restic
 
-```console
-$ apt-get install golang-go
-```
-
-after installation of 'go' go straight forward to 'git clone [...]'
-
-If you are using Linux, BSD or Windows, the only way to install restic on your
-system right now is to compile it from source. restic is written in the Go
-programming language and you need at least Go version 1.6. Building restic may
-also work with older versions of Go, but that's not supported. See the [Getting
-started](https://golang.org/doc/install) guide of the Go project for
+restic is written in the Go programming language and you need at least Go version 1.7.
+Building restic may also work with older versions of Go, but that's not supported.
+See the [Getting started](https://golang.org/doc/install) guide of the Go project for
 instructions how to install Go.
 
 In order to build restic from source, execute the following steps:
@@ -46,6 +46,8 @@ $ go run build.go
 At the moment, the only tested compiler for restic is the official Go compiler.
 Building restic with gccgo may work, but is not supported.
 
+# Usage help
+
 Usage help is available:
 
 ```console
@@ -71,13 +73,15 @@ Available Commands:
   rebuild-index build a new index file
   restore       extract the data from a snapshot
   snapshots     list all snapshots
+  tag           modifies tags on snapshots
   unlock        remove locks other processes created
   version       Print version information
 
 Flags:
+      --json                   set output mode to JSON for commands that support it
       --no-lock                do not lock the repo, this allows some operations on read-only repos
   -p, --password-file string   read the repository password from a file
-  -q, --quiet                  do not outputcomprehensive progress report
+  -q, --quiet                  do not output comprehensive progress report
   -r, --repo string            repository to backup to or restore from (default: $RESTIC_REPOSITORY)
 
 Use "restic [command] --help" for more information about a command.
@@ -108,9 +112,10 @@ Flags:
       --tag tag                 add a tag for the new snapshot (can be specified multiple times)
 
 Global Flags:
+      --json                   set output mode to JSON for commands that support it
       --no-lock                do not lock the repo, this allows some operations on read-only repos
   -p, --password-file string   read the repository password from a file
-  -q, --quiet                  do not outputcomprehensive progress report
+  -q, --quiet                  do not output comprehensive progress report
   -r, --repo string            repository to backup to or restore from (default: $RESTIC_REPOSITORY)
 ```
 
@@ -139,6 +144,8 @@ Please note that knowledge of your password is required to access the repository
 Losing your password means that your data is irrecoverably lost.
 ```
 
+Other backends like sftp and s3 are [described in a later section](#create-an-sftp-repository) of this document.
+
 Remembering your password is important! If you lose it, you won't be able to
 access data stored in the repository.
 
@@ -212,6 +219,15 @@ snapshot 31f7bd63 saved
 In fact several hosts may use the same repository to backup directories and
 files leading to a greater de-duplication.
 
+Please be aware that when you backup different directories (or the directories
+to be saved have a variable name component like a time/date), restic always
+needs to read all files and only afterwards can compute which parts of the
+files need to be saved. When you backup the same directory again (maybe with
+new or changed files) restic will find the old snapshot in the repo and by
+default only reads those files that are new or have been modified since the
+last snapshot. This is decided based on the modify date of the file in the
+file system.
+
 You can exclude folders and files by specifying exclude-patterns.
 Either specify them with multiple `--exclude`'s or one `--exclude-file`
 
@@ -226,7 +242,7 @@ $ restic -r /tmp/backup backup ~/work --exclude=*.c --exclude-file=exclude
 
 Patterns use [`filepath.Glob`](https://golang.org/pkg/path/filepath/#Glob) internally,
 see [`filepath.Match`](https://golang.org/pkg/path/filepath/#Match) for syntax.
-Additionally `**` exludes arbitrary subdirectories.
+Additionally `**` excludes arbitrary subdirectories.
 Environment-variables in exclude-files are expanded with [`os.ExpandEnv`](https://golang.org/pkg/os/#ExpandEnv).
 
 By specifying the option `--one-file-system` you can instruct restic to only
@@ -352,7 +368,6 @@ enter password for repository:
 restoring <Snapshot of [/home/art] at 2015-05-08 21:45:17.884408621 +0200 CEST> to /tmp/restore-work
 ```
 
-
 # Manage repository keys
 
 The `key` command allows you to set multiple access keys or passwords per
@@ -380,6 +395,45 @@ enter password for repository:
 *eb78040b    username    kasimir   2015-08-12 13:29:57
 ```
 
+# Manage tags
+
+Managing tags on snapshots is done with the `tag` command. The existing set of
+tags can be replaced completely, tags can be added to removed. The result is
+directly visible in the `snapshots` command.
+
+Let's say we want to tag snapshot `590c8fc8` with the tags `NL` and `CH` and
+remove all other tags that may be present, the following command does that:
+
+```console
+$ restic -r /tmp/backup tag --set NL,CH 590c8fc8
+Create exclusive lock for repository
+Modified tags on 1 snapshots
+```
+
+Note the snapshot ID has changed, so between each change we need to look up
+the new ID of the snapshot. But there is an even better way, the `tag` command
+accepts `--tag` for a filter, so we can filter snapshots based on the tag we
+just added.
+
+So we can add and remove tags incrementally like this:
+
+```console
+$ restic -r /tmp/backup tag --tag NL --remove CH
+Create exclusive lock for repository
+Modified tags on 1 snapshots
+
+$ restic -r /tmp/backup tag --tag NL --add UK
+Create exclusive lock for repository
+Modified tags on 1 snapshots
+
+$ restic -r /tmp/backup tag --tag NL --remove NL
+Create exclusive lock for repository
+Modified tags on 1 snapshots
+
+$ restic -r /tmp/backup tag --tag NL --add SOMETHING
+No snapshots were modified
+```
+
 # Check integrity and consistency
 
 Imagine your repository is saved on a server that has a faulty hard drive, or
@@ -425,6 +479,11 @@ Don't forget to umount after quitting!
 
 Mounting repositories via FUSE is not possible on Windows and OpenBSD.
 
+Restic supports storage and preservation of hard links. However, since hard links
+exist in the scope of a filesystem by definition, restoring hard links from a fuse
+mount should be done by a program that preserves hard links. A program that does so
+is rsync, used with the option --hard-links.
+
 # Create an SFTP repository
 
 In order to backup data via SFTP, you must first set up a server with SSH and
@@ -448,6 +507,35 @@ You can also specify a relative (read: no slash (`/`) character at the
 beginning) directory, in this case the dir is relative to the remote user's
 home directory.
 
+# Create a REST server repository
+
+In order to backup data to the remote server via HTTP or HTTPS protocol,
+you must first set up a remote [REST server](https://github.com/restic/rest-server)
+instance.  Once the server is configured, accessing it is achieved by changing the
+URL scheme like this:
+
+```console
+$ restic -r rest:http://host:8000/
+```
+
+Depending on your REST server setup, you can use HTTPS protocol, password
+protection, or multiple repositories.  Or any combination of those features, as
+you see fit. TCP/IP port is also configurable. Here are some more examples:
+
+```console
+$ restic -r rest:https://host:8000/
+$ restic -r rest:https://user:pass@host:8000/
+$ restic -r rest:https://user:pass@host:8000/my_backup_repo/
+```
+
+If you use TLS, make sure your certificates are signed, 'cause restic client
+will refuse to communicate otherwise.  It's easy to obtain such certificates
+today, thanks to free certificate authorities like [Let’s
+Encrypt](https://letsencrypt.org/).
+
+REST server uses exactly the same directory structure as local backend, so you
+should be able to access it both locally and via HTTP, even simultaneously.
+
 # Create an Amazon S3 repository
 
 Restic can backup data to any Amazon S3 bucket. However, in this case, changing the URL scheme is not enough since Amazon uses special security credentials to sign HTTP requests. By consequence, you must first setup the following environment variables with the credentials you obtained while creating the bucket.
@@ -480,8 +568,8 @@ only available via HTTP, you can specify the URL to the server like this:
 
 ### Pre-Requisites
 
-* Download and Install [Minio Server](https://minio.io/download/).
-* You can also refer to [https://docs.minio.io](https://docs.minio.io) for step by step guidance on installation and getting started on Minio CLient and Minio Server.
+* Download and Install [Minio Server](https://minio.io/downloads/#minio-server).
+* You can also refer to [https://docs.minio.io](https://docs.minio.io) for step by step guidance on installation and getting started on Minio Client and Minio Server.
 
 You must first setup the following environment variables with the credentials of your running Minio Server.
 
@@ -508,7 +596,8 @@ be done either manually (by specifying a snapshot ID to remove) or by using a
 policy that describes which snapshots to forget. For all remove operations, two
 commands need to be called in sequence: `forget` to remove a snapshot and
 `prune` to actually remove the data that was referenced by the snapshot from
-the repository.
+the repository. This can be automated with the `--prune` option of the `forget`
+command, which runs `prune` automatically if snapshots have been removed.
 
 ## Remove a single snapshot
 
@@ -573,6 +662,41 @@ done
 
 Afterwards the repository is smaller.
 
+You can automate this two-step process by using the `--prune` switch to
+`forget`:
+
+```console
+$ restic forget --keep-last 1 --prune
+snapshots for host mopped, directories /home/user/work:
+
+keep 1 snapshots:
+ID        Date                 Host        Tags        Directory
+----------------------------------------------------------------------
+4bba301e  2017-02-21 10:49:18  mopped                  /home/user/work
+
+remove 1 snapshots:
+ID        Date                 Host        Tags        Directory
+----------------------------------------------------------------------
+8c02b94b  2017-02-21 10:48:33  mopped                  /home/user/work
+
+1 snapshots have been removed, running prune
+counting files in repo
+building new index for repo
+[0:00] 100.00%  37 / 37 packs
+repository contains 37 packs (5521 blobs) with 151.012 MiB bytes
+processed 5521 blobs: 0 duplicate blobs, 0B duplicate
+load all snapshots
+find data that is still in use for 1 snapshots
+[0:00] 100.00%  1 / 1 snapshots
+found 5323 of 5521 data blobs still in use, removing 198 blobs
+will delete 0 packs and rewrite 27 packs, this frees 22.106 MiB
+creating new index
+[0:00] 100.00%  30 / 30 packs
+saved new index as b49f3e68
+done
+```
+
+
 ## Removing snapshots according to a policy
 
 Removing snapshots manually is tedious and error-prone, therefore restic allows
@@ -704,3 +828,10 @@ enter password for repository:
   "gid": 20
 }
 ```
+
+# Scripting restic
+
+Restic supports the output of some commands in JSON format. The JSON flag ```--json``` is currently supported only by ```restic snapshots```.
+
+```console
+$ restic -r /tmp/backup snapshots --json```

doc/REST_backend.md

@@ -6,6 +6,19 @@ following values are valid for `{type}`: `data`, `keys`, `locks`, `snapshots`,
 `index`, `config`. `{path}` is a path to the repository, so that multiple
 different repositories can be accessed. The default path is `/`.
 
+## POST {path}?create=true
+
+This request is used to initially create a new repository. The server responds
+with "200 OK" if the repository structure was created successfully or already
+exists, otherwise an error is returned.
+
+## DELETE {path}
+
+Deletes the repository on the server side. The server responds with "200 OK" if
+the repository was successfully removed. If this function is not implemented
+the server returns "501 Not Implemented", if this it is denied by the server it
+returns "403 Forbidden".
+
 ## HEAD {path}/config
 
 Returns "200 OK" if the repository has a configuration,

run_integration_tests.go

@@ -176,7 +176,7 @@ func (env *TravisEnvironment) Prepare() error {
 				"windows/386", "windows/amd64",
 				"darwin/386", "darwin/amd64",
 				"freebsd/386", "freebsd/amd64",
-				"opendbsd/386", "opendbsd/amd64",
+				"openbsd/386", "openbsd/amd64",
 			}
 			if !strings.HasPrefix(runtime.Version(), "go1.3") {
 				env.goxOSArch = append(env.goxOSArch,

src/cmds/restic-server/.gitignore

@@ -1,24 +0,0 @@
-# Compiled Object files, Static and Dynamic libs (Shared Objects)
-*.o
-*.a
-*.so
-
-# Folders
-_obj
-_test
-
-# Architecture specific extensions/prefixes
-*.[568vq]
-[568vq].out
-
-*.cgo1.go
-*.cgo2.c
-_cgo_defun.c
-_cgo_gotypes.go
-_cgo_export.*
-
-_testmain.go
-
-*.exe
-*.test
-*.prof

src/cmds/restic-server/LICENSE

@@ -1,24 +0,0 @@
-Copyright (c) 2015, Bertil Chapuis
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-* Redistributions of source code must retain the above copyright notice, this
-  list of conditions and the following disclaimer.
-
-* Redistributions in binary form must reproduce the above copyright notice,
-  this list of conditions and the following disclaimer in the documentation
-  and/or other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-

src/cmds/restic-server/README.md

@@ -1,29 +0,0 @@
-# Restic Server
-
-Restic Server is a sample server that implement restic's rest backend api.
-It has been developed for demonstration purpose and is not intented to be used in production.
-
-## Getting started
-
-By default the server persists backup data in `/tmp/restic`. 
-Build and start the server with a custom persistence directory:
-
-```
-go build
-./restic-server -path /user/home/backup
-```
-
-The server use an `.htpasswd` file to specify users. You can create such a file at the root of the persistence directory by executing the following command. In order to append new user to the file, just omit the `-c` argument.
-
-```
-htpasswd -s -c .htpasswd username
-```
-
-By default the server uses http. This is not very secure since with Basic Authentication, username and passwords will be present in every request. In order to enable TLS support just add the `-tls` argument and add a private and public key at the root of your persistence directory. 
-
-Signed certificate are required by the restic backend but if you just want to test the feature you can generate unsigned keys with the following commands:
-
-```
-openssl genrsa -out private_key 2048
-openssl req -new -x509 -key private_key -out public_key -days 365
-```

src/cmds/restic-server/handlers.go

@@ -1,194 +0,0 @@
-// +build go1.4
-
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
-
-	"restic/fs"
-)
-
-// Context contains repository meta-data.
-type Context struct {
-	path string
-}
-
-// AuthHandler wraps h with a http.HandlerFunc that performs basic
-// authentication against the user/passwords pairs stored in f and returns the
-// http.HandlerFunc.
-func AuthHandler(f *HtpasswdFile, h http.Handler) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		username, password, ok := r.BasicAuth()
-		if !ok {
-			http.Error(w, "401 unauthorized", 401)
-			return
-		}
-		if !f.Validate(username, password) {
-			http.Error(w, "401 unauthorized", 401)
-			return
-		}
-		h.ServeHTTP(w, r)
-	}
-}
-
-// CheckConfig returns a http.HandlerFunc that checks whether
-// a configuration exists.
-func CheckConfig(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		config := filepath.Join(c.path, "config")
-		st, err := os.Stat(config)
-		if err != nil {
-			http.Error(w, "404 not found", 404)
-			return
-		}
-		w.Header().Add("Content-Length", fmt.Sprint(st.Size()))
-	}
-}
-
-// GetConfig returns a http.HandlerFunc that allows for a
-// config to be retrieved.
-func GetConfig(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		config := filepath.Join(c.path, "config")
-		bytes, err := ioutil.ReadFile(config)
-		if err != nil {
-			http.Error(w, "404 not found", 404)
-			return
-		}
-		w.Write(bytes)
-	}
-}
-
-// SaveConfig returns a http.HandlerFunc that allows for a
-// config to be saved.
-func SaveConfig(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		config := filepath.Join(c.path, "config")
-		bytes, err := ioutil.ReadAll(r.Body)
-		if err != nil {
-			http.Error(w, "400 bad request", 400)
-			return
-		}
-		errw := ioutil.WriteFile(config, bytes, 0600)
-		if errw != nil {
-			http.Error(w, "500 internal server error", 500)
-			return
-		}
-		w.Write([]byte("200 ok"))
-	}
-}
-
-// ListBlobs returns a http.HandlerFunc that lists
-// all blobs of a given type in an arbitrary order.
-func ListBlobs(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		vars := strings.Split(r.RequestURI, "/")
-		dir := vars[1]
-		path := filepath.Join(c.path, dir)
-		files, err := ioutil.ReadDir(path)
-		if err != nil {
-			http.Error(w, "404 not found", 404)
-			return
-		}
-		names := make([]string, len(files))
-		for i, f := range files {
-			names[i] = f.Name()
-		}
-		data, err := json.Marshal(names)
-		if err != nil {
-			http.Error(w, "500 internal server error", 500)
-			return
-		}
-		w.Write(data)
-	}
-}
-
-// CheckBlob reutrns a http.HandlerFunc that tests whether a blob exists
-// and returns 200, if it does, or 404 otherwise.
-func CheckBlob(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		vars := strings.Split(r.RequestURI, "/")
-		dir := vars[1]
-		name := vars[2]
-		path := filepath.Join(c.path, dir, name)
-		st, err := os.Stat(path)
-		if err != nil {
-			http.Error(w, "404 not found", 404)
-			return
-		}
-		w.Header().Add("Content-Length", fmt.Sprint(st.Size()))
-	}
-}
-
-// GetBlob returns a http.HandlerFunc that retrieves a blob
-// from the repository.
-func GetBlob(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		vars := strings.Split(r.RequestURI, "/")
-		dir := vars[1]
-		name := vars[2]
-		path := filepath.Join(c.path, dir, name)
-		file, err := fs.Open(path)
-		if err != nil {
-			http.Error(w, "404 not found", 404)
-			return
-		}
-		defer file.Close()
-		http.ServeContent(w, r, "", time.Unix(0, 0), file)
-	}
-}
-
-// SaveBlob returns a http.HandlerFunc that saves a blob to the repository.
-func SaveBlob(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		vars := strings.Split(r.RequestURI, "/")
-		dir := vars[1]
-		name := vars[2]
-		path := filepath.Join(c.path, dir, name)
-		tmp := path + "_tmp"
-		tf, err := fs.OpenFile(tmp, os.O_CREATE|os.O_WRONLY, 0600)
-		if err != nil {
-			http.Error(w, "500 internal server error", 500)
-			return
-		}
-		if _, err := io.Copy(tf, r.Body); err != nil {
-			http.Error(w, "400 bad request", 400)
-			tf.Close()
-			os.Remove(tmp)
-			return
-		}
-		if err := tf.Close(); err != nil {
-			http.Error(w, "500 internal server error", 500)
-		}
-		if err := os.Rename(tmp, path); err != nil {
-			http.Error(w, "500 internal server error", 500)
-			return
-		}
-		w.Write([]byte("200 ok"))
-	}
-}
-
-// DeleteBlob returns a http.HandlerFunc that deletes a blob from the
-// repository.
-func DeleteBlob(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		vars := strings.Split(r.RequestURI, "/")
-		dir := vars[1]
-		name := vars[2]
-		path := filepath.Join(c.path, dir, name)
-		err := os.Remove(path)
-		if err != nil {
-			http.Error(w, "500 internal server error", 500)
-			return
-		}
-		w.Write([]byte("200 ok"))
-	}
-}

src/cmds/restic-server/htpasswd.go

@@ -1,97 +0,0 @@
-// +build go1.4
-
-package main
-
-/*
-Copied from: github.com/bitly/oauth2_proxy
-
-MIT License
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-*/
-
-import (
-	"crypto/sha1"
-	"encoding/base64"
-	"encoding/csv"
-	"io"
-	"log"
-
-	"restic/fs"
-)
-
-// lookup passwords in a htpasswd file
-// The entries must have been created with -s for SHA encryption
-
-// HtpasswdFile is a map for usernames to passwords.
-type HtpasswdFile struct {
-	Users map[string]string
-}
-
-// NewHtpasswdFromFile reads the users and passwords from a htpasswd
-// file and returns them. If an error is encountered, it is returned, together
-// with a nil-Pointer for the HtpasswdFile.
-func NewHtpasswdFromFile(path string) (*HtpasswdFile, error) {
-	r, err := fs.Open(path)
-	if err != nil {
-		return nil, err
-	}
-	defer r.Close()
-	return NewHtpasswd(r)
-}
-
-// NewHtpasswd reads the users and passwords from a htpasswd
-// datastream in file and returns them. If an error is encountered,
-// it is returned, together with a nil-Pointer for the HtpasswdFile.
-func NewHtpasswd(file io.Reader) (*HtpasswdFile, error) {
-	cr := csv.NewReader(file)
-	cr.Comma = ':'
-	cr.Comment = '#'
-	cr.TrimLeadingSpace = true
-
-	records, err := cr.ReadAll()
-	if err != nil {
-		return nil, err
-	}
-	h := &HtpasswdFile{Users: make(map[string]string)}
-	for _, record := range records {
-		h.Users[record[0]] = record[1]
-	}
-	return h, nil
-}
-
-// Validate returns true if password matches the stored password
-// for user. If no password for user is stored, or the password
-// is wrong, false is returned.
-func (h *HtpasswdFile) Validate(user string, password string) bool {
-	realPassword, exists := h.Users[user]
-	if !exists {
-		return false
-	}
-	if realPassword[:5] == "{SHA}" {
-		d := sha1.New()
-		d.Write([]byte(password))
-		if realPassword[5:] == base64.StdEncoding.EncodeToString(d.Sum(nil)) {
-			return true
-		}
-	} else {
-		log.Printf("Invalid htpasswd entry for %s. Must be a SHA entry.", user)
-	}
-	return false
-}

src/cmds/restic-server/router.go

@@ -1,137 +0,0 @@
-// +build go1.4
-
-package main
-
-import (
-	"log"
-	"net/http"
-	"strings"
-)
-
-// Route is a handler for a path that was already split.
-type Route struct {
-	path    []string
-	handler http.Handler
-}
-
-// Router maps HTTP methods to a slice of Route handlers.
-type Router struct {
-	routes map[string][]Route
-}
-
-// NewRouter creates a new Router and returns a pointer to it.
-func NewRouter() *Router {
-	return &Router{make(map[string][]Route)}
-}
-
-// Options registers handler for path with method "OPTIONS".
-func (router *Router) Options(path string, handler http.Handler) {
-	router.Handle("OPTIONS", path, handler)
-}
-
-// OptionsFunc registers handler for path with method "OPTIONS".
-func (router *Router) OptionsFunc(path string, handler http.HandlerFunc) {
-	router.Handle("OPTIONS", path, handler)
-}
-
-// Get registers handler for path with method "GET".
-func (router *Router) Get(path string, handler http.Handler) {
-	router.Handle("GET", path, handler)
-}
-
-// GetFunc registers handler for path with method "GET".
-func (router *Router) GetFunc(path string, handler http.HandlerFunc) {
-	router.Handle("GET", path, handler)
-}
-
-// Head registers handler for path with method "HEAD".
-func (router *Router) Head(path string, handler http.Handler) {
-	router.Handle("HEAD", path, handler)
-}
-
-// HeadFunc registers handler for path with method "HEAD".
-func (router *Router) HeadFunc(path string, handler http.HandlerFunc) {
-	router.Handle("HEAD", path, handler)
-}
-
-// Post registers handler for path with method "POST".
-func (router *Router) Post(path string, handler http.Handler) {
-	router.Handle("POST", path, handler)
-}
-
-// PostFunc registers handler for path with method "POST".
-func (router *Router) PostFunc(path string, handler http.HandlerFunc) {
-	router.Handle("POST", path, handler)
-}
-
-// Put registers handler for path with method "PUT".
-func (router *Router) Put(path string, handler http.Handler) {
-	router.Handle("PUT", path, handler)
-}
-
-// PutFunc registers handler for path with method "PUT".
-func (router *Router) PutFunc(path string, handler http.HandlerFunc) {
-	router.Handle("PUT", path, handler)
-}
-
-// Delete registers handler for path with method "DELETE".
-func (router *Router) Delete(path string, handler http.Handler) {
-	router.Handle("DELETE", path, handler)
-}
-
-// DeleteFunc registers handler for path with method "DELETE".
-func (router *Router) DeleteFunc(path string, handler http.HandlerFunc) {
-	router.Handle("DELETE", path, handler)
-}
-
-// Trace registers handler for path with method "TRACE".
-func (router *Router) Trace(path string, handler http.Handler) {
-	router.Handle("TRACE", path, handler)
-}
-
-// TraceFunc registers handler for path with method "TRACE".
-func (router *Router) TraceFunc(path string, handler http.HandlerFunc) {
-	router.Handle("TRACE", path, handler)
-}
-
-// Connect registers handler for path with method "Connect".
-func (router *Router) Connect(path string, handler http.Handler) {
-	router.Handle("Connect", path, handler)
-}
-
-// ConnectFunc registers handler for path with method "Connect".
-func (router *Router) ConnectFunc(path string, handler http.HandlerFunc) {
-	router.Handle("Connect", path, handler)
-}
-
-// Handle registers a http.Handler for method and uri
-func (router *Router) Handle(method string, uri string, handler http.Handler) {
-	routes := router.routes[method]
-	path := strings.Split(uri, "/")
-	routes = append(routes, Route{path, handler})
-	router.routes[method] = routes
-}
-
-func (router *Router) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	method := r.Method
-	uri := r.RequestURI
-	path := strings.Split(uri, "/")
-
-	log.Printf("%s %s", method, uri)
-
-ROUTE:
-	for _, route := range router.routes[method] {
-		if len(route.path) != len(path) {
-			continue
-		}
-		for i := 0; i < len(route.path); i++ {
-			if !strings.HasPrefix(route.path[i], ":") && route.path[i] != path[i] {
-				continue ROUTE
-			}
-		}
-		route.handler.ServeHTTP(w, r)
-		return
-	}
-
-	http.Error(w, "404 not found", 404)
-}

src/cmds/restic-server/router_test.go

@@ -1,74 +0,0 @@
-// +build go1.4
-
-package main
-
-import (
-	"io/ioutil"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-)
-
-func TestRouter(t *testing.T) {
-	router := NewRouter()
-
-	getConfig := []byte("GET /config")
-	router.GetFunc("/config", func(w http.ResponseWriter, r *http.Request) {
-		w.Write(getConfig)
-	})
-
-	postConfig := []byte("POST /config")
-	router.PostFunc("/config", func(w http.ResponseWriter, r *http.Request) {
-		w.Write(postConfig)
-	})
-
-	getBlobs := []byte("GET /blobs/")
-	router.GetFunc("/blobs/", func(w http.ResponseWriter, r *http.Request) {
-		w.Write(getBlobs)
-	})
-
-	getBlob := []byte("GET /blobs/:sha")
-	router.GetFunc("/blobs/:sha", func(w http.ResponseWriter, r *http.Request) {
-		w.Write(getBlob)
-	})
-
-	server := httptest.NewServer(router)
-	defer server.Close()
-
-	getConfigResp, _ := http.Get(server.URL + "/config")
-	getConfigBody, _ := ioutil.ReadAll(getConfigResp.Body)
-	if getConfigResp.StatusCode != 200 {
-		t.Fatalf("Wanted HTTP Status 200, got %d", getConfigResp.StatusCode)
-	}
-	if string(getConfig) != string(getConfigBody) {
-		t.Fatalf("Config wrong:\nWanted '%s'\nGot: '%s'", string(getConfig), string(getConfigBody))
-	}
-
-	postConfigResp, _ := http.Post(server.URL+"/config", "binary/octet-stream", strings.NewReader("post test"))
-	postConfigBody, _ := ioutil.ReadAll(postConfigResp.Body)
-	if postConfigResp.StatusCode != 200 {
-		t.Fatalf("Wanted HTTP Status 200, got %d", postConfigResp.StatusCode)
-	}
-	if string(postConfig) != string(postConfigBody) {
-		t.Fatalf("Config wrong:\nWanted '%s'\nGot: '%s'", string(postConfig), string(postConfigBody))
-	}
-
-	getBlobsResp, _ := http.Get(server.URL + "/blobs/")
-	getBlobsBody, _ := ioutil.ReadAll(getBlobsResp.Body)
-	if getBlobsResp.StatusCode != 200 {
-		t.Fatalf("Wanted HTTP Status 200, got %d", getBlobsResp.StatusCode)
-	}
-	if string(getBlobs) != string(getBlobsBody) {
-		t.Fatalf("Config wrong:\nWanted '%s'\nGot: '%s'", string(getBlobs), string(getBlobsBody))
-	}
-
-	getBlobResp, _ := http.Get(server.URL + "/blobs/test")
-	getBlobBody, _ := ioutil.ReadAll(getBlobResp.Body)
-	if getBlobResp.StatusCode != 200 {
-		t.Fatalf("Wanted HTTP Status 200, got %d", getBlobResp.StatusCode)
-	}
-	if string(getBlob) != string(getBlobBody) {
-		t.Fatalf("Config wrong:\nWanted '%s'\nGot: '%s'", string(getBlob), string(getBlobBody))
-	}
-}

src/cmds/restic-server/server.go

@@ -1,73 +0,0 @@
-// +build go1.4
-
-package main
-
-import (
-	"flag"
-	"log"
-	"net/http"
-	"os"
-	"path/filepath"
-)
-
-const (
-	defaultHTTPPort  = ":8000"
-	defaultHTTPSPort = ":8443"
-)
-
-func main() {
-	// Parse command-line args
-	var path = flag.String("path", "/tmp/restic", "specifies the path of the data directory")
-	var tls = flag.Bool("tls", false, "turns on tls support")
-	flag.Parse()
-
-	// Create the missing directories
-	dirs := []string{
-		"data",
-		"snapshots",
-		"index",
-		"locks",
-		"keys",
-		"tmp",
-	}
-	for _, d := range dirs {
-		os.MkdirAll(filepath.Join(*path, d), 0700)
-	}
-
-	// Define the routes
-	context := &Context{*path}
-	router := NewRouter()
-	router.HeadFunc("/config", CheckConfig(context))
-	router.GetFunc("/config", GetConfig(context))
-	router.PostFunc("/config", SaveConfig(context))
-	router.GetFunc("/:dir/", ListBlobs(context))
-	router.HeadFunc("/:dir/:name", CheckBlob(context))
-	router.GetFunc("/:type/:name", GetBlob(context))
-	router.PostFunc("/:type/:name", SaveBlob(context))
-	router.DeleteFunc("/:type/:name", DeleteBlob(context))
-
-	// Check for a password file
-	var handler http.Handler
-	htpasswdFile, err := NewHtpasswdFromFile(filepath.Join(*path, ".htpasswd"))
-	if err != nil {
-		log.Println("Authentication disabled")
-		handler = router
-	} else {
-		log.Println("Authentication enabled")
-		handler = AuthHandler(htpasswdFile, router)
-	}
-
-	// start the server
-	if !*tls {
-		log.Printf("start server on port %s\n", defaultHTTPPort)
-		http.ListenAndServe(defaultHTTPPort, handler)
-	} else {
-		privateKey := filepath.Join(*path, "private_key")
-		publicKey := filepath.Join(*path, "public_key")
-		log.Println("TLS enabled")
-		log.Printf("private key: %s", privateKey)
-		log.Printf("public key: %s", publicKey)
-		log.Printf("start server on port %s\n", defaultHTTPSPort)
-		http.ListenAndServeTLS(defaultHTTPSPort, publicKey, privateKey, handler)
-	}
-}

src/cmds/restic/cleanup.go

@@ -62,8 +62,13 @@ func CleanupHandler(c <-chan os.Signal) {
 	for s := range c {
 		debug.Log("signal %v received, cleaning up", s)
 		fmt.Printf("%sInterrupt received, cleaning up\n", ClearLine())
-		RunCleanupHandlers()
-		fmt.Println("exiting")
-		os.Exit(0)
+		Exit(0)
 	}
 }
+
+// Exit runs the cleanup handlers and then terminates the process with the
+// given exit code.
+func Exit(code int) {
+	RunCleanupHandlers()
+	os.Exit(code)
+}

src/cmds/restic/cmd_backup.go

@@ -3,14 +3,13 @@ package main
 import (
 	"bufio"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"restic"
 	"strings"
 	"time"
 
-	"golang.org/x/crypto/ssh/terminal"
-
 	"github.com/spf13/cobra"
 
 	"restic/archiver"
@@ -28,6 +27,10 @@ The "backup" command creates a new snapshot and saves the files and directories
 given as the arguments.
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
+		if backupOptions.Stdin && backupOptions.FilesFrom == "-" {
+			return errors.Fatal("cannot use both `--stdin` and `--files-from -`")
+		}
+
 		if backupOptions.Stdin {
 			return readBackupFromStdin(backupOptions, globalOptions, args)
 		}
@@ -46,6 +49,7 @@ type BackupOptions struct {
 	Stdin          bool
 	StdinFilename  string
 	Tags           []string
+	Hostname       string
 	FilesFrom      string
 }
 
@@ -54,15 +58,22 @@ var backupOptions BackupOptions
 func init() {
 	cmdRoot.AddCommand(cmdBackup)
 
+	hostname, err := os.Hostname()
+	if err != nil {
+		debug.Log("os.Hostname() returned err: %v", err)
+		hostname = ""
+	}
+
 	f := cmdBackup.Flags()
 	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent snapshot (default: last snapshot in the repo that has the same target files/directories)")
-	f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories. Overrides the "parent" flag`)
-	f.StringSliceVarP(&backupOptions.Excludes, "exclude", "e", []string{}, "exclude a `pattern` (can be specified multiple times)")
+	f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`)
+	f.StringSliceVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
 	f.StringVar(&backupOptions.ExcludeFile, "exclude-file", "", "read exclude patterns from a file")
-	f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "Exclude other file systems")
+	f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems")
 	f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
-	f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "", "file name to use when reading from stdin")
-	f.StringSliceVar(&backupOptions.Tags, "tag", []string{}, "add a `tag` for the new snapshot (can be specified multiple times)")
+	f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "file name to use when reading from stdin")
+	f.StringSliceVar(&backupOptions.Tags, "tag", nil, "add a `tag` for the new snapshot (can be specified multiple times)")
+	f.StringVar(&backupOptions.Hostname, "hostname", hostname, "set the `hostname` for the snapshot manually")
 	f.StringVar(&backupOptions.FilesFrom, "files-from", "", "read the files to backup from file (can be combined with file args)")
 }
 
@@ -123,8 +134,7 @@ func newArchiveProgress(gopts GlobalOptions, todo restic.Stat) *restic.Progress
 			s.Errors)
 		status2 := fmt.Sprintf("ETA %s ", formatSeconds(eta))
 
-		w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
-		if err == nil {
+		if w := stdoutTerminalWidth(); w > 0 {
 			maxlen := w - len(status2) - 1
 
 			if maxlen < 4 {
@@ -168,8 +178,7 @@ func newArchiveStdinProgress(gopts GlobalOptions) *restic.Progress {
 			formatBytes(s.Bytes),
 			formatBytes(bps))
 
-		w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
-		if err == nil {
+		if w := stdoutTerminalWidth(); w > 0 {
 			maxlen := w - len(status1)
 
 			if maxlen < 4 {
@@ -232,7 +241,15 @@ func gatherDevices(items []string) (deviceMap map[uint64]struct{}, err error) {
 
 func readBackupFromStdin(opts BackupOptions, gopts GlobalOptions, args []string) error {
 	if len(args) != 0 {
-		return errors.Fatalf("when reading from stdin, no additional files can be specified")
+		return errors.Fatal("when reading from stdin, no additional files can be specified")
+	}
+
+	if opts.StdinFilename == "" {
+		return errors.Fatal("filename for backup from stdin must not be empty")
+	}
+
+	if gopts.password == "" && gopts.PasswordFile == "" {
+		return errors.Fatal("unable to read password from stdin when data is to be read from stdin, use --password-file or $RESTIC_PASSWORD")
 	}
 
 	repo, err := OpenRepository(gopts)
@@ -251,7 +268,13 @@ func readBackupFromStdin(opts BackupOptions, gopts GlobalOptions, args []string)
 		return err
 	}
 
-	_, id, err := archiver.ArchiveReader(repo, newArchiveStdinProgress(gopts), os.Stdin, opts.StdinFilename, opts.Tags)
+	r := &archiver.Reader{
+		Repository: repo,
+		Tags:       opts.Tags,
+		Hostname:   opts.Hostname,
+	}
+
+	_, id, err := r.Archive(opts.StdinFilename, os.Stdin, newArchiveStdinProgress(gopts))
 	if err != nil {
 		return err
 	}
@@ -262,23 +285,32 @@ func readBackupFromStdin(opts BackupOptions, gopts GlobalOptions, args []string)
 
 // readFromFile will read all lines from the given filename and write them to a
 // string array, if filename is empty readFromFile returns and empty string
-// array
+// array. If filename is a dash (-), readFromFile will read the lines from
+// the standard input.
 func readLinesFromFile(filename string) ([]string, error) {
 	if filename == "" {
 		return nil, nil
 	}
 
-	file, err := os.Open(filename)
-	if err != nil {
-		return nil, err
+	var r io.Reader = os.Stdin
+	if filename != "-" {
+		f, err := os.Open(filename)
+		if err != nil {
+			return nil, err
+		}
+		defer f.Close()
+		r = f
 	}
-	defer file.Close()
 
 	var lines []string
 
-	scanner := bufio.NewScanner(file)
+	scanner := bufio.NewScanner(r)
 	for scanner.Scan() {
-		lines = append(lines, scanner.Text())
+		line := scanner.Text()
+		if line == "" {
+			continue
+		}
+		lines = append(lines, line)
 	}
 
 	if err := scanner.Err(); err != nil {
@@ -289,7 +321,11 @@ func readLinesFromFile(filename string) ([]string, error) {
 }
 
 func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
-	target, err := readLinesFromFile(opts.FilesFrom)
+	if opts.FilesFrom == "-" && gopts.password == "" && gopts.PasswordFile == "" {
+		return errors.Fatal("no password; either use `--password-file` option or put the password into the RESTIC_PASSWORD environment variable")
+	}
+
+	fromfile, err := readLinesFromFile(opts.FilesFrom)
 	if err != nil {
 		return err
 	}
@@ -297,11 +333,12 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
 	// merge files from files-from into normal args so we can reuse the normal
 	// args checks and have the ability to use both files-from and args at the
 	// same time
-	args = append(args, target...)
+	args = append(args, fromfile...)
 	if len(args) == 0 {
-		return errors.Fatalf("wrong number of parameters")
+		return errors.Fatal("wrong number of parameters")
 	}
 
+	target := make([]string, 0, len(args))
 	for _, d := range args {
 		if a, err := filepath.Abs(d); err == nil {
 			d = a
@@ -354,13 +391,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
 
 	// Find last snapshot to set it as parent, if not already set
 	if !opts.Force && parentSnapshotID == nil {
-		hostname, err := os.Hostname()
-		if err != nil {
-			debug.Log("os.Hostname() returned err: %v", err)
-			hostname = ""
-		}
-
-		id, err := restic.FindLatestSnapshot(repo, target, hostname)
+		id, err := restic.FindLatestSnapshot(repo, target, opts.Tags, opts.Hostname)
 		if err == nil {
 			parentSnapshotID = &id
 		} else if err != restic.ErrNoSnapshotFound {
@@ -436,7 +467,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
 		Warnf("%s\rwarning for %s: %v\n", ClearLine(), dir, err)
 	}
 
-	_, id, err := arch.Snapshot(newArchiveProgress(gopts, stat), target, opts.Tags, parentSnapshotID)
+	_, id, err := arch.Snapshot(newArchiveProgress(gopts, stat), target, opts.Tags, opts.Hostname, parentSnapshotID)
 	if err != nil {
 		return err
 	}

src/cmds/restic/cmd_cat.go

@@ -9,13 +9,12 @@ import (
 
 	"restic"
 	"restic/backend"
-	"restic/debug"
 	"restic/errors"
 	"restic/repository"
 )
 
 var cmdCat = &cobra.Command{
-	Use:   "cat [flags] [pack|blob|tree|snapshot|key|masterkey|config|lock] ID",
+	Use:   "cat [flags] [pack|blob|snapshot|index|key|masterkey|config|lock] ID",
 	Short: "print internal objects to stdout",
 	Long: `
 The "cat" command is used to print internal objects to stdout.
@@ -31,7 +30,7 @@ func init() {
 
 func runCat(gopts GlobalOptions, args []string) error {
 	if len(args) < 1 || (args[0] != "masterkey" && args[0] != "config" && len(args) != 2) {
-		return errors.Fatalf("type or ID not specified")
+		return errors.Fatal("type or ID not specified")
 	}
 
 	repo, err := OpenRepository(gopts)
@@ -99,7 +98,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 		return nil
 	case "key":
 		h := restic.Handle{Type: restic.KeyFile, Name: id.String()}
-		buf, err := backend.LoadAll(repo.Backend(), h, nil)
+		buf, err := backend.LoadAll(repo.Backend(), h)
 		if err != nil {
 			return err
 		}
@@ -150,7 +149,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 	switch tpe {
 	case "pack":
 		h := restic.Handle{Type: restic.DataFile, Name: id.String()}
-		buf, err := backend.LoadAll(repo.Backend(), h, nil)
+		buf, err := backend.LoadAll(repo.Backend(), h)
 		if err != nil {
 			return err
 		}
@@ -172,7 +171,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 			blob := list[0]
 
 			buf := make([]byte, blob.Length)
-			n, err := repo.LoadBlob(restic.DataBlob, id, buf)
+			n, err := repo.LoadBlob(t, id, buf)
 			if err != nil {
 				return err
 			}
@@ -184,23 +183,6 @@ func runCat(gopts GlobalOptions, args []string) error {
 
 		return errors.Fatal("blob not found")
 
-	case "tree":
-		debug.Log("cat tree %v", id.Str())
-		tree, err := repo.LoadTree(id)
-		if err != nil {
-			debug.Log("unable to load tree %v: %v", id.Str(), err)
-			return err
-		}
-
-		buf, err := json.MarshalIndent(&tree, "", "  ")
-		if err != nil {
-			debug.Log("error json.MarshalIndent(): %v", err)
-			return err
-		}
-
-		_, err = os.Stdout.Write(append(buf, '\n'))
-		return nil
-
 	default:
 		return errors.Fatal("invalid type")
 	}

src/cmds/restic/cmd_check.go

@@ -7,8 +7,6 @@ import (
 
 	"github.com/spf13/cobra"
 
-	"golang.org/x/crypto/ssh/terminal"
-
 	"restic"
 	"restic/checker"
 	"restic/errors"
@@ -26,7 +24,7 @@ finds. It can also be used to read all data and therefore simulate a restore.
 	},
 }
 
-// CheckOptions bundle all options for the 'check' command.
+// CheckOptions bundles all options for the 'check' command.
 type CheckOptions struct {
 	ReadData    bool
 	CheckUnused bool
@@ -38,8 +36,8 @@ func init() {
 	cmdRoot.AddCommand(cmdCheck)
 
 	f := cmdCheck.Flags()
-	f.BoolVar(&checkOptions.ReadData, "read-data", false, "Read all data blobs")
-	f.BoolVar(&checkOptions.CheckUnused, "check-unused", false, "Find unused blobs")
+	f.BoolVar(&checkOptions.ReadData, "read-data", false, "read all data blobs")
+	f.BoolVar(&checkOptions.CheckUnused, "check-unused", false, "find unused blobs")
 }
 
 func newReadProgress(gopts GlobalOptions, todo restic.Stat) *restic.Progress {
@@ -55,8 +53,7 @@ func newReadProgress(gopts GlobalOptions, todo restic.Stat) *restic.Progress {
 			formatPercent(s.Blobs, todo.Blobs),
 			s.Blobs, todo.Blobs)
 
-		w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
-		if err == nil {
+		if w := stdoutTerminalWidth(); w > 0 {
 			if len(status) > w {
 				max := w - len(status) - 4
 				status = status[:max] + "... "

src/cmds/restic/cmd_dump.go

@@ -22,7 +22,7 @@ var cmdDump = &cobra.Command{
 	Use:   "dump [indexes|snapshots|trees|all|packs]",
 	Short: "dump data structures",
 	Long: `
-The "dump" command dumps data structures from a repository as JSON objects. It
+The "dump" command dumps data structures from the repository as JSON objects. It
 is used for debugging purposes only.`,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runDump(globalOptions, args)
@@ -168,7 +168,7 @@ func dumpIndexes(repo restic.Repository) error {
 
 func runDump(gopts GlobalOptions, args []string) error {
 	if len(args) != 1 {
-		return errors.Fatalf("type not specified")
+		return errors.Fatal("type not specified")
 	}
 
 	repo, err := OpenRepository(gopts)

src/cmds/restic/cmd_find.go

@@ -1,7 +1,9 @@
 package main
 
 import (
+	"context"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -23,11 +25,16 @@ repo. `,
 	},
 }
 
-// FindOptions bundle all options for the find command.
+// FindOptions bundles all options for the find command.
 type FindOptions struct {
-	Oldest   string
-	Newest   string
-	Snapshot string
+	Oldest          string
+	Newest          string
+	Snapshots       []string
+	CaseInsensitive bool
+	ListLong        bool
+	Host            string
+	Paths           []string
+	Tags            []string
 }
 
 var findOptions FindOptions
@@ -36,19 +43,21 @@ func init() {
 	cmdRoot.AddCommand(cmdFind)
 
 	f := cmdFind.Flags()
-	f.StringVarP(&findOptions.Oldest, "oldest", "o", "", "Oldest modification date/time")
-	f.StringVarP(&findOptions.Newest, "newest", "n", "", "Newest modification date/time")
-	f.StringVarP(&findOptions.Snapshot, "snapshot", "s", "", "Snapshot ID to search in")
+	f.StringVarP(&findOptions.Oldest, "oldest", "o", "", "oldest modification date/time")
+	f.StringVarP(&findOptions.Newest, "newest", "n", "", "newest modification date/time")
+	f.StringSliceVarP(&findOptions.Snapshots, "snapshot", "s", nil, "snapshot `id` to search in (can be given multiple times)")
+	f.BoolVarP(&findOptions.CaseInsensitive, "ignore-case", "i", false, "ignore case for pattern")
+	f.BoolVarP(&findOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
+
+	f.StringVarP(&findOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
+	f.StringSliceVar(&findOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot-ID is given")
+	f.StringSliceVar(&findOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
 }
 
 type findPattern struct {
 	oldest, newest time.Time
 	pattern        string
-}
-
-type findResult struct {
-	node *restic.Node
-	path string
+	ignoreCase     bool
 }
 
 var timeFormats = []string{
@@ -75,20 +84,25 @@ func parseTime(str string) (time.Time, error) {
 	return time.Time{}, errors.Fatalf("unable to parse time: %q", str)
 }
 
-func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, path string) ([]findResult, error) {
+func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, prefix string, snapshotID *string) error {
 	debug.Log("checking tree %v\n", id)
+
 	tree, err := repo.LoadTree(id)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
-	results := []findResult{}
 	for _, node := range tree.Nodes {
 		debug.Log("  testing entry %q\n", node.Name)
 
-		m, err := filepath.Match(pat.pattern, node.Name)
+		name := node.Name
+		if pat.ignoreCase {
+			name = strings.ToLower(name)
+		}
+
+		m, err := filepath.Match(pat.pattern, name)
 		if err != nil {
-			return nil, err
+			return err
 		}
 
 		if m {
@@ -103,69 +117,55 @@ func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, path
 				continue
 			}
 
-			results = append(results, findResult{node: node, path: path})
+			if snapshotID != nil {
+				Verbosef("Found matching entries in snapshot %s\n", *snapshotID)
+				snapshotID = nil
+			}
+			Printf(formatNode(prefix, node, findOptions.ListLong) + "\n")
 		} else {
 			debug.Log("    pattern does not match\n")
 		}
 
 		if node.Type == "dir" {
-			subdirResults, err := findInTree(repo, pat, *node.Subtree, filepath.Join(path, node.Name))
-			if err != nil {
-				return nil, err
+			if err := findInTree(repo, pat, *node.Subtree, filepath.Join(prefix, node.Name), snapshotID); err != nil {
+				return err
 			}
-
-			results = append(results, subdirResults...)
 		}
 	}
 
-	return results, nil
-}
-
-func findInSnapshot(repo *repository.Repository, pat findPattern, id restic.ID) error {
-	debug.Log("searching in snapshot %s\n  for entries within [%s %s]", id.Str(), pat.oldest, pat.newest)
-
-	sn, err := restic.LoadSnapshot(repo, id)
-	if err != nil {
-		return err
-	}
-
-	results, err := findInTree(repo, pat, *sn.Tree, "")
-	if err != nil {
-		return err
-	}
-
-	if len(results) == 0 {
-		return nil
-	}
-	Verbosef("found %d matching entries in snapshot %s\n", len(results), id)
-	for _, res := range results {
-		res.node.Name = filepath.Join(res.path, res.node.Name)
-		Printf("  %s\n", res.node)
-	}
-
+	return nil
+}
+
+func findInSnapshot(repo *repository.Repository, sn *restic.Snapshot, pat findPattern) error {
+	debug.Log("searching in snapshot %s\n  for entries within [%s %s]", sn.ID(), pat.oldest, pat.newest)
+
+	snapshotID := sn.ID().Str()
+	if err := findInTree(repo, pat, *sn.Tree, string(filepath.Separator), &snapshotID); err != nil {
+		return err
+	}
 	return nil
 }
 
 func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
 	if len(args) != 1 {
-		return errors.Fatalf("wrong number of arguments")
+		return errors.Fatal("wrong number of arguments")
 	}
 
-	var (
-		err error
-		pat findPattern
-	)
+	var err error
+	pat := findPattern{pattern: args[0]}
+	if opts.CaseInsensitive {
+		pat.pattern = strings.ToLower(pat.pattern)
+		pat.ignoreCase = true
+	}
 
 	if opts.Oldest != "" {
-		pat.oldest, err = parseTime(opts.Oldest)
-		if err != nil {
+		if pat.oldest, err = parseTime(opts.Oldest); err != nil {
 			return err
 		}
 	}
 
 	if opts.Newest != "" {
-		pat.newest, err = parseTime(opts.Newest)
-		if err != nil {
+		if pat.newest, err = parseTime(opts.Newest); err != nil {
 			return err
 		}
 	}
@@ -183,28 +183,14 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
-	err = repo.LoadIndex()
-	if err != nil {
+	if err = repo.LoadIndex(); err != nil {
 		return err
 	}
 
-	pat.pattern = args[0]
-
-	if opts.Snapshot != "" {
-		snapshotID, err := restic.FindSnapshot(repo, opts.Snapshot)
-		if err != nil {
-			return errors.Fatalf("invalid id %q: %v", args[1], err)
-		}
-
-		return findInSnapshot(repo, pat, snapshotID)
-	}
-
-	done := make(chan struct{})
-	defer close(done)
-	for snapshotID := range repo.List(restic.SnapshotFile, done) {
-		err := findInSnapshot(repo, pat, snapshotID)
-
-		if err != nil {
+	ctx, cancel := context.WithCancel(gopts.ctx)
+	defer cancel()
+	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, opts.Snapshots) {
+		if err = findInSnapshot(repo, sn, pat); err != nil {
 			return err
 		}
 	}

src/cmds/restic/cmd_forget.go

@@ -1,10 +1,10 @@
 package main
 
 import (
-	"encoding/hex"
-	"fmt"
-	"io"
+	"context"
+	"encoding/json"
 	"restic"
+	"sort"
 	"strings"
 
 	"github.com/spf13/cobra"
@@ -25,19 +25,21 @@ data after 'forget' was run successfully, see the 'prune' command. `,
 
 // ForgetOptions collects all options for the forget command.
 type ForgetOptions struct {
-	Last    int
-	Hourly  int
-	Daily   int
-	Weekly  int
-	Monthly int
-	Yearly  int
-
+	Last     int
+	Hourly   int
+	Daily    int
+	Weekly   int
+	Monthly  int
+	Yearly   int
 	KeepTags []string
 
-	Hostname string
-	Tags     []string
+	Host  string
+	Tags  []string
+	Paths []string
 
-	DryRun bool
+	GroupByTags bool
+	DryRun      bool
+	Prune       bool
 }
 
 var forgetOptions ForgetOptions
@@ -53,51 +55,17 @@ func init() {
 	f.IntVarP(&forgetOptions.Monthly, "keep-monthly", "m", 0, "keep the last `n` monthly snapshots")
 	f.IntVarP(&forgetOptions.Yearly, "keep-yearly", "y", 0, "keep the last `n` yearly snapshots")
 
-	f.StringSliceVar(&forgetOptions.KeepTags, "keep-tag", []string{}, "always keep snapshots with this `tag` (can be specified multiple times)")
-	f.StringVar(&forgetOptions.Hostname, "hostname", "", "only forget snapshots for the given hostname")
-	f.StringSliceVar(&forgetOptions.Tags, "tag", []string{}, "only forget snapshots with the `tag` (can be specified multiple times)")
+	f.StringSliceVar(&forgetOptions.KeepTags, "keep-tag", []string{}, "keep snapshots with this `tag` (can be specified multiple times)")
+	f.BoolVarP(&forgetOptions.GroupByTags, "group-by-tags", "G", false, "Group by host,paths,tags instead of just host,paths")
+	// Sadly the commonly used shortcut `H` is already used.
+	f.StringVar(&forgetOptions.Host, "host", "", "only consider snapshots with the given `host`")
+	// Deprecated since 2017-03-07.
+	f.StringVar(&forgetOptions.Host, "hostname", "", "only consider snapshots with the given `hostname` (deprecated)")
+	f.StringSliceVar(&forgetOptions.Tags, "tag", nil, "only consider snapshots which include this `tag` (can be specified multiple times)")
+	f.StringSliceVar(&forgetOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` (can be specified multiple times)")
 
 	f.BoolVarP(&forgetOptions.DryRun, "dry-run", "n", false, "do not delete anything, just print what would be done")
-}
-
-func printSnapshots(w io.Writer, snapshots restic.Snapshots) {
-	tab := NewTable()
-	tab.Header = fmt.Sprintf("%-8s  %-19s  %-10s  %-10s  %s", "ID", "Date", "Host", "Tags", "Directory")
-	tab.RowFormat = "%-8s  %-19s  %-10s  %-10s  %s"
-
-	for _, sn := range snapshots {
-		if len(sn.Paths) == 0 {
-			continue
-		}
-
-		firstTag := ""
-		if len(sn.Tags) > 0 {
-			firstTag = sn.Tags[0]
-		}
-
-		tab.Rows = append(tab.Rows, []interface{}{sn.ID().Str(), sn.Time.Format(TimeFormat), sn.Hostname, firstTag, sn.Paths[0]})
-
-		rows := len(sn.Paths)
-		if len(sn.Tags) > rows {
-			rows = len(sn.Tags)
-		}
-
-		for i := 1; i < rows; i++ {
-			path := ""
-			if len(sn.Paths) > i {
-				path = sn.Paths[i]
-			}
-
-			tag := ""
-			if len(sn.Tags) > i {
-				tag = sn.Tags[i]
-			}
-
-			tab.Rows = append(tab.Rows, []interface{}{"", "", "", tag, path})
-		}
-	}
-
-	tab.Write(w)
+	f.BoolVar(&forgetOptions.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed")
 }
 
 func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
@@ -112,37 +80,45 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	// parse arguments as hex strings
-	var ids []string
-	for _, s := range args {
-		_, err := hex.DecodeString(s)
-		if err != nil {
-			Warnf("argument %q is not a snapshot ID, ignoring\n", s)
-			continue
-		}
-
-		ids = append(ids, s)
+	// group by hostname and dirs
+	type key struct {
+		Hostname string
+		Paths    []string
+		Tags     []string
 	}
+	snapshotGroups := make(map[string]restic.Snapshots)
 
-	// process all snapshot IDs given as arguments
-	for _, s := range ids {
-		id, err := restic.FindSnapshot(repo, s)
-		if err != nil {
-			Warnf("cound not find a snapshot for ID %q, ignoring\n", s)
-			continue
-		}
-
-		if !opts.DryRun {
-			err = repo.Backend().Remove(restic.SnapshotFile, id.String())
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
+		if len(args) > 0 {
+			// When explicit snapshots args are given, remove them immediately.
+			if !opts.DryRun {
+				h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
+				if err = repo.Backend().Remove(h); err != nil {
+					return err
+				}
+				Verbosef("removed snapshot %v\n", sn.ID().Str())
+			} else {
+				Verbosef("would have removed snapshot %v\n", sn.ID().Str())
+			}
+		} else {
+			var tags []string
+			if opts.GroupByTags {
+				tags = sn.Tags
+				sort.StringSlice(tags).Sort()
+			}
+			sort.StringSlice(sn.Paths).Sort()
+			k, err := json.Marshal(key{Hostname: sn.Hostname, Tags: tags, Paths: sn.Paths})
 			if err != nil {
 				return err
 			}
-
-			Verbosef("removed snapshot %v\n", id.Str())
-		} else {
-			Verbosef("would remove snapshot %v\n", id.Str())
+			snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn)
 		}
 	}
+	if len(args) > 0 {
+		return nil
+	}
 
 	policy := restic.ExpirePolicy{
 		Last:    opts.Last,
@@ -155,53 +131,41 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	if policy.Empty() {
+		Verbosef("no policy was specified, no snapshots will be removed\n")
 		return nil
 	}
 
-	// then, load all remaining snapshots
-	snapshots, err := restic.LoadAllSnapshots(repo)
-	if err != nil {
-		return err
-	}
-
-	// group by hostname and dirs
-	type key struct {
-		Hostname string
-		Dirs     string
-	}
-
-	snapshotGroups := make(map[key]restic.Snapshots)
-
-	for _, sn := range snapshots {
-		if opts.Hostname != "" && sn.Hostname != opts.Hostname {
-			continue
+	removeSnapshots := 0
+	for k, snapshotGroup := range snapshotGroups {
+		var key key
+		if json.Unmarshal([]byte(k), &key) != nil {
+			return err
 		}
-
-		if !sn.HasTags(opts.Tags) {
-			continue
+		if opts.GroupByTags {
+			Printf("snapshots for host %v, tags [%v], paths: [%v]:\n\n", key.Hostname, strings.Join(key.Tags, ", "), strings.Join(key.Paths, ", "))
+		} else {
+			Printf("snapshots for host %v, paths: [%v]:\n\n", key.Hostname, strings.Join(key.Paths, ", "))
 		}
-
-		k := key{Hostname: sn.Hostname, Dirs: strings.Join(sn.Paths, ":")}
-		list := snapshotGroups[k]
-		list = append(list, sn)
-		snapshotGroups[k] = list
-	}
-
-	for key, snapshotGroup := range snapshotGroups {
-		Printf("snapshots for host %v, directories %v:\n\n", key.Hostname, key.Dirs)
 		keep, remove := restic.ApplyPolicy(snapshotGroup, policy)
 
-		Printf("keep %d snapshots:\n", len(keep))
-		printSnapshots(globalOptions.stdout, keep)
-		Printf("\n")
+		if len(keep) != 0 {
+			Printf("keep %d snapshots:\n", len(keep))
+			PrintSnapshots(globalOptions.stdout, keep)
+			Printf("\n")
+		}
 
-		Printf("remove %d snapshots:\n", len(remove))
-		printSnapshots(globalOptions.stdout, remove)
-		Printf("\n")
+		if len(remove) != 0 {
+			Printf("remove %d snapshots:\n", len(remove))
+			PrintSnapshots(globalOptions.stdout, remove)
+			Printf("\n")
+		}
+
+		removeSnapshots += len(remove)
 
 		if !opts.DryRun {
 			for _, sn := range remove {
-				err = repo.Backend().Remove(restic.SnapshotFile, sn.ID().String())
+				h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
+				err = repo.Backend().Remove(h)
 				if err != nil {
 					return err
 				}
@@ -209,5 +173,12 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
+	if removeSnapshots > 0 && opts.Prune {
+		Printf("%d snapshots have been removed, running prune\n", removeSnapshots)
+		if !opts.DryRun {
+			return pruneRepository(gopts, repo)
+		}
+	}
+
 	return nil
 }

src/cmds/restic/cmd_key.go

@@ -1,20 +1,20 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"restic"
-
-	"github.com/spf13/cobra"
-
 	"restic/errors"
 	"restic/repository"
+
+	"github.com/spf13/cobra"
 )
 
 var cmdKey = &cobra.Command{
 	Use:   "key [list|add|rm|passwd] [ID]",
 	Short: "manage keys (passwords)",
 	Long: `
-The "key" command manages keys (passwords) for accessing a repository.
+The "key" command manages keys (passwords) for accessing the repository.
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runKey(globalOptions, args)
@@ -25,15 +25,12 @@ func init() {
 	cmdRoot.AddCommand(cmdKey)
 }
 
-func listKeys(s *repository.Repository) error {
+func listKeys(ctx context.Context, s *repository.Repository) error {
 	tab := NewTable()
 	tab.Header = fmt.Sprintf(" %-10s  %-10s  %-10s  %s", "ID", "User", "Host", "Created")
 	tab.RowFormat = "%s%-10s  %-10s  %-10s  %s"
 
-	done := make(chan struct{})
-	defer close(done)
-
-	for id := range s.List(restic.KeyFile, done) {
+	for id := range s.List(restic.KeyFile, ctx.Done()) {
 		k, err := repository.LoadKey(s, id.String())
 		if err != nil {
 			Warnf("LoadKey() failed: %v\n", err)
@@ -87,7 +84,8 @@ func deleteKey(repo *repository.Repository, name string) error {
 		return errors.Fatal("refusing to remove key currently used to access repository")
 	}
 
-	err := repo.Backend().Remove(restic.KeyFile, name)
+	h := restic.Handle{Type: restic.KeyFile, Name: name}
+	err := repo.Backend().Remove(h)
 	if err != nil {
 		return err
 	}
@@ -107,7 +105,8 @@ func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
 		return errors.Fatalf("creating new key failed: %v\n", err)
 	}
 
-	err = repo.Backend().Remove(restic.KeyFile, repo.KeyName())
+	h := restic.Handle{Type: restic.KeyFile, Name: repo.KeyName()}
+	err = repo.Backend().Remove(h)
 	if err != nil {
 		return err
 	}
@@ -118,10 +117,13 @@ func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
 }
 
 func runKey(gopts GlobalOptions, args []string) error {
-	if len(args) < 1 || (args[0] == "rm" && len(args) != 2) {
-		return errors.Fatalf("wrong number of arguments")
+	if len(args) < 1 || (args[0] == "rm" && len(args) != 2) || (args[0] != "rm" && len(args) != 1) {
+		return errors.Fatal("wrong number of arguments")
 	}
 
+	ctx, cancel := context.WithCancel(gopts.ctx)
+	defer cancel()
+
 	repo, err := OpenRepository(gopts)
 	if err != nil {
 		return err
@@ -135,7 +137,7 @@ func runKey(gopts GlobalOptions, args []string) error {
 			return err
 		}
 
-		return listKeys(repo)
+		return listKeys(ctx, repo)
 	case "add":
 		lock, err := lockRepo(repo)
 		defer unlockRepo(lock)

src/cmds/restic/cmd_list.go

@@ -1,17 +1,19 @@
 package main
 
 import (
+	"fmt"
 	"restic"
 	"restic/errors"
+	"restic/index"
 
 	"github.com/spf13/cobra"
 )
 
 var cmdList = &cobra.Command{
 	Use:   "list [blobs|packs|index|snapshots|keys|locks]",
-	Short: "list items in the repository",
+	Short: "list objects in the repository",
 	Long: `
-
+The "list" command allows listing objects in the repository based on type.
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runList(globalOptions, args)
@@ -24,7 +26,7 @@ func init() {
 
 func runList(opts GlobalOptions, args []string) error {
 	if len(args) != 1 {
-		return errors.Fatalf("type not specified")
+		return errors.Fatal("type not specified")
 	}
 
 	repo, err := OpenRepository(opts)
@@ -52,6 +54,19 @@ func runList(opts GlobalOptions, args []string) error {
 		t = restic.KeyFile
 	case "locks":
 		t = restic.LockFile
+	case "blobs":
+		idx, err := index.Load(repo, nil)
+		if err != nil {
+			return err
+		}
+
+		for _, pack := range idx.Packs {
+			for _, entry := range pack.Entries {
+				fmt.Printf("%v %v\n", entry.Type, entry.ID)
+			}
+		}
+
+		return nil
 	default:
 		return errors.Fatal("invalid type")
 	}

src/cmds/restic/cmd_ls.go

@@ -1,8 +1,7 @@
 package main
 
 import (
-	"fmt"
-	"os"
+	"context"
 	"path/filepath"
 
 	"github.com/spf13/cobra"
@@ -13,56 +12,50 @@ import (
 )
 
 var cmdLs = &cobra.Command{
-	Use:   "ls [flags] snapshot-ID",
+	Use:   "ls [flags] [snapshot-ID ...]",
 	Short: "list files in a snapshot",
 	Long: `
 The "ls" command allows listing files and directories in a snapshot.
+
+The special snapshot-ID "latest" can be used to list files and directories of the latest snapshot in the repository.
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runLs(globalOptions, args)
+		return runLs(lsOptions, globalOptions, args)
 	},
 }
 
-var listLong bool
+// LsOptions collects all options for the ls command.
+type LsOptions struct {
+	ListLong bool
+	Host     string
+	Tags     []string
+	Paths    []string
+}
+
+var lsOptions LsOptions
 
 func init() {
 	cmdRoot.AddCommand(cmdLs)
 
-	cmdLs.Flags().BoolVarP(&listLong, "long", "l", false, "use a long listing format showing size and mode")
+	flags := cmdLs.Flags()
+	flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
+
+	flags.StringVarP(&lsOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
+	flags.StringSliceVar(&lsOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot ID is given")
+	flags.StringSliceVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
 }
 
-func printNode(prefix string, n *restic.Node) string {
-	if !listLong {
-		return filepath.Join(prefix, n.Name)
-	}
-
-	switch n.Type {
-	case "file":
-		return fmt.Sprintf("%s %5d %5d %6d %s %s",
-			n.Mode, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name))
-	case "dir":
-		return fmt.Sprintf("%s %5d %5d %6d %s %s",
-			n.Mode|os.ModeDir, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name))
-	case "symlink":
-		return fmt.Sprintf("%s %5d %5d %6d %s %s -> %s",
-			n.Mode|os.ModeSymlink, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name), n.LinkTarget)
-	default:
-		return fmt.Sprintf("<Node(%s) %s>", n.Type, n.Name)
-	}
-}
-
-func printTree(prefix string, repo *repository.Repository, id restic.ID) error {
-	tree, err := repo.LoadTree(id)
+func printTree(repo *repository.Repository, id *restic.ID, prefix string) error {
+	tree, err := repo.LoadTree(*id)
 	if err != nil {
 		return err
 	}
 
 	for _, entry := range tree.Nodes {
-		Printf(printNode(prefix, entry) + "\n")
+		Printf(formatNode(prefix, entry, lsOptions.ListLong) + "\n")
 
 		if entry.Type == "dir" && entry.Subtree != nil {
-			err = printTree(filepath.Join(prefix, entry.Name), repo, *entry.Subtree)
-			if err != nil {
+			if err = printTree(repo, entry.Subtree, filepath.Join(prefix, entry.Name)); err != nil {
 				return err
 			}
 		}
@@ -71,9 +64,9 @@ func printTree(prefix string, repo *repository.Repository, id restic.ID) error {
 	return nil
 }
 
-func runLs(gopts GlobalOptions, args []string) error {
-	if len(args) < 1 || len(args) > 2 {
-		return errors.Fatalf("no snapshot ID given")
+func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
+	if len(args) == 0 && opts.Host == "" && len(opts.Tags) == 0 && len(opts.Paths) == 0 {
+		return errors.Fatal("Invalid arguments, either give one or more snapshot IDs or set filters.")
 	}
 
 	repo, err := OpenRepository(gopts)
@@ -81,22 +74,18 @@ func runLs(gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	err = repo.LoadIndex()
-	if err != nil {
+	if err = repo.LoadIndex(); err != nil {
 		return err
 	}
 
-	id, err := restic.FindSnapshot(repo, args[0])
-	if err != nil {
-		return err
+	ctx, cancel := context.WithCancel(gopts.ctx)
+	defer cancel()
+	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
+		Verbosef("snapshot %s of %v at %s):\n", sn.ID().Str(), sn.Paths, sn.Time)
+
+		if err = printTree(repo, sn.Tree, string(filepath.Separator)); err != nil {
+			return err
+		}
 	}
-
-	sn, err := restic.LoadSnapshot(repo, id)
-	if err != nil {
-		return err
-	}
-
-	Verbosef("snapshot of %v at %s:\n", sn.Paths, sn.Time)
-
-	return printTree("", repo, *sn.Tree)
+	return nil
 }

src/cmds/restic/cmd_mount.go

@@ -32,7 +32,12 @@ read-only mount.
 
 // MountOptions collects all options for the mount command.
 type MountOptions struct {
-	OwnerRoot bool
+	OwnerRoot  bool
+	AllowRoot  bool
+	AllowOther bool
+	Host       string
+	Tags       []string
+	Paths      []string
 }
 
 var mountOptions MountOptions
@@ -40,7 +45,14 @@ var mountOptions MountOptions
 func init() {
 	cmdRoot.AddCommand(cmdMount)
 
-	cmdMount.Flags().BoolVar(&mountOptions.OwnerRoot, "owner-root", false, "use 'root' as the owner of files and dirs")
+	mountFlags := cmdMount.Flags()
+	mountFlags.BoolVar(&mountOptions.OwnerRoot, "owner-root", false, "use 'root' as the owner of files and dirs")
+	mountFlags.BoolVar(&mountOptions.AllowRoot, "allow-root", false, "allow root user to access the data in the mounted directory")
+	mountFlags.BoolVar(&mountOptions.AllowOther, "allow-other", false, "allow other users to access the data in the mounted directory")
+
+	mountFlags.StringVarP(&mountOptions.Host, "host", "H", "", `only consider snapshots for this host`)
+	mountFlags.StringSliceVar(&mountOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`")
+	mountFlags.StringSliceVar(&mountOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`")
 }
 
 func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
@@ -64,11 +76,21 @@ func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
 			return err
 		}
 	}
-	c, err := systemFuse.Mount(
-		mountpoint,
+
+	mountOptions := []systemFuse.MountOption{
 		systemFuse.ReadOnly(),
 		systemFuse.FSName("restic"),
-	)
+	}
+
+	if opts.AllowRoot {
+		mountOptions = append(mountOptions, systemFuse.AllowRoot())
+	}
+
+	if opts.AllowOther {
+		mountOptions = append(mountOptions, systemFuse.AllowOther())
+	}
+
+	c, err := systemFuse.Mount(mountpoint, mountOptions...)
 	if err != nil {
 		return err
 	}
@@ -77,7 +99,7 @@ func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
 	Printf("Don't forget to umount after quitting!\n")
 
 	root := fs.Tree{}
-	root.Add("snapshots", fuse.NewSnapshotsDir(repo, opts.OwnerRoot))
+	root.Add("snapshots", fuse.NewSnapshotsDir(repo, opts.OwnerRoot, opts.Paths, opts.Tags, opts.Host))
 
 	debug.Log("serving mount at %v", mountpoint)
 	err = fs.Serve(c, &root)
@@ -95,7 +117,7 @@ func umount(mountpoint string) error {
 
 func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
 	if len(args) == 0 {
-		return errors.Fatalf("wrong number of parameters")
+		return errors.Fatal("wrong number of parameters")
 	}
 
 	mountpoint := args[0]

src/cmds/restic/cmd_prune.go

@@ -1,8 +1,8 @@
 package main
 
 import (
+	"context"
 	"fmt"
-	"os"
 	"restic"
 	"restic/debug"
 	"restic/errors"
@@ -11,8 +11,6 @@ import (
 	"time"
 
 	"github.com/spf13/cobra"
-
-	"golang.org/x/crypto/ssh/terminal"
 )
 
 var cmdPrune = &cobra.Command{
@@ -45,8 +43,7 @@ func newProgressMax(show bool, max uint64, description string) *restic.Progress
 			formatPercent(s.Blobs, max),
 			s.Blobs, max, description)
 
-		w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
-		if err == nil {
+		if w := stdoutTerminalWidth(); w > 0 {
 			if len(status) > w {
 				max := w - len(status) - 4
 				status = status[:max] + "... "
@@ -75,13 +72,17 @@ func runPrune(gopts GlobalOptions) error {
 		return err
 	}
 
-	err = repo.LoadIndex()
+	return pruneRepository(gopts, repo)
+}
+
+func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
+	err := repo.LoadIndex()
 	if err != nil {
 		return err
 	}
 
-	done := make(chan struct{})
-	defer close(done)
+	ctx, cancel := context.WithCancel(gopts.ctx)
+	defer cancel()
 
 	var stats struct {
 		blobs     int
@@ -91,7 +92,7 @@ func runPrune(gopts GlobalOptions) error {
 	}
 
 	Verbosef("counting files in repo\n")
-	for _ = range repo.List(restic.DataFile, done) {
+	for _ = range repo.List(restic.DataFile, ctx.Done()) {
 		stats.packs++
 	}
 
@@ -103,11 +104,13 @@ func runPrune(gopts GlobalOptions) error {
 		return err
 	}
 
+	blobs := 0
 	for _, pack := range idx.Packs {
 		stats.bytes += pack.Size
+		blobs += len(pack.Entries)
 	}
 	Verbosef("repository contains %v packs (%v blobs) with %v bytes\n",
-		len(idx.Packs), len(idx.Blobs), formatBytes(uint64(stats.bytes)))
+		len(idx.Packs), blobs, formatBytes(uint64(stats.bytes)))
 
 	blobCount := make(map[restic.BlobHandle]int)
 	duplicateBlobs := 0
@@ -164,18 +167,21 @@ func runPrune(gopts GlobalOptions) error {
 
 	// find packs that need a rewrite
 	rewritePacks := restic.NewIDSet()
-	for h, blob := range idx.Blobs {
-		if !usedBlobs.Has(h) {
-			rewritePacks.Merge(blob.Packs)
-			continue
-		}
+	for _, pack := range idx.Packs {
+		for _, blob := range pack.Entries {
+			h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
+			if !usedBlobs.Has(h) {
+				rewritePacks.Insert(pack.ID)
+				continue
+			}
 
-		if blobCount[h] > 1 {
-			rewritePacks.Merge(blob.Packs)
+			if blobCount[h] > 1 {
+				rewritePacks.Insert(pack.ID)
+			}
 		}
 	}
 
-	removeBytes := 0
+	removeBytes := duplicateBytes
 
 	// find packs that are unneeded
 	removePacks := restic.NewIDSet()
@@ -208,46 +214,34 @@ func runPrune(gopts GlobalOptions) error {
 	Verbosef("will delete %d packs and rewrite %d packs, this frees %s\n",
 		len(removePacks), len(rewritePacks), formatBytes(uint64(removeBytes)))
 
-	err = repository.Repack(repo, rewritePacks, usedBlobs)
-	if err != nil {
-		return err
-	}
-
-	for packID := range removePacks {
-		err = repo.Backend().Remove(restic.DataFile, packID.String())
+	if len(rewritePacks) != 0 {
+		bar = newProgressMax(!gopts.Quiet, uint64(len(rewritePacks)), "packs rewritten")
+		bar.Start()
+		err = repository.Repack(repo, rewritePacks, usedBlobs, bar)
 		if err != nil {
-			Warnf("unable to remove file %v from the repository\n", packID.Str())
+			return err
 		}
+		bar.Done()
 	}
 
-	Verbosef("creating new index\n")
-
-	stats.packs = 0
-	for _ = range repo.List(restic.DataFile, done) {
-		stats.packs++
+	if len(removePacks) != 0 {
+		bar = newProgressMax(!gopts.Quiet, uint64(len(removePacks)), "packs deleted")
+		bar.Start()
+		for packID := range removePacks {
+			h := restic.Handle{Type: restic.DataFile, Name: packID.String()}
+			err = repo.Backend().Remove(h)
+			if err != nil {
+				Warnf("unable to remove file %v from the repository\n", packID.Str())
+			}
+			bar.Report(restic.Stat{Blobs: 1})
+		}
+		bar.Done()
 	}
-	bar = newProgressMax(!gopts.Quiet, uint64(stats.packs), "packs")
-	idx, err = index.New(repo, bar)
-	if err != nil {
+
+	if err = rebuildIndex(ctx, repo); err != nil {
 		return err
 	}
 
-	var supersedes restic.IDs
-	for idxID := range repo.List(restic.IndexFile, done) {
-		err := repo.Backend().Remove(restic.IndexFile, idxID.String())
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "unable to remove index %v: %v\n", idxID.Str(), err)
-		}
-
-		supersedes = append(supersedes, idxID)
-	}
-
-	id, err := idx.Save(repo, supersedes)
-	if err != nil {
-		return err
-	}
-	Verbosef("saved new index as %v\n", id.Str())
-
 	Verbosef("done\n")
 	return nil
 }

src/cmds/restic/cmd_rebuild_index.go

@@ -1,7 +1,9 @@
 package main
 
 import (
-	"restic/repository"
+	"context"
+	"restic"
+	"restic/index"
 
 	"github.com/spf13/cobra"
 )
@@ -10,8 +12,8 @@ var cmdRebuildIndex = &cobra.Command{
 	Use:   "rebuild-index [flags]",
 	Short: "build a new index file",
 	Long: `
-The "rebuild-index" command creates a new index by combining the index files
-into a new one.
+The "rebuild-index" command creates a new index based on the pack files in the
+repository.
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runRebuildIndex(globalOptions)
@@ -34,5 +36,49 @@ func runRebuildIndex(gopts GlobalOptions) error {
 		return err
 	}
 
-	return repository.RebuildIndex(repo)
+	ctx, cancel := context.WithCancel(gopts.ctx)
+	defer cancel()
+	return rebuildIndex(ctx, repo)
+}
+
+func rebuildIndex(ctx context.Context, repo restic.Repository) error {
+	Verbosef("counting files in repo\n")
+
+	var packs uint64
+	for _ = range repo.List(restic.DataFile, ctx.Done()) {
+		packs++
+	}
+
+	bar := newProgressMax(!globalOptions.Quiet, packs, "packs")
+	idx, err := index.New(repo, bar)
+	if err != nil {
+		return err
+	}
+
+	Verbosef("finding old index files\n")
+
+	var supersedes restic.IDs
+	for id := range repo.List(restic.IndexFile, ctx.Done()) {
+		supersedes = append(supersedes, id)
+	}
+
+	id, err := idx.Save(repo, supersedes)
+	if err != nil {
+		return err
+	}
+
+	Verbosef("saved new index as %v\n", id.Str())
+
+	Verbosef("remove %d old index files\n", len(supersedes))
+
+	for _, id := range supersedes {
+		if err := repo.Backend().Remove(restic.Handle{
+			Type: restic.IndexFile,
+			Name: id.String(),
+		}); err != nil {
+			Warnf("error removing old index %v: %v\n", id.Str(), err)
+		}
+	}
+
+	return nil
 }

src/cmds/restic/cmd_restore.go

@@ -31,6 +31,7 @@ type RestoreOptions struct {
 	Target  string
 	Host    string
 	Paths   []string
+	Tags    []string
 }
 
 var restoreOptions RestoreOptions
@@ -44,12 +45,13 @@ func init() {
 	flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to")
 
 	flags.StringVarP(&restoreOptions.Host, "host", "H", "", `only consider snapshots for this host when the snapshot ID is "latest"`)
+	flags.StringSliceVar(&restoreOptions.Tags, "tag", nil, "only consider snapshots which include this `tag` for snapshot ID \"latest\"")
 	flags.StringSliceVar(&restoreOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
 }
 
 func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 	if len(args) != 1 {
-		return errors.Fatalf("no snapshot ID specified")
+		return errors.Fatal("no snapshot ID specified")
 	}
 
 	if opts.Target == "" {
@@ -85,7 +87,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 	var id restic.ID
 
 	if snapshotIDString == "latest" {
-		id, err = restic.FindLatestSnapshot(repo, opts.Paths, opts.Host)
+		id, err = restic.FindLatestSnapshot(repo, opts.Paths, opts.Tags, opts.Host)
 		if err != nil {
 			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Host:%v", err, opts.Paths, opts.Host)
 		}

src/cmds/restic/cmd_snapshots.go

@@ -1,9 +1,10 @@
 package main
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
-	"os"
-	"restic/errors"
+	"io"
 	"sort"
 
 	"github.com/spf13/cobra"
@@ -12,19 +13,20 @@ import (
 )
 
 var cmdSnapshots = &cobra.Command{
-	Use:   "snapshots",
+	Use:   "snapshots [snapshotID ...]",
 	Short: "list all snapshots",
 	Long: `
-The "snapshots" command lists all snapshots stored in a repository.
+The "snapshots" command lists all snapshots stored in the repository.
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runSnapshots(snapshotOptions, globalOptions, args)
 	},
 }
 
-// SnapshotOptions bundle all options for the snapshots command.
+// SnapshotOptions bundles all options for the snapshots command.
 type SnapshotOptions struct {
 	Host  string
+	Tags  []string
 	Paths []string
 }
 
@@ -34,15 +36,12 @@ func init() {
 	cmdRoot.AddCommand(cmdSnapshots)
 
 	f := cmdSnapshots.Flags()
-	f.StringVar(&snapshotOptions.Host, "host", "", "only print snapshots for this host")
-	f.StringSliceVar(&snapshotOptions.Paths, "path", []string{}, "only print snapshots for this `path` (can be specified multiple times)")
+	f.StringVarP(&snapshotOptions.Host, "host", "H", "", "only consider snapshots for this `host`")
+	f.StringSliceVar(&snapshotOptions.Tags, "tag", nil, "only consider snapshots which include this `tag` (can be specified multiple times)")
+	f.StringSliceVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)")
 }
 
 func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error {
-	if len(args) != 0 {
-		return errors.Fatalf("wrong number of arguments")
-	}
-
 	repo, err := OpenRepository(gopts)
 	if err != nil {
 		return err
@@ -56,37 +55,47 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 		}
 	}
 
-	tab := NewTable()
-	tab.Header = fmt.Sprintf("%-8s  %-19s  %-10s  %-10s  %s", "ID", "Date", "Host", "Tags", "Directory")
-	tab.RowFormat = "%-8s  %-19s  %-10s  %-10s  %s"
+	ctx, cancel := context.WithCancel(gopts.ctx)
+	defer cancel()
 
-	done := make(chan struct{})
-	defer close(done)
+	var list restic.Snapshots
+	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
+		list = append(list, sn)
+	}
+	sort.Sort(sort.Reverse(list))
 
-	list := []*restic.Snapshot{}
-	for id := range repo.List(restic.SnapshotFile, done) {
-		sn, err := restic.LoadSnapshot(repo, id)
+	if gopts.JSON {
+		err := printSnapshotsJSON(gopts.stdout, list)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "error loading snapshot %s: %v\n", id, err)
-			continue
+			Warnf("error printing snapshot: %v\n", err)
 		}
+		return nil
+	}
+	PrintSnapshots(gopts.stdout, list)
 
-		if restic.SamePaths(sn.Paths, opts.Paths) && (opts.Host == "" || opts.Host == sn.Hostname) {
-			pos := sort.Search(len(list), func(i int) bool {
-				return list[i].Time.After(sn.Time)
-			})
+	return nil
+}
 
-			if pos < len(list) {
-				list = append(list, nil)
-				copy(list[pos+1:], list[pos:])
-				list[pos] = sn
-			} else {
-				list = append(list, sn)
+// PrintSnapshots prints a text table of the snapshots in list to stdout.
+func PrintSnapshots(stdout io.Writer, list restic.Snapshots) {
+
+	// Determine the max widths for host and tag.
+	maxHost, maxTag := 10, 6
+	for _, sn := range list {
+		if len(sn.Hostname) > maxHost {
+			maxHost = len(sn.Hostname)
+		}
+		for _, tag := range sn.Tags {
+			if len(tag) > maxTag {
+				maxTag = len(tag)
 			}
 		}
-
 	}
 
+	tab := NewTable()
+	tab.Header = fmt.Sprintf("%-8s  %-19s  %-*s  %-*s  %-3s %s", "ID", "Date", -maxHost, "Host", -maxTag, "Tags", "", "Directory")
+	tab.RowFormat = fmt.Sprintf("%%-8s  %%-19s  %%%ds  %%%ds  %%-3s %%s", -maxHost, -maxTag)
+
 	for _, sn := range list {
 		if len(sn.Paths) == 0 {
 			continue
@@ -97,9 +106,18 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 			firstTag = sn.Tags[0]
 		}
 
-		tab.Rows = append(tab.Rows, []interface{}{sn.ID().Str(), sn.Time.Format(TimeFormat), sn.Hostname, firstTag, sn.Paths[0]})
-
 		rows := len(sn.Paths)
+		if rows < len(sn.Tags) {
+			rows = len(sn.Tags)
+		}
+
+		treeElement := "   "
+		if rows != 1 {
+			treeElement = "┌──"
+		}
+
+		tab.Rows = append(tab.Rows, []interface{}{sn.ID().Str(), sn.Time.Format(TimeFormat), sn.Hostname, firstTag, treeElement, sn.Paths[0]})
+
 		if len(sn.Tags) > rows {
 			rows = len(sn.Tags)
 		}
@@ -115,11 +133,38 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 				tag = sn.Tags[i]
 			}
 
-			tab.Rows = append(tab.Rows, []interface{}{"", "", "", tag, path})
+			treeElement := "│"
+			if i == (rows - 1) {
+				treeElement = "└──"
+			}
+
+			tab.Rows = append(tab.Rows, []interface{}{"", "", "", tag, treeElement, path})
 		}
 	}
 
-	tab.Write(os.Stdout)
-
-	return nil
+	tab.Write(stdout)
+}
+
+// Snapshot helps to print Snaphots as JSON with their ID included.
+type Snapshot struct {
+	*restic.Snapshot
+
+	ID *restic.ID `json:"id"`
+}
+
+// printSnapshotsJSON writes the JSON representation of list to stdout.
+func printSnapshotsJSON(stdout io.Writer, list restic.Snapshots) error {
+
+	var snapshots []Snapshot
+
+	for _, sn := range list {
+
+		k := Snapshot{
+			Snapshot: sn,
+			ID:       sn.ID(),
+		}
+		snapshots = append(snapshots, k)
+	}
+
+	return json.NewEncoder(stdout).Encode(snapshots)
 }

src/cmds/restic/cmd_tag.go

@@ -0,0 +1,142 @@
+package main
+
+import (
+	"context"
+
+	"github.com/spf13/cobra"
+
+	"restic"
+	"restic/debug"
+	"restic/errors"
+	"restic/repository"
+)
+
+var cmdTag = &cobra.Command{
+	Use:   "tag [flags] [snapshot-ID ...]",
+	Short: "modifies tags on snapshots",
+	Long: `
+The "tag" command allows you to modify tags on exiting snapshots.
+
+You can either set/replace the entire set of tags on a snapshot, or
+add tags to/remove tags from the existing set.
+
+When no snapshot-ID is given, all snapshots matching the host, tag and path filter criteria are modified.
+`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runTag(tagOptions, globalOptions, args)
+	},
+}
+
+// TagOptions bundles all options for the 'tag' command.
+type TagOptions struct {
+	Host       string
+	Paths      []string
+	Tags       []string
+	SetTags    []string
+	AddTags    []string
+	RemoveTags []string
+}
+
+var tagOptions TagOptions
+
+func init() {
+	cmdRoot.AddCommand(cmdTag)
+
+	tagFlags := cmdTag.Flags()
+	tagFlags.StringSliceVar(&tagOptions.SetTags, "set", nil, "`tag` which will replace the existing tags (can be given multiple times)")
+	tagFlags.StringSliceVar(&tagOptions.AddTags, "add", nil, "`tag` which will be added to the existing tags (can be given multiple times)")
+	tagFlags.StringSliceVar(&tagOptions.RemoveTags, "remove", nil, "`tag` which will be removed from the existing tags (can be given multiple times)")
+
+	tagFlags.StringVarP(&tagOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
+	tagFlags.StringSliceVar(&tagOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot-ID is given")
+	tagFlags.StringSliceVar(&tagOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
+}
+
+func changeTags(repo *repository.Repository, sn *restic.Snapshot, setTags, addTags, removeTags []string) (bool, error) {
+	var changed bool
+
+	if len(setTags) != 0 {
+		// Setting the tag to an empty string really means no tags.
+		if len(setTags) == 1 && setTags[0] == "" {
+			setTags = nil
+		}
+		sn.Tags = setTags
+		changed = true
+	} else {
+		changed = sn.AddTags(addTags)
+		if sn.RemoveTags(removeTags) {
+			changed = true
+		}
+	}
+
+	if changed {
+		// Retain the original snapshot id over all tag changes.
+		if sn.Original == nil {
+			sn.Original = sn.ID()
+		}
+
+		// Save the new snapshot.
+		id, err := repo.SaveJSONUnpacked(restic.SnapshotFile, sn)
+		if err != nil {
+			return false, err
+		}
+
+		debug.Log("new snapshot saved as %v", id.Str())
+
+		if err = repo.Flush(); err != nil {
+			return false, err
+		}
+
+		// Remove the old snapshot.
+		h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
+		if err = repo.Backend().Remove(h); err != nil {
+			return false, err
+		}
+
+		debug.Log("old snapshot %v removed", sn.ID())
+	}
+	return changed, nil
+}
+
+func runTag(opts TagOptions, gopts GlobalOptions, args []string) error {
+	if len(opts.SetTags) == 0 && len(opts.AddTags) == 0 && len(opts.RemoveTags) == 0 {
+		return errors.Fatal("nothing to do!")
+	}
+	if len(opts.SetTags) != 0 && (len(opts.AddTags) != 0 || len(opts.RemoveTags) != 0) {
+		return errors.Fatal("--set and --add/--remove cannot be given at the same time")
+	}
+
+	repo, err := OpenRepository(gopts)
+	if err != nil {
+		return err
+	}
+
+	if !gopts.NoLock {
+		Verbosef("Create exclusive lock for repository\n")
+		lock, err := lockRepoExclusive(repo)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+	}
+
+	changeCnt := 0
+	ctx, cancel := context.WithCancel(gopts.ctx)
+	defer cancel()
+	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
+		changed, err := changeTags(repo, sn, opts.SetTags, opts.AddTags, opts.RemoveTags)
+		if err != nil {
+			Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", sn.ID(), err)
+			continue
+		}
+		if changed {
+			changeCnt++
+		}
+	}
+	if changeCnt == 0 {
+		Verbosef("No snapshots were modified\n")
+	} else {
+		Verbosef("Modified tags on %v snapshots\n", changeCnt)
+	}
+	return nil
+}

src/cmds/restic/cmd_unlock.go

@@ -27,7 +27,7 @@ var unlockOptions UnlockOptions
 func init() {
 	cmdRoot.AddCommand(unlockCmd)
 
-	unlockCmd.Flags().BoolVar(&unlockOptions.RemoveAll, "remove-all", false, "Remove all locks, even non-stale ones")
+	unlockCmd.Flags().BoolVar(&unlockOptions.RemoveAll, "remove-all", false, "remove all locks, even non-stale ones")
 }
 
 func runUnlock(opts UnlockOptions, gopts GlobalOptions) error {

src/cmds/restic/cmd_version.go

@@ -9,14 +9,14 @@ import (
 
 var versionCmd = &cobra.Command{
 	Use:   "version",
-	Short: "Print version information",
+	Short: "print version information",
 	Long: `
 The "version" command prints detailed information about the build environment
 and the version of this software.
 `,
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Printf("restic %s\ncompiled at %s with %v on %v/%v\n",
-			version, compiledAt, runtime.Version(), runtime.GOOS, runtime.GOARCH)
+		fmt.Printf("restic %s\ncompiled with %v on %v/%v\n",
+			version, runtime.Version(), runtime.GOOS, runtime.GOARCH)
 	},
 }
 

src/cmds/restic/find.go

@@ -0,0 +1,78 @@
+package main
+
+import (
+	"context"
+
+	"restic"
+	"restic/repository"
+)
+
+// FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots.
+func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, host string, tags []string, paths []string, snapshotIDs []string) <-chan *restic.Snapshot {
+	out := make(chan *restic.Snapshot)
+	go func() {
+		defer close(out)
+		if len(snapshotIDs) != 0 {
+			var (
+				id         restic.ID
+				usedFilter bool
+				err        error
+			)
+			ids := make(restic.IDs, 0, len(snapshotIDs))
+			// Process all snapshot IDs given as arguments.
+			for _, s := range snapshotIDs {
+				if s == "latest" {
+					id, err = restic.FindLatestSnapshot(repo, paths, tags, host)
+					if err != nil {
+						Warnf("Ignoring %q, no snapshot matched given filter (Paths:%v Tags:%v Host:%v)\n", s, paths, tags, host)
+						usedFilter = true
+						continue
+					}
+				} else {
+					id, err = restic.FindSnapshot(repo, s)
+					if err != nil {
+						Warnf("Ignoring %q, it is not a snapshot id\n", s)
+						continue
+					}
+				}
+				ids = append(ids, id)
+			}
+
+			// Give the user some indication their filters are not used.
+			if !usedFilter && (host != "" || len(tags) != 0 || len(paths) != 0) {
+				Warnf("Ignoring filters as there are explicit snapshot ids given\n")
+			}
+
+			for _, id := range ids.Uniq() {
+				sn, err := restic.LoadSnapshot(repo, id)
+				if err != nil {
+					Warnf("Ignoring %q, could not load snapshot: %v\n", id, err)
+					continue
+				}
+				select {
+				case <-ctx.Done():
+					return
+				case out <- sn:
+				}
+			}
+			return
+		}
+
+		for id := range repo.List(restic.SnapshotFile, ctx.Done()) {
+			sn, err := restic.LoadSnapshot(repo, id)
+			if err != nil {
+				Warnf("Ignoring %q, could not load snapshot: %v\n", id, err)
+				continue
+			}
+			if (host != "" && host != sn.Hostname) || !sn.HasTags(tags) || !sn.HasPaths(paths) {
+				continue
+			}
+			select {
+			case <-ctx.Done():
+				return
+			case out <- sn:
+			}
+		}
+	}()
+	return out
+}

src/cmds/restic/format.go

@@ -2,7 +2,11 @@ package main
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
 	"time"
+
+	"restic"
 )
 
 func formatBytes(c uint64) string {
@@ -58,3 +62,23 @@ func formatDuration(d time.Duration) string {
 	sec := uint64(d / time.Second)
 	return formatSeconds(sec)
 }
+
+func formatNode(prefix string, n *restic.Node, long bool) string {
+	if !long {
+		return filepath.Join(prefix, n.Name)
+	}
+
+	switch n.Type {
+	case "file":
+		return fmt.Sprintf("%s %5d %5d %6d %s %s",
+			n.Mode, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name))
+	case "dir":
+		return fmt.Sprintf("%s %5d %5d %6d %s %s",
+			n.Mode|os.ModeDir, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name))
+	case "symlink":
+		return fmt.Sprintf("%s %5d %5d %6d %s %s -> %s",
+			n.Mode|os.ModeSymlink, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name), n.LinkTarget)
+	default:
+		return fmt.Sprintf("<Node(%s) %s>", n.Type, n.Name)
+	}
+}

src/cmds/restic/global.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -10,8 +11,6 @@ import (
 	"strings"
 	"syscall"
 
-	"github.com/spf13/cobra"
-
 	"restic/backend/local"
 	"restic/backend/rest"
 	"restic/backend/s3"
@@ -26,19 +25,6 @@ import (
 )
 
 var version = "compiled manually"
-var compiledAt = "unknown time"
-
-func parseEnvironment(cmd *cobra.Command, args []string) {
-	repo := os.Getenv("RESTIC_REPOSITORY")
-	if repo != "" {
-		globalOptions.Repo = repo
-	}
-
-	pw := os.Getenv("RESTIC_PASSWORD")
-	if pw != "" {
-		globalOptions.password = pw
-	}
-}
 
 // GlobalOptions hold all global options for restic.
 type GlobalOptions struct {
@@ -46,7 +32,9 @@ type GlobalOptions struct {
 	PasswordFile string
 	Quiet        bool
 	NoLock       bool
+	JSON         bool
 
+	ctx      context.Context
 	password string
 	stdout   io.Writer
 	stderr   io.Writer
@@ -58,11 +46,24 @@ var globalOptions = GlobalOptions{
 }
 
 func init() {
+	pw := os.Getenv("RESTIC_PASSWORD")
+	if pw != "" {
+		globalOptions.password = pw
+	}
+
+	var cancel context.CancelFunc
+	globalOptions.ctx, cancel = context.WithCancel(context.Background())
+	AddCleanupHandler(func() error {
+		cancel()
+		return nil
+	})
+
 	f := cmdRoot.PersistentFlags()
-	f.StringVarP(&globalOptions.Repo, "repo", "r", "", "repository to backup to or restore from (default: $RESTIC_REPOSITORY)")
+	f.StringVarP(&globalOptions.Repo, "repo", "r", os.Getenv("RESTIC_REPOSITORY"), "repository to backup to or restore from (default: $RESTIC_REPOSITORY)")
 	f.StringVarP(&globalOptions.PasswordFile, "password-file", "p", "", "read the repository password from a file")
-	f.BoolVarP(&globalOptions.Quiet, "quiet", "q", false, "do not outputcomprehensive progress report")
+	f.BoolVarP(&globalOptions.Quiet, "quiet", "q", false, "do not output comprehensive progress report")
 	f.BoolVar(&globalOptions.NoLock, "no-lock", false, "do not lock the repo, this allows some operations on read-only repos")
+	f.BoolVarP(&globalOptions.JSON, "json", "", false, "set output mode to JSON for commands that support it")
 
 	restoreTerminal()
 }
@@ -90,6 +91,14 @@ func stdoutIsTerminal() bool {
 	return terminal.IsTerminal(int(os.Stdout.Fd()))
 }
 
+func stdoutTerminalWidth() int {
+	w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
+	if err != nil {
+		return 0
+	}
+	return w
+}
+
 // restoreTerminal installs a cleanup handler that restores the previous
 // terminal state on exit.
 func restoreTerminal() {
@@ -118,8 +127,7 @@ func restoreTerminal() {
 // current windows cmd shell.
 func ClearLine() string {
 	if runtime.GOOS == "windows" {
-		w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
-		if err == nil {
+		if w := stdoutTerminalWidth(); w > 0 {
 			return strings.Repeat(" ", w-1) + "\r"
 		}
 		return ""
@@ -132,7 +140,7 @@ func Printf(format string, args ...interface{}) {
 	_, err := fmt.Fprintf(globalOptions.stdout, format, args...)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to write to stdout: %v\n", err)
-		os.Exit(100)
+		Exit(100)
 	}
 }
 
@@ -175,18 +183,19 @@ func Warnf(format string, args ...interface{}) {
 	_, err := fmt.Fprintf(globalOptions.stderr, format, args...)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to write to stderr: %v\n", err)
-		os.Exit(100)
+		Exit(100)
 	}
 }
 
-// Exitf uses Warnf to write the message and then calls os.Exit(exitcode).
+// Exitf uses Warnf to write the message and then terminates the process with
+// the given exit code.
 func Exitf(exitcode int, format string, args ...interface{}) {
 	if format[len(format)-1] != '\n' {
 		format += "\n"
 	}
 
 	Warnf(format, args...)
-	os.Exit(exitcode)
+	Exit(exitcode)
 }
 
 // readPassword reads the password from the given reader directly.

src/cmds/restic/global_debug.go

@@ -0,0 +1,60 @@
+// +build debug
+
+package main
+
+import (
+	"fmt"
+	"net/http"
+	_ "net/http/pprof"
+	"os"
+	"restic/errors"
+
+	"github.com/pkg/profile"
+)
+
+var (
+	listenMemoryProfile string
+	memProfilePath      string
+	cpuProfilePath      string
+
+	prof interface {
+		Stop()
+	}
+)
+
+func init() {
+	f := cmdRoot.PersistentFlags()
+	f.StringVar(&listenMemoryProfile, "listen-profile", "", "listen on this `address:port` for memory profiling")
+	f.StringVar(&memProfilePath, "mem-profile", "", "write memory profile to `dir`")
+	f.StringVar(&cpuProfilePath, "cpu-profile", "", "write cpu profile to `dir`")
+}
+
+func runDebug() error {
+	if listenMemoryProfile != "" {
+		fmt.Fprintf(os.Stderr, "running memory profile HTTP server on %v\n", listenMemoryProfile)
+		go func() {
+			err := http.ListenAndServe(listenMemoryProfile, nil)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "memory profile listen failed: %v\n", err)
+			}
+		}()
+	}
+
+	if memProfilePath != "" && cpuProfilePath != "" {
+		return errors.Fatal("only one profile (memory or CPU) may be activated at the same time")
+	}
+
+	if memProfilePath != "" {
+		prof = profile.Start(profile.Quiet, profile.MemProfile, profile.ProfilePath(memProfilePath))
+	} else if memProfilePath != "" {
+		prof = profile.Start(profile.Quiet, profile.CPUProfile, profile.ProfilePath(memProfilePath))
+	}
+
+	return nil
+}
+
+func shutdownDebug() {
+	if prof != nil {
+		prof.Stop()
+	}
+}

src/cmds/restic/global_release.go

@@ -0,0 +1,9 @@
+// +build !debug
+
+package main
+
+// runDebug is a noop without the debug tag.
+func runDebug() error { return nil }
+
+// shutdownDebug is a noop without the debug tag.
+func shutdownDebug() {}

src/cmds/restic/integration_helpers_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -15,6 +16,7 @@ import (
 type dirEntry struct {
 	path string
 	fi   os.FileInfo
+	link uint64
 }
 
 func walkDir(dir string) <-chan *dirEntry {
@@ -36,6 +38,7 @@ func walkDir(dir string) <-chan *dirEntry {
 			ch <- &dirEntry{
 				path: name,
 				fi:   info,
+				link: nlink(info),
 			}
 
 			return nil
@@ -192,6 +195,7 @@ func withTestEnvironment(t testing.TB, f func(*testEnvironment, GlobalOptions))
 	gopts := GlobalOptions{
 		Repo:     env.repo,
 		Quiet:    true,
+		ctx:      context.Background(),
 		password: TestPassword,
 		stdout:   os.Stdout,
 		stderr:   os.Stderr,

src/cmds/restic/integration_helpers_unix_test.go

@@ -4,7 +4,9 @@ package main
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
+	"path/filepath"
 	"syscall"
 )
 
@@ -37,5 +39,37 @@ func (e *dirEntry) equals(other *dirEntry) bool {
 		return false
 	}
 
+	if stat.Nlink != stat2.Nlink {
+		fmt.Fprintf(os.Stderr, "%v: Number of links do not match (%v != %v)\n", e.path, stat.Nlink, stat2.Nlink)
+		return false
+	}
+
 	return true
 }
+
+func nlink(info os.FileInfo) uint64 {
+	stat, _ := info.Sys().(*syscall.Stat_t)
+	return uint64(stat.Nlink)
+}
+
+func inode(info os.FileInfo) uint64 {
+	stat, _ := info.Sys().(*syscall.Stat_t)
+	return uint64(stat.Ino)
+}
+
+func createFileSetPerHardlink(dir string) map[uint64][]string {
+	var stat syscall.Stat_t
+	linkTests := make(map[uint64][]string)
+	files, err := ioutil.ReadDir(dir)
+	if err != nil {
+		return nil
+	}
+	for _, f := range files {
+
+		if err := syscall.Stat(filepath.Join(dir, f.Name()), &stat); err != nil {
+			return nil
+		}
+		linkTests[uint64(stat.Ino)] = append(linkTests[uint64(stat.Ino)], f.Name())
+	}
+	return linkTests
+}

src/cmds/restic/integration_helpers_windows_test.go

@@ -4,6 +4,7 @@ package main
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
 )
 
@@ -25,3 +26,24 @@ func (e *dirEntry) equals(other *dirEntry) bool {
 
 	return true
 }
+
+func nlink(info os.FileInfo) uint64 {
+	return 1
+}
+
+func inode(info os.FileInfo) uint64 {
+	return uint64(0)
+}
+
+func createFileSetPerHardlink(dir string) map[uint64][]string {
+	linkTests := make(map[uint64][]string)
+	files, err := ioutil.ReadDir(dir)
+	if err != nil {
+		return nil
+	}
+	for i, f := range files {
+		linkTests[uint64(i)] = append(linkTests[uint64(i)], f.Name())
+		i++
+	}
+	return linkTests
+}

src/cmds/restic/integration_test.go

@@ -4,9 +4,11 @@ import (
 	"bufio"
 	"bytes"
 	"crypto/rand"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
+	mrand "math/rand"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -140,7 +142,9 @@ func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
 		globalOptions.Quiet = quiet
 	}()
 
-	OK(t, runLs(gopts, []string{snapshotID}))
+	opts := LsOptions{}
+
+	OK(t, runLs(opts, gopts, []string{snapshotID}))
 
 	return strings.Split(string(buf.Bytes()), "\n")
 }
@@ -159,6 +163,41 @@ func testRunFind(t testing.TB, gopts GlobalOptions, pattern string) []string {
 	return strings.Split(string(buf.Bytes()), "\n")
 }
 
+func testRunSnapshots(t testing.TB, gopts GlobalOptions) (newest *Snapshot, snapmap map[restic.ID]Snapshot) {
+	buf := bytes.NewBuffer(nil)
+	globalOptions.stdout = buf
+	globalOptions.JSON = true
+	defer func() {
+		globalOptions.stdout = os.Stdout
+		globalOptions.JSON = gopts.JSON
+	}()
+
+	opts := SnapshotOptions{}
+
+	OK(t, runSnapshots(opts, globalOptions, []string{}))
+
+	snapshots := []Snapshot{}
+	OK(t, json.Unmarshal(buf.Bytes(), &snapshots))
+
+	snapmap = make(map[restic.ID]Snapshot, len(snapshots))
+	for _, sn := range snapshots {
+		snapmap[*sn.ID] = sn
+		if newest == nil || sn.Time.After(newest.Time) {
+			newest = &sn
+		}
+	}
+	return
+}
+
+func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) {
+	opts := ForgetOptions{}
+	OK(t, runForget(opts, gopts, args))
+}
+
+func testRunPrune(t testing.TB, gopts GlobalOptions) {
+	OK(t, runPrune(gopts))
+}
+
 func TestBackup(t *testing.T) {
 	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
 		datafile := filepath.Join("testdata", "backup-data.tar.gz")
@@ -506,23 +545,23 @@ func TestBackupExclude(t *testing.T) {
 		testRunBackup(t, []string{datadir}, opts, gopts)
 		snapshots, snapshotID := lastSnapshot(snapshots, loadSnapshotMap(t, gopts))
 		files := testRunLs(t, gopts, snapshotID)
-		Assert(t, includes(files, filepath.Join("testdata", "foo.tar.gz")),
+		Assert(t, includes(files, filepath.Join(string(filepath.Separator), "testdata", "foo.tar.gz")),
 			"expected file %q in first snapshot, but it's not included", "foo.tar.gz")
 
 		opts.Excludes = []string{"*.tar.gz"}
 		testRunBackup(t, []string{datadir}, opts, gopts)
 		snapshots, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, gopts))
 		files = testRunLs(t, gopts, snapshotID)
-		Assert(t, !includes(files, filepath.Join("testdata", "foo.tar.gz")),
+		Assert(t, !includes(files, filepath.Join(string(filepath.Separator), "testdata", "foo.tar.gz")),
 			"expected file %q not in first snapshot, but it's included", "foo.tar.gz")
 
 		opts.Excludes = []string{"*.tar.gz", "private/secret"}
 		testRunBackup(t, []string{datadir}, opts, gopts)
 		snapshots, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, gopts))
 		files = testRunLs(t, gopts, snapshotID)
-		Assert(t, !includes(files, filepath.Join("testdata", "foo.tar.gz")),
+		Assert(t, !includes(files, filepath.Join(string(filepath.Separator), "testdata", "foo.tar.gz")),
 			"expected file %q not in first snapshot, but it's included", "foo.tar.gz")
-		Assert(t, !includes(files, filepath.Join("testdata", "private", "secret", "passwords.txt")),
+		Assert(t, !includes(files, filepath.Join(string(filepath.Separator), "testdata", "private", "secret", "passwords.txt")),
 			"expected file %q not in first snapshot, but it's included", "passwords.txt")
 	})
 }
@@ -592,6 +631,105 @@ func TestIncrementalBackup(t *testing.T) {
 	})
 }
 
+func TestBackupTags(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		testRunInit(t, gopts)
+		SetupTarTestFixture(t, env.testdata, datafile)
+
+		opts := BackupOptions{}
+
+		testRunBackup(t, []string{env.testdata}, opts, gopts)
+		testRunCheck(t, gopts)
+		newest, _ := testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 0,
+			"expected no tags, got %v", newest.Tags)
+
+		opts.Tags = []string{"NL"}
+		testRunBackup(t, []string{env.testdata}, opts, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 1 && newest.Tags[0] == "NL",
+			"expected one NL tag, got %v", newest.Tags)
+	})
+}
+
+func testRunTag(t testing.TB, opts TagOptions, gopts GlobalOptions) {
+	OK(t, runTag(opts, gopts, []string{}))
+}
+
+func TestTag(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		testRunInit(t, gopts)
+		SetupTarTestFixture(t, env.testdata, datafile)
+
+		testRunBackup(t, []string{env.testdata}, BackupOptions{}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ := testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 0,
+			"expected no tags, got %v", newest.Tags)
+		Assert(t, newest.Original == nil,
+			"expected original ID to be nil, got %v", newest.Original)
+		originalID := *newest.ID
+
+		testRunTag(t, TagOptions{SetTags: []string{"NL"}}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 1 && newest.Tags[0] == "NL",
+			"set failed, expected one NL tag, got %v", newest.Tags)
+		Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+		Assert(t, *newest.Original == originalID,
+			"expected original ID to be set to the first snapshot id")
+
+		testRunTag(t, TagOptions{AddTags: []string{"CH"}}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 2 && newest.Tags[0] == "NL" && newest.Tags[1] == "CH",
+			"add failed, expected CH,NL tags, got %v", newest.Tags)
+		Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+		Assert(t, *newest.Original == originalID,
+			"expected original ID to be set to the first snapshot id")
+
+		testRunTag(t, TagOptions{RemoveTags: []string{"NL"}}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 1 && newest.Tags[0] == "CH",
+			"remove failed, expected one CH tag, got %v", newest.Tags)
+		Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+		Assert(t, *newest.Original == originalID,
+			"expected original ID to be set to the first snapshot id")
+
+		testRunTag(t, TagOptions{AddTags: []string{"US", "RU"}}, gopts)
+		testRunTag(t, TagOptions{RemoveTags: []string{"CH", "US", "RU"}}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 0,
+			"expected no tags, got %v", newest.Tags)
+		Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+		Assert(t, *newest.Original == originalID,
+			"expected original ID to be set to the first snapshot id")
+
+		// Check special case of removing all tags.
+		testRunTag(t, TagOptions{SetTags: []string{""}}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 0,
+			"expected no tags, got %v", newest.Tags)
+		Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+		Assert(t, *newest.Original == originalID,
+			"expected original ID to be set to the first snapshot id")
+	})
+}
+
 func testRunKeyListOtherIDs(t testing.TB, gopts GlobalOptions) []string {
 	buf := bytes.NewBuffer(nil)
 
@@ -730,6 +868,30 @@ func TestRestoreFilter(t *testing.T) {
 	})
 }
 
+func TestRestore(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+		testRunInit(t, gopts)
+
+		for i := 0; i < 10; i++ {
+			p := filepath.Join(env.testdata, fmt.Sprintf("foo/bar/testfile%v", i))
+			OK(t, os.MkdirAll(filepath.Dir(p), 0755))
+			OK(t, appendRandomData(p, uint(mrand.Intn(5<<21))))
+		}
+
+		opts := BackupOptions{}
+
+		testRunBackup(t, []string{env.testdata}, opts, gopts)
+		testRunCheck(t, gopts)
+
+		// Restore latest without any filters
+		restoredir := filepath.Join(env.base, "restore")
+		testRunRestoreLatest(t, gopts, restoredir, nil, "")
+
+		Assert(t, directoriesEqualContents(env.testdata, filepath.Join(restoredir, filepath.Base(env.testdata))),
+			"directories are not equal")
+	})
+}
+
 func TestRestoreLatest(t *testing.T) {
 
 	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
@@ -802,7 +964,7 @@ func TestRestoreWithPermissionFailure(t *testing.T) {
 
 		testRunRestore(t, gopts, filepath.Join(env.base, "restore"), snapshots[0])
 
-		// make sure that all files have been restored, regardeless of any
+		// make sure that all files have been restored, regardless of any
 		// permission errors
 		files := testRunLs(t, gopts, snapshots[0].String())
 		for _, filename := range files {
@@ -901,7 +1063,7 @@ func TestRebuildIndex(t *testing.T) {
 		}
 
 		if !strings.Contains(out, "restic rebuild-index") {
-			t.Fatalf("did not find hint for rebuild-index comman")
+			t.Fatalf("did not find hint for rebuild-index command")
 		}
 
 		testRunRebuildIndex(t, gopts)
@@ -947,3 +1109,130 @@ func TestCheckRestoreNoLock(t *testing.T) {
 		testRunRestore(t, gopts, filepath.Join(env.base, "restore"), snapshotIDs[0])
 	})
 }
+
+func TestPrune(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		fd, err := os.Open(datafile)
+		if os.IsNotExist(errors.Cause(err)) {
+			t.Skipf("unable to find data file %q, skipping", datafile)
+			return
+		}
+		OK(t, err)
+		OK(t, fd.Close())
+
+		testRunInit(t, gopts)
+
+		SetupTarTestFixture(t, env.testdata, datafile)
+		opts := BackupOptions{}
+
+		testRunBackup(t, []string{filepath.Join(env.testdata, "0", "0", "1")}, opts, gopts)
+		testRunBackup(t, []string{filepath.Join(env.testdata, "0", "0", "2")}, opts, gopts)
+		testRunBackup(t, []string{filepath.Join(env.testdata, "0", "0", "3")}, opts, gopts)
+
+		snapshotIDs := testRunList(t, "snapshots", gopts)
+		Assert(t, len(snapshotIDs) == 3,
+			"expected one snapshot, got %v", snapshotIDs)
+
+		testRunForget(t, gopts, snapshotIDs[0].String())
+		testRunPrune(t, gopts)
+		testRunCheck(t, gopts)
+	})
+}
+
+func TestHardLink(t *testing.T) {
+	// this test assumes a test set with a single directory containing hard linked files
+	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+		datafile := filepath.Join("testdata", "test.hl.tar.gz")
+		fd, err := os.Open(datafile)
+		if os.IsNotExist(errors.Cause(err)) {
+			t.Skipf("unable to find data file %q, skipping", datafile)
+			return
+		}
+		OK(t, err)
+		OK(t, fd.Close())
+
+		testRunInit(t, gopts)
+
+		SetupTarTestFixture(t, env.testdata, datafile)
+
+		linkTests := createFileSetPerHardlink(env.testdata)
+
+		opts := BackupOptions{}
+
+		// first backup
+		testRunBackup(t, []string{env.testdata}, opts, gopts)
+		snapshotIDs := testRunList(t, "snapshots", gopts)
+		Assert(t, len(snapshotIDs) == 1,
+			"expected one snapshot, got %v", snapshotIDs)
+
+		testRunCheck(t, gopts)
+
+		// restore all backups and compare
+		for i, snapshotID := range snapshotIDs {
+			restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i))
+			t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir)
+			testRunRestore(t, gopts, restoredir, snapshotIDs[0])
+			Assert(t, directoriesEqualContents(env.testdata, filepath.Join(restoredir, "testdata")),
+				"directories are not equal")
+
+			linkResults := createFileSetPerHardlink(filepath.Join(restoredir, "testdata"))
+			Assert(t, linksEqual(linkTests, linkResults),
+				"links are not equal")
+		}
+
+		testRunCheck(t, gopts)
+	})
+}
+
+func linksEqual(source, dest map[uint64][]string) bool {
+	for _, vs := range source {
+		found := false
+		for kd, vd := range dest {
+			if linkEqual(vs, vd) {
+				delete(dest, kd)
+				found = true
+				break
+			}
+		}
+		if !found {
+			return false
+		}
+	}
+
+	if len(dest) != 0 {
+		return false
+	}
+
+	return true
+}
+
+func linkEqual(source, dest []string) bool {
+	// equal if sliced are equal without considering order
+	if source == nil && dest == nil {
+		return true
+	}
+
+	if source == nil || dest == nil {
+		return false
+	}
+
+	if len(source) != len(dest) {
+		return false
+	}
+
+	for i := range source {
+		found := false
+		for j := range dest {
+			if source[i] == dest[j] {
+				found = true
+				break
+			}
+		}
+		if !found {
+			return false
+		}
+	}
+
+	return true
+}

src/cmds/restic/lock.go

@@ -36,6 +36,7 @@ func lockRepository(repo *repository.Repository, exclusive bool) (*restic.Lock,
 	if err != nil {
 		return nil, err
 	}
+	debug.Log("create lock %p (exclusive %v)", lock, exclusive)
 
 	globalLocks.Lock()
 	if globalLocks.cancelRefresh == nil {
@@ -88,7 +89,7 @@ func unlockRepo(lock *restic.Lock) error {
 	globalLocks.Lock()
 	defer globalLocks.Unlock()
 
-	debug.Log("unlocking repository")
+	debug.Log("unlocking repository with lock %p", lock)
 	if err := lock.Unlock(); err != nil {
 		debug.Log("error while unlocking: %v", err)
 		return err

src/cmds/restic/main.go

@@ -5,7 +5,6 @@ import (
 	"os"
 	"restic"
 	"restic/debug"
-	"runtime"
 
 	"github.com/spf13/cobra"
 
@@ -20,20 +19,17 @@ var cmdRoot = &cobra.Command{
 restic is a backup program which allows saving multiple revisions of files and
 directories in an encrypted repository stored on different backends.
 `,
-	SilenceErrors:    true,
-	SilenceUsage:     true,
-	PersistentPreRun: parseEnvironment,
-}
+	SilenceErrors: true,
+	SilenceUsage:  true,
 
-func init() {
-	// set GOMAXPROCS to number of CPUs
-	if runtime.Version() < "go1.5" {
-		gomaxprocs := os.Getenv("GOMAXPROCS")
-		debug.Log("read GOMAXPROCS from env variable, value: %s", gomaxprocs)
-		if gomaxprocs == "" {
-			runtime.GOMAXPROCS(runtime.NumCPU())
-		}
-	}
+	// run the debug functions for all subcommands (if build tag "debug" is
+	// enabled)
+	PersistentPreRunE: func(*cobra.Command, []string) error {
+		return runDebug()
+	},
+	PersistentPostRun: func(*cobra.Command, []string) {
+		shutdownDebug()
+	},
 }
 
 func main() {
@@ -49,9 +45,10 @@ func main() {
 		fmt.Fprintf(os.Stderr, "%+v\n", err)
 	}
 
-	RunCleanupHandlers()
-
+	var exitCode int
 	if err != nil {
-		os.Exit(1)
+		exitCode = 1
 	}
+
+	Exit(exitCode)
 }

src/restic/archiver/archive_reader.go

@@ -11,11 +11,22 @@ import (
 	"github.com/restic/chunker"
 )
 
-// ArchiveReader reads from the reader and archives the data. Returned is the
-// resulting snapshot and its ID.
-func ArchiveReader(repo restic.Repository, p *restic.Progress, rd io.Reader, name string, tags []string) (*restic.Snapshot, restic.ID, error) {
+// Reader allows saving a stream of data to the repository.
+type Reader struct {
+	restic.Repository
+
+	Tags     []string
+	Hostname string
+}
+
+// Archive reads data from the reader and saves it to the repo.
+func (r *Reader) Archive(name string, rd io.Reader, p *restic.Progress) (*restic.Snapshot, restic.ID, error) {
+	if name == "" {
+		return nil, restic.ID{}, errors.New("no filename given")
+	}
+
 	debug.Log("start archiving %s", name)
-	sn, err := restic.NewSnapshot([]string{name}, tags)
+	sn, err := restic.NewSnapshot([]string{name}, r.Tags, r.Hostname)
 	if err != nil {
 		return nil, restic.ID{}, err
 	}
@@ -23,9 +34,10 @@ func ArchiveReader(repo restic.Repository, p *restic.Progress, rd io.Reader, nam
 	p.Start()
 	defer p.Done()
 
+	repo := r.Repository
 	chnker := chunker.New(rd, repo.Config().ChunkerPolynomial)
 
-	var ids restic.IDs
+	ids := restic.IDs{}
 	var fileSize uint64
 
 	for {

src/restic/archiver/archive_reader_test.go

@@ -2,9 +2,11 @@ package archiver
 
 import (
 	"bytes"
+	"errors"
 	"io"
 	"math/rand"
 	"restic"
+	"restic/checker"
 	"restic/repository"
 	"testing"
 )
@@ -44,7 +46,7 @@ func checkSavedFile(t *testing.T, repo restic.Repository, treeID restic.ID, name
 			t.Fatal(err)
 		}
 
-		buf := make([]byte, int(size))
+		buf := restic.NewBlobBuffer(int(size))
 		n := loadBlob(t, repo, id, buf)
 		if n != len(buf) {
 			t.Errorf("wrong number of bytes read, want %d, got %d", len(buf), n)
@@ -77,7 +79,13 @@ func TestArchiveReader(t *testing.T) {
 
 	f := fakeFile(t, seed, size)
 
-	sn, id, err := ArchiveReader(repo, nil, f, "fakefile", []string{"test"})
+	r := &Reader{
+		Repository: repo,
+		Hostname:   "localhost",
+		Tags:       []string{"test"},
+	}
+
+	sn, id, err := r.Archive("fakefile", f, nil)
 	if err != nil {
 		t.Fatalf("ArchiveReader() returned error %v", err)
 	}
@@ -89,6 +97,80 @@ func TestArchiveReader(t *testing.T) {
 	t.Logf("snapshot saved as %v, tree is %v", id.Str(), sn.Tree.Str())
 
 	checkSavedFile(t, repo, *sn.Tree, "fakefile", fakeFile(t, seed, size))
+
+	checker.TestCheckRepo(t, repo)
+}
+
+func TestArchiveReaderNull(t *testing.T) {
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
+
+	r := &Reader{
+		Repository: repo,
+		Hostname:   "localhost",
+		Tags:       []string{"test"},
+	}
+
+	sn, id, err := r.Archive("fakefile", bytes.NewReader(nil), nil)
+	if err != nil {
+		t.Fatalf("ArchiveReader() returned error %v", err)
+	}
+
+	if id.IsNull() {
+		t.Fatalf("ArchiveReader() returned null ID")
+	}
+
+	t.Logf("snapshot saved as %v, tree is %v", id.Str(), sn.Tree.Str())
+
+	checker.TestCheckRepo(t, repo)
+}
+
+type errReader string
+
+func (e errReader) Read([]byte) (int, error) {
+	return 0, errors.New(string(e))
+}
+
+func countSnapshots(t testing.TB, repo restic.Repository) int {
+	done := make(chan struct{})
+	defer close(done)
+
+	snapshots := 0
+	for range repo.List(restic.SnapshotFile, done) {
+		snapshots++
+	}
+	return snapshots
+}
+
+func TestArchiveReaderError(t *testing.T) {
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
+
+	r := &Reader{
+		Repository: repo,
+		Hostname:   "localhost",
+		Tags:       []string{"test"},
+	}
+
+	sn, id, err := r.Archive("fakefile", errReader("error returned by reading stdin"), nil)
+	if err == nil {
+		t.Errorf("expected error not returned")
+	}
+
+	if sn != nil {
+		t.Errorf("Snapshot should be nil, but isn't")
+	}
+
+	if !id.IsNull() {
+		t.Errorf("id should be null, but %v returned", id.Str())
+	}
+
+	n := countSnapshots(t, repo)
+	if n > 0 {
+		t.Errorf("expected zero snapshots, but got %d", n)
+	}
+
+	checker.TestCheckRepo(t, repo)
 }
 
 func BenchmarkArchiveReader(t *testing.B) {
@@ -103,11 +185,17 @@ func BenchmarkArchiveReader(t *testing.B) {
 		t.Fatal(err)
 	}
 
+	r := &Reader{
+		Repository: repo,
+		Hostname:   "localhost",
+		Tags:       []string{"test"},
+	}
+
 	t.SetBytes(size)
 	t.ResetTimer()
 
 	for i := 0; i < t.N; i++ {
-		_, _, err := ArchiveReader(repo, nil, bytes.NewReader(buf), "fakefile", []string{"test"})
+		_, _, err := r.Archive("fakefile", bytes.NewReader(buf), nil)
 		if err != nil {
 			t.Fatal(err)
 		}

src/restic/archiver/archiver.go

@@ -142,7 +142,7 @@ func (arch *Archiver) reloadFileIfChanged(node *restic.Node, file fs.File) (*res
 	node, err = restic.NodeFromFileInfo(node.Path, fi)
 	if err != nil {
 		debug.Log("restic.NodeFromFileInfo returned error for %v: %v", node.Path, err)
-		return nil, err
+		arch.Warn(node.Path, fi, err)
 	}
 
 	return node, nil
@@ -210,14 +210,14 @@ func (arch *Archiver) SaveFile(p *restic.Progress, node *restic.Node) (*restic.N
 	file, err := fs.Open(node.Path)
 	defer file.Close()
 	if err != nil {
-		return nil, errors.Wrap(err, "Open")
+		return node, errors.Wrap(err, "Open")
 	}
 
 	debug.RunHook("archiver.SaveFile", node.Path)
 
 	node, err = arch.reloadFileIfChanged(node, file)
 	if err != nil {
-		return nil, err
+		return node, err
 	}
 
 	chnker := chunker.New(file, arch.repo.Config().ChunkerPolynomial)
@@ -230,7 +230,7 @@ func (arch *Archiver) SaveFile(p *restic.Progress, node *restic.Node) (*restic.N
 		}
 
 		if err != nil {
-			return nil, errors.Wrap(err, "chunker.Next")
+			return node, errors.Wrap(err, "chunker.Next")
 		}
 
 		resCh := make(chan saveResult, 1)
@@ -240,7 +240,7 @@ func (arch *Archiver) SaveFile(p *restic.Progress, node *restic.Node) (*restic.N
 
 	results, err := waitForResults(resultChannels)
 	if err != nil {
-		return nil, err
+		return node, err
 	}
 	err = updateNodeContent(node, results)
 
@@ -275,11 +275,8 @@ func (arch *Archiver) fileWorker(wg *sync.WaitGroup, p *restic.Progress, done <-
 
 			node, err := restic.NodeFromFileInfo(e.Fullpath(), e.Info())
 			if err != nil {
-				// TODO: integrate error reporting
 				debug.Log("restic.NodeFromFileInfo returned error for %v: %v", node.Path, err)
-				e.Result() <- nil
-				p.Report(restic.Stat{Errors: 1})
-				continue
+				arch.Warn(e.Fullpath(), e.Info(), err)
 			}
 
 			// try to use old node, if present
@@ -307,11 +304,11 @@ func (arch *Archiver) fileWorker(wg *sync.WaitGroup, p *restic.Progress, done <-
 
 			// otherwise read file normally
 			if node.Type == "file" && len(node.Content) == 0 {
-				debug.Log("   read and save %v, content: %v", e.Path(), node.Content)
+				debug.Log("   read and save %v", e.Path())
 				node, err = arch.SaveFile(p, node)
 				if err != nil {
-					// TODO: integrate error reporting
 					fmt.Fprintf(os.Stderr, "error for %v: %v\n", node.Path, err)
+					arch.Warn(e.Path(), nil, err)
 					// ignore this file
 					e.Result() <- nil
 					p.Report(restic.Stat{Errors: 1})
@@ -371,25 +368,28 @@ func (arch *Archiver) dirWorker(wg *sync.WaitGroup, p *restic.Progress, done <-c
 
 				// else insert node
 				node := res.(*restic.Node)
-				tree.Insert(node)
 
 				if node.Type == "dir" {
 					debug.Log("got tree node for %s: %v", node.Path, node.Subtree)
 
+					if node.Subtree == nil {
+						debug.Log("subtree is nil for node %v", node.Path)
+						continue
+					}
+
 					if node.Subtree.IsNull() {
 						panic("invalid null subtree restic.ID")
 					}
 				}
+				tree.Insert(node)
 			}
 
 			node := &restic.Node{}
 
 			if dir.Path() != "" && dir.Info() != nil {
-				n, err := restic.NodeFromFileInfo(dir.Path(), dir.Info())
+				n, err := restic.NodeFromFileInfo(dir.Fullpath(), dir.Info())
 				if err != nil {
-					n.Error = err.Error()
-					dir.Result() <- n
-					continue
+					arch.Warn(dir.Path(), dir.Info(), err)
 				}
 				node = n
 			}
@@ -634,7 +634,7 @@ func (p baseNameSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
 // Snapshot creates a snapshot of the given paths. If parentrestic.ID is set, this is
 // used to compare the files to the ones archived at the time this snapshot was
 // taken.
-func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, parentID *restic.ID) (*restic.Snapshot, restic.ID, error) {
+func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, hostname string, parentID *restic.ID) (*restic.Snapshot, restic.ID, error) {
 	paths = unique(paths)
 	sort.Sort(baseNameSlice(paths))
 
@@ -650,7 +650,7 @@ func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, parentI
 	defer p.Done()
 
 	// create new snapshot
-	sn, err := restic.NewSnapshot(paths, tags)
+	sn, err := restic.NewSnapshot(paths, tags, hostname)
 	if err != nil {
 		return nil, restic.ID{}, err
 	}
@@ -734,6 +734,21 @@ func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, parentI
 		return nil, restic.ID{}, err
 	}
 
+	// receive the top-level tree
+	root := (<-resCh).(*restic.Node)
+	debug.Log("root node received: %v", root.Subtree.Str())
+	sn.Tree = root.Subtree
+
+	// load top-level tree again to see if it is empty
+	toptree, err := arch.repo.LoadTree(*root.Subtree)
+	if err != nil {
+		return nil, restic.ID{}, err
+	}
+
+	if len(toptree.Nodes) == 0 {
+		return nil, restic.ID{}, errors.Fatal("no files/dirs saved, refusing to create empty snapshot")
+	}
+
 	// save index
 	err = arch.repo.SaveIndex()
 	if err != nil {
@@ -743,11 +758,6 @@ func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, parentI
 
 	debug.Log("saved indexes")
 
-	// receive the top-level tree
-	root := (<-resCh).(*restic.Node)
-	debug.Log("root node received: %v", root.Subtree.Str())
-	sn.Tree = root.Subtree
-
 	// save snapshot
 	id, err := arch.repo.SaveJSONUnpacked(restic.SnapshotFile, sn)
 	if err != nil {

src/restic/archiver/archiver_duplication_test.go

@@ -39,15 +39,15 @@ func randomID() restic.ID {
 func forgetfulBackend() restic.Backend {
 	be := &mock.Backend{}
 
-	be.TestFn = func(t restic.FileType, name string) (bool, error) {
+	be.TestFn = func(h restic.Handle) (bool, error) {
 		return false, nil
 	}
 
-	be.LoadFn = func(h restic.Handle, p []byte, off int64) (int, error) {
-		return 0, errors.New("not found")
+	be.LoadFn = func(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+		return nil, errors.New("not found")
 	}
 
-	be.SaveFn = func(h restic.Handle, p []byte) error {
+	be.SaveFn = func(h restic.Handle, rd io.Reader) error {
 		return nil
 	}
 
@@ -55,7 +55,7 @@ func forgetfulBackend() restic.Backend {
 		return restic.FileInfo{}, errors.New("not found")
 	}
 
-	be.RemoveFn = func(t restic.FileType, name string) error {
+	be.RemoveFn = func(h restic.Handle) error {
 		return nil
 	}
 
@@ -142,6 +142,11 @@ func testArchiverDuplication(t *testing.T) {
 	close(done)
 
 	wg.Wait()
+
+	err = repo.Flush()
+	if err != nil {
+		t.Fatal(err)
+	}
 }
 
 func TestArchiverDuplication(t *testing.T) {

src/restic/archiver/archiver_test.go

@@ -104,7 +104,7 @@ func archiveDirectory(b testing.TB) {
 
 	arch := archiver.New(repo)
 
-	_, id, err := arch.Snapshot(nil, []string{BenchArchiveDirectory}, nil, nil)
+	_, id, err := arch.Snapshot(nil, []string{BenchArchiveDirectory}, nil, "localhost", nil)
 	OK(b, err)
 
 	b.Logf("snapshot archived as %v", id)
@@ -220,7 +220,7 @@ func testParallelSaveWithDuplication(t *testing.T, seed int) {
 
 	errChannels := [](<-chan error){}
 
-	// interweaved processing of subsequent chunks
+	// interwoven processing of subsequent chunks
 	maxParallel := 2*duplication - 1
 	barrier := make(chan struct{}, maxParallel)
 
@@ -294,3 +294,23 @@ func assertNoUnreferencedPacks(t *testing.T, chkr *checker.Checker) {
 		OK(t, err)
 	}
 }
+
+func TestArchiveEmptySnapshot(t *testing.T) {
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
+
+	arch := archiver.New(repo)
+
+	sn, id, err := arch.Snapshot(nil, []string{"file-does-not-exist-123123213123", "file2-does-not-exist-too-123123123"}, nil, "localhost", nil)
+	if err == nil {
+		t.Errorf("expected error for empty snapshot, got nil")
+	}
+
+	if !id.IsNull() {
+		t.Errorf("expected null ID for empty snapshot, got %v", id.Str())
+	}
+
+	if sn != nil {
+		t.Errorf("expected null snapshot for empty snapshot, got %v", sn)
+	}
+}

src/restic/archiver/testing.go

@@ -8,7 +8,7 @@ import (
 // TestSnapshot creates a new snapshot of path.
 func TestSnapshot(t testing.TB, repo restic.Repository, path string, parent *restic.ID) *restic.Snapshot {
 	arch := New(repo)
-	sn, _, err := arch.Snapshot(nil, []string{path}, []string{"test"}, parent)
+	sn, _, err := arch.Snapshot(nil, []string{path}, []string{"test"}, "localhost", parent)
 	if err != nil {
 		t.Fatal(err)
 	}

src/restic/backend.go

@@ -1,5 +1,7 @@
 package restic
 
+import "io"
+
 // Backend is used to store and access data.
 type Backend interface {
 	// Location returns a string that describes the type and location of the
@@ -7,22 +9,22 @@ type Backend interface {
 	Location() string
 
 	// Test a boolean value whether a File with the name and type exists.
-	Test(t FileType, name string) (bool, error)
+	Test(h Handle) (bool, error)
 
 	// Remove removes a File with type t and name.
-	Remove(t FileType, name string) error
+	Remove(h Handle) error
 
 	// Close the backend
 	Close() error
 
-	// Load returns the data stored in the backend for h at the given offset
-	// and saves it in p. Load has the same semantics as io.ReaderAt, except
-	// that a negative offset is also allowed. In this case it references a
-	// position relative to the end of the file (similar to Seek()).
-	Load(h Handle, p []byte, off int64) (int, error)
-
 	// Save stores the data in the backend under the given handle.
-	Save(h Handle, p []byte) error
+	Save(h Handle, rd io.Reader) error
+
+	// Load returns a reader that yields the contents of the file at h at the
+	// given offset. If length is larger than zero, only a portion of the file
+	// is returned. rd must be closed after use. If an error is returned, the
+	// ReadCloser must be nil.
+	Load(h Handle, length int, offset int64) (io.ReadCloser, error)
 
 	// Stat returns information about the File identified by h.
 	Stat(h Handle) (FileInfo, error)

src/restic/backend/local/backend_test.go

@@ -51,13 +51,6 @@ func TestLocalBackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestLocalBackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestLocalBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/local/local.go

@@ -101,58 +101,18 @@ func dirname(base string, t restic.FileType, name string) string {
 	return filepath.Join(base, n)
 }
 
-// Load returns the data stored in the backend for h at the given offset and
-// saves it in p. Load has the same semantics as io.ReaderAt, with one
-// exception: when off is lower than zero, it is treated as an offset relative
-// to the end of the file.
-func (b *Local) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
-	debug.Log("Load %v, length %v at %v", h, len(p), off)
-	if err := h.Valid(); err != nil {
-		return 0, err
-	}
-
-	f, err := fs.Open(filename(b.p, h.Type, h.Name))
-	if err != nil {
-		return 0, errors.Wrap(err, "Open")
-	}
-
-	defer func() {
-		e := f.Close()
-		if err == nil {
-			err = errors.Wrap(e, "Close")
-		}
-	}()
-
-	switch {
-	case off > 0:
-		_, err = f.Seek(off, 0)
-	case off < 0:
-		_, err = f.Seek(off, 2)
-	}
-
-	if err != nil {
-		return 0, errors.Wrap(err, "Seek")
-	}
-
-	return io.ReadFull(f, p)
-}
-
-// writeToTempfile saves p into a tempfile in tempdir.
-func writeToTempfile(tempdir string, p []byte) (filename string, err error) {
+// copyToTempfile saves p into a tempfile in tempdir.
+func copyToTempfile(tempdir string, rd io.Reader) (filename string, err error) {
 	tmpfile, err := ioutil.TempFile(tempdir, "temp-")
 	if err != nil {
 		return "", errors.Wrap(err, "TempFile")
 	}
 
-	n, err := tmpfile.Write(p)
+	_, err = io.Copy(tmpfile, rd)
 	if err != nil {
 		return "", errors.Wrap(err, "Write")
 	}
 
-	if n != len(p) {
-		return "", errors.New("not all bytes writen")
-	}
-
 	if err = tmpfile.Sync(); err != nil {
 		return "", errors.Wrap(err, "Syncn")
 	}
@@ -166,14 +126,14 @@ func writeToTempfile(tempdir string, p []byte) (filename string, err error) {
 }
 
 // Save stores data in the backend at the handle.
-func (b *Local) Save(h restic.Handle, p []byte) (err error) {
-	debug.Log("Save %v, length %v", h, len(p))
+func (b *Local) Save(h restic.Handle, rd io.Reader) (err error) {
+	debug.Log("Save %v", h)
 	if err := h.Valid(); err != nil {
 		return err
 	}
 
-	tmpfile, err := writeToTempfile(filepath.Join(b.p, backend.Paths.Temp), p)
-	debug.Log("saved %v (%d bytes) to %v", h, len(p), tmpfile)
+	tmpfile, err := copyToTempfile(filepath.Join(b.p, backend.Paths.Temp), rd)
+	debug.Log("saved %v to %v", h, tmpfile)
 	if err != nil {
 		return err
 	}
@@ -210,6 +170,39 @@ func (b *Local) Save(h restic.Handle, p []byte) (err error) {
 	return setNewFileMode(filename, fi)
 }
 
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (b *Local) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	f, err := os.Open(filename(b.p, h.Type, h.Name))
+	if err != nil {
+		return nil, err
+	}
+
+	if offset > 0 {
+		_, err = f.Seek(offset, 0)
+		if err != nil {
+			f.Close()
+			return nil, err
+		}
+	}
+
+	if length > 0 {
+		return backend.LimitReadCloser(f, int64(length)), nil
+	}
+
+	return f, nil
+}
+
 // Stat returns information about a blob.
 func (b *Local) Stat(h restic.Handle) (restic.FileInfo, error) {
 	debug.Log("Stat %v", h)
@@ -226,9 +219,9 @@ func (b *Local) Stat(h restic.Handle) (restic.FileInfo, error) {
 }
 
 // Test returns true if a blob of the given type and name exists in the backend.
-func (b *Local) Test(t restic.FileType, name string) (bool, error) {
-	debug.Log("Test %v %v", t, name)
-	_, err := fs.Stat(filename(b.p, t, name))
+func (b *Local) Test(h restic.Handle) (bool, error) {
+	debug.Log("Test %v", h)
+	_, err := fs.Stat(filename(b.p, h.Type, h.Name))
 	if err != nil {
 		if os.IsNotExist(errors.Cause(err)) {
 			return false, nil
@@ -240,9 +233,9 @@ func (b *Local) Test(t restic.FileType, name string) (bool, error) {
 }
 
 // Remove removes the blob with the given name and type.
-func (b *Local) Remove(t restic.FileType, name string) error {
-	debug.Log("Remove %v %v", t, name)
-	fn := filename(b.p, t, name)
+func (b *Local) Remove(h restic.Handle) error {
+	debug.Log("Remove %v", h)
+	fn := filename(b.p, h.Type, h.Name)
 
 	// reset read-only flag
 	err := fs.Chmod(fn, 0666)

src/restic/backend/mem/backend_test.go

@@ -51,13 +51,6 @@ func TestMemBackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestMemBackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestMemBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/mem/mem_backend.go

@@ -1,21 +1,19 @@
 package mem
 
 import (
+	"bytes"
 	"io"
+	"io/ioutil"
 	"restic"
 	"sync"
 
+	"restic/backend"
 	"restic/errors"
 
 	"restic/debug"
 )
 
-type entry struct {
-	Type restic.FileType
-	Name string
-}
-
-type memMap map[entry][]byte
+type memMap map[restic.Handle][]byte
 
 // make sure that MemoryBackend implements backend.Backend
 var _ restic.Backend = &MemoryBackend{}
@@ -39,61 +37,21 @@ func New() *MemoryBackend {
 }
 
 // Test returns whether a file exists.
-func (be *MemoryBackend) Test(t restic.FileType, name string) (bool, error) {
+func (be *MemoryBackend) Test(h restic.Handle) (bool, error) {
 	be.m.Lock()
 	defer be.m.Unlock()
 
-	debug.Log("test %v %v", t, name)
+	debug.Log("Test %v", h)
 
-	if _, ok := be.data[entry{t, name}]; ok {
+	if _, ok := be.data[h]; ok {
 		return true, nil
 	}
 
 	return false, nil
 }
 
-// Load reads data from the backend.
-func (be *MemoryBackend) Load(h restic.Handle, p []byte, off int64) (int, error) {
-	if err := h.Valid(); err != nil {
-		return 0, err
-	}
-
-	be.m.Lock()
-	defer be.m.Unlock()
-
-	if h.Type == restic.ConfigFile {
-		h.Name = ""
-	}
-
-	debug.Log("get %v offset %v len %v", h, off, len(p))
-
-	if _, ok := be.data[entry{h.Type, h.Name}]; !ok {
-		return 0, errors.New("no such data")
-	}
-
-	buf := be.data[entry{h.Type, h.Name}]
-	switch {
-	case off > int64(len(buf)):
-		return 0, errors.New("offset beyond end of file")
-	case off < -int64(len(buf)):
-		off = 0
-	case off < 0:
-		off = int64(len(buf)) + off
-	}
-
-	buf = buf[off:]
-
-	n := copy(p, buf)
-
-	if len(p) > len(buf) {
-		return n, io.ErrUnexpectedEOF
-	}
-
-	return n, nil
-}
-
 // Save adds new Data to the backend.
-func (be *MemoryBackend) Save(h restic.Handle, p []byte) error {
+func (be *MemoryBackend) Save(h restic.Handle, rd io.Reader) error {
 	if err := h.Valid(); err != nil {
 		return err
 	}
@@ -105,18 +63,59 @@ func (be *MemoryBackend) Save(h restic.Handle, p []byte) error {
 		h.Name = ""
 	}
 
-	if _, ok := be.data[entry{h.Type, h.Name}]; ok {
+	if _, ok := be.data[h]; ok {
 		return errors.New("file already exists")
 	}
 
-	debug.Log("save %v bytes at %v", len(p), h)
-	buf := make([]byte, len(p))
-	copy(buf, p)
-	be.data[entry{h.Type, h.Name}] = buf
+	buf, err := ioutil.ReadAll(rd)
+	if err != nil {
+		return err
+	}
+
+	be.data[h] = buf
+	debug.Log("saved %v bytes at %v", len(buf), h)
 
 	return nil
 }
 
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (be *MemoryBackend) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	be.m.Lock()
+	defer be.m.Unlock()
+
+	if h.Type == restic.ConfigFile {
+		h.Name = ""
+	}
+
+	debug.Log("Load %v offset %v len %v", h, offset, length)
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	if _, ok := be.data[h]; !ok {
+		return nil, errors.New("no such data")
+	}
+
+	buf := be.data[h]
+	if offset > int64(len(buf)) {
+		return nil, errors.New("offset beyond end of file")
+	}
+
+	buf = buf[offset:]
+	if length > 0 && len(buf) > length {
+		buf = buf[:length]
+	}
+
+	return backend.Closer{Reader: bytes.NewReader(buf)}, nil
+}
+
 // Stat returns information about a file in the backend.
 func (be *MemoryBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 	be.m.Lock()
@@ -132,7 +131,7 @@ func (be *MemoryBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 
 	debug.Log("stat %v", h)
 
-	e, ok := be.data[entry{h.Type, h.Name}]
+	e, ok := be.data[h]
 	if !ok {
 		return restic.FileInfo{}, errors.New("no such data")
 	}
@@ -141,17 +140,17 @@ func (be *MemoryBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 }
 
 // Remove deletes a file from the backend.
-func (be *MemoryBackend) Remove(t restic.FileType, name string) error {
+func (be *MemoryBackend) Remove(h restic.Handle) error {
 	be.m.Lock()
 	defer be.m.Unlock()
 
-	debug.Log("get %v %v", t, name)
+	debug.Log("Remove %v", h)
 
-	if _, ok := be.data[entry{t, name}]; !ok {
+	if _, ok := be.data[h]; !ok {
 		return errors.New("no such data")
 	}
 
-	delete(be.data, entry{t, name})
+	delete(be.data, h)
 
 	return nil
 }

src/restic/backend/rest/backend_test.go

@@ -51,13 +51,6 @@ func TestRestBackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestRestBackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestRestBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/rest/rest.go

@@ -1,7 +1,6 @@
 package rest
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -12,12 +11,16 @@ import (
 	"restic"
 	"strings"
 
+	"restic/debug"
 	"restic/errors"
 
 	"restic/backend"
 )
 
-const connLimit = 10
+const connLimit = 40
+
+// make sure the rest backend implements restic.Backend
+var _ restic.Backend = &restBackend{}
 
 // restPath returns the path to the given resource.
 func restPath(url *url.URL, h restic.Handle) string {
@@ -71,65 +74,18 @@ func (b *restBackend) Location() string {
 	return b.url.String()
 }
 
-// Load returns the data stored in the backend for h at the given offset
-// and saves it in p. Load has the same semantics as io.ReaderAt.
-func (b *restBackend) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
-	if err := h.Valid(); err != nil {
-		return 0, err
-	}
-
-	// invert offset
-	if off < 0 {
-		info, err := b.Stat(h)
-		if err != nil {
-			return 0, errors.Wrap(err, "Stat")
-		}
-
-		if -off > info.Size {
-			off = 0
-		} else {
-			off = info.Size + off
-		}
-	}
-
-	req, err := http.NewRequest("GET", restPath(b.url, h), nil)
-	if err != nil {
-		return 0, errors.Wrap(err, "http.NewRequest")
-	}
-	req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", off, off+int64(len(p))))
-	<-b.connChan
-	resp, err := b.client.Do(req)
-	b.connChan <- struct{}{}
-
-	if resp != nil {
-		defer func() {
-			io.Copy(ioutil.Discard, resp.Body)
-			e := resp.Body.Close()
-
-			if err == nil {
-				err = errors.Wrap(e, "Close")
-			}
-		}()
-	}
-
-	if err != nil {
-		return 0, errors.Wrap(err, "client.Do")
-	}
-	if resp.StatusCode != 200 && resp.StatusCode != 206 {
-		return 0, errors.Errorf("unexpected HTTP response code %v", resp.StatusCode)
-	}
-
-	return io.ReadFull(resp.Body, p)
-}
-
 // Save stores data in the backend at the handle.
-func (b *restBackend) Save(h restic.Handle, p []byte) (err error) {
+func (b *restBackend) Save(h restic.Handle, rd io.Reader) (err error) {
 	if err := h.Valid(); err != nil {
 		return err
 	}
 
+	// make sure that client.Post() cannot close the reader by wrapping it in
+	// backend.Closer, which has a noop method.
+	rd = backend.Closer{Reader: rd}
+
 	<-b.connChan
-	resp, err := b.client.Post(restPath(b.url, h), "binary/octet-stream", bytes.NewReader(p))
+	resp, err := b.client.Post(restPath(b.url, h), "binary/octet-stream", rd)
 	b.connChan <- struct{}{}
 
 	if resp != nil {
@@ -154,6 +110,56 @@ func (b *restBackend) Save(h restic.Handle, p []byte) (err error) {
 	return nil
 }
 
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (b *restBackend) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	if length < 0 {
+		return nil, errors.Errorf("invalid length %d", length)
+	}
+
+	req, err := http.NewRequest("GET", restPath(b.url, h), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "http.NewRequest")
+	}
+
+	byteRange := fmt.Sprintf("bytes=%d-", offset)
+	if length > 0 {
+		byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1)
+	}
+	req.Header.Add("Range", byteRange)
+	debug.Log("Load(%v) send range %v", h, byteRange)
+
+	<-b.connChan
+	resp, err := b.client.Do(req)
+	b.connChan <- struct{}{}
+
+	if err != nil {
+		if resp != nil {
+			io.Copy(ioutil.Discard, resp.Body)
+			resp.Body.Close()
+		}
+		return nil, errors.Wrap(err, "client.Do")
+	}
+
+	if resp.StatusCode != 200 && resp.StatusCode != 206 {
+		io.Copy(ioutil.Discard, resp.Body)
+		resp.Body.Close()
+		return nil, errors.Errorf("unexpected HTTP response code %v", resp.StatusCode)
+	}
+
+	return resp.Body, nil
+}
+
 // Stat returns information about a blob.
 func (b *restBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 	if err := h.Valid(); err != nil {
@@ -188,8 +194,8 @@ func (b *restBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 }
 
 // Test returns true if a blob of the given type and name exists in the backend.
-func (b *restBackend) Test(t restic.FileType, name string) (bool, error) {
-	_, err := b.Stat(restic.Handle{Type: t, Name: name})
+func (b *restBackend) Test(h restic.Handle) (bool, error) {
+	_, err := b.Stat(h)
 	if err != nil {
 		return false, nil
 	}
@@ -198,8 +204,7 @@ func (b *restBackend) Test(t restic.FileType, name string) (bool, error) {
 }
 
 // Remove removes the blob with the given name and type.
-func (b *restBackend) Remove(t restic.FileType, name string) error {
-	h := restic.Handle{Type: t, Name: name}
+func (b *restBackend) Remove(h restic.Handle) error {
 	if err := h.Valid(); err != nil {
 		return err
 	}

src/restic/backend/rest/rest_test.go

@@ -37,7 +37,7 @@ func init() {
 			return nil, err
 		}
 
-		exists, err := be.Test(restic.ConfigFile, "")
+		exists, err := be.Test(restic.Handle{Type: restic.ConfigFile, Name: ""})
 		if err != nil {
 			return nil, err
 		}

src/restic/backend/s3/backend_test.go

@@ -51,13 +51,6 @@ func TestS3BackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestS3BackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestS3BackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/s3/s3.go

@@ -3,10 +3,12 @@ package s3
 import (
 	"bytes"
 	"io"
+	"net/http"
 	"path"
 	"restic"
 	"strings"
 
+	"restic/backend"
 	"restic/errors"
 
 	"github.com/minio/minio-go"
@@ -14,7 +16,7 @@ import (
 	"restic/debug"
 )
 
-const connLimit = 10
+const connLimit = 40
 
 // s3 is a backend which stores the data on an S3 endpoint.
 type s3 struct {
@@ -35,6 +37,10 @@ func Open(cfg Config) (restic.Backend, error) {
 	}
 
 	be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix}
+
+	tr := &http.Transport{MaxIdleConnsPerHost: connLimit}
+	client.SetCustomTransport(tr)
+
 	be.createConnections()
 
 	found, err := client.BucketExists(cfg.Bucket)
@@ -54,11 +60,11 @@ func Open(cfg Config) (restic.Backend, error) {
 	return be, nil
 }
 
-func (be *s3) s3path(t restic.FileType, name string) string {
-	if t == restic.ConfigFile {
-		return path.Join(be.prefix, string(t))
+func (be *s3) s3path(h restic.Handle) string {
+	if h.Type == restic.ConfigFile {
+		return path.Join(be.prefix, string(h.Type))
 	}
-	return path.Join(be.prefix, string(t), name)
+	return path.Join(be.prefix, string(h.Type), h.Name)
 }
 
 func (be *s3) createConnections() {
@@ -73,88 +79,15 @@ func (be *s3) Location() string {
 	return be.bucketname
 }
 
-// Load returns the data stored in the backend for h at the given offset
-// and saves it in p. Load has the same semantics as io.ReaderAt.
-func (be s3) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
-	var obj *minio.Object
-
-	debug.Log("%v, offset %v, len %v", h, off, len(p))
-	objName := be.s3path(h.Type, h.Name)
-
-	<-be.connChan
-	defer func() {
-		be.connChan <- struct{}{}
-	}()
-
-	obj, err = be.client.GetObject(be.bucketname, objName)
-	if err != nil {
-		debug.Log("  err %v", err)
-		return 0, errors.Wrap(err, "client.GetObject")
-	}
-
-	// make sure that the object is closed properly.
-	defer func() {
-		e := obj.Close()
-		if err == nil {
-			err = errors.Wrap(e, "Close")
-		}
-	}()
-
-	info, err := obj.Stat()
-	if err != nil {
-		return 0, errors.Wrap(err, "obj.Stat")
-	}
-
-	// handle negative offsets
-	if off < 0 {
-		// if the negative offset is larger than the object itself, read from
-		// the beginning.
-		if -off > info.Size {
-			off = 0
-		} else {
-			// otherwise compute the offset from the end of the file.
-			off = info.Size + off
-		}
-	}
-
-	// return an error if the offset is beyond the end of the file
-	if off > info.Size {
-		return 0, errors.Wrap(io.EOF, "")
-	}
-
-	var nextError error
-
-	// manually create an io.ErrUnexpectedEOF
-	if off+int64(len(p)) > info.Size {
-		newlen := info.Size - off
-		p = p[:newlen]
-
-		nextError = io.ErrUnexpectedEOF
-
-		debug.Log("    capped buffer to %v byte", len(p))
-	}
-
-	n, err = obj.ReadAt(p, off)
-	if int64(n) == info.Size-off && errors.Cause(err) == io.EOF {
-		err = nil
-	}
-
-	if err == nil {
-		err = nextError
-	}
-
-	return n, err
-}
-
 // Save stores data in the backend at the handle.
-func (be s3) Save(h restic.Handle, p []byte) (err error) {
+func (be *s3) Save(h restic.Handle, rd io.Reader) (err error) {
 	if err := h.Valid(); err != nil {
 		return err
 	}
 
-	debug.Log("%v with %d bytes", h, len(p))
+	debug.Log("Save %v", h)
 
-	objName := be.s3path(h.Type, h.Name)
+	objName := be.s3path(h)
 
 	// Check key does not already exist
 	_, err = be.client.StatObject(be.bucketname, objName)
@@ -168,19 +101,129 @@ func (be s3) Save(h restic.Handle, p []byte) (err error) {
 		be.connChan <- struct{}{}
 	}()
 
-	debug.Log("PutObject(%v, %v, %v, %v)",
-		be.bucketname, objName, int64(len(p)), "binary/octet-stream")
-	n, err := be.client.PutObject(be.bucketname, objName, bytes.NewReader(p), "binary/octet-stream")
+	debug.Log("PutObject(%v, %v)",
+		be.bucketname, objName)
+	n, err := be.client.PutObject(be.bucketname, objName, rd, "binary/octet-stream")
 	debug.Log("%v -> %v bytes, err %#v", objName, n, err)
 
 	return errors.Wrap(err, "client.PutObject")
 }
 
+// wrapReader wraps an io.ReadCloser to run an additional function on Close.
+type wrapReader struct {
+	io.ReadCloser
+	f func()
+}
+
+func (wr wrapReader) Close() error {
+	err := wr.ReadCloser.Close()
+	wr.f()
+	return err
+}
+
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	if length < 0 {
+		return nil, errors.Errorf("invalid length %d", length)
+	}
+
+	var obj *minio.Object
+
+	objName := be.s3path(h)
+
+	// get token for connection
+	<-be.connChan
+
+	obj, err := be.client.GetObject(be.bucketname, objName)
+	if err != nil {
+		debug.Log("  err %v", err)
+
+		// return token
+		be.connChan <- struct{}{}
+
+		return nil, errors.Wrap(err, "client.GetObject")
+	}
+
+	// if we're going to read the whole object, just pass it on.
+	if length == 0 {
+		debug.Log("Load %v: pass on object", h)
+
+		_, err = obj.Seek(offset, 0)
+		if err != nil {
+			_ = obj.Close()
+
+			// return token
+			be.connChan <- struct{}{}
+
+			return nil, errors.Wrap(err, "obj.Seek")
+		}
+
+		rd := wrapReader{
+			ReadCloser: obj,
+			f: func() {
+				debug.Log("Close()")
+				// return token
+				be.connChan <- struct{}{}
+			},
+		}
+		return rd, nil
+	}
+
+	defer func() {
+		// return token
+		be.connChan <- struct{}{}
+	}()
+
+	// otherwise use a buffer with ReadAt
+	info, err := obj.Stat()
+	if err != nil {
+		_ = obj.Close()
+		return nil, errors.Wrap(err, "obj.Stat")
+	}
+
+	if offset > info.Size {
+		_ = obj.Close()
+		return nil, errors.New("offset larger than file size")
+	}
+
+	l := int64(length)
+	if offset+l > info.Size {
+		l = info.Size - offset
+	}
+
+	buf := make([]byte, l)
+	n, err := obj.ReadAt(buf, offset)
+	debug.Log("Load %v: use buffer with ReadAt: %v, %v", h, n, err)
+	if err == io.EOF {
+		debug.Log("Load %v: shorten buffer %v -> %v", h, len(buf), n)
+		buf = buf[:n]
+		err = nil
+	}
+
+	if err != nil {
+		_ = obj.Close()
+		return nil, errors.Wrap(err, "obj.ReadAt")
+	}
+
+	return backend.Closer{Reader: bytes.NewReader(buf)}, nil
+}
+
 // Stat returns information about a blob.
-func (be s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) {
+func (be *s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) {
 	debug.Log("%v", h)
 
-	objName := be.s3path(h.Type, h.Name)
+	objName := be.s3path(h)
 	var obj *minio.Object
 
 	obj, err = be.client.GetObject(be.bucketname, objName)
@@ -207,9 +250,9 @@ func (be s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) {
 }
 
 // Test returns true if a blob of the given type and name exists in the backend.
-func (be *s3) Test(t restic.FileType, name string) (bool, error) {
+func (be *s3) Test(h restic.Handle) (bool, error) {
 	found := false
-	objName := be.s3path(t, name)
+	objName := be.s3path(h)
 	_, err := be.client.StatObject(be.bucketname, objName)
 	if err == nil {
 		found = true
@@ -220,10 +263,10 @@ func (be *s3) Test(t restic.FileType, name string) (bool, error) {
 }
 
 // Remove removes the blob with the given name and type.
-func (be *s3) Remove(t restic.FileType, name string) error {
-	objName := be.s3path(t, name)
+func (be *s3) Remove(h restic.Handle) error {
+	objName := be.s3path(h)
 	err := be.client.RemoveObject(be.bucketname, objName)
-	debug.Log("%v %v -> err %v", t, name, err)
+	debug.Log("Remove(%v) -> err %v", h, err)
 	return errors.Wrap(err, "client.RemoveObject")
 }
 
@@ -234,7 +277,7 @@ func (be *s3) List(t restic.FileType, done <-chan struct{}) <-chan string {
 	debug.Log("listing %v", t)
 	ch := make(chan string)
 
-	prefix := be.s3path(t, "") + "/"
+	prefix := be.s3path(restic.Handle{Type: t}) + "/"
 
 	listresp := be.client.ListObjects(be.bucketname, prefix, true, done)
 
@@ -262,7 +305,7 @@ func (be *s3) removeKeys(t restic.FileType) error {
 	done := make(chan struct{})
 	defer close(done)
 	for key := range be.List(restic.DataFile, done) {
-		err := be.Remove(restic.DataFile, key)
+		err := be.Remove(restic.Handle{Type: restic.DataFile, Name: key})
 		if err != nil {
 			return err
 		}
@@ -287,7 +330,7 @@ func (be *s3) Delete() error {
 		}
 	}
 
-	return be.Remove(restic.ConfigFile, "")
+	return be.Remove(restic.Handle{Type: restic.ConfigFile})
 }
 
 // Close does nothing

src/restic/backend/s3/s3_test.go

@@ -44,7 +44,7 @@ func init() {
 			return nil, err
 		}
 
-		exists, err := be.Test(restic.ConfigFile, "")
+		exists, err := be.Test(restic.Handle{Type: restic.ConfigFile})
 		if err != nil {
 			return nil, err
 		}

src/restic/backend/sftp/backend_test.go

@@ -51,13 +51,6 @@ func TestSftpBackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestSftpBackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestSftpBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/sftp/sftp.go

@@ -153,7 +153,7 @@ func buildSSHCommand(cfg Config) []string {
 }
 
 // OpenWithConfig opens an sftp backend as described by the config by running
-// "ssh" with the appropiate arguments.
+// "ssh" with the appropriate arguments.
 func OpenWithConfig(cfg Config) (*SFTP, error) {
 	debug.Log("open with config %v", cfg)
 	return Open(cfg.Dir, "ssh", buildSSHCommand(cfg)...)
@@ -193,7 +193,7 @@ func Create(dir string, program string, args ...string) (*SFTP, error) {
 }
 
 // CreateWithConfig creates an sftp backend as described by the config by running
-// "ssh" with the appropiate arguments.
+// "ssh" with the appropriate arguments.
 func CreateWithConfig(cfg Config) (*SFTP, error) {
 	debug.Log("config %v", cfg)
 	return Create(cfg.Dir, "ssh", buildSSHCommand(cfg)...)
@@ -259,11 +259,11 @@ func (r *SFTP) mkdirAll(dir string, mode os.FileMode) error {
 }
 
 // Rename temp file to final name according to type and name.
-func (r *SFTP) renameFile(oldname string, t restic.FileType, name string) error {
-	filename := r.filename(t, name)
+func (r *SFTP) renameFile(oldname string, h restic.Handle) error {
+	filename := r.filename(h)
 
 	// create directories if necessary
-	if t == restic.DataFile {
+	if h.Type == restic.DataFile {
 		err := r.mkdirAll(path.Dir(filename), backend.Modes.Dir)
 		if err != nil {
 			return err
@@ -297,22 +297,22 @@ func Join(parts ...string) string {
 }
 
 // Construct path for given restic.Type and name.
-func (r *SFTP) filename(t restic.FileType, name string) string {
-	if t == restic.ConfigFile {
+func (r *SFTP) filename(h restic.Handle) string {
+	if h.Type == restic.ConfigFile {
 		return Join(r.p, "config")
 	}
 
-	return Join(r.dirname(t, name), name)
+	return Join(r.dirname(h), h.Name)
 }
 
 // Construct directory for given backend.Type.
-func (r *SFTP) dirname(t restic.FileType, name string) string {
+func (r *SFTP) dirname(h restic.Handle) string {
 	var n string
-	switch t {
+	switch h.Type {
 	case restic.DataFile:
 		n = backend.Paths.Data
-		if len(name) > 2 {
-			n = Join(n, name[:2])
+		if len(h.Name) > 2 {
+			n = Join(n, h.Name[:2])
 		}
 	case restic.SnapshotFile:
 		n = backend.Paths.Snapshots
@@ -326,47 +326,9 @@ func (r *SFTP) dirname(t restic.FileType, name string) string {
 	return Join(r.p, n)
 }
 
-// Load returns the data stored in the backend for h at the given offset
-// and saves it in p. Load has the same semantics as io.ReaderAt.
-func (r *SFTP) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
-	debug.Log("load %v, %d bytes, offset %v", h, len(p), off)
-	if err := r.clientError(); err != nil {
-		return 0, err
-	}
-
-	if err := h.Valid(); err != nil {
-		return 0, err
-	}
-
-	f, err := r.c.Open(r.filename(h.Type, h.Name))
-	if err != nil {
-		return 0, errors.Wrap(err, "Open")
-	}
-
-	defer func() {
-		e := f.Close()
-		if err == nil {
-			err = errors.Wrap(e, "Close")
-		}
-	}()
-
-	switch {
-	case off > 0:
-		_, err = f.Seek(off, 0)
-	case off < 0:
-		_, err = f.Seek(off, 2)
-	}
-
-	if err != nil {
-		return 0, errors.Wrap(err, "Seek")
-	}
-
-	return io.ReadFull(f, p)
-}
-
 // Save stores data in the backend at the handle.
-func (r *SFTP) Save(h restic.Handle, p []byte) (err error) {
-	debug.Log("save %v bytes to %v", h, len(p))
+func (r *SFTP) Save(h restic.Handle, rd io.Reader) (err error) {
+	debug.Log("save to %v", h)
 	if err := r.clientError(); err != nil {
 		return err
 	}
@@ -380,31 +342,60 @@ func (r *SFTP) Save(h restic.Handle, p []byte) (err error) {
 		return err
 	}
 
-	debug.Log("save %v (%d bytes) to %v", h, len(p), filename)
-
-	n, err := tmpfile.Write(p)
+	n, err := io.Copy(tmpfile, rd)
 	if err != nil {
 		return errors.Wrap(err, "Write")
 	}
 
-	if n != len(p) {
-		return errors.New("not all bytes writen")
-	}
+	debug.Log("saved %v (%d bytes) to %v", h, n, filename)
 
 	err = tmpfile.Close()
 	if err != nil {
 		return errors.Wrap(err, "Close")
 	}
 
-	err = r.renameFile(filename, h.Type, h.Name)
+	err = r.renameFile(filename, h)
 	debug.Log("save %v: rename %v: %v",
 		h, path.Base(filename), err)
 	return err
 }
 
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (r *SFTP) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	f, err := r.c.Open(r.filename(h))
+	if err != nil {
+		return nil, err
+	}
+
+	if offset > 0 {
+		_, err = f.Seek(offset, 0)
+		if err != nil {
+			_ = f.Close()
+			return nil, err
+		}
+	}
+
+	if length > 0 {
+		return backend.LimitReadCloser(f, int64(length)), nil
+	}
+
+	return f, nil
+}
+
 // Stat returns information about a blob.
 func (r *SFTP) Stat(h restic.Handle) (restic.FileInfo, error) {
-	debug.Log("stat %v", h)
+	debug.Log("Stat(%v)", h)
 	if err := r.clientError(); err != nil {
 		return restic.FileInfo{}, err
 	}
@@ -413,7 +404,7 @@ func (r *SFTP) Stat(h restic.Handle) (restic.FileInfo, error) {
 		return restic.FileInfo{}, err
 	}
 
-	fi, err := r.c.Lstat(r.filename(h.Type, h.Name))
+	fi, err := r.c.Lstat(r.filename(h))
 	if err != nil {
 		return restic.FileInfo{}, errors.Wrap(err, "Lstat")
 	}
@@ -422,13 +413,13 @@ func (r *SFTP) Stat(h restic.Handle) (restic.FileInfo, error) {
 }
 
 // Test returns true if a blob of the given type and name exists in the backend.
-func (r *SFTP) Test(t restic.FileType, name string) (bool, error) {
-	debug.Log("type %v, name %v", t, name)
+func (r *SFTP) Test(h restic.Handle) (bool, error) {
+	debug.Log("Test(%v)", h)
 	if err := r.clientError(); err != nil {
 		return false, err
 	}
 
-	_, err := r.c.Lstat(r.filename(t, name))
+	_, err := r.c.Lstat(r.filename(h))
 	if os.IsNotExist(errors.Cause(err)) {
 		return false, nil
 	}
@@ -441,13 +432,13 @@ func (r *SFTP) Test(t restic.FileType, name string) (bool, error) {
 }
 
 // Remove removes the content stored at name.
-func (r *SFTP) Remove(t restic.FileType, name string) error {
-	debug.Log("type %v, name %v", t, name)
+func (r *SFTP) Remove(h restic.Handle) error {
+	debug.Log("Remove(%v)", h)
 	if err := r.clientError(); err != nil {
 		return err
 	}
 
-	return r.c.Remove(r.filename(t, name))
+	return r.c.Remove(r.filename(h))
 }
 
 // List returns a channel that yields all names of blobs of type t. A
@@ -462,7 +453,7 @@ func (r *SFTP) List(t restic.FileType, done <-chan struct{}) <-chan string {
 
 		if t == restic.DataFile {
 			// read first level
-			basedir := r.dirname(t, "")
+			basedir := r.dirname(restic.Handle{Type: t})
 
 			list1, err := r.c.ReadDir(basedir)
 			if err != nil {
@@ -495,7 +486,7 @@ func (r *SFTP) List(t restic.FileType, done <-chan struct{}) <-chan string {
 				}
 			}
 		} else {
-			entries, err := r.c.ReadDir(r.dirname(t, ""))
+			entries, err := r.c.ReadDir(r.dirname(restic.Handle{Type: t}))
 			if err != nil {
 				return
 			}

src/restic/backend/test/backend_test.go

@@ -51,13 +51,6 @@ func TestTestBackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestTestBackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestTestBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/test/tests.go

@@ -4,13 +4,16 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"math/rand"
+	"os"
 	"reflect"
 	"restic"
+	"restic/errors"
 	"sort"
+	"strings"
 	"testing"
 
-	"restic/errors"
 	"restic/test"
 
 	"restic/backend"
@@ -127,7 +130,7 @@ func TestCreateWithConfig(t testing.TB) {
 	}
 
 	// remove config
-	err = b.Remove(restic.ConfigFile, "")
+	err = b.Remove(restic.Handle{Type: restic.ConfigFile, Name: ""})
 	if err != nil {
 		t.Fatalf("unexpected error removing config: %v", err)
 	}
@@ -152,12 +155,12 @@ func TestConfig(t testing.TB) {
 	var testString = "Config"
 
 	// create config and read it back
-	_, err := backend.LoadAll(b, restic.Handle{Type: restic.ConfigFile}, nil)
+	_, err := backend.LoadAll(b, restic.Handle{Type: restic.ConfigFile})
 	if err == nil {
 		t.Fatalf("did not get expected error for non-existing config")
 	}
 
-	err = b.Save(restic.Handle{Type: restic.ConfigFile}, []byte(testString))
+	err = b.Save(restic.Handle{Type: restic.ConfigFile}, strings.NewReader(testString))
 	if err != nil {
 		t.Fatalf("Save() error: %v", err)
 	}
@@ -166,7 +169,7 @@ func TestConfig(t testing.TB) {
 	// same config
 	for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} {
 		h := restic.Handle{Type: restic.ConfigFile, Name: name}
-		buf, err := backend.LoadAll(b, h, nil)
+		buf, err := backend.LoadAll(b, h)
 		if err != nil {
 			t.Fatalf("unable to read config with name %q: %v", name, err)
 		}
@@ -182,12 +185,12 @@ func TestLoad(t testing.TB) {
 	b := open(t)
 	defer close(t)
 
-	_, err := b.Load(restic.Handle{}, nil, 0)
+	_, err := b.Load(restic.Handle{}, 0, 0)
 	if err == nil {
 		t.Fatalf("Load() did not return an error for invalid handle")
 	}
 
-	_, err = b.Load(restic.Handle{Type: restic.DataFile, Name: "foobar"}, nil, 0)
+	_, err = b.Load(restic.Handle{Type: restic.DataFile, Name: "foobar"}, 0, 0)
 	if err == nil {
 		t.Fatalf("Load() did not return an error for non-existing blob")
 	}
@@ -198,11 +201,20 @@ func TestLoad(t testing.TB) {
 	id := restic.Hash(data)
 
 	handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
-	err = b.Save(handle, data)
+	err = b.Save(handle, bytes.NewReader(data))
 	if err != nil {
 		t.Fatalf("Save() error: %v", err)
 	}
 
+	rd, err := b.Load(handle, 100, -1)
+	if err == nil {
+		t.Fatalf("Load() returned no error for negative offset!")
+	}
+
+	if rd != nil {
+		t.Fatalf("Load() returned a non-nil reader for negative offset!")
+	}
+
 	for i := 0; i < 50; i++ {
 		l := rand.Intn(length + 2000)
 		o := rand.Intn(length + 2000)
@@ -215,157 +227,64 @@ func TestLoad(t testing.TB) {
 			d = d[:0]
 		}
 
-		if l > 0 && l < len(d) {
-			d = d[:l]
-		}
-
-		buf := make([]byte, l)
-		n, err := b.Load(handle, buf, int64(o))
-
-		// if we requested data beyond the end of the file, require
-		// ErrUnexpectedEOF error
-		if l > len(d) {
-			if errors.Cause(err) != io.ErrUnexpectedEOF {
-				t.Errorf("Load(%d, %d) did not return io.ErrUnexpectedEOF", len(buf), int64(o))
-			}
-			err = nil
-			buf = buf[:len(d)]
-		}
-
-		if err != nil {
-			t.Errorf("Load(%d, %d): unexpected error: %v", len(buf), int64(o), err)
-			continue
-		}
-
-		if n != len(buf) {
-			t.Errorf("Load(%d, %d): wrong length returned, want %d, got %d",
-				len(buf), int64(o), len(buf), n)
-			continue
-		}
-
-		buf = buf[:n]
-		if !bytes.Equal(buf, d) {
-			t.Errorf("Load(%d, %d) returned wrong bytes", len(buf), int64(o))
-			continue
-		}
-	}
-
-	// test with negative offset
-	for i := 0; i < 50; i++ {
-		l := rand.Intn(length + 2000)
-		o := rand.Intn(length + 2000)
-
-		d := data
-		if o < len(d) {
-			d = d[len(d)-o:]
-		} else {
-			o = 0
+		getlen := l
+		if l >= len(d) && rand.Float32() >= 0.5 {
+			getlen = 0
 		}
 
 		if l > 0 && l < len(d) {
 			d = d[:l]
 		}
 
-		buf := make([]byte, l)
-		n, err := b.Load(handle, buf, -int64(o))
-
-		// if we requested data beyond the end of the file, require
-		// ErrUnexpectedEOF error
-		if l > len(d) {
-			if errors.Cause(err) != io.ErrUnexpectedEOF {
-				t.Errorf("Load(%d, %d) did not return io.ErrUnexpectedEOF", len(buf), int64(o))
-				continue
-			}
-			err = nil
-			buf = buf[:len(d)]
-		}
-
+		rd, err := b.Load(handle, getlen, int64(o))
 		if err != nil {
-			t.Errorf("Load(%d, %d): unexpected error: %v", len(buf), int64(o), err)
+			t.Errorf("Load(%d, %d) returned unexpected error: %v", l, o, err)
 			continue
 		}
 
-		if n != len(buf) {
-			t.Errorf("Load(%d, %d): wrong length returned, want %d, got %d",
-				len(buf), int64(o), len(buf), n)
+		buf, err := ioutil.ReadAll(rd)
+		if err != nil {
+			t.Errorf("Load(%d, %d) ReadAll() returned unexpected error: %v", l, o, err)
+			rd.Close()
+			continue
+		}
+
+		if l <= len(d) && len(buf) != l {
+			t.Errorf("Load(%d, %d) wrong number of bytes read: want %d, got %d", l, o, l, len(buf))
+			rd.Close()
+			continue
+		}
+
+		if l > len(d) && len(buf) != len(d) {
+			t.Errorf("Load(%d, %d) wrong number of bytes read for overlong read: want %d, got %d", l, o, l, len(buf))
+			rd.Close()
 			continue
 		}
 
-		buf = buf[:n]
 		if !bytes.Equal(buf, d) {
-			t.Errorf("Load(%d, %d) returned wrong bytes", len(buf), int64(o))
+			t.Errorf("Load(%d, %d) returned wrong bytes", l, o)
+			rd.Close()
+			continue
+		}
+
+		err = rd.Close()
+		if err != nil {
+			t.Errorf("Load(%d, %d) rd.Close() returned unexpected error: %v", l, o, err)
 			continue
 		}
 	}
 
-	// load with a too-large buffer, this should return io.ErrUnexpectedEOF
-	buf := make([]byte, length+100)
-	n, err := b.Load(handle, buf, 0)
-	if n != length {
-		t.Errorf("wrong length for larger buffer returned, want %d, got %d", length, n)
-	}
-
-	if errors.Cause(err) != io.ErrUnexpectedEOF {
-		t.Errorf("wrong error returned for larger buffer: want io.ErrUnexpectedEOF, got %#v", err)
-	}
-
-	test.OK(t, b.Remove(restic.DataFile, id.String()))
+	test.OK(t, b.Remove(handle))
 }
 
-// TestLoadNegativeOffset tests the backend's Load function with negative offsets.
-func TestLoadNegativeOffset(t testing.TB) {
-	b := open(t)
-	defer close(t)
+type errorCloser struct {
+	io.Reader
+	t testing.TB
+}
 
-	length := rand.Intn(1<<24) + 2000
-
-	data := test.Random(23, length)
-	id := restic.Hash(data)
-
-	handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
-	err := b.Save(handle, data)
-	if err != nil {
-		t.Fatalf("Save() error: %v", err)
-	}
-
-	// test normal reads
-	for i := 0; i < 50; i++ {
-		l := rand.Intn(length + 2000)
-		o := -rand.Intn(length + 2000)
-
-		buf := make([]byte, l)
-		n, err := b.Load(handle, buf, int64(o))
-
-		// if we requested data beyond the end of the file, require
-		// ErrUnexpectedEOF error
-		if len(buf) > -o {
-			if errors.Cause(err) != io.ErrUnexpectedEOF {
-				t.Errorf("Load(%d, %d) did not return io.ErrUnexpectedEOF", len(buf), o)
-				continue
-			}
-			err = nil
-			buf = buf[:-o]
-		}
-
-		if err != nil {
-			t.Errorf("Load(%d, %d) returned error: %v", len(buf), o, err)
-			continue
-		}
-
-		if n != len(buf) {
-			t.Errorf("Load(%d, %d) returned short read, only got %d bytes", len(buf), o, n)
-			continue
-		}
-
-		p := len(data) + o
-		if !bytes.Equal(buf, data[p:p+len(buf)]) {
-			t.Errorf("Load(%d, %d) returned wrong bytes", len(buf), o)
-			continue
-		}
-
-	}
-
-	test.OK(t, b.Remove(restic.DataFile, id.String()))
+func (ec errorCloser) Close() error {
+	ec.t.Error("forbidden method close was called")
+	return errors.New("forbidden method close was called")
 }
 
 // TestSave tests saving data in the backend.
@@ -384,10 +303,10 @@ func TestSave(t testing.TB) {
 			Type: restic.DataFile,
 			Name: fmt.Sprintf("%s-%d", id, i),
 		}
-		err := b.Save(h, data)
+		err := b.Save(h, bytes.NewReader(data))
 		test.OK(t, err)
 
-		buf, err := backend.LoadAll(b, h, nil)
+		buf, err := backend.LoadAll(b, h)
 		test.OK(t, err)
 		if len(buf) != len(data) {
 			t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf))
@@ -404,11 +323,51 @@ func TestSave(t testing.TB) {
 			t.Fatalf("Stat() returned different size, want %q, got %d", len(data), fi.Size)
 		}
 
-		err = b.Remove(h.Type, h.Name)
+		err = b.Remove(h)
 		if err != nil {
 			t.Fatalf("error removing item: %v", err)
 		}
 	}
+
+	// test saving from a tempfile
+	tmpfile, err := ioutil.TempFile("", "restic-backend-save-test-")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	length := rand.Intn(1<<23) + 200000
+	data := test.Random(23, length)
+	copy(id[:], data)
+
+	if _, err = tmpfile.Write(data); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err = tmpfile.Seek(0, 0); err != nil {
+		t.Fatal(err)
+	}
+
+	h := restic.Handle{Type: restic.DataFile, Name: id.String()}
+
+	// wrap the tempfile in an errorCloser, so we can detect if the backend
+	// closes the reader
+	err = b.Save(h, errorCloser{t: t, Reader: tmpfile})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err = tmpfile.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	if err = os.Remove(tmpfile.Name()); err != nil {
+		t.Fatal(err)
+	}
+
+	err = b.Remove(h)
+	if err != nil {
+		t.Fatalf("error removing item: %v", err)
+	}
 }
 
 var filenameTests = []struct {
@@ -430,13 +389,13 @@ func TestSaveFilenames(t testing.TB) {
 
 	for i, test := range filenameTests {
 		h := restic.Handle{Name: test.name, Type: restic.DataFile}
-		err := b.Save(h, []byte(test.data))
+		err := b.Save(h, strings.NewReader(test.data))
 		if err != nil {
 			t.Errorf("test %d failed: Save() returned %v", i, err)
 			continue
 		}
 
-		buf, err := backend.LoadAll(b, h, nil)
+		buf, err := backend.LoadAll(b, h)
 		if err != nil {
 			t.Errorf("test %d failed: Load() returned %v", i, err)
 			continue
@@ -446,7 +405,7 @@ func TestSaveFilenames(t testing.TB) {
 			t.Errorf("test %d: returned wrong bytes", i)
 		}
 
-		err = b.Remove(h.Type, h.Name)
+		err = b.Remove(h)
 		if err != nil {
 			t.Errorf("test %d failed: Remove() returned %v", i, err)
 			continue
@@ -464,10 +423,12 @@ var testStrings = []struct {
 	{"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"},
 }
 
-func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) {
+func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) restic.Handle {
 	id := restic.Hash(data)
-	err := b.Save(restic.Handle{Name: id.String(), Type: tpe}, data)
+	h := restic.Handle{Name: id.String(), Type: tpe}
+	err := b.Save(h, bytes.NewReader(data))
 	test.OK(t, err)
+	return h
 }
 
 // TestBackend tests all functions of the backend.
@@ -485,21 +446,21 @@ func TestBackend(t testing.TB) {
 			test.OK(t, err)
 
 			// test if blob is already in repository
-			ret, err := b.Test(tpe, id.String())
+			h := restic.Handle{Type: tpe, Name: id.String()}
+			ret, err := b.Test(h)
 			test.OK(t, err)
 			test.Assert(t, !ret, "blob was found to exist before creating")
 
 			// try to stat a not existing blob
-			h := restic.Handle{Type: tpe, Name: id.String()}
 			_, err = b.Stat(h)
 			test.Assert(t, err != nil, "blob data could be extracted before creation")
 
 			// try to read not existing blob
-			_, err = b.Load(h, nil, 0)
+			_, err = b.Load(h, 0, 0)
 			test.Assert(t, err != nil, "blob reader could be obtained before creation")
 
 			// try to get string out, should fail
-			ret, err = b.Test(tpe, id.String())
+			ret, err = b.Test(h)
 			test.OK(t, err)
 			test.Assert(t, !ret, "id %q was found (but should not have)", ts.id)
 		}
@@ -510,7 +471,7 @@ func TestBackend(t testing.TB) {
 
 			// test Load()
 			h := restic.Handle{Type: tpe, Name: ts.id}
-			buf, err := backend.LoadAll(b, h, nil)
+			buf, err := backend.LoadAll(b, h)
 			test.OK(t, err)
 			test.Equals(t, ts.data, string(buf))
 
@@ -520,9 +481,18 @@ func TestBackend(t testing.TB) {
 			length := end - start
 
 			buf2 := make([]byte, length)
-			n, err := b.Load(h, buf2, int64(start))
+			rd, err := b.Load(h, len(buf2), int64(start))
 			test.OK(t, err)
-			test.Equals(t, length, n)
+			n, err := io.ReadFull(rd, buf2)
+			test.OK(t, err)
+			test.Equals(t, len(buf2), n)
+
+			remaining, err := io.Copy(ioutil.Discard, rd)
+			test.OK(t, err)
+			test.Equals(t, int64(0), remaining)
+
+			test.OK(t, rd.Close())
+
 			test.Equals(t, ts.data[start:end], string(buf2))
 		}
 
@@ -530,20 +500,21 @@ func TestBackend(t testing.TB) {
 		ts := testStrings[0]
 
 		// create blob
-		err := b.Save(restic.Handle{Type: tpe, Name: ts.id}, []byte(ts.data))
+		err := b.Save(restic.Handle{Type: tpe, Name: ts.id}, strings.NewReader(ts.data))
 		test.Assert(t, err != nil, "expected error, got %v", err)
 
 		// remove and recreate
-		err = b.Remove(tpe, ts.id)
+		h := restic.Handle{Type: tpe, Name: ts.id}
+		err = b.Remove(h)
 		test.OK(t, err)
 
 		// test that the blob is gone
-		ok, err := b.Test(tpe, ts.id)
+		ok, err := b.Test(h)
 		test.OK(t, err)
 		test.Assert(t, ok == false, "removed blob still present")
 
 		// create blob
-		err = b.Save(restic.Handle{Type: tpe, Name: ts.id}, []byte(ts.data))
+		err = b.Save(h, strings.NewReader(ts.data))
 		test.OK(t, err)
 
 		// list items
@@ -578,12 +549,14 @@ func TestBackend(t testing.TB) {
 				id, err := restic.ParseID(ts.id)
 				test.OK(t, err)
 
-				found, err := b.Test(tpe, id.String())
+				h := restic.Handle{Type: tpe, Name: id.String()}
+
+				found, err := b.Test(h)
 				test.OK(t, err)
 
-				test.OK(t, b.Remove(tpe, id.String()))
+				test.OK(t, b.Remove(h))
 
-				found, err = b.Test(tpe, id.String())
+				found, err = b.Test(h)
 				test.OK(t, err)
 				test.Assert(t, !found, fmt.Sprintf("id %q not found after removal", id))
 			}

src/restic/backend/utils.go

@@ -2,29 +2,51 @@ package backend
 
 import (
 	"io"
+	"io/ioutil"
 	"restic"
-
-	"restic/errors"
 )
 
-// LoadAll reads all data stored in the backend for the handle. The buffer buf
-// is resized to accomodate all data in the blob. Errors returned by be.Load()
-// are passed on, except io.ErrUnexpectedEOF is silenced and nil returned
-// instead, since it means this function is working properly.
-func LoadAll(be restic.Backend, h restic.Handle, buf []byte) ([]byte, error) {
-	fi, err := be.Stat(h)
+// LoadAll reads all data stored in the backend for the handle.
+func LoadAll(be restic.Backend, h restic.Handle) (buf []byte, err error) {
+	rd, err := be.Load(h, 0, 0)
 	if err != nil {
-		return nil, errors.Wrap(err, "Stat")
+		return nil, err
 	}
 
-	if fi.Size > int64(len(buf)) {
-		buf = make([]byte, int(fi.Size))
-	}
+	defer func() {
+		io.Copy(ioutil.Discard, rd)
+		e := rd.Close()
+		if err == nil {
+			err = e
+		}
+	}()
 
-	n, err := be.Load(h, buf, 0)
-	if errors.Cause(err) == io.ErrUnexpectedEOF {
-		err = nil
-	}
-	buf = buf[:n]
-	return buf, err
+	return ioutil.ReadAll(rd)
+}
+
+// Closer wraps an io.Reader and adds a Close() method that does nothing.
+type Closer struct {
+	io.Reader
+}
+
+// Close is a no-op.
+func (c Closer) Close() error {
+	return nil
+}
+
+// LimitedReadCloser wraps io.LimitedReader and exposes the Close() method.
+type LimitedReadCloser struct {
+	io.ReadCloser
+	io.Reader
+}
+
+// Read reads data from the limited reader.
+func (l *LimitedReadCloser) Read(p []byte) (int, error) {
+	return l.Reader.Read(p)
+}
+
+// LimitReadCloser returns a new reader wraps r in an io.LimitReader, but also
+// exposes the Close() method.
+func LimitReadCloser(r io.ReadCloser, n int64) *LimitedReadCloser {
+	return &LimitedReadCloser{ReadCloser: r, Reader: io.LimitReader(r, n)}
 }

src/restic/backend/utils_test.go

@@ -21,10 +21,10 @@ func TestLoadAll(t *testing.T) {
 		data := Random(23+i, rand.Intn(MiB)+500*KiB)
 
 		id := restic.Hash(data)
-		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, data)
+		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data))
 		OK(t, err)
 
-		buf, err := backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()}, nil)
+		buf, err := backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()})
 		OK(t, err)
 
 		if len(buf) != len(data) {
@@ -46,11 +46,11 @@ func TestLoadSmallBuffer(t *testing.T) {
 		data := Random(23+i, rand.Intn(MiB)+500*KiB)
 
 		id := restic.Hash(data)
-		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, data)
+		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data))
 		OK(t, err)
 
 		buf := make([]byte, len(data)-23)
-		buf, err = backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()}, buf)
+		buf, err = backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()})
 		OK(t, err)
 
 		if len(buf) != len(data) {
@@ -72,11 +72,11 @@ func TestLoadLargeBuffer(t *testing.T) {
 		data := Random(23+i, rand.Intn(MiB)+500*KiB)
 
 		id := restic.Hash(data)
-		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, data)
+		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data))
 		OK(t, err)
 
 		buf := make([]byte, len(data)+100)
-		buf, err = backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()}, buf)
+		buf, err = backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()})
 		OK(t, err)
 
 		if len(buf) != len(data) {

src/restic/blob.go

@@ -14,6 +14,11 @@ type Blob struct {
 	Offset uint
 }
 
+func (b Blob) String() string {
+	return fmt.Sprintf("<Blob (%v) %v, offset %v, length %v>",
+		b.Type, b.ID.Str(), b.Offset, b.Length)
+}
+
 // PackedBlob is a blob stored within a file.
 type PackedBlob struct {
 	Blob

src/restic/buffer.go

@@ -0,0 +1,21 @@
+package restic
+
+import "restic/crypto"
+
+// NewBlobBuffer returns a buffer that is large enough to hold a blob of size
+// plaintext bytes, including the crypto overhead.
+func NewBlobBuffer(size int) []byte {
+	return make([]byte, size, size+crypto.Extension)
+}
+
+// PlaintextLength returns the plaintext length of a blob with ciphertextSize
+// bytes.
+func PlaintextLength(ciphertextSize int) int {
+	return ciphertextSize - crypto.Extension
+}
+
+// CiphertextLength returns the encrypted length of a blob with plaintextSize
+// bytes.
+func CiphertextLength(plaintextSize int) int {
+	return plaintextSize + crypto.Extension
+}

src/restic/checker/checker.go

@@ -1,14 +1,17 @@
 package checker
 
 import (
-	"bytes"
+	"crypto/sha256"
 	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
 	"sync"
 
 	"restic/errors"
+	"restic/hashing"
 
 	"restic"
-	"restic/backend"
 	"restic/crypto"
 	"restic/debug"
 	"restic/pack"
@@ -77,6 +80,7 @@ func (c *Checker) LoadIndex() (hints []error, errs []error) {
 	debug.Log("Start")
 	type indexRes struct {
 		Index *repository.Index
+		err   error
 		ID    string
 	}
 
@@ -92,39 +96,40 @@ func (c *Checker) LoadIndex() (hints []error, errs []error) {
 			idx, err = repository.LoadIndexWithDecoder(c.repo, id, repository.DecodeOldIndex)
 		}
 
-		if err != nil {
-			return err
-		}
+		err = errors.Wrapf(err, "error loading index %v", id.Str())
 
 		select {
-		case indexCh <- indexRes{Index: idx, ID: id.String()}:
+		case indexCh <- indexRes{Index: idx, ID: id.String(), err: err}:
 		case <-done:
 		}
 
 		return nil
 	}
 
-	var perr error
 	go func() {
 		defer close(indexCh)
 		debug.Log("start loading indexes in parallel")
-		perr = repository.FilesInParallel(c.repo.Backend(), restic.IndexFile, defaultParallelism,
+		err := repository.FilesInParallel(c.repo.Backend(), restic.IndexFile, defaultParallelism,
 			repository.ParallelWorkFuncParseID(worker))
-		debug.Log("loading indexes finished, error: %v", perr)
+		debug.Log("loading indexes finished, error: %v", err)
+		if err != nil {
+			panic(err)
+		}
 	}()
 
 	done := make(chan struct{})
 	defer close(done)
 
-	if perr != nil {
-		errs = append(errs, perr)
-		return hints, errs
-	}
-
 	packToIndex := make(map[restic.ID]restic.IDSet)
 
 	for res := range indexCh {
-		debug.Log("process index %v", res.ID)
+		debug.Log("process index %v, err %v", res.ID, res.err)
+
+		if res.err != nil {
+			errs = append(errs, res.err)
+			continue
+		}
+
 		idxID, err := restic.ParseID(res.ID)
 		if err != nil {
 			errs = append(errs, errors.Errorf("unable to parse as index ID: %v", res.ID))
@@ -151,8 +156,6 @@ func (c *Checker) LoadIndex() (hints []error, errs []error) {
 		debug.Log("%d blobs processed", cnt)
 	}
 
-	debug.Log("done, error %v", perr)
-
 	debug.Log("checking for duplicate packs")
 	for packID := range c.packs {
 		debug.Log("  check pack %v: contained in %d indexes", packID.Str(), len(packToIndex[packID]))
@@ -187,7 +190,8 @@ func packIDTester(repo restic.Repository, inChan <-chan restic.ID, errChan chan<
 	defer wg.Done()
 
 	for id := range inChan {
-		ok, err := repo.Backend().Test(restic.DataFile, id.String())
+		h := restic.Handle{Type: restic.DataFile, Name: id.String()}
+		ok, err := repo.Backend().Test(h)
 		if err != nil {
 			err = PackError{ID: id, Err: err}
 		} else {
@@ -658,36 +662,77 @@ func (c *Checker) CountPacks() uint64 {
 func checkPack(r restic.Repository, id restic.ID) error {
 	debug.Log("checking pack %v", id.Str())
 	h := restic.Handle{Type: restic.DataFile, Name: id.String()}
-	buf, err := backend.LoadAll(r.Backend(), h, nil)
+
+	rd, err := r.Backend().Load(h, 0, 0)
 	if err != nil {
 		return err
 	}
 
-	hash := restic.Hash(buf)
+	packfile, err := ioutil.TempFile("", "restic-temp-check-")
+	if err != nil {
+		return errors.Wrap(err, "TempFile")
+	}
+
+	defer func() {
+		packfile.Close()
+		os.Remove(packfile.Name())
+	}()
+
+	hrd := hashing.NewReader(rd, sha256.New())
+	size, err := io.Copy(packfile, hrd)
+	if err != nil {
+		return errors.Wrap(err, "Copy")
+	}
+
+	if err = rd.Close(); err != nil {
+		return err
+	}
+
+	hash := restic.IDFromHash(hrd.Sum(nil))
+	debug.Log("hash for pack %v is %v", id.Str(), hash.Str())
+
 	if !hash.Equal(id) {
 		debug.Log("Pack ID does not match, want %v, got %v", id.Str(), hash.Str())
 		return errors.Errorf("Pack ID does not match, want %v, got %v", id.Str(), hash.Str())
 	}
 
-	blobs, err := pack.List(r.Key(), bytes.NewReader(buf), int64(len(buf)))
+	blobs, err := pack.List(r.Key(), packfile, size)
 	if err != nil {
 		return err
 	}
 
 	var errs []error
+	var buf []byte
 	for i, blob := range blobs {
-		debug.Log("  check blob %d: %v", i, blob.ID.Str())
+		debug.Log("  check blob %d: %v", i, blob)
 
-		plainBuf := make([]byte, blob.Length)
-		n, err := crypto.Decrypt(r.Key(), plainBuf, buf[blob.Offset:blob.Offset+blob.Length])
+		buf = buf[:cap(buf)]
+		if uint(len(buf)) < blob.Length {
+			buf = make([]byte, blob.Length)
+		}
+		buf = buf[:blob.Length]
+
+		_, err := packfile.Seek(int64(blob.Offset), 0)
+		if err != nil {
+			return errors.Errorf("Seek(%v): %v", blob.Offset, err)
+		}
+
+		_, err = io.ReadFull(packfile, buf)
+		if err != nil {
+			debug.Log("  error loading blob %v: %v", blob.ID.Str(), err)
+			errs = append(errs, errors.Errorf("blob %v: %v", i, err))
+			continue
+		}
+
+		n, err := crypto.Decrypt(r.Key(), buf, buf)
 		if err != nil {
 			debug.Log("  error decrypting blob %v: %v", blob.ID.Str(), err)
 			errs = append(errs, errors.Errorf("blob %v: %v", i, err))
 			continue
 		}
-		plainBuf = plainBuf[:n]
+		buf = buf[:n]
 
-		hash := restic.Hash(plainBuf)
+		hash := restic.Hash(buf)
 		if !hash.Equal(blob.ID) {
 			debug.Log("  Blob ID does not match, want %v, got %v", blob.ID.Str(), hash.Str())
 			errs = append(errs, errors.Errorf("Blob ID does not match, want %v, got %v", blob.ID.Str(), hash.Str()))

src/restic/checker/checker_test.go

@@ -1,6 +1,7 @@
 package checker_test
 
 import (
+	"io"
 	"math/rand"
 	"path/filepath"
 	"sort"
@@ -8,7 +9,6 @@ import (
 
 	"restic"
 	"restic/archiver"
-	"restic/backend/mem"
 	"restic/checker"
 	"restic/repository"
 	"restic/test"
@@ -73,8 +73,11 @@ func TestMissingPack(t *testing.T) {
 
 	repo := repository.TestOpenLocal(t, repodir)
 
-	packID := "657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6"
-	test.OK(t, repo.Backend().Remove(restic.DataFile, packID))
+	packHandle := restic.Handle{
+		Type: restic.DataFile,
+		Name: "657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6",
+	}
+	test.OK(t, repo.Backend().Remove(packHandle))
 
 	chkr := checker.New(repo)
 	hints, errs := chkr.LoadIndex()
@@ -92,7 +95,7 @@ func TestMissingPack(t *testing.T) {
 		"expected exactly one error, got %v", len(errs))
 
 	if err, ok := errs[0].(checker.PackError); ok {
-		test.Equals(t, packID, err.ID.String())
+		test.Equals(t, packHandle.Name, err.ID.String())
 	} else {
 		t.Errorf("expected error returned by checker.Packs() to be PackError, got %v", err)
 	}
@@ -105,9 +108,12 @@ func TestUnreferencedPack(t *testing.T) {
 	repo := repository.TestOpenLocal(t, repodir)
 
 	// index 3f1a only references pack 60e0
-	indexID := "3f1abfcb79c6f7d0a3be517d2c83c8562fba64ef2c8e9a3544b4edaf8b5e3b44"
 	packID := "60e0438dcb978ec6860cc1f8c43da648170ee9129af8f650f876bad19f8f788e"
-	test.OK(t, repo.Backend().Remove(restic.IndexFile, indexID))
+	indexHandle := restic.Handle{
+		Type: restic.IndexFile,
+		Name: "3f1abfcb79c6f7d0a3be517d2c83c8562fba64ef2c8e9a3544b4edaf8b5e3b44",
+	}
+	test.OK(t, repo.Backend().Remove(indexHandle))
 
 	chkr := checker.New(repo)
 	hints, errs := chkr.LoadIndex()
@@ -137,8 +143,11 @@ func TestUnreferencedBlobs(t *testing.T) {
 
 	repo := repository.TestOpenLocal(t, repodir)
 
-	snID := "51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02"
-	test.OK(t, repo.Backend().Remove(restic.SnapshotFile, snID))
+	snapshotHandle := restic.Handle{
+		Type: restic.SnapshotFile,
+		Name: "51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02",
+	}
+	test.OK(t, repo.Backend().Remove(snapshotHandle))
 
 	unusedBlobsBySnapshot := restic.IDs{
 		restic.TestParseID("58c748bbe2929fdf30c73262bd8313fe828f8925b05d1d4a87fe109082acb849"),
@@ -170,6 +179,48 @@ func TestUnreferencedBlobs(t *testing.T) {
 	test.Equals(t, unusedBlobsBySnapshot, blobs)
 }
 
+func TestModifiedIndex(t *testing.T) {
+	repodir, cleanup := test.Env(t, checkerTestData)
+	defer cleanup()
+
+	repo := repository.TestOpenLocal(t, repodir)
+
+	done := make(chan struct{})
+	defer close(done)
+
+	h := restic.Handle{
+		Type: restic.IndexFile,
+		Name: "90f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
+	}
+	f, err := repo.Backend().Load(h, 0, 0)
+	test.OK(t, err)
+
+	// save the index again with a modified name so that the hash doesn't match
+	// the content any more
+	h2 := restic.Handle{
+		Type: restic.IndexFile,
+		Name: "80f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
+	}
+	err = repo.Backend().Save(h2, f)
+	test.OK(t, err)
+
+	test.OK(t, f.Close())
+
+	chkr := checker.New(repo)
+	hints, errs := chkr.LoadIndex()
+	if len(errs) == 0 {
+		t.Fatalf("expected errors not found")
+	}
+
+	for _, err := range errs {
+		t.Logf("found expected error %v", err)
+	}
+
+	if len(hints) > 0 {
+		t.Errorf("expected no hints, got %v: %v", len(hints), hints)
+	}
+}
+
 var checkerDuplicateIndexTestData = filepath.Join("testdata", "duplicate-packs-in-index-test-repo.tar.gz")
 
 func TestDuplicatePacksInIndex(t *testing.T) {
@@ -208,15 +259,35 @@ type errorBackend struct {
 	ProduceErrors bool
 }
 
-func (b errorBackend) Load(h restic.Handle, p []byte, off int64) (int, error) {
-	n, err := b.Backend.Load(h, p, off)
+func (b errorBackend) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	rd, err := b.Backend.Load(h, length, offset)
+	if err != nil {
+		return rd, err
+	}
 
 	if b.ProduceErrors {
-		induceError(p)
+		return errorReadCloser{rd}, err
+	}
+
+	return rd, nil
+}
+
+type errorReadCloser struct {
+	io.ReadCloser
+}
+
+func (erd errorReadCloser) Read(p []byte) (int, error) {
+	n, err := erd.ReadCloser.Read(p)
+	if n > 0 {
+		induceError(p[:n])
 	}
 	return n, err
 }
 
+func (erd errorReadCloser) Close() error {
+	return erd.ReadCloser.Close()
+}
+
 // induceError flips a bit in the slice.
 func induceError(data []byte) {
 	if rand.Float32() < 0.2 {
@@ -228,19 +299,15 @@ func induceError(data []byte) {
 }
 
 func TestCheckerModifiedData(t *testing.T) {
-	be := mem.New()
-
-	repository.TestUseLowSecurityKDFParameters(t)
-
-	repo := repository.New(be)
-	test.OK(t, repo.Init(test.TestPassword))
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
 
 	arch := archiver.New(repo)
-	_, id, err := arch.Snapshot(nil, []string{"."}, nil, nil)
+	_, id, err := arch.Snapshot(nil, []string{"."}, nil, "localhost", nil)
 	test.OK(t, err)
 	t.Logf("archived as %v", id.Str())
 
-	beError := &errorBackend{Backend: be}
+	beError := &errorBackend{Backend: repo.Backend()}
 	checkRepo := repository.New(beError)
 	test.OK(t, checkRepo.SearchKey(test.TestPassword, 5))
 
@@ -266,7 +333,7 @@ func TestCheckerModifiedData(t *testing.T) {
 	}
 
 	for _, err := range checkData(chkr) {
-		t.Logf("struct error: %v", err)
+		t.Logf("data error: %v", err)
 		errFound = true
 	}
 
@@ -274,3 +341,28 @@ func TestCheckerModifiedData(t *testing.T) {
 		t.Fatal("no error found, checker is broken")
 	}
 }
+
+func BenchmarkChecker(t *testing.B) {
+	repodir, cleanup := test.Env(t, checkerTestData)
+	defer cleanup()
+
+	repo := repository.TestOpenLocal(t, repodir)
+
+	chkr := checker.New(repo)
+	hints, errs := chkr.LoadIndex()
+	if len(errs) > 0 {
+		t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
+	}
+
+	if len(hints) > 0 {
+		t.Errorf("expected no hints, got %v: %v", len(hints), hints)
+	}
+
+	t.ResetTimer()
+
+	for i := 0; i < t.N; i++ {
+		test.OKs(t, checkPacks(chkr))
+		test.OKs(t, checkStruct(chkr))
+		test.OKs(t, checkData(chkr))
+	}
+}

src/restic/errors/wrap.go

@@ -18,3 +18,7 @@ var Errorf = errors.Errorf
 // Wrap wraps an error retrieved from outside of restic. Wrapped so that this
 // package does not appear in the stack trace.
 var Wrap = errors.Wrap
+
+// Wrapf returns an error annotating err with the format specifier. If err is
+// nil, Wrapf returns nil.
+var Wrapf = errors.Wrapf

src/restic/fs/doc.go

@@ -1,3 +1,3 @@
-// Package fs implements an OS independend abstraction of a file system
+// Package fs implements an OS independent abstraction of a file system
 // suitable for backup purposes.
 package fs

src/restic/fs/file.go

@@ -102,6 +102,12 @@ func Symlink(oldname, newname string) error {
 	return os.Symlink(fixpath(oldname), fixpath(newname))
 }
 
+// Link creates newname as a hard link to oldname.
+// If there is an error, it will be of type *LinkError.
+func Link(oldname, newname string) error {
+	return os.Link(fixpath(oldname), fixpath(newname))
+}
+
 // Stat returns a FileInfo structure describing the named file.
 // If there is an error, it will be of type *PathError.
 func Stat(name string) (os.FileInfo, error) {

src/restic/fuse/dir.go

@@ -114,9 +114,25 @@ func (d *dir) Attr(ctx context.Context, a *fuse.Attr) error {
 	a.Atime = d.node.AccessTime
 	a.Ctime = d.node.ChangeTime
 	a.Mtime = d.node.ModTime
+
+	a.Nlink = d.calcNumberOfLinks()
+
 	return nil
 }
 
+func (d *dir) calcNumberOfLinks() uint32 {
+	// a directory d has 2 hardlinks + the number
+	// of directories contained by d
+	var count uint32
+	count = 2
+	for _, node := range d.items {
+		if node.Type == "dir" {
+			count++
+		}
+	}
+	return count
+}
+
 func (d *dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
 	debug.Log("called")
 	ret := make([]fuse.Dirent, 0, len(d.items))
@@ -161,3 +177,21 @@ func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
 		return nil, fuse.ENOENT
 	}
 }
+
+func (d *dir) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error {
+	debug.Log("Listxattr(%v, %v)", d.node.Name, req.Size)
+	for _, attr := range d.node.ExtendedAttributes {
+		resp.Append(attr.Name)
+	}
+	return nil
+}
+
+func (d *dir) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
+	debug.Log("Getxattr(%v, %v, %v)", d.node.Name, req.Name, req.Size)
+	attrval := d.node.GetExtendedAttribute(req.Name)
+	if attrval != nil {
+		resp.Xattr = attrval
+		return nil
+	}
+	return fuse.ErrNoXattr
+}

src/restic/fuse/file.go

@@ -4,8 +4,6 @@
 package fuse
 
 import (
-	"sync"
-
 	"restic/errors"
 
 	"restic"
@@ -35,29 +33,23 @@ type file struct {
 	node        *restic.Node
 	ownerIsRoot bool
 
-	sizes []uint
+	sizes []int
 	blobs [][]byte
 }
 
 const defaultBlobSize = 128 * 1024
 
-var blobPool = sync.Pool{
-	New: func() interface{} {
-		return make([]byte, defaultBlobSize)
-	},
-}
-
 func newFile(repo BlobLoader, node *restic.Node, ownerIsRoot bool) (*file, error) {
 	debug.Log("create new file for %v with %d blobs", node.Name, len(node.Content))
 	var bytes uint64
-	sizes := make([]uint, len(node.Content))
+	sizes := make([]int, len(node.Content))
 	for i, id := range node.Content {
 		size, err := repo.LookupBlobSize(id, restic.DataBlob)
 		if err != nil {
 			return nil, err
 		}
 
-		sizes[i] = size
+		sizes[i] = int(size)
 		bytes += uint64(size)
 	}
 
@@ -82,6 +74,7 @@ func (f *file) Attr(ctx context.Context, a *fuse.Attr) error {
 	a.Size = f.node.Size
 	a.Blocks = (f.node.Size / blockSize) + 1
 	a.BlockSize = blockSize
+	a.Nlink = uint32(f.node.Links)
 
 	if !f.ownerIsRoot {
 		a.Uid = f.node.UID
@@ -90,7 +83,9 @@ func (f *file) Attr(ctx context.Context, a *fuse.Attr) error {
 	a.Atime = f.node.AccessTime
 	a.Ctime = f.node.ChangeTime
 	a.Mtime = f.node.ModTime
+
 	return nil
+
 }
 
 func (f *file) getBlobAt(i int) (blob []byte, err error) {
@@ -99,16 +94,12 @@ func (f *file) getBlobAt(i int) (blob []byte, err error) {
 		return f.blobs[i], nil
 	}
 
-	buf := blobPool.Get().([]byte)
-	buf = buf[:cap(buf)]
-
-	if uint(len(buf)) < f.sizes[i] {
-		if len(buf) > defaultBlobSize {
-			blobPool.Put(buf)
-		}
-		buf = make([]byte, f.sizes[i])
+	// release earlier blobs
+	for j := 0; j < i; j++ {
+		f.blobs[j] = nil
 	}
 
+	buf := restic.NewBlobBuffer(f.sizes[i])
 	n, err := f.repo.LoadBlob(restic.DataBlob, f.node.Content[i], buf)
 	if err != nil {
 		debug.Log("LoadBlob(%v, %v) failed: %v", f.node.Name, f.node.Content[i], err)
@@ -169,10 +160,25 @@ func (f *file) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadR
 
 func (f *file) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
 	for i := range f.blobs {
-		if f.blobs[i] != nil {
-			blobPool.Put(f.blobs[i])
-			f.blobs[i] = nil
-		}
+		f.blobs[i] = nil
 	}
 	return nil
 }
+
+func (f *file) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error {
+	debug.Log("Listxattr(%v, %v)", f.node.Name, req.Size)
+	for _, attr := range f.node.ExtendedAttributes {
+		resp.Append(attr.Name)
+	}
+	return nil
+}
+
+func (f *file) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
+	debug.Log("Getxattr(%v, %v, %v)", f.node.Name, req.Name, req.Size)
+	attrval := f.node.GetExtendedAttribute(req.Name)
+	if attrval != nil {
+		resp.Xattr = attrval
+		return nil
+	}
+	return fuse.ErrNoXattr
+}

src/restic/fuse/file_test.go

@@ -9,7 +9,9 @@ import (
 	"testing"
 	"time"
 
-	"restic/errors"
+	"golang.org/x/net/context"
+
+	"restic/repository"
 
 	"bazil.org/fuse"
 
@@ -17,108 +19,96 @@ import (
 	. "restic/test"
 )
 
-type MockRepo struct {
-	blobs map[restic.ID][]byte
-}
-
-func NewMockRepo(content map[restic.ID][]byte) *MockRepo {
-	return &MockRepo{blobs: content}
-}
-
-func (m *MockRepo) LookupBlobSize(id restic.ID, t restic.BlobType) (uint, error) {
-	buf, ok := m.blobs[id]
-	if !ok {
-		return 0, errors.New("blob not found")
-	}
-
-	return uint(len(buf)), nil
-}
-
-func (m *MockRepo) LoadBlob(t restic.BlobType, id restic.ID, buf []byte) (int, error) {
-	size, err := m.LookupBlobSize(id, t)
-	if err != nil {
-		return 0, err
-	}
-
-	if uint(len(buf)) < size {
-		return 0, errors.New("buffer too small")
-	}
-
-	buf = buf[:size]
-	copy(buf, m.blobs[id])
-	return int(size), nil
-}
-
-type MockContext struct{}
-
-func (m MockContext) Deadline() (time.Time, bool)       { return time.Now(), false }
-func (m MockContext) Done() <-chan struct{}             { return nil }
-func (m MockContext) Err() error                        { return nil }
-func (m MockContext) Value(key interface{}) interface{} { return nil }
-
-var testContent = genTestContent()
-var testContentLengths = []uint{
-	4646 * 1024,
-	655 * 1024,
-	378 * 1024,
-	8108 * 1024,
-	558 * 1024,
-}
-var testMaxFileSize uint
-
-func genTestContent() map[restic.ID][]byte {
-	m := make(map[restic.ID][]byte)
-
-	for _, length := range testContentLengths {
-		buf := Random(int(length), int(length))
-		id := restic.Hash(buf)
-		m[id] = buf
-		testMaxFileSize += length
-	}
-
-	return m
-}
-
-const maxBufSize = 20 * 1024 * 1024
-
-func testRead(t *testing.T, f *file, offset, length int, data []byte) {
-	ctx := MockContext{}
+func testRead(t testing.TB, f *file, offset, length int, data []byte) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
 
 	req := &fuse.ReadRequest{
 		Offset: int64(offset),
 		Size:   length,
 	}
 	resp := &fuse.ReadResponse{
-		Data: make([]byte, length),
+		Data: data,
 	}
 	OK(t, f.Read(ctx, req, resp))
 }
 
-var offsetReadsTests = []struct {
-	offset, length int
-}{
-	{0, 5 * 1024 * 1024},
-	{4000 * 1024, 1000 * 1024},
+func firstSnapshotID(t testing.TB, repo restic.Repository) (first restic.ID) {
+	done := make(chan struct{})
+	defer close(done)
+	for id := range repo.List(restic.SnapshotFile, done) {
+		if first.IsNull() {
+			first = id
+		}
+	}
+	return first
+}
+
+func loadFirstSnapshot(t testing.TB, repo restic.Repository) *restic.Snapshot {
+	id := firstSnapshotID(t, repo)
+	sn, err := restic.LoadSnapshot(repo, id)
+	OK(t, err)
+	return sn
+}
+
+func loadTree(t testing.TB, repo restic.Repository, id restic.ID) *restic.Tree {
+	tree, err := repo.LoadTree(id)
+	OK(t, err)
+	return tree
 }
 
 func TestFuseFile(t *testing.T) {
-	repo := NewMockRepo(testContent)
-	ctx := MockContext{}
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
 
-	memfile := make([]byte, 0, maxBufSize)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	timestamp, err := time.Parse(time.RFC3339, "2017-01-24T10:42:56+01:00")
+	OK(t, err)
+	restic.TestCreateSnapshot(t, repo, timestamp, 2, 0.1)
+
+	sn := loadFirstSnapshot(t, repo)
+	tree := loadTree(t, repo, *sn.Tree)
+
+	var content restic.IDs
+	for _, node := range tree.Nodes {
+		content = append(content, node.Content...)
+	}
+	t.Logf("tree loaded, content: %v", content)
+
+	var (
+		filesize uint64
+		memfile  []byte
+	)
+	for _, id := range content {
+		size, err := repo.LookupBlobSize(id, restic.DataBlob)
+		OK(t, err)
+		filesize += uint64(size)
+
+		buf := restic.NewBlobBuffer(int(size))
+		n, err := repo.LoadBlob(restic.DataBlob, id, buf)
+		OK(t, err)
+
+		if uint(n) != size {
+			t.Fatalf("not enough bytes read for id %v: want %v, got %v", id.Str(), size, n)
+		}
+
+		if uint(len(buf)) != size {
+			t.Fatalf("buffer has wrong length for id %v: want %v, got %v", id.Str(), size, len(buf))
+		}
 
-	var ids restic.IDs
-	for id, buf := range repo.blobs {
-		ids = append(ids, id)
 		memfile = append(memfile, buf...)
 	}
 
+	t.Logf("filesize is %v, memfile has size %v", filesize, len(memfile))
+
 	node := &restic.Node{
 		Name:    "foo",
 		Inode:   23,
 		Mode:    0742,
-		Size:    42,
-		Content: ids,
+		Size:    filesize,
+		Content: content,
 	}
 	f, err := newFile(repo, node, false)
 	OK(t, err)
@@ -131,28 +121,19 @@ func TestFuseFile(t *testing.T) {
 	Equals(t, node.Size, attr.Size)
 	Equals(t, (node.Size/uint64(attr.BlockSize))+1, attr.Blocks)
 
-	for i, test := range offsetReadsTests {
-		b := memfile[test.offset : test.offset+test.length]
-		buf := make([]byte, test.length)
-		testRead(t, f, test.offset, test.length, buf)
-		if !bytes.Equal(b, buf) {
-			t.Errorf("test %d failed, wrong data returned", i)
-		}
-	}
-
 	for i := 0; i < 200; i++ {
-		length := rand.Intn(int(testMaxFileSize) / 2)
-		offset := rand.Intn(int(testMaxFileSize))
-		if length+offset > int(testMaxFileSize) {
-			diff := length + offset - int(testMaxFileSize)
-			length -= diff
-		}
+		offset := rand.Intn(int(filesize))
+		length := rand.Intn(int(filesize)-offset) + 100
 
 		b := memfile[offset : offset+length]
+
 		buf := make([]byte, length)
+
 		testRead(t, f, offset, length, buf)
 		if !bytes.Equal(b, buf) {
-			t.Errorf("test %d failed (offset %d, length %d), wrong data returned", i, offset, length)
+			t.Errorf("test %d failed, wrong data returned (offset %v, length %v)", i, offset, length)
 		}
 	}
+
+	OK(t, f.Release(ctx, nil))
 }

src/restic/fuse/link.go

@@ -38,5 +38,8 @@ func (l *link) Attr(ctx context.Context, a *fuse.Attr) error {
 	a.Atime = l.node.AccessTime
 	a.Ctime = l.node.ChangeTime
 	a.Mtime = l.node.ModTime
+
+	a.Nlink = uint32(l.node.Links)
+
 	return nil
 }

src/restic/fuse/snapshot.go

@@ -32,6 +32,9 @@ var _ = fs.NodeStringLookuper(&SnapshotsDir{})
 type SnapshotsDir struct {
 	repo        restic.Repository
 	ownerIsRoot bool
+	paths       []string
+	tags        []string
+	host        string
 
 	// knownSnapshots maps snapshot timestamp to the snapshot
 	sync.RWMutex
@@ -40,12 +43,15 @@ type SnapshotsDir struct {
 }
 
 // NewSnapshotsDir returns a new dir object for the snapshots.
-func NewSnapshotsDir(repo restic.Repository, ownerIsRoot bool) *SnapshotsDir {
+func NewSnapshotsDir(repo restic.Repository, ownerIsRoot bool, paths []string, tags []string, host string) *SnapshotsDir {
 	debug.Log("fuse mount initiated")
 	return &SnapshotsDir{
 		repo:           repo,
-		knownSnapshots: make(map[string]SnapshotWithId),
 		ownerIsRoot:    ownerIsRoot,
+		paths:          paths,
+		tags:           tags,
+		host:           host,
+		knownSnapshots: make(map[string]SnapshotWithId),
 		processed:      restic.NewIDSet(),
 	}
 }
@@ -79,6 +85,13 @@ func (sn *SnapshotsDir) updateCache(ctx context.Context) error {
 			return err
 		}
 
+		// Filter snapshots we don't care for.
+		if (sn.host != "" && sn.host != snapshot.Hostname) ||
+			!snapshot.HasTags(sn.tags) ||
+			!snapshot.HasPaths(sn.paths) {
+			continue
+		}
+
 		timestamp := snapshot.Time.Format(time.RFC3339)
 		for i := 1; ; i++ {
 			if _, ok := sn.knownSnapshots[timestamp]; !ok {

src/restic/hardlinks_index.go

@@ -0,0 +1,57 @@
+package restic
+
+import (
+	"sync"
+)
+
+// HardlinkKey is a composed key for finding inodes on a specific device.
+type HardlinkKey struct {
+	Inode, Device uint64
+}
+
+// HardlinkIndex contains a list of inodes, devices these inodes are one, and associated file names.
+type HardlinkIndex struct {
+	m     sync.Mutex
+	Index map[HardlinkKey]string
+}
+
+// NewHardlinkIndex create a new index for hard links
+func NewHardlinkIndex() *HardlinkIndex {
+	return &HardlinkIndex{
+		Index: make(map[HardlinkKey]string),
+	}
+}
+
+// Has checks wether the link already exist in the index.
+func (idx *HardlinkIndex) Has(inode uint64, device uint64) bool {
+	idx.m.Lock()
+	defer idx.m.Unlock()
+	_, ok := idx.Index[HardlinkKey{inode, device}]
+
+	return ok
+}
+
+// Add adds a link to the index.
+func (idx *HardlinkIndex) Add(inode uint64, device uint64, name string) {
+	idx.m.Lock()
+	defer idx.m.Unlock()
+	_, ok := idx.Index[HardlinkKey{inode, device}]
+
+	if !ok {
+		idx.Index[HardlinkKey{inode, device}] = name
+	}
+}
+
+// GetFilename obtains the filename from the index.
+func (idx *HardlinkIndex) GetFilename(inode uint64, device uint64) string {
+	idx.m.Lock()
+	defer idx.m.Unlock()
+	return idx.Index[HardlinkKey{inode, device}]
+}
+
+// Remove removes a link from the index.
+func (idx *HardlinkIndex) Remove(inode uint64, device uint64) {
+	idx.m.Lock()
+	defer idx.m.Unlock()
+	delete(idx.Index, HardlinkKey{inode, device})
+}

src/restic/hardlinks_index_test.go

@@ -0,0 +1,35 @@
+package restic_test
+
+import (
+	"testing"
+
+	"restic"
+	. "restic/test"
+)
+
+// TestHardLinks contains various tests for HardlinkIndex.
+func TestHardLinks(t *testing.T) {
+
+	idx := restic.NewHardlinkIndex()
+
+	idx.Add(1, 2, "inode1-file1-on-device2")
+	idx.Add(2, 3, "inode2-file2-on-device3")
+
+	var sresult string
+	sresult = idx.GetFilename(1, 2)
+	Equals(t, sresult, "inode1-file1-on-device2")
+
+	sresult = idx.GetFilename(2, 3)
+	Equals(t, sresult, "inode2-file2-on-device3")
+
+	var bresult bool
+	bresult = idx.Has(1, 2)
+	Equals(t, bresult, true)
+
+	bresult = idx.Has(1, 3)
+	Equals(t, bresult, false)
+
+	idx.Remove(1, 2)
+	bresult = idx.Has(1, 2)
+	Equals(t, bresult, false)
+}

src/restic/hashing/reader.go

@@ -0,0 +1,29 @@
+package hashing
+
+import (
+	"hash"
+	"io"
+)
+
+// Reader hashes all data read from the underlying reader.
+type Reader struct {
+	r io.Reader
+	h hash.Hash
+}
+
+// NewReader returns a new Reader that uses the hash h.
+func NewReader(r io.Reader, h hash.Hash) *Reader {
+	return &Reader{
+		h: h,
+		r: io.TeeReader(r, h),
+	}
+}
+
+func (h *Reader) Read(p []byte) (int, error) {
+	return h.r.Read(p)
+}
+
+// Sum returns the hash of the data read so far.
+func (h *Reader) Sum(d []byte) []byte {
+	return h.h.Sum(d)
+}

src/restic/hashing/reader_test.go

@@ -0,0 +1,73 @@
+package hashing
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/sha256"
+	"io"
+	"io/ioutil"
+	"testing"
+)
+
+func TestReader(t *testing.T) {
+	tests := []int{5, 23, 2<<18 + 23, 1 << 20}
+
+	for _, size := range tests {
+		data := make([]byte, size)
+		_, err := io.ReadFull(rand.Reader, data)
+		if err != nil {
+			t.Fatalf("ReadFull: %v", err)
+		}
+
+		expectedHash := sha256.Sum256(data)
+
+		rd := NewReader(bytes.NewReader(data), sha256.New())
+		n, err := io.Copy(ioutil.Discard, rd)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if n != int64(size) {
+			t.Errorf("Reader: invalid number of bytes written: got %d, expected %d",
+				n, size)
+		}
+
+		resultingHash := rd.Sum(nil)
+
+		if !bytes.Equal(expectedHash[:], resultingHash) {
+			t.Errorf("Reader: hashes do not match: expected %02x, got %02x",
+				expectedHash, resultingHash)
+		}
+	}
+}
+
+func BenchmarkReader(b *testing.B) {
+	buf := make([]byte, 1<<22)
+	_, err := io.ReadFull(rand.Reader, buf)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	expectedHash := sha256.Sum256(buf)
+
+	b.SetBytes(int64(len(buf)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		rd := NewReader(bytes.NewReader(buf), sha256.New())
+		n, err := io.Copy(ioutil.Discard, rd)
+		if err != nil {
+			b.Fatal(err)
+		}
+
+		if n != int64(len(buf)) {
+			b.Errorf("Reader: invalid number of bytes written: got %d, expected %d",
+				n, len(buf))
+		}
+
+		resultingHash := rd.Sum(nil)
+		if !bytes.Equal(expectedHash[:], resultingHash) {
+			b.Errorf("Reader: hashes do not match: expected %02x, got %02x",
+				expectedHash, resultingHash)
+		}
+	}
+}

src/restic/hashing/writer.go

@@ -0,0 +1,31 @@
+package hashing
+
+import (
+	"hash"
+	"io"
+)
+
+// Writer transparently hashes all data while writing it to the underlying writer.
+type Writer struct {
+	w io.Writer
+	h hash.Hash
+}
+
+// NewWriter wraps the writer w and feeds all data written to the hash h.
+func NewWriter(w io.Writer, h hash.Hash) *Writer {
+	return &Writer{
+		h: h,
+		w: io.MultiWriter(w, h),
+	}
+}
+
+// Write wraps the write method of the underlying writer and also hashes all data.
+func (h *Writer) Write(p []byte) (int, error) {
+	n, err := h.w.Write(p)
+	return n, err
+}
+
+// Sum returns the hash of all data written so far.
+func (h *Writer) Sum(d []byte) []byte {
+	return h.h.Sum(d)
+}

src/restic/hashing/writer_test.go

@@ -0,0 +1,74 @@
+package hashing
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/sha256"
+	"io"
+	"io/ioutil"
+	"testing"
+)
+
+func TestWriter(t *testing.T) {
+	tests := []int{5, 23, 2<<18 + 23, 1 << 20}
+
+	for _, size := range tests {
+		data := make([]byte, size)
+		_, err := io.ReadFull(rand.Reader, data)
+		if err != nil {
+			t.Fatalf("ReadFull: %v", err)
+		}
+
+		expectedHash := sha256.Sum256(data)
+
+		wr := NewWriter(ioutil.Discard, sha256.New())
+
+		n, err := io.Copy(wr, bytes.NewReader(data))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if n != int64(size) {
+			t.Errorf("Writer: invalid number of bytes written: got %d, expected %d",
+				n, size)
+		}
+
+		resultingHash := wr.Sum(nil)
+
+		if !bytes.Equal(expectedHash[:], resultingHash) {
+			t.Errorf("Writer: hashes do not match: expected %02x, got %02x",
+				expectedHash, resultingHash)
+		}
+	}
+}
+
+func BenchmarkWriter(b *testing.B) {
+	buf := make([]byte, 1<<22)
+	_, err := io.ReadFull(rand.Reader, buf)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	expectedHash := sha256.Sum256(buf)
+
+	b.SetBytes(int64(len(buf)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		wr := NewWriter(ioutil.Discard, sha256.New())
+		n, err := io.Copy(wr, bytes.NewReader(buf))
+		if err != nil {
+			b.Fatal(err)
+		}
+
+		if n != int64(len(buf)) {
+			b.Errorf("Writer: invalid number of bytes written: got %d, expected %d",
+				n, len(buf))
+		}
+
+		resultingHash := wr.Sum(nil)
+		if !bytes.Equal(expectedHash[:], resultingHash) {
+			b.Errorf("Writer: hashes do not match: expected %02x, got %02x",
+				expectedHash, resultingHash)
+		}
+	}
+}

src/restic/id.go

@@ -43,7 +43,7 @@ func (id ID) String() string {
 	return hex.EncodeToString(id[:])
 }
 
-// NewRandomID retuns a randomly generated ID. When reading from rand fails,
+// NewRandomID returns a randomly generated ID. When reading from rand fails,
 // the function panics.
 func NewRandomID() ID {
 	id := ID{}
@@ -114,3 +114,13 @@ func (id *ID) UnmarshalJSON(b []byte) error {
 
 	return nil
 }
+
+// IDFromHash returns the ID for the hash.
+func IDFromHash(hash []byte) (id ID) {
+	if len(hash) != idSize {
+		panic("invalid hash type, not enough/too many bytes")
+	}
+
+	copy(id[:], hash)
+	return id
+}

src/restic/index/index.go

@@ -14,27 +14,20 @@ import (
 
 // Pack contains information about the contents of a pack.
 type Pack struct {
+	ID      restic.ID
 	Size    int64
 	Entries []restic.Blob
 }
 
-// Blob contains information about a blob.
-type Blob struct {
-	Size  int64
-	Packs restic.IDSet
-}
-
 // Index contains information about blobs and packs stored in a repo.
 type Index struct {
 	Packs    map[restic.ID]Pack
-	Blobs    map[restic.BlobHandle]Blob
 	IndexIDs restic.IDSet
 }
 
 func newIndex() *Index {
 	return &Index{
 		Packs:    make(map[restic.ID]Pack),
-		Blobs:    make(map[restic.BlobHandle]Blob),
 		IndexIDs: restic.NewIDSet(),
 	}
 }
@@ -69,9 +62,6 @@ func New(repo restic.Repository, p *restic.Progress) (*Index, error) {
 		if err != nil {
 			return nil, err
 		}
-
-		p := Pack{Entries: j.Entries(), Size: j.Size()}
-		idx.Packs[packID] = p
 	}
 
 	return idx, nil
@@ -179,19 +169,7 @@ func (idx *Index) AddPack(id restic.ID, size int64, entries []restic.Blob) error
 		return errors.Errorf("pack %v already present in the index", id.Str())
 	}
 
-	idx.Packs[id] = Pack{Size: size, Entries: entries}
-
-	for _, entry := range entries {
-		h := restic.BlobHandle{ID: entry.ID, Type: entry.Type}
-		if _, ok := idx.Blobs[h]; !ok {
-			idx.Blobs[h] = Blob{
-				Size:  int64(entry.Length),
-				Packs: restic.NewIDSet(),
-			}
-		}
-
-		idx.Blobs[h].Packs.Insert(id)
-	}
+	idx.Packs[id] = Pack{ID: id, Size: size, Entries: entries}
 
 	return nil
 }
@@ -202,15 +180,6 @@ func (idx *Index) RemovePack(id restic.ID) error {
 		return errors.Errorf("pack %v not found in the index", id.Str())
 	}
 
-	for _, blob := range idx.Packs[id].Entries {
-		h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
-		idx.Blobs[h].Packs.Delete(id)
-
-		if len(idx.Blobs[h].Packs) == 0 {
-			delete(idx.Blobs, h)
-		}
-	}
-
 	delete(idx.Packs, id)
 
 	return nil
@@ -239,14 +208,11 @@ func (idx *Index) DuplicateBlobs() (dups restic.BlobSet) {
 func (idx *Index) PacksForBlobs(blobs restic.BlobSet) (packs restic.IDSet) {
 	packs = restic.NewIDSet()
 
-	for h := range blobs {
-		blob, ok := idx.Blobs[h]
-		if !ok {
-			continue
-		}
-
-		for id := range blob.Packs {
-			packs.Insert(id)
+	for id, p := range idx.Packs {
+		for _, entry := range p.Entries {
+			if blobs.Has(restic.BlobHandle{ID: entry.ID, Type: entry.Type}) {
+				packs.Insert(id)
+			}
 		}
 	}
 
@@ -264,31 +230,20 @@ type Location struct {
 var ErrBlobNotFound = errors.New("blob not found in index")
 
 // FindBlob returns a list of packs and positions the blob can be found in.
-func (idx *Index) FindBlob(h restic.BlobHandle) ([]Location, error) {
-	blob, ok := idx.Blobs[h]
-	if !ok {
-		return nil, ErrBlobNotFound
+func (idx *Index) FindBlob(h restic.BlobHandle) (result []Location, err error) {
+	for id, p := range idx.Packs {
+		for _, entry := range p.Entries {
+			if entry.ID.Equal(h.ID) && entry.Type == h.Type {
+				result = append(result, Location{
+					PackID: id,
+					Blob:   entry,
+				})
+			}
+		}
 	}
 
-	result := make([]Location, 0, len(blob.Packs))
-	for packID := range blob.Packs {
-		pack, ok := idx.Packs[packID]
-		if !ok {
-			return nil, errors.Errorf("pack %v not found in index", packID.Str())
-		}
-
-		for _, entry := range pack.Entries {
-			if entry.Type != h.Type {
-				continue
-			}
-
-			if !entry.ID.Equal(h.ID) {
-				continue
-			}
-
-			loc := Location{PackID: packID, Blob: entry}
-			result = append(result, loc)
-		}
+	if len(result) == 0 {
+		return nil, ErrBlobNotFound
 	}
 
 	return result, nil

src/restic/index/index_test.go

@@ -3,7 +3,9 @@ package index
 import (
 	"math/rand"
 	"restic"
+	"restic/checker"
 	"restic/repository"
+	"restic/test"
 	"testing"
 	"time"
 )
@@ -25,9 +27,14 @@ func createFilledRepo(t testing.TB, snapshots int, dup float32) (restic.Reposito
 
 func validateIndex(t testing.TB, repo restic.Repository, idx *Index) {
 	for id := range repo.List(restic.DataFile, nil) {
-		if _, ok := idx.Packs[id]; !ok {
+		p, ok := idx.Packs[id]
+		if !ok {
 			t.Errorf("pack %v missing from index", id.Str())
 		}
+
+		if !p.ID.Equal(id) {
+			t.Errorf("pack %v has invalid ID: want %v, got %v", id.Str(), id, p.ID)
+		}
 	}
 }
 
@@ -135,6 +142,40 @@ func BenchmarkIndexNew(b *testing.B) {
 		if idx == nil {
 			b.Fatalf("New() returned nil index")
 		}
+		b.Logf("idx %v packs", len(idx.Packs))
+	}
+}
+
+func BenchmarkIndexSave(b *testing.B) {
+	repo, cleanup := repository.TestRepository(b)
+	defer cleanup()
+
+	idx, err := New(repo, nil)
+	test.OK(b, err)
+
+	for i := 0; i < 8000; i++ {
+		entries := make([]restic.Blob, 0, 200)
+		for j := 0; j < cap(entries); j++ {
+			entries = append(entries, restic.Blob{
+				ID:     restic.NewRandomID(),
+				Length: 1000,
+				Offset: 5,
+				Type:   restic.DataBlob,
+			})
+		}
+
+		idx.AddPack(restic.NewRandomID(), 10000, entries)
+	}
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		id, err := idx.Save(repo, nil)
+		if err != nil {
+			b.Fatalf("New() returned error %v", err)
+		}
+
+		b.Logf("saved as %v", id.Str())
 	}
 }
 
@@ -151,7 +192,7 @@ func TestIndexDuplicateBlobs(t *testing.T) {
 	if len(dups) == 0 {
 		t.Errorf("no duplicate blobs found")
 	}
-	t.Logf("%d packs, %d unique blobs", len(idx.Packs), len(idx.Blobs))
+	t.Logf("%d packs, %d duplicate blobs", len(idx.Packs), len(dups))
 
 	packs := idx.PacksForBlobs(dups)
 	if len(packs) == 0 {
@@ -169,7 +210,7 @@ func loadIndex(t testing.TB, repo restic.Repository) *Index {
 	return idx
 }
 
-func TestIndexSave(t *testing.T) {
+func TestSave(t *testing.T) {
 	repo, cleanup := createFilledRepo(t, 3, 0)
 	defer cleanup()
 
@@ -193,7 +234,8 @@ func TestIndexSave(t *testing.T) {
 
 	for id := range idx.IndexIDs {
 		t.Logf("remove index %v", id.Str())
-		err = repo.Backend().Remove(restic.IndexFile, id.String())
+		h := restic.Handle{Type: restic.IndexFile, Name: id.String()}
+		err = repo.Backend().Remove(h)
 		if err != nil {
 			t.Errorf("error removing index %v: %v", id, err)
 		}
@@ -219,6 +261,42 @@ func TestIndexSave(t *testing.T) {
 	}
 }
 
+func TestIndexSave(t *testing.T) {
+	repo, cleanup := createFilledRepo(t, 3, 0)
+	defer cleanup()
+
+	idx := loadIndex(t, repo)
+
+	id, err := idx.Save(repo, idx.IndexIDs.List())
+	if err != nil {
+		t.Fatalf("unable to save new index: %v", err)
+	}
+
+	t.Logf("new index saved as %v", id.Str())
+
+	for id := range idx.IndexIDs {
+		t.Logf("remove index %v", id.Str())
+		h := restic.Handle{Type: restic.IndexFile, Name: id.String()}
+		err = repo.Backend().Remove(h)
+		if err != nil {
+			t.Errorf("error removing index %v: %v", id, err)
+		}
+	}
+
+	idx2 := loadIndex(t, repo)
+	t.Logf("load new index with %d packs", len(idx2.Packs))
+
+	checker := checker.New(repo)
+	hints, errs := checker.LoadIndex()
+	for _, h := range hints {
+		t.Logf("hint: %v\n", h)
+	}
+
+	for _, err := range errs {
+		t.Errorf("checker found error: %v", err)
+	}
+}
+
 func TestIndexAddRemovePack(t *testing.T) {
 	repo, cleanup := createFilledRepo(t, 3, 0)
 	defer cleanup()
@@ -249,12 +327,7 @@ func TestIndexAddRemovePack(t *testing.T) {
 		if err == nil {
 			t.Errorf("removed blob %v found in index", h)
 		}
-
-		if _, ok := idx.Blobs[h]; ok {
-			t.Errorf("removed blob %v found in index.Blobs", h)
-		}
 	}
-
 }
 
 // example index serialization from doc/Design.md

src/restic/list/list.go

@@ -25,7 +25,7 @@ func (l Result) PackID() restic.ID {
 	return l.packID
 }
 
-// Size ruturns the size of the pack.
+// Size returns the size of the pack.
 func (l Result) Size() int64 {
 	return l.size
 }

src/restic/lock.go

@@ -186,7 +186,7 @@ func (l *Lock) Unlock() error {
 		return nil
 	}
 
-	return l.repo.Backend().Remove(LockFile, l.lockID.String())
+	return l.repo.Backend().Remove(Handle{Type: LockFile, Name: l.lockID.String()})
 }
 
 var staleTimeout = 30 * time.Minute
@@ -203,7 +203,7 @@ func (l *Lock) Stale() bool {
 
 	hn, err := os.Hostname()
 	if err != nil {
-		debug.Log("unable to find current hostnanme: %v", err)
+		debug.Log("unable to find current hostname: %v", err)
 		// since we cannot find the current hostname, assume that the lock is
 		// not stale.
 		return false
@@ -234,7 +234,7 @@ func (l *Lock) Refresh() error {
 		return err
 	}
 
-	err = l.repo.Backend().Remove(LockFile, l.lockID.String())
+	err = l.repo.Backend().Remove(Handle{Type: LockFile, Name: l.lockID.String()})
 	if err != nil {
 		return err
 	}
@@ -289,7 +289,7 @@ func RemoveStaleLocks(repo Repository) error {
 		}
 
 		if lock.Stale() {
-			return repo.Backend().Remove(LockFile, id.String())
+			return repo.Backend().Remove(Handle{Type: LockFile, Name: id.String()})
 		}
 
 		return nil
@@ -299,6 +299,6 @@ func RemoveStaleLocks(repo Repository) error {
 // RemoveAllLocks removes all locks forcefully.
 func RemoveAllLocks(repo Repository) error {
 	return eachLock(repo, func(id ID, lock *Lock, err error) error {
-		return repo.Backend().Remove(LockFile, id.String())
+		return repo.Backend().Remove(Handle{Type: LockFile, Name: id.String()})
 	})
 }

src/restic/lock_test.go

@@ -102,7 +102,8 @@ func createFakeLock(repo restic.Repository, t time.Time, pid int) (restic.ID, er
 }
 
 func removeLock(repo restic.Repository, id restic.ID) error {
-	return repo.Backend().Remove(restic.LockFile, id.String())
+	h := restic.Handle{Type: restic.LockFile, Name: id.String()}
+	return repo.Backend().Remove(h)
 }
 
 var staleLockTests = []struct {
@@ -162,7 +163,8 @@ func TestLockStale(t *testing.T) {
 }
 
 func lockExists(repo restic.Repository, t testing.TB, id restic.ID) bool {
-	exists, err := repo.Backend().Test(restic.LockFile, id.String())
+	h := restic.Handle{Type: restic.LockFile, Name: id.String()}
+	exists, err := repo.Backend().Test(h)
 	OK(t, err)
 
 	return exists

src/restic/mock/backend.go

@@ -1,6 +1,7 @@
 package mock
 
 import (
+	"io"
 	"restic"
 
 	"restic/errors"
@@ -9,12 +10,12 @@ import (
 // Backend implements a mock backend.
 type Backend struct {
 	CloseFn    func() error
-	LoadFn     func(h restic.Handle, p []byte, off int64) (int, error)
-	SaveFn     func(h restic.Handle, p []byte) error
+	SaveFn     func(h restic.Handle, rd io.Reader) error
+	LoadFn     func(h restic.Handle, length int, offset int64) (io.ReadCloser, error)
 	StatFn     func(h restic.Handle) (restic.FileInfo, error)
 	ListFn     func(restic.FileType, <-chan struct{}) <-chan string
-	RemoveFn   func(restic.FileType, string) error
-	TestFn     func(restic.FileType, string) (bool, error)
+	RemoveFn   func(h restic.Handle) error
+	TestFn     func(h restic.Handle) (bool, error)
 	DeleteFn   func() error
 	LocationFn func() string
 }
@@ -37,22 +38,22 @@ func (m *Backend) Location() string {
 	return m.LocationFn()
 }
 
-// Load loads data from the backend.
-func (m *Backend) Load(h restic.Handle, p []byte, off int64) (int, error) {
-	if m.LoadFn == nil {
-		return 0, errors.New("not implemented")
-	}
-
-	return m.LoadFn(h, p, off)
-}
-
 // Save data in the backend.
-func (m *Backend) Save(h restic.Handle, p []byte) error {
+func (m *Backend) Save(h restic.Handle, rd io.Reader) error {
 	if m.SaveFn == nil {
 		return errors.New("not implemented")
 	}
 
-	return m.SaveFn(h, p)
+	return m.SaveFn(h, rd)
+}
+
+// Load loads data from the backend.
+func (m *Backend) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	if m.LoadFn == nil {
+		return nil, errors.New("not implemented")
+	}
+
+	return m.LoadFn(h, length, offset)
 }
 
 // Stat an object in the backend.
@@ -76,21 +77,21 @@ func (m *Backend) List(t restic.FileType, done <-chan struct{}) <-chan string {
 }
 
 // Remove data from the backend.
-func (m *Backend) Remove(t restic.FileType, name string) error {
+func (m *Backend) Remove(h restic.Handle) error {
 	if m.RemoveFn == nil {
 		return errors.New("not implemented")
 	}
 
-	return m.RemoveFn(t, name)
+	return m.RemoveFn(h)
 }
 
 // Test for the existence of a specific item.
-func (m *Backend) Test(t restic.FileType, name string) (bool, error) {
+func (m *Backend) Test(h restic.Handle) (bool, error) {
 	if m.TestFn == nil {
 		return false, errors.New("not implemented")
 	}
 
-	return m.TestFn(t, name)
+	return m.TestFn(h)
 }
 
 // Delete all data.