Add VERSION file for 0.5.0-rc.1

Add --tag filtering to every command, where applicable

.envrc

@@ -0,0 +1 @@
+GOPATH=$PWD:$PWD/vendor

.travis.yml

@@ -2,8 +2,9 @@ language: go
 sudo: false
 
 go:
-  - 1.6.3
-  - 1.7.1
+  - 1.7.5
+  - 1.8
+  - tip
 
 os:
   - linux
@@ -16,16 +17,23 @@ env:
 matrix:
   exclude:
     - os: osx
-      go: 1.6.3
+      go: 1.7.5
+    - os: osx
+      go: tip
     - os: linux
-      go: 1.7.1
+      go: 1.8
   include:
     - os: linux
-      go: 1.7.1
+      go: 1.8
       sudo: true
       env:
         RESTIC_TEST_FUSE=1
+  allow_failures:
+    - go: tip
 
+branches:
+  only:
+    - master
 
 notifications:
   irc:
@@ -40,11 +48,9 @@ install:
   - export GOBIN="$GOPATH/bin"
   - export PATH="$PATH:$GOBIN"
   - go env
-  - ulimit -n 2048
 
 script:
   - go run run_integration_tests.go
 
 after_success:
-  - GOPATH=$PWD:$PWD/vendor goveralls -coverprofile=all.cov -service=travis-ci -repotoken "$COVERALLS_TOKEN"
   - bash <(curl -s https://codecov.io/bash) -f all.cov

CONTRIBUTING.md

@@ -3,7 +3,10 @@ This document describes the way you can contribute to the restic project.
 Ways to Help Out
 ================
 
-Thank you for your contribution!
+Thank you for your contribution! Please **open an issue first** (or add a
+comment to an existing issue) if you plan to work on any code or add a new
+feature. This way, duplicate work is prevented and we can discuss your ideas
+and design first.
 
 There are several ways you can help us out. First of all code contributions and
 bug fixes are most welcome. However even "minor" details as fixing spelling
@@ -74,7 +77,7 @@ Just clone the repository, `cd` to it and run `gb build` to build the binary:
     [...]
     $ bin/restic version
     restic compiled manually
-    compiled at unknown time with go1.6
+    compiled at unknown time with go1.7
 
 The following commands can be used to run all the tests:
 
@@ -83,7 +86,7 @@ The following commands can be used to run all the tests:
     [...]
 
 If you want to run your tests on Linux, OpenBSD or FreeBSD, you can use
-[vagrant](https://www.vagrantup.com/) with the proveded `Vagrantfile` to
+[vagrant](https://www.vagrantup.com/) with the provided `Vagrantfile` to
 quickly set up VMs and run the tests, e.g.:
 
     $ vagrant up freebsd
@@ -92,6 +95,16 @@ quickly set up VMs and run the tests, e.g.:
     $ vagrant ssh freebsd -c 'cd restic/restic; go test -v ./...'
     [...]
 
+The default `go` tool can also be used by setting the environment variable
+`GOPATH` to the following value while being in the top level directory in the
+git repository:
+
+    $ export GOPATH=$PWD:$PWD/vendor
+
+The file `.envrc` allows automatic `GOPATH` configuration with
+[direnv](https://direnv.net/), inspect the file and then allow automatic
+configuration by running `direnv allow`.
+
 Providing Patches
 =================
 

Dockerfile

@@ -14,11 +14,11 @@
 #   docker run --rm -v $PWD:/home/travis/restic restic/test gb test -v ./backend
 #
 # build the image for an older version of Go:
-#   docker build --build-arg GOVERSION=1.3.3 -t restic/test:go1.3.3 .
+#   docker build --build-arg GOVERSION=1.6.4 -t restic/test:go1.6.4 .
 
 FROM ubuntu:14.04
 
-ARG GOVERSION=1.7
+ARG GOVERSION=1.7.5
 ARG GOARCH=amd64
 
 # install dependencies

README.md

@@ -11,17 +11,30 @@ restic is a backup program that is fast, efficient and secure. Detailed
 information can be found in [the documentation](doc/index.md) and [the user
 manual](doc/Manual.md). The [design document](doc/Design.md) lists the
 technical background and gives detailed information about the structure of the
-repository and the data saved therein.
+repository and the data saved therein. The file [FAQ.md](doc/FAQ.md) lists the
+most frequently asked questions.
 
 The latest documentation can be viewed online at
 <https://restic.readthedocs.io/en/latest>. On the bottom left corner there is
 a menu that allows switching to the documentation and user manual for the
 latest released version.
 
+News
+====
+
+You can follow the restic project on Twitter
+[@resticbackup](https://twitter.com/resticbackup) or by subscribing to the
+[development blog](https://restic.github.io/blog/).
+
+Install restic
+==============
+
+You can download the latest pre-compiled binary from the [restic release page](https://github.com/restic/restic/releases/latest).
+
 Build restic
 ============
 
-Install Go/Golang (at least version 1.6), then run `go run build.go`,
+Install Go/Golang (at least version 1.7), then run `go run build.go`,
 afterwards you'll find the binary in the current directory:
 
     $ go run build.go
@@ -39,10 +52,15 @@ Building restic with gccgo may work, but is not supported.
 Contribute and Documentation
 ============================
 
-Contributions are welcome! More information and a description of the
-development environment can be found in [`CONTRIBUTING.md`](CONTRIBUTING.md). A
-document describing the design of restic and the data structures stored on the
-back end is contained in [`doc/Design.md`](doc/Design.md).
+Contributions are welcome! Please **open an issue first** (or add a comment to
+an existing issue) if you plan to work on any code or add a new feature. This
+way, duplicate work is prevented and we can discuss your ideas and design
+first.
+
+More information and a description of the development environment can be found
+in [`CONTRIBUTING.md`](CONTRIBUTING.md). A document describing the design of
+restic and the data structures stored on the back end is contained in
+[`doc/Design.md`](doc/Design.md).
 
 If you'd like to start contributing to restic, but don't know exactly what do
 to, have a look at this great article by Dave Cheney:
@@ -67,7 +85,6 @@ alexander@bumpern.de. If possible, please encrypt your email using the following
 pub   4096R/91A6868BD3F7A907 2014-11-01
       Key fingerprint = CF8F 18F2 8445 7597 3F79  D4E1 91A6 868B D3F7 A907
       uid                          Alexander Neumann <alexander@bumpern.de>
-      uid                          Alexander Neumann <alexander@debian.org>
       sub   4096R/D5FC2ACF4043FDF1 2014-11-01
 ```
 

VERSION

@@ -1 +1 @@
-0.3.0
+0.5.0-rc.1

Vagrantfile

@@ -1,7 +1,7 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 
-GO_VERSION = '1.6'
+GO_VERSION = '1.7'
 
 def packages_freebsd
   return <<-EOF

appveyor.yml

@@ -3,6 +3,10 @@ clone_folder: c:\restic
 environment:
   GOPATH: c:\gopath
 
+branches:
+  only:
+    - master
+
 init:
   - ps: >-
       $app = Get-WmiObject -Class Win32_Product -Filter "Vendor = 'http://golang.org'"
@@ -13,8 +17,8 @@ init:
 
 install:
   - rmdir c:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.7.windows-amd64.msi
-  - msiexec /i go1.7.windows-amd64.msi /q
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.8.windows-amd64.msi
+  - msiexec /i go1.8.windows-amd64.msi /q
   - go version
   - go env
   - appveyor DownloadFile http://sourceforge.netcologne.de/project/gnuwin32/tar/1.13-1/tar-1.13-1-bin.zip -FileName tar.zip

build.go

@@ -12,7 +12,6 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
-	"time"
 )
 
 var (
@@ -21,7 +20,17 @@ var (
 	runTests   bool
 )
 
-const timeFormat = "2006-01-02 15:04:05"
+var config = struct {
+	Name      string
+	Namespace string
+	Main      string
+	Tests     []string
+}{
+	Name:      "restic",                           // name of the program executable and directory
+	Namespace: "",                                 // subdir of GOPATH, e.g. "github.com/foo/bar"
+	Main:      "cmds/restic",                      // package name for the main package
+	Tests:     []string{"restic/...", "cmds/..."}, // tests to run
+}
 
 // specialDir returns true if the file begins with a special character ('.' or '_').
 func specialDir(name string) bool {
@@ -96,6 +105,15 @@ func updateGopath(dst, src, prefix string) error {
 	})
 }
 
+func directoryExists(dirname string) bool {
+	stat, err := os.Stat(dirname)
+	if err != nil && os.IsNotExist(err) {
+		return false
+	}
+
+	return stat.IsDir()
+}
+
 // copyFile creates dst from src, preserving file attributes and timestamps.
 func copyFile(dst, src string) error {
 	fi, err := os.Stat(src)
@@ -156,6 +174,7 @@ func showUsage(output io.Writer) {
 	fmt.Fprintf(output, "  -t     --tags          specify additional build tags\n")
 	fmt.Fprintf(output, "  -k     --keep-gopath   do not remove the GOPATH after build\n")
 	fmt.Fprintf(output, "  -T     --test          run tests\n")
+	fmt.Fprintf(output, "  -o     --output        set output file name\n")
 	fmt.Fprintf(output, "         --goos value    set GOOS for cross-compilation\n")
 	fmt.Fprintf(output, "         --goarch value  set GOARCH for cross-compilation\n")
 }
@@ -186,7 +205,7 @@ func cleanEnv() (env []string) {
 func build(cwd, goos, goarch, gopath string, args ...string) error {
 	args = append([]string{"build"}, args...)
 	cmd := exec.Command("go", args...)
-	cmd.Env = append(cleanEnv(), "GOPATH="+gopath, "GOARCH="+goarch, "GOOS="+goos)
+	cmd.Env = append(cleanEnv(), "GOPATH="+gopath, "GOARCH="+goarch, "GOOS="+goos, "CGO_ENABLED=0")
 	cmd.Dir = cwd
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
@@ -264,24 +283,14 @@ type Constants map[string]string
 func (cs Constants) LDFlags() string {
 	l := make([]string, 0, len(cs))
 
-	if runtime.Version() < "go1.5" {
-		for k, v := range cs {
-			l = append(l, fmt.Sprintf(`-X %q %q`, k, v))
-		}
-	} else {
-		for k, v := range cs {
-			l = append(l, fmt.Sprintf(`-X "%s=%s"`, k, v))
-		}
+	for k, v := range cs {
+		l = append(l, fmt.Sprintf(`-X "%s=%s"`, k, v))
 	}
 
 	return strings.Join(l, " ")
 }
 
 func main() {
-	if runtime.Version() < "go1.6" {
-		fmt.Fprintf(os.Stderr, "old version of Go detected (%v), I'll try but no guarantees\n", runtime.Version())
-	}
-
 	buildTags := []string{}
 
 	skipNext := false
@@ -290,6 +299,8 @@ func main() {
 	targetGOOS := runtime.GOOS
 	targetGOARCH := runtime.GOARCH
 
+	var outputFilename string
+
 	for i, arg := range params {
 		if skipNext {
 			skipNext = false
@@ -302,8 +313,14 @@ func main() {
 		case "-k", "--keep-gopath":
 			keepGopath = true
 		case "-t", "-tags", "--tags":
+			if i+1 >= len(params) {
+				die("-t given but no tag specified")
+			}
 			skipNext = true
 			buildTags = strings.Split(params[i+1], " ")
+		case "-o", "--output":
+			skipNext = true
+			outputFilename = params[i+1]
 		case "-T", "--test":
 			runTests = true
 		case "--goos":
@@ -338,23 +355,21 @@ func main() {
 		die("Getwd(): %v\n", err)
 	}
 
-	gopath, err := ioutil.TempDir("", "restic-build-")
+	gopath, err := ioutil.TempDir("", fmt.Sprintf("%v-build-", config.Name))
 	if err != nil {
 		die("TempDir(): %v\n", err)
 	}
 
 	verbosePrintf("create GOPATH at %v\n", gopath)
-	if err = updateGopath(gopath, filepath.Join(root, "src", "restic"), "restic"); err != nil {
-		die("copying files from %v/src/restic to %v/src/restic failed: %v\n", root, gopath, err)
-	}
-
-	if err = updateGopath(gopath, filepath.Join(root, "src", "cmds"), "cmds"); err != nil {
-		die("copying files from %v/src/cmds to %v/src/restic/cmds failed: %v\n", root, gopath, err)
+	if err = updateGopath(gopath, filepath.Join(root, "src"), config.Namespace); err != nil {
+		die("copying files from %v/src to %v/src failed: %v\n", root, gopath, err)
 	}
 
 	vendor := filepath.Join(root, "vendor", "src")
-	if err = updateGopath(gopath, vendor, ""); err != nil {
-		die("copying files from %v to %v/src failed: %v\n", vendor, gopath, err)
+	if directoryExists(vendor) {
+		if err = updateGopath(gopath, vendor, ""); err != nil {
+			die("copying files from %v to %v failed: %v\n", root, gopath, err)
+		}
 	}
 
 	defer func() {
@@ -368,9 +383,11 @@ func main() {
 		}
 	}()
 
-	outputFilename := "restic"
-	if targetGOOS == "windows" {
-		outputFilename = "restic.exe"
+	if outputFilename == "" {
+		outputFilename = config.Name
+		if targetGOOS == "windows" {
+			outputFilename += ".exe"
+		}
 	}
 
 	cwd, err := os.Getwd()
@@ -380,8 +397,7 @@ func main() {
 	output := filepath.Join(cwd, outputFilename)
 
 	version := getVersion()
-	compileTime := time.Now().Format(timeFormat)
-	constants := Constants{`main.compiledAt`: compileTime}
+	constants := Constants{}
 	if version != "" {
 		constants["main.version"] = version
 	}
@@ -391,7 +407,7 @@ func main() {
 	args := []string{
 		"-tags", strings.Join(buildTags, " "),
 		"-ldflags", ldflags,
-		"-o", output, "cmds/restic",
+		"-o", output, config.Main,
 	}
 
 	err = build(filepath.Join(gopath, "src"), targetGOOS, targetGOARCH, gopath, args...)
@@ -402,7 +418,7 @@ func main() {
 	if runTests {
 		verbosePrintf("running tests\n")
 
-		err = test(filepath.Join(gopath, "src"), gopath, "restic/...")
+		err = test(cwd, gopath, config.Tests...)
 		if err != nil {
 			die("running tests failed: %v\n", err)
 		}

build_release_binaries.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+set -e
+
+if [[ -z "$VERSION" ]]; then
+    echo '$VERSION unset'
+    exit 1
+fi
+
+dir=$(mktemp -d --tmpdir restic-release-XXXXXX)
+echo "path is ${dir}"
+
+for R in       \
+    darwin/386     \
+    darwin/amd64   \
+    freebsd/386    \
+    freebsd/amd64  \
+    freebsd/arm    \
+    linux/386      \
+    linux/amd64    \
+    linux/arm      \
+    linux/arm64    \
+    openbsd/386   \
+    openbsd/amd64 \
+    windows/386    \
+    windows/amd64  \
+    ; do \
+
+    OS=$(dirname $R)
+    ARCH=$(basename $R)
+    filename=restic_${VERSION}_${OS}_${ARCH}
+
+    if [[ "$OS" == "windows" ]]; then
+        filename="${filename}.exe"
+    fi
+
+    echo $filename
+
+    go run build.go --goos $OS --goarch $ARCH --output ${filename}
+    if [[ "$OS" == "windows" ]]; then
+        zip ${filename%.exe}.zip ${filename}
+        rm ${filename}
+        mv ${filename%.exe}.zip ${dir}
+    else
+        bzip2 ${filename}
+        mv ${filename}.bz2 ${dir}
+    fi
+done
+
+echo "packing sources"
+git archive --format=tar --prefix=restic-$VERSION/ v$VERSION | gzip -n > restic-$VERSION.tar.gz
+mv restic-$VERSION.tar.gz ${dir}
+
+echo "creating checksums"
+pushd ${dir}
+sha256sum restic_*.{zip,bz2} restic-$VERSION.tar.gz > SHA256SUMS
+gpg --armor --detach-sign SHA256SUMS
+popd
+
+echo "creating source signature file"
+gpg --armor --detach-sign ${dir}/restic-$VERSION.tar.gz
+
+echo
+echo "done, path is ${dir}"

doc/Design.md

@@ -62,11 +62,13 @@ overhead is 32 bytes. For each file, a new random IV is selected.
 The file `config` is encrypted this way and contains a JSON document like the
 following:
 
-    {
-      "version": 1,
-      "id": "5956a3f67a6230d4a92cefb29529f10196c7d92582ec305fd71ff6d331d6271b",
-      "chunker_polynomial": "25b468838dcb75"
-    }
+```json
+{
+  "version": 1,
+  "id": "5956a3f67a6230d4a92cefb29529f10196c7d92582ec305fd71ff6d331d6271b",
+  "chunker_polynomial": "25b468838dcb75"
+}
+```
 
 After decryption, restic first checks that the version field contains a version
 number that it understands, otherwise it aborts. At the moment, the version is
@@ -102,7 +104,9 @@ The basic layout of a sample restic repository is shown here:
 
 A repository can be initialized with the `restic init` command, e.g.:
 
-    $ restic -r /tmp/restic-repo init
+```console
+$ restic -r /tmp/restic-repo init
+```
 
 Pack Format
 -----------
@@ -163,35 +167,37 @@ used to reconstruct the index. The files are encrypted and authenticated like
 Data and Tree Blobs, so the outer structure is `IV || Ciphertext || MAC` again.
 The plaintext consists of a JSON document like the following:
 
+```json
+{
+  "supersedes": [
+    "ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452"
+  ],
+  "packs": [
     {
-      "supersedes": [
-        "ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452"
-      ],
-      "packs": [
+      "id": "73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c",
+      "blobs": [
         {
-          "id": "73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c",
-          "blobs": [
-            {
-              "id": "3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce",
-              "type": "data",
-              "offset": 0,
-              "length": 25
-            },{
-              "id": "9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae",
-              "type": "tree",
-              "offset": 38,
-              "length": 100
-            },
-            {
-              "id": "d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66",
-              "type": "data",
-              "offset": 150,
-              "length": 123
-            }
-          ]
-        }, [...]
+          "id": "3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce",
+          "type": "data",
+          "offset": 0,
+          "length": 25
+        },{
+          "id": "9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae",
+          "type": "tree",
+          "offset": 38,
+          "length": 100
+        },
+        {
+          "id": "d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66",
+          "type": "data",
+          "offset": 150,
+          "length": 123
+        }
       ]
-    }
+    }, [...]
+  ]
+}
+```
 
 This JSON document lists Packs and the blobs contained therein. In this
 example, the Pack `73d04e61` contains two data Blobs and one Tree blob, the
@@ -258,14 +264,16 @@ document which contains the master encryption and message authentication keys
 for this repository (encoded in Base64). The command `restic cat masterkey` can
 be used as follows to decrypt and pretty-print the master key:
 
-    $ restic -r /tmp/restic-repo cat masterkey
-    {
-        "mac": {
-          "k": "evFWd9wWlndL9jc501268g==",
-          "r": "E9eEDnSJZgqwTOkDtOp+Dw=="
-        },
-        "encrypt": "UQCqa0lKZ94PygPxMRqkePTZnHRYh1k1pX2k2lM2v3Q=",
-    }
+```console
+$ restic -r /tmp/restic-repo cat masterkey
+{
+    "mac": {
+      "k": "evFWd9wWlndL9jc501268g==",
+      "r": "E9eEDnSJZgqwTOkDtOp+Dw=="
+    },
+    "encrypt": "UQCqa0lKZ94PygPxMRqkePTZnHRYh1k1pX2k2lM2v3Q=",
+}
+```
 
 All data in the repository is encrypted and authenticated with these master keys.
 For encryption, the AES-256 algorithm in Counter mode is used. For message
@@ -277,7 +285,7 @@ This way, the password can be changed without having to re-encrypt all data.
 Snapshots
 ---------
 
-A snapshots represents a directory with all files and sub-directories at a
+A snapshot represents a directory with all files and sub-directories at a
 given point in time. For each backup that is made, a new snapshot is created. A
 snapshot is a JSON document that is stored in an encrypted file below the
 directory `snapshots` in the repository. The filename is the storage ID. This
@@ -286,20 +294,52 @@ string is unique and used within restic to uniquely identify a snapshot.
 The command `restic cat snapshot` can be used as follows to decrypt and
 pretty-print the contents of a snapshot file:
 
-    $ restic -r /tmp/restic-repo cat snapshot 22a5af1b
-    enter password for repository:
-    {
-      "time": "2015-01-02T18:10:50.895208559+01:00",
-      "tree": "2da81727b6585232894cfbb8f8bdab8d1eccd3d8f7c92bc934d62e62e618ffdf",
-      "dir": "/tmp/testdata",
-      "hostname": "kasimir",
-      "username": "fd0",
-      "uid": 1000,
-      "gid": 100
-    }
+```console
+$ restic -r /tmp/restic-repo cat snapshot 251c2e58
+enter password for repository:
+{
+  "time": "2015-01-02T18:10:50.895208559+01:00",
+  "tree": "2da81727b6585232894cfbb8f8bdab8d1eccd3d8f7c92bc934d62e62e618ffdf",
+  "dir": "/tmp/testdata",
+  "hostname": "kasimir",
+  "username": "fd0",
+  "uid": 1000,
+  "gid": 100,
+  "tags": [
+    "NL"
+  ]
+}
+```
 
 Here it can be seen that this snapshot represents the contents of the directory
-`/tmp/testdata`. The most important field is `tree`.
+`/tmp/testdata`. The most important field is `tree`. When the meta data (e.g.
+the tags) of a snapshot change, the snapshot needs to be re-encrypted and saved.
+This will change the storage ID, so in order to relate these seemingly
+different snapshots, a field `original` is introduced which contains the ID of
+the original snapshot, e.g. after adding the tag `DE` to the snapshot above it
+becomes:
+
+```console
+$ restic -r /tmp/restic-repo cat snapshot 22a5af1b
+enter password for repository:
+{
+  "time": "2015-01-02T18:10:50.895208559+01:00",
+  "tree": "2da81727b6585232894cfbb8f8bdab8d1eccd3d8f7c92bc934d62e62e618ffdf",
+  "dir": "/tmp/testdata",
+  "hostname": "kasimir",
+  "username": "fd0",
+  "uid": 1000,
+  "gid": 100,
+  "tags": [
+    "NL",
+    "DE"
+  ],
+  "original": "251c2e5841355f743f9d4ffd3260bee765acee40a6229857e32b60446991b837"
+}
+```
+
+Once introduced, the `original` field is not modified when the snapshot's meta
+data is changed again.
 
 All content within a restic repository is referenced according to its SHA-256
 hash. Before saving, each file is split into variable sized Blobs of data. The
@@ -317,70 +357,77 @@ A snapshot references a tree by the SHA-256 hash of the JSON string
 representation of its contents. Trees and data are saved in pack files in a
 subdirectory of the directory `data`.
 
-The command `restic cat tree` can be used to inspect the tree referenced above:
+The command `restic cat blob` can be used to inspect the tree referenced above
+(piping the output of the command to `jq .` so that the JSON is indented):
 
-    $ restic -r /tmp/restic-repo cat tree b8138ab08a4722596ac89c917827358da4672eac68e3c03a8115b88dbf4bfb59
-    enter password for repository:
+```console
+$ restic -r /tmp/restic-repo cat blob b8138ab08a4722596ac89c917827358da4672eac68e3c03a8115b88dbf4bfb59 | jq .
+enter password for repository:
+{
+  "nodes": [
     {
-      "nodes": [
-        {
-          "name": "testdata",
-          "type": "dir",
-          "mode": 493,
-          "mtime": "2014-12-22T14:47:59.912418701+01:00",
-          "atime": "2014-12-06T17:49:21.748468803+01:00",
-          "ctime": "2014-12-22T14:47:59.912418701+01:00",
-          "uid": 1000,
-          "gid": 100,
-          "user": "fd0",
-          "inode": 409704562,
-          "content": null,
-          "subtree": "b26e315b0988ddcd1cee64c351d13a100fedbc9fdbb144a67d1b765ab280b4dc"
-        }
-      ]
+      "name": "testdata",
+      "type": "dir",
+      "mode": 493,
+      "mtime": "2014-12-22T14:47:59.912418701+01:00",
+      "atime": "2014-12-06T17:49:21.748468803+01:00",
+      "ctime": "2014-12-22T14:47:59.912418701+01:00",
+      "uid": 1000,
+      "gid": 100,
+      "user": "fd0",
+      "inode": 409704562,
+      "content": null,
+      "subtree": "b26e315b0988ddcd1cee64c351d13a100fedbc9fdbb144a67d1b765ab280b4dc"
     }
+  ]
+}
+```
 
 A tree contains a list of entries (in the field `nodes`) which contain meta
 data like a name and timestamps. When the entry references a directory, the
 field `subtree` contains the plain text ID of another tree object.
 
-When the command `restic cat tree` is used, the storage hash is needed to print
+When the command `restic cat blob` is used, the plaintext ID is needed to print
 a tree. The tree referenced above can be dumped as follows:
 
-    $ restic -r /tmp/restic-repo cat tree 8b238c8811cc362693e91a857460c78d3acf7d9edb2f111048691976803cf16e
-    enter password for repository:
+```console
+$ restic -r /tmp/restic-repo cat blob 8b238c8811cc362693e91a857460c78d3acf7d9edb2f111048691976803cf16e
+enter password for repository:
+{
+  "nodes": [
     {
-      "nodes": [
-        {
-          "name": "testfile",
-          "type": "file",
-          "mode": 420,
-          "mtime": "2014-12-06T17:50:23.34513538+01:00",
-          "atime": "2014-12-06T17:50:23.338468713+01:00",
-          "ctime": "2014-12-06T17:50:23.34513538+01:00",
-          "uid": 1000,
-          "gid": 100,
-          "user": "fd0",
-          "inode": 416863351,
-          "size": 1234,
-          "links": 1,
-          "content": [
-            "50f77b3b4291e8411a027b9f9b9e64658181cc676ce6ba9958b95f268cb1109d"
-          ]
-        },
-        [...]
+      "name": "testfile",
+      "type": "file",
+      "mode": 420,
+      "mtime": "2014-12-06T17:50:23.34513538+01:00",
+      "atime": "2014-12-06T17:50:23.338468713+01:00",
+      "ctime": "2014-12-06T17:50:23.34513538+01:00",
+      "uid": 1000,
+      "gid": 100,
+      "user": "fd0",
+      "inode": 416863351,
+      "size": 1234,
+      "links": 1,
+      "content": [
+        "50f77b3b4291e8411a027b9f9b9e64658181cc676ce6ba9958b95f268cb1109d"
       ]
-    }
+    },
+    [...]
+  ]
+}
+```
 
 This tree contains a file entry. This time, the `subtree` field is not present
 and the `content` field contains a list with one plain text SHA-256 hash.
 
-The command `restic cat data` can be used to extract and decrypt data given a
-plaintext ID, e.g. for the data mentioned above:
+The command `restic cat blob` can also be used to extract and decrypt data
+given a plaintext ID, e.g. for the data mentioned above:
 
-    $ restic -r /tmp/restic-repo cat blob 50f77b3b4291e8411a027b9f9b9e64658181cc676ce6ba9958b95f268cb1109d | sha256sum
-    enter password for repository:
-    50f77b3b4291e8411a027b9f9b9e64658181cc676ce6ba9958b95f268cb1109d  -
+```console
+$ restic -r /tmp/restic-repo cat blob 50f77b3b4291e8411a027b9f9b9e64658181cc676ce6ba9958b95f268cb1109d | sha256sum
+enter password for repository:
+50f77b3b4291e8411a027b9f9b9e64658181cc676ce6ba9958b95f268cb1109d  -
+```
 
 As can be seen from the output of the program `sha256sum`, the hash matches the
 plaintext hash from the map included in the tree above, so the correct data has
@@ -404,15 +451,17 @@ A lock is a file in the subdir `locks` whose filename is the storage ID of
 the contents. It is encrypted and authenticated the same way as other files
 in the repository and contains the following JSON structure:
 
-    {
-      "time": "2015-06-27T12:18:51.759239612+02:00",
-      "exclusive": false,
-      "hostname": "kasimir",
-      "username": "fd0",
-      "pid": 13607,
-      "uid": 1000,
-      "gid": 100
-    }
+```json
+{
+  "time": "2015-06-27T12:18:51.759239612+02:00",
+  "exclusive": false,
+  "hostname": "kasimir",
+  "username": "fd0",
+  "pid": 13607,
+  "uid": 1000,
+  "gid": 100
+}
+```
 
 The field `exclusive` defines the type of lock. When a new lock is to be
 created, restic checks all locks in the repository. When a lock is found, it

doc/FAQ.md

@@ -0,0 +1,20 @@
+FAQ
+===
+
+This is the list of Frequently Asked Questions for restic.
+
+`restic check` reports packs that aren't referenced in any index, is my repository broken?
+------------------------------------------------------------------------------------------
+
+When `restic check` reports that there are pack files in the repository that are not referenced in any index, that's (in contrast to what restic reports at the moment) not a source for concern. The output looks like this:
+
+    $ restic check
+    Create exclusive lock for repository
+    Load indexes
+    Check all packs
+    pack 819a9a52e4f51230afa89aefbf90df37fb70996337ae57e6f7a822959206a85e: not referenced in any index
+    pack de299e69fb075354a3775b6b045d152387201f1cdc229c31d1caa34c3b340141: not referenced in any index
+    Check snapshots, trees and blobs
+    Fatal: repository contains errors
+
+The message means that there is more data stored in the repo than strictly necessary. With high probability this is duplicate data. In order to clean it up, the command `restic prune` can be used. The cause of this bug is not yet known.

doc/Manual.md

@@ -1,107 +1,123 @@
 Thanks for using restic. This document will give you an overview of the basic
 functionality provided by restic.
 
-# Building/installing restic
+# Installing restic
+
+## from pre-compiled binary
+
+You can download the latest pre-compiled binary from the [restic release page](https://github.com/restic/restic/releases/latest).
+
+## Mac OS X
 
 If you are using Mac OS X, you can install restic using the
 [homebrew](http://brew.sh/) packet manager:
 
-    $ brew tap restic/restic
-    $ brew install restic
+```console
+$ brew tap restic/restic
+$ brew install restic
+```
+
+## archlinux
 
 On archlinux, there is a package called `restic-git` which can be installed from AUR, e.g. with `pacaur`:
 
-    $ pacaur -S restic-git
+```console
+$ pacaur -S restic-git
+```
 
-At debian stable you can install 'go' directly from the repositories (as root):
+# Building restic
 
-     $ apt-get install golang-go
-
-after installation of 'go' go straight forward to 'git clone [...]'
-
-If you are using Linux, BSD or Windows, the only way to install restic on your
-system right now is to compile it from source. restic is written in the Go
-programming language and you need at least Go version 1.6. Building restic may
-also work with older versions of Go, but that's not supported. See the [Getting
-started](https://golang.org/doc/install) guide of the Go project for
+restic is written in the Go programming language and you need at least Go version 1.7.
+Building restic may also work with older versions of Go, but that's not supported.
+See the [Getting started](https://golang.org/doc/install) guide of the Go project for
 instructions how to install Go.
 
 In order to build restic from source, execute the following steps:
 
-    $ git clone https://github.com/restic/restic
-    [...]
+```console
+$ git clone https://github.com/restic/restic
+[...]
 
-    $ cd restic
+$ cd restic
 
-    $ go run build.go
+$ go run build.go
+```
 
 At the moment, the only tested compiler for restic is the official Go compiler.
 Building restic with gccgo may work, but is not supported.
 
+# Usage help
+
 Usage help is available:
 
-    $ ./restic --help
-    Usage:
-      restic [OPTIONS] <command>
+```console
+$ ./restic --help
+restic is a backup program which allows saving multiple revisions of files and
+directories in an encrypted repository stored on different backends.
 
-    Application Options:
-      -r, --repo=      Repository directory to backup to/restore from
-          --cache-dir= Directory to use as a local cache
-      -q, --quiet      Do not output comprehensive progress report (false)
-          --no-lock    Do not lock the repo, this allows some operations on read-only repos. (false)
-      -o, --option=    Specify options in the form 'foo.key=value'
+Usage:
+  restic [command]
 
-    Help Options:
-      -h, --help       Show this help message
+Available Commands:
+  backup        create a new backup of files and/or directories
+  cat           print internal objects to stdout
+  check         check the repository for errors
+  find          find a file or directory
+  forget        forget removes snapshots from the repository
+  init          initialize a new repository
+  key           manage keys (passwords)
+  list          list items in the repository
+  ls            list files in a snapshot
+  mount         mount the repository
+  prune         remove unneeded data from the repository
+  rebuild-index build a new index file
+  restore       extract the data from a snapshot
+  snapshots     list all snapshots
+  tag           modifies tags on snapshots
+  unlock        remove locks other processes created
+  version       Print version information
 
-    Available commands:
-      backup         save file/directory
-      cat            dump something
-      check          check the repository
-      find           find a file/directory
-      forget         removes snapshots from a repository
-      init           create repository
-      key            manage keys
-      list           lists data
-      ls             list files
-      mount          mount a repository
-      prune          removes content from a repository
-      rebuild-index  rebuild the index
-      restore        restore a snapshot
-      snapshots      show snapshots
-      unlock         remove locks
-      version        display version
+Flags:
+      --json                   set output mode to JSON for commands that support it
+      --no-lock                do not lock the repo, this allows some operations on read-only repos
+  -p, --password-file string   read the repository password from a file
+  -q, --quiet                  do not output comprehensive progress report
+  -r, --repo string            repository to backup to or restore from (default: $RESTIC_REPOSITORY)
+
+Use "restic [command] --help" for more information about a command.
+```
 
 Similar to programs such as `git`, restic has a number of sub-commands. You can
 see these commands in the listing above. Each sub-command may have own
 command-line options, and there is a help option for each command which lists
 them, e.g. for the `backup` command:
 
-    $ ./restic backup --help
-    Usage:
-      restic [OPTIONS] backup DIR/FILE [DIR/FILE] [...]
+```console
+$ ./restic backup --help
+The "backup" command creates a new snapshot and saves the files and directories
+given as the arguments.
 
-    The backup command creates a snapshot of a file or directory
+Usage:
+  restic backup [flags] FILE/DIR [FILE/DIR] ...
 
-    Application Options:
-      -r, --repo=               Repository directory to backup to/restore from (/tmp/repo)
-      -p, --password-file=      Read the repository password from a file
-          --cache-dir=          Directory to use as a local cache
-      -q, --quiet               Do not output comprehensive progress report (false)
-          --no-lock             Do not lock the repo, this allows some operations on read-only repos. (false)
-      -o, --option=             Specify options in the form 'foo.key=value'
+Flags:
+  -e, --exclude pattern         exclude a pattern (can be specified multiple times)
+      --exclude-file string     read exclude patterns from a file
+      --files-from string       read the files to backup from file (can be combined with file args)
+  -f, --force                   force re-reading the target files/directories. Overrides the "parent" flag
+  -x, --one-file-system         Exclude other file systems
+      --parent string           use this parent snapshot (default: last snapshot in the repo that has the same target files/directories)
+      --stdin                   read backup from stdin
+      --stdin-filename string   file name to use when reading from stdin
+      --tag tag                 add a tag for the new snapshot (can be specified multiple times)
 
-    Help Options:
-      -h, --help                Show this help message
-
-    [backup command options]
-          -p, --parent=         use this parent snapshot (default: last snapshot in repo that has the same target)
-          -f, --force           Force re-reading the target. Overrides the "parent" flag
-          -e, --exclude=        Exclude a pattern (can be specified multiple times)
-              --exclude-file=   Read exclude-patterns from file
-              --stdin           read backup data from stdin
-              --stdin-filename= file name to use when reading from stdin (stdin)
-              --tag=            Add a tag (can be specified multiple times)
+Global Flags:
+      --json                   set output mode to JSON for commands that support it
+      --no-lock                do not lock the repo, this allows some operations on read-only repos
+  -p, --password-file string   read the repository password from a file
+  -q, --quiet                  do not output comprehensive progress report
+  -r, --repo string            repository to backup to or restore from (default: $RESTIC_REPOSITORY)
+```
 
 Subcommand that support showing progress information such as `backup`, `check` and `prune` will do so unless
 the quiet flag `-q` or `--quiet` is set. When running from a non-interactive console progress reporting will
@@ -119,12 +135,16 @@ will be saved at.
 In order to create a repository at `/tmp/backup`, run the following command and
 enter the same password twice:
 
-    $ restic init --repo /tmp/backup
-    enter password for new backend:
-    enter password again:
-    created restic backend 085b3c76b9 at /tmp/backup
-    Please note that knowledge of your password is required to access the repository.
-    Losing your password means that your data is irrecoverably lost.
+```console
+$ restic init --repo /tmp/backup
+enter password for new backend:
+enter password again:
+created restic backend 085b3c76b9 at /tmp/backup
+Please note that knowledge of your password is required to access the repository.
+Losing your password means that your data is irrecoverably lost.
+```
+
+Other backends like sftp and s3 are [described in a later section](#create-an-sftp-repository) of this document.
 
 Remembering your password is important! If you lose it, you won't be able to
 access data stored in the repository.
@@ -146,8 +166,10 @@ You can workaround this by using a special tool called `winpty` (look
 [here](https://github.com/rprichard/winpty) for detail information). On MSYS2,
 you can install `winpty` as follows:
 
-    $ pacman -S winpty
-    $ winpty restic -r /tmp/backup init
+```console
+$ pacman -S winpty
+$ winpty restic -r /tmp/backup init
+```
 
 # Create a snapshot
 
@@ -155,13 +177,15 @@ Now we're ready to backup some data. The contents of a directory at a specific
 point in time is called a "snapshot" in restic. Run the following command and
 enter the repository password you chose above again:
 
-    $ restic -r /tmp/backup backup ~/work
-    enter password for repository:
-    scan [/home/user/work]
-    scanned 764 directories, 1816 files in 0:00
-    [0:29] 100.00%  54.732 MiB/s  1.582 GiB / 1.582 GiB  2580 / 2580 items  0 errors  ETA 0:00
-    duration: 0:29, 54.47MiB/s
-    snapshot 40dc1520 saved
+```console
+$ restic -r /tmp/backup backup ~/work
+enter password for repository:
+scan [/home/user/work]
+scanned 764 directories, 1816 files in 0:00
+[0:29] 100.00%  54.732 MiB/s  1.582 GiB / 1.582 GiB  2580 / 2580 items  0 errors  ETA 0:00
+duration: 0:29, 54.47MiB/s
+snapshot 40dc1520 saved
+```
 
 As you can see, restic created a backup of the directory and was pretty fast!
 The specific snapshot just created is identified by a sequence of hexadecimal
@@ -170,40 +194,55 @@ characters, `40dc1520` in this case.
 If you run the command again, restic will create another snapshot of your data,
 but this time it's even faster. This is de-duplication at work!
 
-    $ restic -r /tmp/backup backup ~/shared/work/web
-    enter password for repository:
-    using parent snapshot 40dc1520aa6a07b7b3ae561786770a01951245d2367241e71e9485f18ae8228c
-    scan [/home/user/work]
-    scanned 764 directories, 1816 files in 0:00
-    [0:00] 100.00%  0B/s  1.582 GiB / 1.582 GiB  2580 / 2580 items  0 errors  ETA 0:00
-    duration: 0:00, 6572.38MiB/s
-    snapshot 79766175 saved
+```console
+$ restic -r /tmp/backup backup ~/shared/work/web
+enter password for repository:
+using parent snapshot 40dc1520aa6a07b7b3ae561786770a01951245d2367241e71e9485f18ae8228c
+scan [/home/user/work]
+scanned 764 directories, 1816 files in 0:00
+[0:00] 100.00%  0B/s  1.582 GiB / 1.582 GiB  2580 / 2580 items  0 errors  ETA 0:00
+duration: 0:00, 6572.38MiB/s
+snapshot 79766175 saved
+```
 
 You can even backup individual files in the same repository.
 
-    $ restic -r /tmp/backup backup ~/work.txt
-    scan [~/work.txt]
-    scanned 0 directories, 1 files in 0:00
-    [0:00] 100.00%  0B/s  220B / 220B  1 / 1 items  0 errors  ETA 0:00
-    duration: 0:00, 0.03MiB/s
-    snapshot 31f7bd63 saved
+```console
+$ restic -r /tmp/backup backup ~/work.txt
+scan [~/work.txt]
+scanned 0 directories, 1 files in 0:00
+[0:00] 100.00%  0B/s  220B / 220B  1 / 1 items  0 errors  ETA 0:00
+duration: 0:00, 0.03MiB/s
+snapshot 31f7bd63 saved
+```
 
 In fact several hosts may use the same repository to backup directories and
 files leading to a greater de-duplication.
 
-You can exclude folders and files by specifying exclude-patterns.  
+Please be aware that when you backup different directories (or the directories
+to be saved have a variable name component like a time/date), restic always
+needs to read all files and only afterwards can compute which parts of the
+files need to be saved. When you backup the same directory again (maybe with
+new or changed files) restic will find the old snapshot in the repo and by
+default only reads those files that are new or have been modified since the
+last snapshot. This is decided based on the modify date of the file in the
+file system.
+
+You can exclude folders and files by specifying exclude-patterns.
 Either specify them with multiple `--exclude`'s or one `--exclude-file`
 
-    $ cat exclude
-    # exclude go-files
-    *.go
-    # exclude foo/x/y/z/bar foo/x/bar foo/bar
-    foo/**/bar
-    $ restic -r /tmp/backup backup ~/work --exclude=*.c --exclude-file=exclude
+```console
+$ cat exclude
+# exclude go-files
+*.go
+# exclude foo/x/y/z/bar foo/x/bar foo/bar
+foo/**/bar
+$ restic -r /tmp/backup backup ~/work --exclude=*.c --exclude-file=exclude
+```
 
 Patterns use [`filepath.Glob`](https://golang.org/pkg/path/filepath/#Glob) internally,
 see [`filepath.Match`](https://golang.org/pkg/path/filepath/#Match) for syntax.
-Additionally `**` exludes arbitrary subdirectories.  
+Additionally `**` excludes arbitrary subdirectories.
 Environment-variables in exclude-files are expanded with [`os.ExpandEnv`](https://golang.org/pkg/os/#ExpandEnv).
 
 By specifying the option `--one-file-system` you can instruct restic to only
@@ -211,7 +250,30 @@ backup files from the file systems the initially specified files or directories
 reside on. For example, calling restic like this won't backup `/sys` or
 `/dev` on a Linux system:
 
-    $ restic -r /tmp/backup backup --one-file-system /
+```console
+$ restic -r /tmp/backup backup --one-file-system /
+```
+By using the `--files-from` option you can read the files you want to backup
+from a file. This is especially useful if a lot of files have to be backed up
+that are not in the same folder or are maybe pre-filtered by other software.
+
+For example maybe you want to backup files that have a certain filename in them:
+
+```console
+$ find /tmp/somefiles | grep 'PATTERN' > /tmp/files_to_backup
+```
+
+You can then use restic to backup the filtered files:
+
+```console
+$ restic -r /tmp/backup backup --files-from /tmp/files_to_backup
+```
+
+Incidentally you can also combine `--files-from` with the normal files args:
+
+```console
+$ restic -r /tmp/backup backup --files-from /tmp/files_to_backup /tmp/some_additional_file
+```
 
 ## Reading data from stdin
 
@@ -219,7 +281,9 @@ Sometimes it can be nice to directly save the output of a program, e.g.
 `mysqldump` so that the SQL can later be restored. Restic supports this mode of
 operation, just supply the option `--stdin` to the `backup` command like this:
 
-    $ mysqldump [...] | restic -r /tmp/backup backup --stdin
+```console
+$ mysqldump [...] | restic -r /tmp/backup backup --stdin
+```
 
 This creates a new snapshot of the output of `mysqldump`. You can then use e.g.
 the fuse mounting option (see below) to mount the repository and read the file.
@@ -227,15 +291,19 @@ the fuse mounting option (see below) to mount the repository and read the file.
 By default, the file name `stdin` is used, a different name can be specified
 with `--stdin-filename`, e.g. like this:
 
-    $ mysqldump [...] | restic -r /tmp/backup backup --stdin --stdin-filename production.sql
+```console
+$ mysqldump [...] | restic -r /tmp/backup backup --stdin --stdin-filename production.sql
+```
 
 ## Tags
 
 Snapshots can have one or more tags, short strings which add identifying
 information. Just specify the tags for a snapshot with `--tag`:
 
-    $ restic -r /tmp/backup backup --tag projectX ~/shared/work/web
-    [...]
+```console
+$ restic -r /tmp/backup backup --tag projectX ~/shared/work/web
+[...]
+```
 
 The tags can later be used to keep (or forget) snapshots.
 
@@ -243,53 +311,62 @@ The tags can later be used to keep (or forget) snapshots.
 
 Now, you can list all the snapshots stored in the repository:
 
-    $ restic -r /tmp/backup snapshots
-    enter password for repository:
-    ID        Date                 Host    Tags   Directory
-    ----------------------------------------------------------------------
-    40dc1520  2015-05-08 21:38:30  kasimir        /home/user/work
-    79766175  2015-05-08 21:40:19  kasimir        /home/user/work
-    bdbd3439  2015-05-08 21:45:17  luigi          /home/art
-    590c8fc8  2015-05-08 21:47:38  kazik          /srv
-    9f0bc19e  2015-05-08 21:46:11  luigi          /srv
+```console
+$ restic -r /tmp/backup snapshots
+enter password for repository:
+ID        Date                 Host    Tags   Directory
+----------------------------------------------------------------------
+40dc1520  2015-05-08 21:38:30  kasimir        /home/user/work
+79766175  2015-05-08 21:40:19  kasimir        /home/user/work
+bdbd3439  2015-05-08 21:45:17  luigi          /home/art
+590c8fc8  2015-05-08 21:47:38  kazik          /srv
+9f0bc19e  2015-05-08 21:46:11  luigi          /srv
+```
 
 You can filter the listing by directory path:
 
-    $ restic -r /tmp/backup snapshots --path="/srv"
-    enter password for repository:
-    ID        Date                 Host    Tags   Directory
-    ----------------------------------------------------------------------
-    590c8fc8  2015-05-08 21:47:38  kazik          /srv
-    9f0bc19e  2015-05-08 21:46:11  luigi          /srv
+```console
+$ restic -r /tmp/backup snapshots --path="/srv"
+enter password for repository:
+ID        Date                 Host    Tags   Directory
+----------------------------------------------------------------------
+590c8fc8  2015-05-08 21:47:38  kazik          /srv
+9f0bc19e  2015-05-08 21:46:11  luigi          /srv
+```
 
 Or filter by host:
 
-    $ restic -r /tmp/backup snapshots --host luigi
-    enter password for repository:
-    ID        Date                 Host    Tags   Directory
-    ----------------------------------------------------------------------
-    bdbd3439  2015-05-08 21:45:17  luigi          /home/art
-    9f0bc19e  2015-05-08 21:46:11  luigi          /srv
+```console
+$ restic -r /tmp/backup snapshots --host luigi
+enter password for repository:
+ID        Date                 Host    Tags   Directory
+----------------------------------------------------------------------
+bdbd3439  2015-05-08 21:45:17  luigi          /home/art
+9f0bc19e  2015-05-08 21:46:11  luigi          /srv
+```
 
-Combining filters is also possible.    
+Combining filters is also possible.
 
 # Restore a snapshot
 
 Restoring a snapshot is as easy as it sounds, just use the following command to
 restore the contents of the latest snapshot to `/tmp/restore-work`:
 
-    $ restic -r /tmp/backup restore 79766175 --target ~/tmp/restore-work
-    enter password for repository:
-    restoring <Snapshot of [/home/user/work] at 2015-05-08 21:40:19.884408621 +0200 CEST> to /tmp/restore-work
+```console
+$ restic -r /tmp/backup restore 79766175 --target ~/tmp/restore-work
+enter password for repository:
+restoring <Snapshot of [/home/user/work] at 2015-05-08 21:40:19.884408621 +0200 CEST> to /tmp/restore-work
+```
 
 Use the word `latest` to restore the last backup. You can also combine `latest`
 with the `--host` and `--path` filters to choose the last backup for a specific
 host, path or both.
 
-    $ restic -r /tmp/backup restore latest --target ~/tmp/restore-work --path "/home/art" --host luigi
-    enter password for repository:
-    restoring <Snapshot of [/home/art] at 2015-05-08 21:45:17.884408621 +0200 CEST> to /tmp/restore-work
-
+```console
+$ restic -r /tmp/backup restore latest --target ~/tmp/restore-work --path "/home/art" --host luigi
+enter password for repository:
+restoring <Snapshot of [/home/art] at 2015-05-08 21:45:17.884408621 +0200 CEST> to /tmp/restore-work
+```
 
 # Manage repository keys
 
@@ -297,24 +374,65 @@ The `key` command allows you to set multiple access keys or passwords per
 repository. In fact, you can use the `list`, `add`, `remove` and `passwd`
 sub-commands to manage these keys very precisely:
 
-    $ restic -r /tmp/backup key list
-    enter password for repository:
-     ID          User        Host        Created
-    ----------------------------------------------------------------------
-    *eb78040b    username    kasimir   2015-08-12 13:29:57
+```console
+$ restic -r /tmp/backup key list
+enter password for repository:
+ ID          User        Host        Created
+----------------------------------------------------------------------
+*eb78040b    username    kasimir   2015-08-12 13:29:57
 
-    $ restic -r /tmp/backup key add
-    enter password for repository:
-    enter password for new key:
-    enter password again:
-    saved new key as <Key of username@kasimir, created on 2015-08-12 13:35:05.316831933 +0200 CEST>
+$ restic -r /tmp/backup key add
+enter password for repository:
+enter password for new key:
+enter password again:
+saved new key as <Key of username@kasimir, created on 2015-08-12 13:35:05.316831933 +0200 CEST>
 
-    $ restic -r backup key list
-    enter password for repository:
-     ID          User        Host        Created
-    ----------------------------------------------------------------------
-     5c657874    username    kasimir   2015-08-12 13:35:05
-    *eb78040b    username    kasimir   2015-08-12 13:29:57
+$ restic -r backup key list
+enter password for repository:
+ ID          User        Host        Created
+----------------------------------------------------------------------
+ 5c657874    username    kasimir   2015-08-12 13:35:05
+*eb78040b    username    kasimir   2015-08-12 13:29:57
+```
+
+# Manage tags
+
+Managing tags on snapshots is done with the `tag` command. The existing set of
+tags can be replaced completely, tags can be added to removed. The result is
+directly visible in the `snapshots` command.
+
+Let's say we want to tag snapshot `590c8fc8` with the tags `NL` and `CH` and
+remove all other tags that may be present, the following command does that:
+
+```console
+$ restic -r /tmp/backup tag --set NL,CH 590c8fc8
+Create exclusive lock for repository
+Modified tags on 1 snapshots
+```
+
+Note the snapshot ID has changed, so between each change we need to look up
+the new ID of the snapshot. But there is an even better way, the `tag` command
+accepts `--tag` for a filter, so we can filter snapshots based on the tag we
+just added.
+
+So we can add and remove tags incrementally like this:
+
+```console
+$ restic -r /tmp/backup tag --tag NL --remove CH
+Create exclusive lock for repository
+Modified tags on 1 snapshots
+
+$ restic -r /tmp/backup tag --tag NL --add UK
+Create exclusive lock for repository
+Modified tags on 1 snapshots
+
+$ restic -r /tmp/backup tag --tag NL --remove NL
+Create exclusive lock for repository
+Modified tags on 1 snapshots
+
+$ restic -r /tmp/backup tag --tag NL --add SOMETHING
+No snapshots were modified
+```
 
 # Check integrity and consistency
 
@@ -322,22 +440,28 @@ Imagine your repository is saved on a server that has a faulty hard drive, or
 even worse, attackers get privileged access and modify your backup with the
 intention to make you restore malicious data:
 
-    $ sudo echo "boom" >> backup/index/d795ffa99a8ab8f8e42cec1f814df4e48b8f49129360fb57613df93739faee97
+```console
+$ sudo echo "boom" >> backup/index/d795ffa99a8ab8f8e42cec1f814df4e48b8f49129360fb57613df93739faee97
+```
 
 In order to detect these things, it is a good idea to regularly use the `check`
 command to test whether everything is alright, your precious backup data is
 consistent and the integrity is unharmed:
 
-    $ restic -r /tmp/backup check
-    Load indexes
-    ciphertext verification failed
+```console
+$ restic -r /tmp/backup check
+Load indexes
+ciphertext verification failed
+```
 
 Trying to restore a snapshot which has been modified as shown above will yield
 the same error:
 
-    $ restic -r /tmp/backup restore 79766175 --target ~/tmp/restore-work
-    Load indexes
-    ciphertext verification failed
+```console
+$ restic -r /tmp/backup restore 79766175 --target ~/tmp/restore-work
+Load indexes
+ciphertext verification failed
+```
 
 # Mount a repository
 
@@ -345,14 +469,21 @@ Browsing your backup as a regular file system is also very easy. First, create
 a mount point such as `/mnt/restic` and then use the following command to serve
 the repository with FUSE:
 
-    $ mkdir /mnt/restic
-    $ restic -r /tmp/backup mount /mnt/restic
-    enter password for repository:
-    Now serving /tmp/backup at /tmp/restic
-    Don't forget to umount after quitting!
+```console
+$ mkdir /mnt/restic
+$ restic -r /tmp/backup mount /mnt/restic
+enter password for repository:
+Now serving /tmp/backup at /tmp/restic
+Don't forget to umount after quitting!
+```
 
 Mounting repositories via FUSE is not possible on Windows and OpenBSD.
 
+Restic supports storage and preservation of hard links. However, since hard links
+exist in the scope of a filesystem by definition, restoring hard links from a fuse
+mount should be done by a program that preserves hard links. A program that does so
+is rsync, used with the option --hard-links.
+
 # Create an SFTP repository
 
 In order to backup data via SFTP, you must first set up a server with SSH and
@@ -363,34 +494,71 @@ credentials.
 Once the server is configured, the setup of the SFTP repository can simply be
 achieved by changing the URL scheme in the `init` command:
 
-    $ restic -r sftp:user@host:/tmp/backup init
-    enter password for new backend:
-    enter password again:
-    created restic backend f1c6108821 at sftp:user@host:/tmp/backup
-    Please note that knowledge of your password is required to access the repository.
-    Losing your password means that your data is irrecoverably lost.
+```console
+$ restic -r sftp:user@host:/tmp/backup init
+enter password for new backend:
+enter password again:
+created restic backend f1c6108821 at sftp:user@host:/tmp/backup
+Please note that knowledge of your password is required to access the repository.
+Losing your password means that your data is irrecoverably lost.
+```
 
 You can also specify a relative (read: no slash (`/`) character at the
 beginning) directory, in this case the dir is relative to the remote user's
 home directory.
 
+# Create a REST server repository
+
+In order to backup data to the remote server via HTTP or HTTPS protocol,
+you must first set up a remote [REST server](https://github.com/restic/rest-server)
+instance.  Once the server is configured, accessing it is achieved by changing the
+URL scheme like this:
+
+```console
+$ restic -r rest:http://host:8000/
+```
+
+Depending on your REST server setup, you can use HTTPS protocol, password
+protection, or multiple repositories.  Or any combination of those features, as
+you see fit. TCP/IP port is also configurable. Here are some more examples:
+
+```console
+$ restic -r rest:https://host:8000/
+$ restic -r rest:https://user:pass@host:8000/
+$ restic -r rest:https://user:pass@host:8000/my_backup_repo/
+```
+
+If you use TLS, make sure your certificates are signed, 'cause restic client
+will refuse to communicate otherwise.  It's easy to obtain such certificates
+today, thanks to free certificate authorities like [Let’s
+Encrypt](https://letsencrypt.org/).
+
+REST server uses exactly the same directory structure as local backend, so you
+should be able to access it both locally and via HTTP, even simultaneously.
+
 # Create an Amazon S3 repository
 
 Restic can backup data to any Amazon S3 bucket. However, in this case, changing the URL scheme is not enough since Amazon uses special security credentials to sign HTTP requests. By consequence, you must first setup the following environment variables with the credentials you obtained while creating the bucket.
 
-    $ export AWS_ACCESS_KEY_ID=<MY_ACCESS_KEY>
-    $ export AWS_SECRET_ACCESS_KEY=<MY_SECRET_ACCESS_KEY>
+```console
+$ export AWS_ACCESS_KEY_ID=<MY_ACCESS_KEY>
+$ export AWS_SECRET_ACCESS_KEY=<MY_SECRET_ACCESS_KEY>
+```
 
-You can then easily initialize a repository that uses your Amazon S3 as a backend.
+You can then easily initialize a repository that uses your Amazon S3 as a backend, if the bucket does not exist yet it will be created in the default location:
 
-    $ restic -r s3:eu-central-1/bucket_name init
-    enter password for new backend:
-    enter password again:
-    created restic backend eefee03bbd at s3:eu-central-1/bucket_name
-    Please note that knowledge of your password is required to access the repository.
-    Losing your password means that your data is irrecoverably lost.
+```console
+$ restic -r s3:s3.amazonaws.com/bucket_name init
+enter password for new backend:
+enter password again:
+created restic backend eefee03bbd at s3:s3.amazonaws.com/bucket_name
+Please note that knowledge of your password is required to access the repository.
+Losing your password means that your data is irrecoverably lost.
+```
 
-Fro an s3-compatible server that is not Amazon (like Minio, see below), or is
+It is not possible at the moment to have restic create a new bucket in a different location, so you need to create it using a different program. Afterwards, the S3 server (`s3.amazonaws.com`) will redirect restic to the correct endpoint.
+
+For an S3-compatible server that is not Amazon (like Minio, see below), or is
 only available via HTTP, you can specify the URL to the server like this:
 `s3:http://server:port/bucket_name`.
 
@@ -400,22 +568,26 @@ only available via HTTP, you can specify the URL to the server like this:
 
 ### Pre-Requisites
 
-* Download and Install [Minio Server](https://minio.io/download/). 
-* You can also refer to [https://docs.minio.io](https://docs.minio.io) for step by step guidance on installation and getting started on Minio CLient and Minio Server.
+* Download and Install [Minio Server](https://minio.io/downloads/#minio-server).
+* You can also refer to [https://docs.minio.io](https://docs.minio.io) for step by step guidance on installation and getting started on Minio Client and Minio Server.
 
 You must first setup the following environment variables with the credentials of your running Minio Server.
 
-    $ export AWS_ACCESS_KEY_ID=<YOUR-MINIO-ACCESS-KEY-ID>
-    $ export AWS_SECRET_ACCESS_KEY= <YOUR-MINIO-SECRET-ACCESS-KEY>
+```console
+$ export AWS_ACCESS_KEY_ID=<YOUR-MINIO-ACCESS-KEY-ID>
+$ export AWS_SECRET_ACCESS_KEY= <YOUR-MINIO-SECRET-ACCESS-KEY>
+```
 
 Now you can easily initialize restic to use Minio server as backend with this command.
 
-    $ ./restic -r s3:http://localhost:9000/restic init
-    enter password for new backend: 
-    enter password again: 
-    created restic backend 6ad29560f5 at s3:http://localhost:9000/restic1
-    Please note that knowledge of your password is required to access
-    the repository. Losing your password means that your data is irrecoverably lost. 
+```console
+$ ./restic -r s3:http://localhost:9000/restic init
+enter password for new backend:
+enter password again:
+created restic backend 6ad29560f5 at s3:http://localhost:9000/restic1
+Please note that knowledge of your password is required to access
+the repository. Losing your password means that your data is irrecoverably lost.
+```
 
 # Removing old snapshots
 
@@ -424,63 +596,107 @@ be done either manually (by specifying a snapshot ID to remove) or by using a
 policy that describes which snapshots to forget. For all remove operations, two
 commands need to be called in sequence: `forget` to remove a snapshot and
 `prune` to actually remove the data that was referenced by the snapshot from
-the repository.
+the repository. This can be automated with the `--prune` option of the `forget`
+command, which runs `prune` automatically if snapshots have been removed.
 
 ## Remove a single snapshot
 
 The command `snapshots` can be used to list all snapshots in a repository like this:
 
-    $ restic -r /tmp/backup snapshots
-    enter password for repository:
-    ID        Date                 Host      Tags  Directory
-    ----------------------------------------------------------------------
-    40dc1520  2015-05-08 21:38:30  kasimir         /home/user/work
-    79766175  2015-05-08 21:40:19  kasimir         /home/user/work
-    bdbd3439  2015-05-08 21:45:17  luigi           /home/art
-    590c8fc8  2015-05-08 21:47:38  kazik           /srv
-    9f0bc19e  2015-05-08 21:46:11  luigi           /srv
+```console
+$ restic -r /tmp/backup snapshots
+enter password for repository:
+ID        Date                 Host      Tags  Directory
+----------------------------------------------------------------------
+40dc1520  2015-05-08 21:38:30  kasimir         /home/user/work
+79766175  2015-05-08 21:40:19  kasimir         /home/user/work
+bdbd3439  2015-05-08 21:45:17  luigi           /home/art
+590c8fc8  2015-05-08 21:47:38  kazik           /srv
+9f0bc19e  2015-05-08 21:46:11  luigi           /srv
+```
 
 In order to remove the snapshot of `/home/art`, use the `forget` command and
 specify the snapshot ID on the command line:
 
-    $ restic -r /tmp/backup forget bdbd3439
-    enter password for repository:
-    removed snapshot d3f01f63
+```console
+$ restic -r /tmp/backup forget bdbd3439
+enter password for repository:
+removed snapshot d3f01f63
+```
 
 Afterwards this snapshot is removed:
 
-    $ restic -r /tmp/backup snapshots
-    enter password for repository:
-    ID        Date                 Host     Tags  Directory
-    ----------------------------------------------------------------------
-    40dc1520  2015-05-08 21:38:30  kasimir        /home/user/work
-    79766175  2015-05-08 21:40:19  kasimir        /home/user/work
-    590c8fc8  2015-05-08 21:47:38  kazik          /srv
-    9f0bc19e  2015-05-08 21:46:11  luigi          /srv
+```console
+$ restic -r /tmp/backup snapshots
+enter password for repository:
+ID        Date                 Host     Tags  Directory
+----------------------------------------------------------------------
+40dc1520  2015-05-08 21:38:30  kasimir        /home/user/work
+79766175  2015-05-08 21:40:19  kasimir        /home/user/work
+590c8fc8  2015-05-08 21:47:38  kazik          /srv
+9f0bc19e  2015-05-08 21:46:11  luigi          /srv
+```
 
 But the data that was referenced by files in this snapshot is still stored in
 the repository. To cleanup unreferenced data, the `prune` command must be run:
 
-    $ restic -r /tmp/backup prune
-    enter password for repository:
+```console
+$ restic -r /tmp/backup prune
+enter password for repository:
 
-    counting files in repo
-    building new index for repo
-    [0:00] 100.00%  22 / 22 files
-    repository contains 22 packs (8512 blobs) with 100.092 MiB bytes
-    processed 8512 blobs: 0 duplicate blobs, 0B duplicate
-    load all snapshots
-    find data that is still in use for 1 snapshots
-    [0:00] 100.00%  1 / 1 snapshots
-    found 8433 of 8512 data blobs still in use
-    will rewrite 3 packs
-    creating new index
-    [0:00] 86.36%  19 / 22 files
-    saved new index as 544a5084
-    done
+counting files in repo
+building new index for repo
+[0:00] 100.00%  22 / 22 files
+repository contains 22 packs (8512 blobs) with 100.092 MiB bytes
+processed 8512 blobs: 0 duplicate blobs, 0B duplicate
+load all snapshots
+find data that is still in use for 1 snapshots
+[0:00] 100.00%  1 / 1 snapshots
+found 8433 of 8512 data blobs still in use
+will rewrite 3 packs
+creating new index
+[0:00] 86.36%  19 / 22 files
+saved new index as 544a5084
+done
+```
 
 Afterwards the repository is smaller.
 
+You can automate this two-step process by using the `--prune` switch to
+`forget`:
+
+```console
+$ restic forget --keep-last 1 --prune
+snapshots for host mopped, directories /home/user/work:
+
+keep 1 snapshots:
+ID        Date                 Host        Tags        Directory
+----------------------------------------------------------------------
+4bba301e  2017-02-21 10:49:18  mopped                  /home/user/work
+
+remove 1 snapshots:
+ID        Date                 Host        Tags        Directory
+----------------------------------------------------------------------
+8c02b94b  2017-02-21 10:48:33  mopped                  /home/user/work
+
+1 snapshots have been removed, running prune
+counting files in repo
+building new index for repo
+[0:00] 100.00%  37 / 37 packs
+repository contains 37 packs (5521 blobs) with 151.012 MiB bytes
+processed 5521 blobs: 0 duplicate blobs, 0B duplicate
+load all snapshots
+find data that is still in use for 1 snapshots
+[0:00] 100.00%  1 / 1 snapshots
+found 5323 of 5521 data blobs still in use, removing 198 blobs
+will delete 0 packs and rewrite 27 packs, this frees 22.106 MiB
+creating new index
+[0:00] 100.00%  30 / 30 packs
+saved new index as b49f3e68
+done
+```
+
+
 ## Removing snapshots according to a policy
 
 Removing snapshots manually is tedious and error-prone, therefore restic allows
@@ -530,12 +746,16 @@ all snapshots!
 
 The program can be built with debug support like this:
 
-    $ go run build.go -tags debug
+```console
+$ go run build.go -tags debug
+```
 
 Afterwards, extensive debug messages are written to the file in environment
 variable `DEBUG_LOG`, e.g.:
 
-    $ DEBUG_LOG=/tmp/restic-debug.log restic backup ~/work
+```console
+$ DEBUG_LOG=/tmp/restic-debug.log restic backup ~/work
+```
 
 If you suspect that there is a bug, you can have a look at the debug log.
 Please be aware that the debug log might contain sensitive information such as
@@ -551,23 +771,29 @@ separated by commas. Patterns are case sensitive.
 Printing all log messages to the console can be achieved by setting the file
 filter to `*`:
 
-    $ DEBUG_FILES=* restic check
+```console
+$ DEBUG_FILES=* restic check
+```
 
 If you want restic to just print all debug log messages from the files
-`main.go` and `lock.go`, set the environment variable `DEBUG_FILES` like this: 
+`main.go` and `lock.go`, set the environment variable `DEBUG_FILES` like this:
 
-    $ DEBUG_FILES=main.go,lock.go restic check
+```console
+$ DEBUG_FILES=main.go,lock.go restic check
+```
 
 The following command line instructs restic to only print debug statements
 originating in functions that match the pattern `*unlock*` (case sensitive):
 
-    $ DEBUG_FUNCS=*unlock* restic check
+```console
+$ DEBUG_FUNCS=*unlock* restic check
+```
 
 # Under the hood: Browse repository objects
 
 Internally, a repository stores data of several different types described in the [design documentation](https://github.com/restic/restic/blob/master/doc/Design.md). You can `list` objects such as blobs, packs, index, snapshots, keys or locks with the following command:
 
-```shell
+```console
 $ restic -r /tmp/backup list snapshots
 d369ccc7d126594950bf74f0a348d5d98d9e99f3215082eb69bf02dc9b3e464c
 ```
@@ -575,26 +801,37 @@ d369ccc7d126594950bf74f0a348d5d98d9e99f3215082eb69bf02dc9b3e464c
 The `find` command searches for a given
 [pattern](http://golang.org/pkg/path/filepath/#Match) in the repository.
 
-    $ restic -r backup find test.txt
-    debug log file restic.log
-    debug enabled
-    enter password for repository:
-    found 1 matching entries in snapshot 196bc5760c909a7681647949e80e5448e276521489558525680acf1bd428af36
-      -rw-r--r--   501    20      5 2015-08-26 14:09:57 +0200 CEST path/to/test.txt
+```console
+$ restic -r backup find test.txt
+debug log file restic.log
+debug enabled
+enter password for repository:
+found 1 matching entries in snapshot 196bc5760c909a7681647949e80e5448e276521489558525680acf1bd428af36
+  -rw-r--r--   501    20      5 2015-08-26 14:09:57 +0200 CEST path/to/test.txt
+```
 
 The `cat` command allows you to display the JSON representation of the objects
 or its raw content.
 
-    $ restic -r /tmp/backup cat snapshot d369ccc7d126594950bf74f0a348d5d98d9e99f3215082eb69bf02dc9b3e464c
-    enter password for repository:
-    {
-      "time": "2015-08-12T12:52:44.091448856+02:00",
-      "tree": "05cec17e8d3349f402576d02576a2971fc0d9f9776ce2f441c7010849c4ff5af",
-      "paths": [
-        "/home/user/work"
-      ],
-      "hostname": "kasimir",
-      "username": "username",
-      "uid": 501,
-      "gid": 20
-    }
+```console
+$ restic -r /tmp/backup cat snapshot d369ccc7d126594950bf74f0a348d5d98d9e99f3215082eb69bf02dc9b3e464c
+enter password for repository:
+{
+  "time": "2015-08-12T12:52:44.091448856+02:00",
+  "tree": "05cec17e8d3349f402576d02576a2971fc0d9f9776ce2f441c7010849c4ff5af",
+  "paths": [
+    "/home/user/work"
+  ],
+  "hostname": "kasimir",
+  "username": "username",
+  "uid": 501,
+  "gid": 20
+}
+```
+
+# Scripting restic
+
+Restic supports the output of some commands in JSON format. The JSON flag ```--json``` is currently supported only by ```restic snapshots```.
+
+```console
+$ restic -r /tmp/backup snapshots --json```

doc/REST_backend.md

@@ -6,6 +6,19 @@ following values are valid for `{type}`: `data`, `keys`, `locks`, `snapshots`,
 `index`, `config`. `{path}` is a path to the repository, so that multiple
 different repositories can be accessed. The default path is `/`.
 
+## POST {path}?create=true
+
+This request is used to initially create a new repository. The server responds
+with "200 OK" if the repository structure was created successfully or already
+exists, otherwise an error is returned.
+
+## DELETE {path}
+
+Deletes the repository on the server side. The server responds with "200 OK" if
+the repository was successfully removed. If this function is not implemented
+the server returns "501 Not Implemented", if this it is denied by the server it
+returns "403 Forbidden".
+
 ## HEAD {path}/config
 
 Returns "200 OK" if the repository has a configuration,

doc/code.css

@@ -0,0 +1,124 @@
+code {
+    font-size: 90%;
+}
+
+/* based on https://github.com/mkdocs/mkdocs/issues/1019 */
+
+.codehilite code, .codehilite pre {
+    color:#3F3F3F;background-color:#F7F7F7;
+    overflow: auto;
+    box-sizing: border-box;
+
+    padding: 0.01em 4px;
+    padding-top: 0.01em;
+    padding-right-value: 4px;
+    padding-bottom: 0.01em;
+    padding-left-value: 4px;
+    padding-left-ltr-source: physical;
+    padding-left-rtl-source: physical;
+    padding-right-ltr-source: physical;
+    padding-right-rtl-source: physical;
+
+    border-radius: 4px !important;
+    border-top-left-radius: 4px;
+    border-top-right-radius: 4px;
+    border-bottom-right-radius: 4px;
+    border-bottom-left-radius: 4px;
+
+    border: 1px solid #CCC !important;
+    border-top-width: 1px;
+    border-right-width-value: 1px;
+    border-right-width-ltr-source: physical;
+    border-right-width-rtl-source: physical;
+    border-bottom-width: 1px;
+    border-left-width-value: 1px;
+    border-left-width-ltr-source: physical;
+    border-left-width-rtl-source: physical;
+    border-top-style: solid;
+    border-right-style-value: solid;
+    border-right-style-ltr-source: physical;
+    border-right-style-rtl-source: physical;
+    border-bottom-style: solid;
+    border-left-style-value: solid;
+    border-left-style-ltr-source: physical;
+    border-left-style-rtl-source: physical;
+    border-top-color: #CCC;
+    border-right-color-value: #CCC;
+    border-right-color-ltr-source: physical;
+    border-right-color-rtl-source: physical;
+    border-bottom-color: #CCC;
+    border-left-color-value: #CCC;
+    border-left-color-ltr-source: physical;
+    border-left-color-rtl-source: physical;
+    -moz-border-top-colors: none;
+    -moz-border-right-colors: none;
+    -moz-border-bottom-colors: none;
+    -moz-border-left-colors: none;
+    border-image-source: none;
+    border-image-slice: 100% 100% 100% 100%;
+    border-image-width: 1 1 1 1;
+    border-image-outset: 0 0 0 0;
+    border-image-repeat: stretch stretch;
+}
+
+.codehilite .hll { background-color: #ffffcc  }
+.codehilite .c { color: #999988; font-style: italic  } /* Comment */
+.codehilite .err { color: #a61717; background-color: #e3d2d2  } /* Error */
+.codehilite .k { color: #000000; font-weight: bold  } /* Keyword */
+.codehilite .o { color: #000000; font-weight: bold  } /* Operator */
+.codehilite .cm { color: #999988; font-style: italic  } /* Comment.Multiline */
+.codehilite .cp { color: #999999; font-weight: bold; font-style: italic  } /* Comment.Preproc */
+.codehilite .c1 { color: #999988; font-style: italic  } /* Comment.Single */
+.codehilite .cs { color: #999999; font-weight: bold; font-style: italic  } /* Comment.Special */
+.codehilite .gd { color: #000000; background-color: #ffdddd  } /* Generic.Deleted */
+.codehilite .ge { color: #000000; font-style: italic  } /* Generic.Emph */
+.codehilite .gr { color: #aa0000  } /* Generic.Error */
+.codehilite .gh { color: #999999  } /* Generic.Heading */
+.codehilite .gi { color: #000000; background-color: #ddffdd  } /* Generic.Inserted */
+.codehilite .go { color: #888888  } /* Generic.Output */
+.codehilite .gp { color: #555555  } /* Generic.Prompt */
+.codehilite .gs { font-weight: bold  } /* Generic.Strong */
+.codehilite .gu { color: #aaaaaa  } /* Generic.Subheading */
+.codehilite .gt { color: #aa0000  } /* Generic.Traceback */
+.codehilite .kc { color: #000000; font-weight: bold  } /* Keyword.Constant */
+.codehilite .kd { color: #000000; font-weight: bold  } /* Keyword.Declaration */
+.codehilite .kn { color: #000000; font-weight: bold  } /* Keyword.Namespace */
+.codehilite .kp { color: #000000; font-weight: bold  } /* Keyword.Pseudo */
+.codehilite .kr { color: #000000; font-weight: bold  } /* Keyword.Reserved */
+.codehilite .kt { color: #445588; font-weight: bold  } /* Keyword.Type */
+.codehilite .m { color: #009999  } /* Literal.Number */
+.codehilite .s { color: #d01040  } /* Literal.String */
+.codehilite .na { color: #008080  } /* Name.Attribute */
+.codehilite .nb { color: #0086B3  } /* Name.Builtin */
+.codehilite .nc { color: #445588; font-weight: bold  } /* Name.Class */
+.codehilite .no { color: #008080  } /* Name.Constant */
+.codehilite .nd { color: #3c5d5d; font-weight: bold  } /* Name.Decorator */
+.codehilite .ni { color: #800080  } /* Name.Entity */
+.codehilite .ne { color: #990000; font-weight: bold  } /* Name.Exception */
+.codehilite .nf { color: #990000; font-weight: bold  } /* Name.Function */
+.codehilite .nl { color: #990000; font-weight: bold  } /* Name.Label */
+.codehilite .nn { color: #555555  } /* Name.Namespace */
+.codehilite .nt { color: #000080  } /* Name.Tag */
+.codehilite .nv { color: #008080  } /* Name.Variable */
+.codehilite .ow { color: #000000; font-weight: bold  } /* Operator.Word */
+.codehilite .w { color: #bbbbbb  } /* Text.Whitespace */
+.codehilite .mf { color: #009999  } /* Literal.Number.Float */
+.codehilite .mh { color: #009999  } /* Literal.Number.Hex */
+.codehilite .mi { color: #009999  } /* Literal.Number.Integer */
+.codehilite .mo { color: #009999  } /* Literal.Number.Oct */
+.codehilite .sb { color: #d01040  } /* Literal.String.Backtick */
+.codehilite .sc { color: #d01040  } /* Literal.String.Char */
+.codehilite .sd { color: #d01040  } /* Literal.String.Doc */
+.codehilite .s2 { color: #d01040  } /* Literal.String.Double */
+.codehilite .se { color: #d01040  } /* Literal.String.Escape */
+.codehilite .sh { color: #d01040  } /* Literal.String.Heredoc */
+.codehilite .si { color: #d01040  } /* Literal.String.Interpol */
+.codehilite .sx { color: #d01040  } /* Literal.String.Other */
+.codehilite .sr { color: #009926  } /* Literal.String.Regex */
+.codehilite .s1 { color: #d01040  } /* Literal.String.Single */
+.codehilite .ss { color: #990073  } /* Literal.String.Symbol */
+.codehilite .bp { color: #999999  } /* Name.Builtin.Pseudo */
+.codehilite .vc { color: #008080  } /* Name.Variable.Class */
+.codehilite .vg { color: #008080  } /* Name.Variable.Global */
+.codehilite .vi { color: #008080  } /* Name.Variable.Instance */
+.codehilite .il { color: #009999  } /* Literal.Number.Integer.Long */

doc/index.md

@@ -20,10 +20,12 @@ this page, where you can select the version.
 The restic documentation is built with [MkDocs](http://www.mkdocs.org). After
 installing it, you can edit and view the documentation locally by running:
 
-    $ mkdocs serve
-    INFO    -  Building documentation...
-    INFO    -  Cleaning site directory
-    [I 160221 12:33:57 server:271] Serving on http://127.0.0.1:8000
+```console
+$ mkdocs serve
+INFO    -  Building documentation...
+INFO    -  Cleaning site directory
+[I 160221 12:33:57 server:271] Serving on http://127.0.0.1:8000
+```
 
 Afterwards visit the URL with a browser.
 

mkdocs.yml

@@ -1,5 +1,9 @@
 site_name: Documentation for restic
 theme: readthedocs
+markdown_extensions:
+  - codehilite:
+extra_css:
+  - code.css
 docs_dir: doc
 pages:
   - Getting Started: index.md

run_integration_tests.go

@@ -176,7 +176,7 @@ func (env *TravisEnvironment) Prepare() error {
 				"windows/386", "windows/amd64",
 				"darwin/386", "darwin/amd64",
 				"freebsd/386", "freebsd/amd64",
-				"opendbsd/386", "opendbsd/amd64",
+				"openbsd/386", "openbsd/amd64",
 			}
 			if !strings.HasPrefix(runtime.Version(), "go1.3") {
 				env.goxOSArch = append(env.goxOSArch,

src/cmds/restic-server/.gitignore

@@ -1,24 +0,0 @@
-# Compiled Object files, Static and Dynamic libs (Shared Objects)
-*.o
-*.a
-*.so
-
-# Folders
-_obj
-_test
-
-# Architecture specific extensions/prefixes
-*.[568vq]
-[568vq].out
-
-*.cgo1.go
-*.cgo2.c
-_cgo_defun.c
-_cgo_gotypes.go
-_cgo_export.*
-
-_testmain.go
-
-*.exe
-*.test
-*.prof

src/cmds/restic-server/LICENSE

@@ -1,24 +0,0 @@
-Copyright (c) 2015, Bertil Chapuis
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-* Redistributions of source code must retain the above copyright notice, this
-  list of conditions and the following disclaimer.
-
-* Redistributions in binary form must reproduce the above copyright notice,
-  this list of conditions and the following disclaimer in the documentation
-  and/or other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-

src/cmds/restic-server/README.md

@@ -1,29 +0,0 @@
-# Restic Server
-
-Restic Server is a sample server that implement restic's rest backend api.
-It has been developed for demonstration purpose and is not intented to be used in production.
-
-## Getting started
-
-By default the server persists backup data in `/tmp/restic`. 
-Build and start the server with a custom persistence directory:
-
-```
-go build
-./restic-server -path /user/home/backup
-```
-
-The server use an `.htpasswd` file to specify users. You can create such a file at the root of the persistence directory by executing the following command. In order to append new user to the file, just omit the `-c` argument.
-
-```
-htpasswd -s -c .htpasswd username
-```
-
-By default the server uses http. This is not very secure since with Basic Authentication, username and passwords will be present in every request. In order to enable TLS support just add the `-tls` argument and add a private and public key at the root of your persistence directory. 
-
-Signed certificate are required by the restic backend but if you just want to test the feature you can generate unsigned keys with the following commands:
-
-```
-openssl genrsa -out private_key 2048
-openssl req -new -x509 -key private_key -out public_key -days 365
-```

src/cmds/restic-server/handlers.go

@@ -1,194 +0,0 @@
-// +build go1.4
-
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
-
-	"restic/fs"
-)
-
-// Context contains repository meta-data.
-type Context struct {
-	path string
-}
-
-// AuthHandler wraps h with a http.HandlerFunc that performs basic
-// authentication against the user/passwords pairs stored in f and returns the
-// http.HandlerFunc.
-func AuthHandler(f *HtpasswdFile, h http.Handler) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		username, password, ok := r.BasicAuth()
-		if !ok {
-			http.Error(w, "401 unauthorized", 401)
-			return
-		}
-		if !f.Validate(username, password) {
-			http.Error(w, "401 unauthorized", 401)
-			return
-		}
-		h.ServeHTTP(w, r)
-	}
-}
-
-// CheckConfig returns a http.HandlerFunc that checks whether
-// a configuration exists.
-func CheckConfig(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		config := filepath.Join(c.path, "config")
-		st, err := os.Stat(config)
-		if err != nil {
-			http.Error(w, "404 not found", 404)
-			return
-		}
-		w.Header().Add("Content-Length", fmt.Sprint(st.Size()))
-	}
-}
-
-// GetConfig returns a http.HandlerFunc that allows for a
-// config to be retrieved.
-func GetConfig(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		config := filepath.Join(c.path, "config")
-		bytes, err := ioutil.ReadFile(config)
-		if err != nil {
-			http.Error(w, "404 not found", 404)
-			return
-		}
-		w.Write(bytes)
-	}
-}
-
-// SaveConfig returns a http.HandlerFunc that allows for a
-// config to be saved.
-func SaveConfig(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		config := filepath.Join(c.path, "config")
-		bytes, err := ioutil.ReadAll(r.Body)
-		if err != nil {
-			http.Error(w, "400 bad request", 400)
-			return
-		}
-		errw := ioutil.WriteFile(config, bytes, 0600)
-		if errw != nil {
-			http.Error(w, "500 internal server error", 500)
-			return
-		}
-		w.Write([]byte("200 ok"))
-	}
-}
-
-// ListBlobs returns a http.HandlerFunc that lists
-// all blobs of a given type in an arbitrary order.
-func ListBlobs(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		vars := strings.Split(r.RequestURI, "/")
-		dir := vars[1]
-		path := filepath.Join(c.path, dir)
-		files, err := ioutil.ReadDir(path)
-		if err != nil {
-			http.Error(w, "404 not found", 404)
-			return
-		}
-		names := make([]string, len(files))
-		for i, f := range files {
-			names[i] = f.Name()
-		}
-		data, err := json.Marshal(names)
-		if err != nil {
-			http.Error(w, "500 internal server error", 500)
-			return
-		}
-		w.Write(data)
-	}
-}
-
-// CheckBlob reutrns a http.HandlerFunc that tests whether a blob exists
-// and returns 200, if it does, or 404 otherwise.
-func CheckBlob(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		vars := strings.Split(r.RequestURI, "/")
-		dir := vars[1]
-		name := vars[2]
-		path := filepath.Join(c.path, dir, name)
-		st, err := os.Stat(path)
-		if err != nil {
-			http.Error(w, "404 not found", 404)
-			return
-		}
-		w.Header().Add("Content-Length", fmt.Sprint(st.Size()))
-	}
-}
-
-// GetBlob returns a http.HandlerFunc that retrieves a blob
-// from the repository.
-func GetBlob(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		vars := strings.Split(r.RequestURI, "/")
-		dir := vars[1]
-		name := vars[2]
-		path := filepath.Join(c.path, dir, name)
-		file, err := fs.Open(path)
-		if err != nil {
-			http.Error(w, "404 not found", 404)
-			return
-		}
-		defer file.Close()
-		http.ServeContent(w, r, "", time.Unix(0, 0), file)
-	}
-}
-
-// SaveBlob returns a http.HandlerFunc that saves a blob to the repository.
-func SaveBlob(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		vars := strings.Split(r.RequestURI, "/")
-		dir := vars[1]
-		name := vars[2]
-		path := filepath.Join(c.path, dir, name)
-		tmp := path + "_tmp"
-		tf, err := fs.OpenFile(tmp, os.O_CREATE|os.O_WRONLY, 0600)
-		if err != nil {
-			http.Error(w, "500 internal server error", 500)
-			return
-		}
-		if _, err := io.Copy(tf, r.Body); err != nil {
-			http.Error(w, "400 bad request", 400)
-			tf.Close()
-			os.Remove(tmp)
-			return
-		}
-		if err := tf.Close(); err != nil {
-			http.Error(w, "500 internal server error", 500)
-		}
-		if err := os.Rename(tmp, path); err != nil {
-			http.Error(w, "500 internal server error", 500)
-			return
-		}
-		w.Write([]byte("200 ok"))
-	}
-}
-
-// DeleteBlob returns a http.HandlerFunc that deletes a blob from the
-// repository.
-func DeleteBlob(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		vars := strings.Split(r.RequestURI, "/")
-		dir := vars[1]
-		name := vars[2]
-		path := filepath.Join(c.path, dir, name)
-		err := os.Remove(path)
-		if err != nil {
-			http.Error(w, "500 internal server error", 500)
-			return
-		}
-		w.Write([]byte("200 ok"))
-	}
-}

src/cmds/restic-server/htpasswd.go

@@ -1,97 +0,0 @@
-// +build go1.4
-
-package main
-
-/*
-Copied from: github.com/bitly/oauth2_proxy
-
-MIT License
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-*/
-
-import (
-	"crypto/sha1"
-	"encoding/base64"
-	"encoding/csv"
-	"io"
-	"log"
-
-	"restic/fs"
-)
-
-// lookup passwords in a htpasswd file
-// The entries must have been created with -s for SHA encryption
-
-// HtpasswdFile is a map for usernames to passwords.
-type HtpasswdFile struct {
-	Users map[string]string
-}
-
-// NewHtpasswdFromFile reads the users and passwords from a htpasswd
-// file and returns them. If an error is encountered, it is returned, together
-// with a nil-Pointer for the HtpasswdFile.
-func NewHtpasswdFromFile(path string) (*HtpasswdFile, error) {
-	r, err := fs.Open(path)
-	if err != nil {
-		return nil, err
-	}
-	defer r.Close()
-	return NewHtpasswd(r)
-}
-
-// NewHtpasswd reads the users and passwords from a htpasswd
-// datastream in file and returns them. If an error is encountered,
-// it is returned, together with a nil-Pointer for the HtpasswdFile.
-func NewHtpasswd(file io.Reader) (*HtpasswdFile, error) {
-	cr := csv.NewReader(file)
-	cr.Comma = ':'
-	cr.Comment = '#'
-	cr.TrimLeadingSpace = true
-
-	records, err := cr.ReadAll()
-	if err != nil {
-		return nil, err
-	}
-	h := &HtpasswdFile{Users: make(map[string]string)}
-	for _, record := range records {
-		h.Users[record[0]] = record[1]
-	}
-	return h, nil
-}
-
-// Validate returns true if password matches the stored password
-// for user. If no password for user is stored, or the password
-// is wrong, false is returned.
-func (h *HtpasswdFile) Validate(user string, password string) bool {
-	realPassword, exists := h.Users[user]
-	if !exists {
-		return false
-	}
-	if realPassword[:5] == "{SHA}" {
-		d := sha1.New()
-		d.Write([]byte(password))
-		if realPassword[5:] == base64.StdEncoding.EncodeToString(d.Sum(nil)) {
-			return true
-		}
-	} else {
-		log.Printf("Invalid htpasswd entry for %s. Must be a SHA entry.", user)
-	}
-	return false
-}

src/cmds/restic-server/router.go

@@ -1,137 +0,0 @@
-// +build go1.4
-
-package main
-
-import (
-	"log"
-	"net/http"
-	"strings"
-)
-
-// Route is a handler for a path that was already split.
-type Route struct {
-	path    []string
-	handler http.Handler
-}
-
-// Router maps HTTP methods to a slice of Route handlers.
-type Router struct {
-	routes map[string][]Route
-}
-
-// NewRouter creates a new Router and returns a pointer to it.
-func NewRouter() *Router {
-	return &Router{make(map[string][]Route)}
-}
-
-// Options registers handler for path with method "OPTIONS".
-func (router *Router) Options(path string, handler http.Handler) {
-	router.Handle("OPTIONS", path, handler)
-}
-
-// OptionsFunc registers handler for path with method "OPTIONS".
-func (router *Router) OptionsFunc(path string, handler http.HandlerFunc) {
-	router.Handle("OPTIONS", path, handler)
-}
-
-// Get registers handler for path with method "GET".
-func (router *Router) Get(path string, handler http.Handler) {
-	router.Handle("GET", path, handler)
-}
-
-// GetFunc registers handler for path with method "GET".
-func (router *Router) GetFunc(path string, handler http.HandlerFunc) {
-	router.Handle("GET", path, handler)
-}
-
-// Head registers handler for path with method "HEAD".
-func (router *Router) Head(path string, handler http.Handler) {
-	router.Handle("HEAD", path, handler)
-}
-
-// HeadFunc registers handler for path with method "HEAD".
-func (router *Router) HeadFunc(path string, handler http.HandlerFunc) {
-	router.Handle("HEAD", path, handler)
-}
-
-// Post registers handler for path with method "POST".
-func (router *Router) Post(path string, handler http.Handler) {
-	router.Handle("POST", path, handler)
-}
-
-// PostFunc registers handler for path with method "POST".
-func (router *Router) PostFunc(path string, handler http.HandlerFunc) {
-	router.Handle("POST", path, handler)
-}
-
-// Put registers handler for path with method "PUT".
-func (router *Router) Put(path string, handler http.Handler) {
-	router.Handle("PUT", path, handler)
-}
-
-// PutFunc registers handler for path with method "PUT".
-func (router *Router) PutFunc(path string, handler http.HandlerFunc) {
-	router.Handle("PUT", path, handler)
-}
-
-// Delete registers handler for path with method "DELETE".
-func (router *Router) Delete(path string, handler http.Handler) {
-	router.Handle("DELETE", path, handler)
-}
-
-// DeleteFunc registers handler for path with method "DELETE".
-func (router *Router) DeleteFunc(path string, handler http.HandlerFunc) {
-	router.Handle("DELETE", path, handler)
-}
-
-// Trace registers handler for path with method "TRACE".
-func (router *Router) Trace(path string, handler http.Handler) {
-	router.Handle("TRACE", path, handler)
-}
-
-// TraceFunc registers handler for path with method "TRACE".
-func (router *Router) TraceFunc(path string, handler http.HandlerFunc) {
-	router.Handle("TRACE", path, handler)
-}
-
-// Connect registers handler for path with method "Connect".
-func (router *Router) Connect(path string, handler http.Handler) {
-	router.Handle("Connect", path, handler)
-}
-
-// ConnectFunc registers handler for path with method "Connect".
-func (router *Router) ConnectFunc(path string, handler http.HandlerFunc) {
-	router.Handle("Connect", path, handler)
-}
-
-// Handle registers a http.Handler for method and uri
-func (router *Router) Handle(method string, uri string, handler http.Handler) {
-	routes := router.routes[method]
-	path := strings.Split(uri, "/")
-	routes = append(routes, Route{path, handler})
-	router.routes[method] = routes
-}
-
-func (router *Router) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	method := r.Method
-	uri := r.RequestURI
-	path := strings.Split(uri, "/")
-
-	log.Printf("%s %s", method, uri)
-
-ROUTE:
-	for _, route := range router.routes[method] {
-		if len(route.path) != len(path) {
-			continue
-		}
-		for i := 0; i < len(route.path); i++ {
-			if !strings.HasPrefix(route.path[i], ":") && route.path[i] != path[i] {
-				continue ROUTE
-			}
-		}
-		route.handler.ServeHTTP(w, r)
-		return
-	}
-
-	http.Error(w, "404 not found", 404)
-}

src/cmds/restic-server/router_test.go

@@ -1,74 +0,0 @@
-// +build go1.4
-
-package main
-
-import (
-	"io/ioutil"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-)
-
-func TestRouter(t *testing.T) {
-	router := NewRouter()
-
-	getConfig := []byte("GET /config")
-	router.GetFunc("/config", func(w http.ResponseWriter, r *http.Request) {
-		w.Write(getConfig)
-	})
-
-	postConfig := []byte("POST /config")
-	router.PostFunc("/config", func(w http.ResponseWriter, r *http.Request) {
-		w.Write(postConfig)
-	})
-
-	getBlobs := []byte("GET /blobs/")
-	router.GetFunc("/blobs/", func(w http.ResponseWriter, r *http.Request) {
-		w.Write(getBlobs)
-	})
-
-	getBlob := []byte("GET /blobs/:sha")
-	router.GetFunc("/blobs/:sha", func(w http.ResponseWriter, r *http.Request) {
-		w.Write(getBlob)
-	})
-
-	server := httptest.NewServer(router)
-	defer server.Close()
-
-	getConfigResp, _ := http.Get(server.URL + "/config")
-	getConfigBody, _ := ioutil.ReadAll(getConfigResp.Body)
-	if getConfigResp.StatusCode != 200 {
-		t.Fatalf("Wanted HTTP Status 200, got %d", getConfigResp.StatusCode)
-	}
-	if string(getConfig) != string(getConfigBody) {
-		t.Fatalf("Config wrong:\nWanted '%s'\nGot: '%s'", string(getConfig), string(getConfigBody))
-	}
-
-	postConfigResp, _ := http.Post(server.URL+"/config", "binary/octet-stream", strings.NewReader("post test"))
-	postConfigBody, _ := ioutil.ReadAll(postConfigResp.Body)
-	if postConfigResp.StatusCode != 200 {
-		t.Fatalf("Wanted HTTP Status 200, got %d", postConfigResp.StatusCode)
-	}
-	if string(postConfig) != string(postConfigBody) {
-		t.Fatalf("Config wrong:\nWanted '%s'\nGot: '%s'", string(postConfig), string(postConfigBody))
-	}
-
-	getBlobsResp, _ := http.Get(server.URL + "/blobs/")
-	getBlobsBody, _ := ioutil.ReadAll(getBlobsResp.Body)
-	if getBlobsResp.StatusCode != 200 {
-		t.Fatalf("Wanted HTTP Status 200, got %d", getBlobsResp.StatusCode)
-	}
-	if string(getBlobs) != string(getBlobsBody) {
-		t.Fatalf("Config wrong:\nWanted '%s'\nGot: '%s'", string(getBlobs), string(getBlobsBody))
-	}
-
-	getBlobResp, _ := http.Get(server.URL + "/blobs/test")
-	getBlobBody, _ := ioutil.ReadAll(getBlobResp.Body)
-	if getBlobResp.StatusCode != 200 {
-		t.Fatalf("Wanted HTTP Status 200, got %d", getBlobResp.StatusCode)
-	}
-	if string(getBlob) != string(getBlobBody) {
-		t.Fatalf("Config wrong:\nWanted '%s'\nGot: '%s'", string(getBlob), string(getBlobBody))
-	}
-}

src/cmds/restic-server/server.go

@@ -1,73 +0,0 @@
-// +build go1.4
-
-package main
-
-import (
-	"flag"
-	"log"
-	"net/http"
-	"os"
-	"path/filepath"
-)
-
-const (
-	defaultHTTPPort  = ":8000"
-	defaultHTTPSPort = ":8443"
-)
-
-func main() {
-	// Parse command-line args
-	var path = flag.String("path", "/tmp/restic", "specifies the path of the data directory")
-	var tls = flag.Bool("tls", false, "turns on tls support")
-	flag.Parse()
-
-	// Create the missing directories
-	dirs := []string{
-		"data",
-		"snapshots",
-		"index",
-		"locks",
-		"keys",
-		"tmp",
-	}
-	for _, d := range dirs {
-		os.MkdirAll(filepath.Join(*path, d), 0700)
-	}
-
-	// Define the routes
-	context := &Context{*path}
-	router := NewRouter()
-	router.HeadFunc("/config", CheckConfig(context))
-	router.GetFunc("/config", GetConfig(context))
-	router.PostFunc("/config", SaveConfig(context))
-	router.GetFunc("/:dir/", ListBlobs(context))
-	router.HeadFunc("/:dir/:name", CheckBlob(context))
-	router.GetFunc("/:type/:name", GetBlob(context))
-	router.PostFunc("/:type/:name", SaveBlob(context))
-	router.DeleteFunc("/:type/:name", DeleteBlob(context))
-
-	// Check for a password file
-	var handler http.Handler
-	htpasswdFile, err := NewHtpasswdFromFile(filepath.Join(*path, ".htpasswd"))
-	if err != nil {
-		log.Println("Authentication disabled")
-		handler = router
-	} else {
-		log.Println("Authentication enabled")
-		handler = AuthHandler(htpasswdFile, router)
-	}
-
-	// start the server
-	if !*tls {
-		log.Printf("start server on port %s\n", defaultHTTPPort)
-		http.ListenAndServe(defaultHTTPPort, handler)
-	} else {
-		privateKey := filepath.Join(*path, "private_key")
-		publicKey := filepath.Join(*path, "public_key")
-		log.Println("TLS enabled")
-		log.Printf("private key: %s", privateKey)
-		log.Printf("public key: %s", publicKey)
-		log.Printf("start server on port %s\n", defaultHTTPSPort)
-		http.ListenAndServeTLS(defaultHTTPSPort, publicKey, privateKey, handler)
-	}
-}

src/cmds/restic/background.go

@@ -0,0 +1,9 @@
+// +build !linux
+
+package main
+
+// IsProcessBackground should return true if it is running in the background or false if not
+func IsProcessBackground() bool {
+	//TODO: Check if the process are running in the background in other OS than linux
+	return false
+}

src/cmds/restic/background_linux.go

@@ -0,0 +1,21 @@
+package main
+
+import (
+	"syscall"
+	"unsafe"
+
+	"restic/debug"
+)
+
+// IsProcessBackground returns true if it is running in the background or false if not
+func IsProcessBackground() bool {
+	var pid int
+	_, _, err := syscall.Syscall(syscall.SYS_IOCTL, uintptr(syscall.Stdin), syscall.TIOCGPGRP, uintptr(unsafe.Pointer(&pid)))
+
+	if err != 0 {
+		debug.Log("Can't check if we are in the background. Using default behaviour. Error: %s\n", err.Error())
+		return false
+	}
+
+	return pid != syscall.Getpgrp()
+}

src/cmds/restic/cleanup.go

@@ -62,8 +62,13 @@ func CleanupHandler(c <-chan os.Signal) {
 	for s := range c {
 		debug.Log("signal %v received, cleaning up", s)
 		fmt.Printf("%sInterrupt received, cleaning up\n", ClearLine())
-		RunCleanupHandlers()
-		fmt.Println("exiting")
-		os.Exit(0)
+		Exit(0)
 	}
 }
+
+// Exit runs the cleanup handlers and then terminates the process with the
+// given exit code.
+func Exit(code int) {
+	RunCleanupHandlers()
+	os.Exit(code)
+}

src/cmds/restic/cmd_backup.go

@@ -3,14 +3,13 @@ package main
 import (
 	"bufio"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"restic"
 	"strings"
 	"time"
 
-	"golang.org/x/crypto/ssh/terminal"
-
 	"github.com/spf13/cobra"
 
 	"restic/archiver"
@@ -28,6 +27,10 @@ The "backup" command creates a new snapshot and saves the files and directories
 given as the arguments.
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
+		if backupOptions.Stdin && backupOptions.FilesFrom == "-" {
+			return errors.Fatal("cannot use both `--stdin` and `--files-from -`")
+		}
+
 		if backupOptions.Stdin {
 			return readBackupFromStdin(backupOptions, globalOptions, args)
 		}
@@ -46,6 +49,8 @@ type BackupOptions struct {
 	Stdin          bool
 	StdinFilename  string
 	Tags           []string
+	Hostname       string
+	FilesFrom      string
 }
 
 var backupOptions BackupOptions
@@ -53,15 +58,23 @@ var backupOptions BackupOptions
 func init() {
 	cmdRoot.AddCommand(cmdBackup)
 
+	hostname, err := os.Hostname()
+	if err != nil {
+		debug.Log("os.Hostname() returned err: %v", err)
+		hostname = ""
+	}
+
 	f := cmdBackup.Flags()
 	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent snapshot (default: last snapshot in the repo that has the same target files/directories)")
-	f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories. Overrides the "parent" flag`)
-	f.StringSliceVarP(&backupOptions.Excludes, "exclude", "e", []string{}, "exclude a `pattern` (can be specified multiple times)")
+	f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`)
+	f.StringSliceVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
 	f.StringVar(&backupOptions.ExcludeFile, "exclude-file", "", "read exclude patterns from a file")
-	f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "Exclude other file systems")
+	f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems")
 	f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
-	f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "", "file name to use when reading from stdin")
-	f.StringSliceVar(&backupOptions.Tags, "tag", []string{}, "add a `tag` for the new snapshot (can be specified multiple times)")
+	f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "file name to use when reading from stdin")
+	f.StringSliceVar(&backupOptions.Tags, "tag", nil, "add a `tag` for the new snapshot (can be specified multiple times)")
+	f.StringVar(&backupOptions.Hostname, "hostname", hostname, "set the `hostname` for the snapshot manually")
+	f.StringVar(&backupOptions.FilesFrom, "files-from", "", "read the files to backup from file (can be combined with file args)")
 }
 
 func newScanProgress(gopts GlobalOptions) *restic.Progress {
@@ -71,8 +84,13 @@ func newScanProgress(gopts GlobalOptions) *restic.Progress {
 
 	p := restic.NewProgress()
 	p.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
+		if IsProcessBackground() {
+			return
+		}
+
 		PrintProgress("[%s] %d directories, %d files, %s", formatDuration(d), s.Dirs, s.Files, formatBytes(s.Bytes))
 	}
+
 	p.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
 		PrintProgress("scanned %d directories, %d files in %s\n", s.Dirs, s.Files, formatDuration(d))
 	}
@@ -91,6 +109,10 @@ func newArchiveProgress(gopts GlobalOptions, todo restic.Stat) *restic.Progress
 	itemsTodo := todo.Files + todo.Dirs
 
 	archiveProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
+		if IsProcessBackground() {
+			return
+		}
+
 		sec := uint64(d / time.Second)
 		if todo.Bytes > 0 && sec > 0 && ticker {
 			bps = s.Bytes / sec
@@ -112,8 +134,7 @@ func newArchiveProgress(gopts GlobalOptions, todo restic.Stat) *restic.Progress
 			s.Errors)
 		status2 := fmt.Sprintf("ETA %s ", formatSeconds(eta))
 
-		w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
-		if err == nil {
+		if w := stdoutTerminalWidth(); w > 0 {
 			maxlen := w - len(status2) - 1
 
 			if maxlen < 4 {
@@ -144,6 +165,10 @@ func newArchiveStdinProgress(gopts GlobalOptions) *restic.Progress {
 	var bps uint64
 
 	archiveProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
+		if IsProcessBackground() {
+			return
+		}
+
 		sec := uint64(d / time.Second)
 		if s.Bytes > 0 && sec > 0 && ticker {
 			bps = s.Bytes / sec
@@ -153,8 +178,7 @@ func newArchiveStdinProgress(gopts GlobalOptions) *restic.Progress {
 			formatBytes(s.Bytes),
 			formatBytes(bps))
 
-		w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
-		if err == nil {
+		if w := stdoutTerminalWidth(); w > 0 {
 			maxlen := w - len(status1)
 
 			if maxlen < 4 {
@@ -217,7 +241,15 @@ func gatherDevices(items []string) (deviceMap map[uint64]struct{}, err error) {
 
 func readBackupFromStdin(opts BackupOptions, gopts GlobalOptions, args []string) error {
 	if len(args) != 0 {
-		return errors.Fatalf("when reading from stdin, no additional files can be specified")
+		return errors.Fatal("when reading from stdin, no additional files can be specified")
+	}
+
+	if opts.StdinFilename == "" {
+		return errors.Fatal("filename for backup from stdin must not be empty")
+	}
+
+	if gopts.password == "" && gopts.PasswordFile == "" {
+		return errors.Fatal("unable to read password from stdin when data is to be read from stdin, use --password-file or $RESTIC_PASSWORD")
 	}
 
 	repo, err := OpenRepository(gopts)
@@ -236,18 +268,74 @@ func readBackupFromStdin(opts BackupOptions, gopts GlobalOptions, args []string)
 		return err
 	}
 
-	_, id, err := archiver.ArchiveReader(repo, newArchiveStdinProgress(gopts), os.Stdin, opts.StdinFilename, opts.Tags)
+	r := &archiver.Reader{
+		Repository: repo,
+		Tags:       opts.Tags,
+		Hostname:   opts.Hostname,
+	}
+
+	_, id, err := r.Archive(opts.StdinFilename, os.Stdin, newArchiveStdinProgress(gopts))
 	if err != nil {
 		return err
 	}
 
-	fmt.Printf("archived as %v\n", id.Str())
+	Verbosef("archived as %v\n", id.Str())
 	return nil
 }
 
+// readFromFile will read all lines from the given filename and write them to a
+// string array, if filename is empty readFromFile returns and empty string
+// array. If filename is a dash (-), readFromFile will read the lines from
+// the standard input.
+func readLinesFromFile(filename string) ([]string, error) {
+	if filename == "" {
+		return nil, nil
+	}
+
+	var r io.Reader = os.Stdin
+	if filename != "-" {
+		f, err := os.Open(filename)
+		if err != nil {
+			return nil, err
+		}
+		defer f.Close()
+		r = f
+	}
+
+	var lines []string
+
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if line == "" {
+			continue
+		}
+		lines = append(lines, line)
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+
+	return lines, nil
+}
+
 func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
+	if opts.FilesFrom == "-" && gopts.password == "" && gopts.PasswordFile == "" {
+		return errors.Fatal("no password; either use `--password-file` option or put the password into the RESTIC_PASSWORD environment variable")
+	}
+
+	fromfile, err := readLinesFromFile(opts.FilesFrom)
+	if err != nil {
+		return err
+	}
+
+	// merge files from files-from into normal args so we can reuse the normal
+	// args checks and have the ability to use both files-from and args at the
+	// same time
+	args = append(args, fromfile...)
 	if len(args) == 0 {
-		return errors.Fatalf("wrong number of parameters")
+		return errors.Fatal("wrong number of parameters")
 	}
 
 	target := make([]string, 0, len(args))
@@ -258,7 +346,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
 		target = append(target, d)
 	}
 
-	target, err := filterExisting(target)
+	target, err = filterExisting(target)
 	if err != nil {
 		return err
 	}
@@ -303,7 +391,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
 
 	// Find last snapshot to set it as parent, if not already set
 	if !opts.Force && parentSnapshotID == nil {
-		id, err := restic.FindLatestSnapshot(repo, target, "")
+		id, err := restic.FindLatestSnapshot(repo, target, opts.Tags, opts.Hostname)
 		if err == nil {
 			parentSnapshotID = &id
 		} else if err != restic.ErrNoSnapshotFound {
@@ -346,7 +434,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
 			return false
 		}
 
-		if !opts.ExcludeOtherFS {
+		if !opts.ExcludeOtherFS || fi == nil {
 			return true
 		}
 
@@ -374,13 +462,12 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
 	arch.Excludes = opts.Excludes
 	arch.SelectFilter = selectFilter
 
-	arch.Error = func(dir string, fi os.FileInfo, err error) error {
+	arch.Warn = func(dir string, fi os.FileInfo, err error) {
 		// TODO: make ignoring errors configurable
-		Warnf("%s\rerror for %s: %v\n", ClearLine(), dir, err)
-		return nil
+		Warnf("%s\rwarning for %s: %v\n", ClearLine(), dir, err)
 	}
 
-	_, id, err := arch.Snapshot(newArchiveProgress(gopts, stat), target, opts.Tags, parentSnapshotID)
+	_, id, err := arch.Snapshot(newArchiveProgress(gopts, stat), target, opts.Tags, opts.Hostname, parentSnapshotID)
 	if err != nil {
 		return err
 	}

src/cmds/restic/cmd_cat.go

@@ -9,13 +9,12 @@ import (
 
 	"restic"
 	"restic/backend"
-	"restic/debug"
 	"restic/errors"
 	"restic/repository"
 )
 
 var cmdCat = &cobra.Command{
-	Use:   "cat [flags] [pack|blob|tree|snapshot|key|masterkey|config|lock] ID",
+	Use:   "cat [flags] [pack|blob|snapshot|index|key|masterkey|config|lock] ID",
 	Short: "print internal objects to stdout",
 	Long: `
 The "cat" command is used to print internal objects to stdout.
@@ -31,7 +30,7 @@ func init() {
 
 func runCat(gopts GlobalOptions, args []string) error {
 	if len(args) < 1 || (args[0] != "masterkey" && args[0] != "config" && len(args) != 2) {
-		return errors.Fatalf("type or ID not specified")
+		return errors.Fatal("type or ID not specified")
 	}
 
 	repo, err := OpenRepository(gopts)
@@ -99,7 +98,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 		return nil
 	case "key":
 		h := restic.Handle{Type: restic.KeyFile, Name: id.String()}
-		buf, err := backend.LoadAll(repo.Backend(), h, nil)
+		buf, err := backend.LoadAll(repo.Backend(), h)
 		if err != nil {
 			return err
 		}
@@ -150,7 +149,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 	switch tpe {
 	case "pack":
 		h := restic.Handle{Type: restic.DataFile, Name: id.String()}
-		buf, err := backend.LoadAll(repo.Backend(), h, nil)
+		buf, err := backend.LoadAll(repo.Backend(), h)
 		if err != nil {
 			return err
 		}
@@ -172,7 +171,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 			blob := list[0]
 
 			buf := make([]byte, blob.Length)
-			n, err := repo.LoadBlob(restic.DataBlob, id, buf)
+			n, err := repo.LoadBlob(t, id, buf)
 			if err != nil {
 				return err
 			}
@@ -184,23 +183,6 @@ func runCat(gopts GlobalOptions, args []string) error {
 
 		return errors.Fatal("blob not found")
 
-	case "tree":
-		debug.Log("cat tree %v", id.Str())
-		tree, err := repo.LoadTree(id)
-		if err != nil {
-			debug.Log("unable to load tree %v: %v", id.Str(), err)
-			return err
-		}
-
-		buf, err := json.MarshalIndent(&tree, "", "  ")
-		if err != nil {
-			debug.Log("error json.MarshalIndent(): %v", err)
-			return err
-		}
-
-		_, err = os.Stdout.Write(append(buf, '\n'))
-		return nil
-
 	default:
 		return errors.Fatal("invalid type")
 	}

src/cmds/restic/cmd_check.go

@@ -7,8 +7,6 @@ import (
 
 	"github.com/spf13/cobra"
 
-	"golang.org/x/crypto/ssh/terminal"
-
 	"restic"
 	"restic/checker"
 	"restic/errors"
@@ -26,7 +24,7 @@ finds. It can also be used to read all data and therefore simulate a restore.
 	},
 }
 
-// CheckOptions bundle all options for the 'check' command.
+// CheckOptions bundles all options for the 'check' command.
 type CheckOptions struct {
 	ReadData    bool
 	CheckUnused bool
@@ -38,8 +36,8 @@ func init() {
 	cmdRoot.AddCommand(cmdCheck)
 
 	f := cmdCheck.Flags()
-	f.BoolVar(&checkOptions.ReadData, "read-data", false, "Read all data blobs")
-	f.BoolVar(&checkOptions.CheckUnused, "check-unused", false, "Find unused blobs")
+	f.BoolVar(&checkOptions.ReadData, "read-data", false, "read all data blobs")
+	f.BoolVar(&checkOptions.CheckUnused, "check-unused", false, "find unused blobs")
 }
 
 func newReadProgress(gopts GlobalOptions, todo restic.Stat) *restic.Progress {
@@ -55,8 +53,7 @@ func newReadProgress(gopts GlobalOptions, todo restic.Stat) *restic.Progress {
 			formatPercent(s.Blobs, todo.Blobs),
 			s.Blobs, todo.Blobs)
 
-		w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
-		if err == nil {
+		if w := stdoutTerminalWidth(); w > 0 {
 			if len(status) > w {
 				max := w - len(status) - 4
 				status = status[:max] + "... "

src/cmds/restic/cmd_dump.go

@@ -22,7 +22,7 @@ var cmdDump = &cobra.Command{
 	Use:   "dump [indexes|snapshots|trees|all|packs]",
 	Short: "dump data structures",
 	Long: `
-The "dump" command dumps data structures from a repository as JSON objects. It
+The "dump" command dumps data structures from the repository as JSON objects. It
 is used for debugging purposes only.`,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runDump(globalOptions, args)
@@ -168,7 +168,7 @@ func dumpIndexes(repo restic.Repository) error {
 
 func runDump(gopts GlobalOptions, args []string) error {
 	if len(args) != 1 {
-		return errors.Fatalf("type not specified")
+		return errors.Fatal("type not specified")
 	}
 
 	repo, err := OpenRepository(gopts)

src/cmds/restic/cmd_find.go

@@ -1,7 +1,9 @@
 package main
 
 import (
+	"context"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -23,11 +25,16 @@ repo. `,
 	},
 }
 
-// FindOptions bundle all options for the find command.
+// FindOptions bundles all options for the find command.
 type FindOptions struct {
-	Oldest   string
-	Newest   string
-	Snapshot string
+	Oldest          string
+	Newest          string
+	Snapshots       []string
+	CaseInsensitive bool
+	ListLong        bool
+	Host            string
+	Paths           []string
+	Tags            []string
 }
 
 var findOptions FindOptions
@@ -36,19 +43,21 @@ func init() {
 	cmdRoot.AddCommand(cmdFind)
 
 	f := cmdFind.Flags()
-	f.StringVarP(&findOptions.Oldest, "oldest", "o", "", "Oldest modification date/time")
-	f.StringVarP(&findOptions.Newest, "newest", "n", "", "Newest modification date/time")
-	f.StringVarP(&findOptions.Snapshot, "snapshot", "s", "", "Snapshot ID to search in")
+	f.StringVarP(&findOptions.Oldest, "oldest", "o", "", "oldest modification date/time")
+	f.StringVarP(&findOptions.Newest, "newest", "n", "", "newest modification date/time")
+	f.StringSliceVarP(&findOptions.Snapshots, "snapshot", "s", nil, "snapshot `id` to search in (can be given multiple times)")
+	f.BoolVarP(&findOptions.CaseInsensitive, "ignore-case", "i", false, "ignore case for pattern")
+	f.BoolVarP(&findOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
+
+	f.StringVarP(&findOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
+	f.StringSliceVar(&findOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot-ID is given")
+	f.StringSliceVar(&findOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
 }
 
 type findPattern struct {
 	oldest, newest time.Time
 	pattern        string
-}
-
-type findResult struct {
-	node *restic.Node
-	path string
+	ignoreCase     bool
 }
 
 var timeFormats = []string{
@@ -75,20 +84,25 @@ func parseTime(str string) (time.Time, error) {
 	return time.Time{}, errors.Fatalf("unable to parse time: %q", str)
 }
 
-func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, path string) ([]findResult, error) {
+func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, prefix string, snapshotID *string) error {
 	debug.Log("checking tree %v\n", id)
+
 	tree, err := repo.LoadTree(id)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
-	results := []findResult{}
 	for _, node := range tree.Nodes {
 		debug.Log("  testing entry %q\n", node.Name)
 
-		m, err := filepath.Match(pat.pattern, node.Name)
+		name := node.Name
+		if pat.ignoreCase {
+			name = strings.ToLower(name)
+		}
+
+		m, err := filepath.Match(pat.pattern, name)
 		if err != nil {
-			return nil, err
+			return err
 		}
 
 		if m {
@@ -103,69 +117,55 @@ func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, path
 				continue
 			}
 
-			results = append(results, findResult{node: node, path: path})
+			if snapshotID != nil {
+				Verbosef("Found matching entries in snapshot %s\n", *snapshotID)
+				snapshotID = nil
+			}
+			Printf(formatNode(prefix, node, findOptions.ListLong) + "\n")
 		} else {
 			debug.Log("    pattern does not match\n")
 		}
 
 		if node.Type == "dir" {
-			subdirResults, err := findInTree(repo, pat, *node.Subtree, filepath.Join(path, node.Name))
-			if err != nil {
-				return nil, err
+			if err := findInTree(repo, pat, *node.Subtree, filepath.Join(prefix, node.Name), snapshotID); err != nil {
+				return err
 			}
-
-			results = append(results, subdirResults...)
 		}
 	}
 
-	return results, nil
-}
-
-func findInSnapshot(repo *repository.Repository, pat findPattern, id restic.ID) error {
-	debug.Log("searching in snapshot %s\n  for entries within [%s %s]", id.Str(), pat.oldest, pat.newest)
-
-	sn, err := restic.LoadSnapshot(repo, id)
-	if err != nil {
-		return err
-	}
-
-	results, err := findInTree(repo, pat, *sn.Tree, "")
-	if err != nil {
-		return err
-	}
-
-	if len(results) == 0 {
-		return nil
-	}
-	Verbosef("found %d matching entries in snapshot %s\n", len(results), id)
-	for _, res := range results {
-		res.node.Name = filepath.Join(res.path, res.node.Name)
-		Printf("  %s\n", res.node)
-	}
-
+	return nil
+}
+
+func findInSnapshot(repo *repository.Repository, sn *restic.Snapshot, pat findPattern) error {
+	debug.Log("searching in snapshot %s\n  for entries within [%s %s]", sn.ID(), pat.oldest, pat.newest)
+
+	snapshotID := sn.ID().Str()
+	if err := findInTree(repo, pat, *sn.Tree, string(filepath.Separator), &snapshotID); err != nil {
+		return err
+	}
 	return nil
 }
 
 func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
 	if len(args) != 1 {
-		return errors.Fatalf("wrong number of arguments")
+		return errors.Fatal("wrong number of arguments")
 	}
 
-	var (
-		err error
-		pat findPattern
-	)
+	var err error
+	pat := findPattern{pattern: args[0]}
+	if opts.CaseInsensitive {
+		pat.pattern = strings.ToLower(pat.pattern)
+		pat.ignoreCase = true
+	}
 
 	if opts.Oldest != "" {
-		pat.oldest, err = parseTime(opts.Oldest)
-		if err != nil {
+		if pat.oldest, err = parseTime(opts.Oldest); err != nil {
 			return err
 		}
 	}
 
 	if opts.Newest != "" {
-		pat.newest, err = parseTime(opts.Newest)
-		if err != nil {
+		if pat.newest, err = parseTime(opts.Newest); err != nil {
 			return err
 		}
 	}
@@ -183,28 +183,14 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
-	err = repo.LoadIndex()
-	if err != nil {
+	if err = repo.LoadIndex(); err != nil {
 		return err
 	}
 
-	pat.pattern = args[0]
-
-	if opts.Snapshot != "" {
-		snapshotID, err := restic.FindSnapshot(repo, opts.Snapshot)
-		if err != nil {
-			return errors.Fatalf("invalid id %q: %v", args[1], err)
-		}
-
-		return findInSnapshot(repo, pat, snapshotID)
-	}
-
-	done := make(chan struct{})
-	defer close(done)
-	for snapshotID := range repo.List(restic.SnapshotFile, done) {
-		err := findInSnapshot(repo, pat, snapshotID)
-
-		if err != nil {
+	ctx, cancel := context.WithCancel(gopts.ctx)
+	defer cancel()
+	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, opts.Snapshots) {
+		if err = findInSnapshot(repo, sn, pat); err != nil {
 			return err
 		}
 	}

src/cmds/restic/cmd_forget.go

@@ -1,9 +1,10 @@
 package main
 
 import (
-	"fmt"
-	"io"
+	"context"
+	"encoding/json"
 	"restic"
+	"sort"
 	"strings"
 
 	"github.com/spf13/cobra"
@@ -24,19 +25,21 @@ data after 'forget' was run successfully, see the 'prune' command. `,
 
 // ForgetOptions collects all options for the forget command.
 type ForgetOptions struct {
-	Last    int
-	Hourly  int
-	Daily   int
-	Weekly  int
-	Monthly int
-	Yearly  int
-
+	Last     int
+	Hourly   int
+	Daily    int
+	Weekly   int
+	Monthly  int
+	Yearly   int
 	KeepTags []string
 
-	Hostname string
-	Tags     []string
+	Host  string
+	Tags  []string
+	Paths []string
 
-	DryRun bool
+	GroupByTags bool
+	DryRun      bool
+	Prune       bool
 }
 
 var forgetOptions ForgetOptions
@@ -52,51 +55,17 @@ func init() {
 	f.IntVarP(&forgetOptions.Monthly, "keep-monthly", "m", 0, "keep the last `n` monthly snapshots")
 	f.IntVarP(&forgetOptions.Yearly, "keep-yearly", "y", 0, "keep the last `n` yearly snapshots")
 
-	f.StringSliceVar(&forgetOptions.KeepTags, "keep-tag", []string{}, "always keep snapshots with this `tag` (can be specified multiple times)")
-	f.StringVar(&forgetOptions.Hostname, "hostname", "", "only forget snapshots for the given hostname")
-	f.StringSliceVar(&forgetOptions.Tags, "tag", []string{}, "only forget snapshots with the `tag` (can be specified multiple times)")
+	f.StringSliceVar(&forgetOptions.KeepTags, "keep-tag", []string{}, "keep snapshots with this `tag` (can be specified multiple times)")
+	f.BoolVarP(&forgetOptions.GroupByTags, "group-by-tags", "G", false, "Group by host,paths,tags instead of just host,paths")
+	// Sadly the commonly used shortcut `H` is already used.
+	f.StringVar(&forgetOptions.Host, "host", "", "only consider snapshots with the given `host`")
+	// Deprecated since 2017-03-07.
+	f.StringVar(&forgetOptions.Host, "hostname", "", "only consider snapshots with the given `hostname` (deprecated)")
+	f.StringSliceVar(&forgetOptions.Tags, "tag", nil, "only consider snapshots which include this `tag` (can be specified multiple times)")
+	f.StringSliceVar(&forgetOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` (can be specified multiple times)")
 
 	f.BoolVarP(&forgetOptions.DryRun, "dry-run", "n", false, "do not delete anything, just print what would be done")
-}
-
-func printSnapshots(w io.Writer, snapshots restic.Snapshots) {
-	tab := NewTable()
-	tab.Header = fmt.Sprintf("%-8s  %-19s  %-10s  %-10s  %s", "ID", "Date", "Host", "Tags", "Directory")
-	tab.RowFormat = "%-8s  %-19s  %-10s  %-10s  %s"
-
-	for _, sn := range snapshots {
-		if len(sn.Paths) == 0 {
-			continue
-		}
-
-		firstTag := ""
-		if len(sn.Tags) > 0 {
-			firstTag = sn.Tags[0]
-		}
-
-		tab.Rows = append(tab.Rows, []interface{}{sn.ID().Str(), sn.Time.Format(TimeFormat), sn.Hostname, firstTag, sn.Paths[0]})
-
-		rows := len(sn.Paths)
-		if len(sn.Tags) > rows {
-			rows = len(sn.Tags)
-		}
-
-		for i := 1; i < rows; i++ {
-			path := ""
-			if len(sn.Paths) > i {
-				path = sn.Paths[i]
-			}
-
-			tag := ""
-			if len(sn.Tags) > i {
-				tag = sn.Tags[i]
-			}
-
-			tab.Rows = append(tab.Rows, []interface{}{"", "", "", tag, path})
-		}
-	}
-
-	tab.Write(w)
+	f.BoolVar(&forgetOptions.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed")
 }
 
 func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
@@ -111,29 +80,45 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	err = repo.LoadIndex()
-	if err != nil {
-		return err
+	// group by hostname and dirs
+	type key struct {
+		Hostname string
+		Paths    []string
+		Tags     []string
 	}
+	snapshotGroups := make(map[string]restic.Snapshots)
 
-	// first, process all snapshot IDs given as arguments
-	for _, s := range args {
-		id, err := restic.FindSnapshot(repo, s)
-		if err != nil {
-			return err
-		}
-
-		if !opts.DryRun {
-			err = repo.Backend().Remove(restic.SnapshotFile, id.String())
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
+		if len(args) > 0 {
+			// When explicit snapshots args are given, remove them immediately.
+			if !opts.DryRun {
+				h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
+				if err = repo.Backend().Remove(h); err != nil {
+					return err
+				}
+				Verbosef("removed snapshot %v\n", sn.ID().Str())
+			} else {
+				Verbosef("would have removed snapshot %v\n", sn.ID().Str())
+			}
+		} else {
+			var tags []string
+			if opts.GroupByTags {
+				tags = sn.Tags
+				sort.StringSlice(tags).Sort()
+			}
+			sort.StringSlice(sn.Paths).Sort()
+			k, err := json.Marshal(key{Hostname: sn.Hostname, Tags: tags, Paths: sn.Paths})
 			if err != nil {
 				return err
 			}
-
-			Verbosef("removed snapshot %v\n", id.Str())
-		} else {
-			Verbosef("would removed snapshot %v\n", id.Str())
+			snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn)
 		}
 	}
+	if len(args) > 0 {
+		return nil
+	}
 
 	policy := restic.ExpirePolicy{
 		Last:    opts.Last,
@@ -146,53 +131,41 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	if policy.Empty() {
+		Verbosef("no policy was specified, no snapshots will be removed\n")
 		return nil
 	}
 
-	// then, load all remaining snapshots
-	snapshots, err := restic.LoadAllSnapshots(repo)
-	if err != nil {
-		return err
-	}
-
-	// group by hostname and dirs
-	type key struct {
-		Hostname string
-		Dirs     string
-	}
-
-	snapshotGroups := make(map[key]restic.Snapshots)
-
-	for _, sn := range snapshots {
-		if opts.Hostname != "" && sn.Hostname != opts.Hostname {
-			continue
+	removeSnapshots := 0
+	for k, snapshotGroup := range snapshotGroups {
+		var key key
+		if json.Unmarshal([]byte(k), &key) != nil {
+			return err
 		}
-
-		if !sn.HasTags(opts.Tags) {
-			continue
+		if opts.GroupByTags {
+			Printf("snapshots for host %v, tags [%v], paths: [%v]:\n\n", key.Hostname, strings.Join(key.Tags, ", "), strings.Join(key.Paths, ", "))
+		} else {
+			Printf("snapshots for host %v, paths: [%v]:\n\n", key.Hostname, strings.Join(key.Paths, ", "))
 		}
-
-		k := key{Hostname: sn.Hostname, Dirs: strings.Join(sn.Paths, ":")}
-		list := snapshotGroups[k]
-		list = append(list, sn)
-		snapshotGroups[k] = list
-	}
-
-	for key, snapshotGroup := range snapshotGroups {
-		Printf("snapshots for host %v, directories %v:\n\n", key.Hostname, key.Dirs)
 		keep, remove := restic.ApplyPolicy(snapshotGroup, policy)
 
-		Printf("keep %d snapshots:\n", len(keep))
-		printSnapshots(globalOptions.stdout, keep)
-		Printf("\n")
+		if len(keep) != 0 {
+			Printf("keep %d snapshots:\n", len(keep))
+			PrintSnapshots(globalOptions.stdout, keep)
+			Printf("\n")
+		}
 
-		Printf("remove %d snapshots:\n", len(remove))
-		printSnapshots(globalOptions.stdout, remove)
-		Printf("\n")
+		if len(remove) != 0 {
+			Printf("remove %d snapshots:\n", len(remove))
+			PrintSnapshots(globalOptions.stdout, remove)
+			Printf("\n")
+		}
+
+		removeSnapshots += len(remove)
 
 		if !opts.DryRun {
 			for _, sn := range remove {
-				err = repo.Backend().Remove(restic.SnapshotFile, sn.ID().String())
+				h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
+				err = repo.Backend().Remove(h)
 				if err != nil {
 					return err
 				}
@@ -200,5 +173,12 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
+	if removeSnapshots > 0 && opts.Prune {
+		Printf("%d snapshots have been removed, running prune\n", removeSnapshots)
+		if !opts.DryRun {
+			return pruneRepository(gopts, repo)
+		}
+	}
+
 	return nil
 }

src/cmds/restic/cmd_key.go

@@ -1,20 +1,20 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"restic"
-
-	"github.com/spf13/cobra"
-
 	"restic/errors"
 	"restic/repository"
+
+	"github.com/spf13/cobra"
 )
 
 var cmdKey = &cobra.Command{
 	Use:   "key [list|add|rm|passwd] [ID]",
 	Short: "manage keys (passwords)",
 	Long: `
-The "key" command manages keys (passwords) for accessing a repository.
+The "key" command manages keys (passwords) for accessing the repository.
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runKey(globalOptions, args)
@@ -25,15 +25,12 @@ func init() {
 	cmdRoot.AddCommand(cmdKey)
 }
 
-func listKeys(s *repository.Repository) error {
+func listKeys(ctx context.Context, s *repository.Repository) error {
 	tab := NewTable()
 	tab.Header = fmt.Sprintf(" %-10s  %-10s  %-10s  %s", "ID", "User", "Host", "Created")
 	tab.RowFormat = "%s%-10s  %-10s  %-10s  %s"
 
-	done := make(chan struct{})
-	defer close(done)
-
-	for id := range s.List(restic.KeyFile, done) {
+	for id := range s.List(restic.KeyFile, ctx.Done()) {
 		k, err := repository.LoadKey(s, id.String())
 		if err != nil {
 			Warnf("LoadKey() failed: %v\n", err)
@@ -87,7 +84,8 @@ func deleteKey(repo *repository.Repository, name string) error {
 		return errors.Fatal("refusing to remove key currently used to access repository")
 	}
 
-	err := repo.Backend().Remove(restic.KeyFile, name)
+	h := restic.Handle{Type: restic.KeyFile, Name: name}
+	err := repo.Backend().Remove(h)
 	if err != nil {
 		return err
 	}
@@ -107,7 +105,8 @@ func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
 		return errors.Fatalf("creating new key failed: %v\n", err)
 	}
 
-	err = repo.Backend().Remove(restic.KeyFile, repo.KeyName())
+	h := restic.Handle{Type: restic.KeyFile, Name: repo.KeyName()}
+	err = repo.Backend().Remove(h)
 	if err != nil {
 		return err
 	}
@@ -118,10 +117,13 @@ func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
 }
 
 func runKey(gopts GlobalOptions, args []string) error {
-	if len(args) < 1 || (args[0] == "rm" && len(args) != 2) {
-		return errors.Fatalf("wrong number of arguments")
+	if len(args) < 1 || (args[0] == "rm" && len(args) != 2) || (args[0] != "rm" && len(args) != 1) {
+		return errors.Fatal("wrong number of arguments")
 	}
 
+	ctx, cancel := context.WithCancel(gopts.ctx)
+	defer cancel()
+
 	repo, err := OpenRepository(gopts)
 	if err != nil {
 		return err
@@ -135,7 +137,7 @@ func runKey(gopts GlobalOptions, args []string) error {
 			return err
 		}
 
-		return listKeys(repo)
+		return listKeys(ctx, repo)
 	case "add":
 		lock, err := lockRepo(repo)
 		defer unlockRepo(lock)

src/cmds/restic/cmd_list.go

@@ -1,17 +1,19 @@
 package main
 
 import (
+	"fmt"
 	"restic"
 	"restic/errors"
+	"restic/index"
 
 	"github.com/spf13/cobra"
 )
 
 var cmdList = &cobra.Command{
 	Use:   "list [blobs|packs|index|snapshots|keys|locks]",
-	Short: "list items in the repository",
+	Short: "list objects in the repository",
 	Long: `
-
+The "list" command allows listing objects in the repository based on type.
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runList(globalOptions, args)
@@ -24,7 +26,7 @@ func init() {
 
 func runList(opts GlobalOptions, args []string) error {
 	if len(args) != 1 {
-		return errors.Fatalf("type not specified")
+		return errors.Fatal("type not specified")
 	}
 
 	repo, err := OpenRepository(opts)
@@ -52,6 +54,19 @@ func runList(opts GlobalOptions, args []string) error {
 		t = restic.KeyFile
 	case "locks":
 		t = restic.LockFile
+	case "blobs":
+		idx, err := index.Load(repo, nil)
+		if err != nil {
+			return err
+		}
+
+		for _, pack := range idx.Packs {
+			for _, entry := range pack.Entries {
+				fmt.Printf("%v %v\n", entry.Type, entry.ID)
+			}
+		}
+
+		return nil
 	default:
 		return errors.Fatal("invalid type")
 	}

src/cmds/restic/cmd_ls.go

@@ -1,8 +1,7 @@
 package main
 
 import (
-	"fmt"
-	"os"
+	"context"
 	"path/filepath"
 
 	"github.com/spf13/cobra"
@@ -13,56 +12,50 @@ import (
 )
 
 var cmdLs = &cobra.Command{
-	Use:   "ls [flags] snapshot-ID",
+	Use:   "ls [flags] [snapshot-ID ...]",
 	Short: "list files in a snapshot",
 	Long: `
 The "ls" command allows listing files and directories in a snapshot.
+
+The special snapshot-ID "latest" can be used to list files and directories of the latest snapshot in the repository.
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runLs(globalOptions, args)
+		return runLs(lsOptions, globalOptions, args)
 	},
 }
 
-var listLong bool
+// LsOptions collects all options for the ls command.
+type LsOptions struct {
+	ListLong bool
+	Host     string
+	Tags     []string
+	Paths    []string
+}
+
+var lsOptions LsOptions
 
 func init() {
 	cmdRoot.AddCommand(cmdLs)
 
-	cmdLs.Flags().BoolVarP(&listLong, "long", "l", false, "use a long listing format showing size and mode")
+	flags := cmdLs.Flags()
+	flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
+
+	flags.StringVarP(&lsOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
+	flags.StringSliceVar(&lsOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot ID is given")
+	flags.StringSliceVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
 }
 
-func printNode(prefix string, n *restic.Node) string {
-	if !listLong {
-		return filepath.Join(prefix, n.Name)
-	}
-
-	switch n.Type {
-	case "file":
-		return fmt.Sprintf("%s %5d %5d %6d %s %s",
-			n.Mode, n.UID, n.GID, n.Size, n.ModTime, filepath.Join(prefix, n.Name))
-	case "dir":
-		return fmt.Sprintf("%s %5d %5d %6d %s %s",
-			n.Mode|os.ModeDir, n.UID, n.GID, n.Size, n.ModTime, filepath.Join(prefix, n.Name))
-	case "symlink":
-		return fmt.Sprintf("%s %5d %5d %6d %s %s -> %s",
-			n.Mode|os.ModeSymlink, n.UID, n.GID, n.Size, n.ModTime, filepath.Join(prefix, n.Name), n.LinkTarget)
-	default:
-		return fmt.Sprintf("<Node(%s) %s>", n.Type, n.Name)
-	}
-}
-
-func printTree(prefix string, repo *repository.Repository, id restic.ID) error {
-	tree, err := repo.LoadTree(id)
+func printTree(repo *repository.Repository, id *restic.ID, prefix string) error {
+	tree, err := repo.LoadTree(*id)
 	if err != nil {
 		return err
 	}
 
 	for _, entry := range tree.Nodes {
-		Printf(printNode(prefix, entry) + "\n")
+		Printf(formatNode(prefix, entry, lsOptions.ListLong) + "\n")
 
 		if entry.Type == "dir" && entry.Subtree != nil {
-			err = printTree(filepath.Join(prefix, entry.Name), repo, *entry.Subtree)
-			if err != nil {
+			if err = printTree(repo, entry.Subtree, filepath.Join(prefix, entry.Name)); err != nil {
 				return err
 			}
 		}
@@ -71,9 +64,9 @@ func printTree(prefix string, repo *repository.Repository, id restic.ID) error {
 	return nil
 }
 
-func runLs(gopts GlobalOptions, args []string) error {
-	if len(args) < 1 || len(args) > 2 {
-		return errors.Fatalf("no snapshot ID given")
+func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
+	if len(args) == 0 && opts.Host == "" && len(opts.Tags) == 0 && len(opts.Paths) == 0 {
+		return errors.Fatal("Invalid arguments, either give one or more snapshot IDs or set filters.")
 	}
 
 	repo, err := OpenRepository(gopts)
@@ -81,22 +74,18 @@ func runLs(gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	err = repo.LoadIndex()
-	if err != nil {
+	if err = repo.LoadIndex(); err != nil {
 		return err
 	}
 
-	id, err := restic.FindSnapshot(repo, args[0])
-	if err != nil {
-		return err
+	ctx, cancel := context.WithCancel(gopts.ctx)
+	defer cancel()
+	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
+		Verbosef("snapshot %s of %v at %s):\n", sn.ID().Str(), sn.Paths, sn.Time)
+
+		if err = printTree(repo, sn.Tree, string(filepath.Separator)); err != nil {
+			return err
+		}
 	}
-
-	sn, err := restic.LoadSnapshot(repo, id)
-	if err != nil {
-		return err
-	}
-
-	Verbosef("snapshot of %v at %s:\n", sn.Paths, sn.Time)
-
-	return printTree("", repo, *sn.Tree)
+	return nil
 }

src/cmds/restic/cmd_mount.go

@@ -32,7 +32,12 @@ read-only mount.
 
 // MountOptions collects all options for the mount command.
 type MountOptions struct {
-	OwnerRoot bool
+	OwnerRoot  bool
+	AllowRoot  bool
+	AllowOther bool
+	Host       string
+	Tags       []string
+	Paths      []string
 }
 
 var mountOptions MountOptions
@@ -40,7 +45,14 @@ var mountOptions MountOptions
 func init() {
 	cmdRoot.AddCommand(cmdMount)
 
-	cmdMount.Flags().BoolVar(&mountOptions.OwnerRoot, "owner-root", false, "use 'root' as the owner of files and dirs")
+	mountFlags := cmdMount.Flags()
+	mountFlags.BoolVar(&mountOptions.OwnerRoot, "owner-root", false, "use 'root' as the owner of files and dirs")
+	mountFlags.BoolVar(&mountOptions.AllowRoot, "allow-root", false, "allow root user to access the data in the mounted directory")
+	mountFlags.BoolVar(&mountOptions.AllowOther, "allow-other", false, "allow other users to access the data in the mounted directory")
+
+	mountFlags.StringVarP(&mountOptions.Host, "host", "H", "", `only consider snapshots for this host`)
+	mountFlags.StringSliceVar(&mountOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`")
+	mountFlags.StringSliceVar(&mountOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`")
 }
 
 func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
@@ -64,11 +76,21 @@ func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
 			return err
 		}
 	}
-	c, err := systemFuse.Mount(
-		mountpoint,
+
+	mountOptions := []systemFuse.MountOption{
 		systemFuse.ReadOnly(),
 		systemFuse.FSName("restic"),
-	)
+	}
+
+	if opts.AllowRoot {
+		mountOptions = append(mountOptions, systemFuse.AllowRoot())
+	}
+
+	if opts.AllowOther {
+		mountOptions = append(mountOptions, systemFuse.AllowOther())
+	}
+
+	c, err := systemFuse.Mount(mountpoint, mountOptions...)
 	if err != nil {
 		return err
 	}
@@ -77,7 +99,7 @@ func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
 	Printf("Don't forget to umount after quitting!\n")
 
 	root := fs.Tree{}
-	root.Add("snapshots", fuse.NewSnapshotsDir(repo, opts.OwnerRoot))
+	root.Add("snapshots", fuse.NewSnapshotsDir(repo, opts.OwnerRoot, opts.Paths, opts.Tags, opts.Host))
 
 	debug.Log("serving mount at %v", mountpoint)
 	err = fs.Serve(c, &root)
@@ -95,7 +117,7 @@ func umount(mountpoint string) error {
 
 func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
 	if len(args) == 0 {
-		return errors.Fatalf("wrong number of parameters")
+		return errors.Fatal("wrong number of parameters")
 	}
 
 	mountpoint := args[0]

src/cmds/restic/cmd_prune.go

@@ -1,8 +1,8 @@
 package main
 
 import (
+	"context"
 	"fmt"
-	"os"
 	"restic"
 	"restic/debug"
 	"restic/errors"
@@ -11,8 +11,6 @@ import (
 	"time"
 
 	"github.com/spf13/cobra"
-
-	"golang.org/x/crypto/ssh/terminal"
 )
 
 var cmdPrune = &cobra.Command{
@@ -45,8 +43,7 @@ func newProgressMax(show bool, max uint64, description string) *restic.Progress
 			formatPercent(s.Blobs, max),
 			s.Blobs, max, description)
 
-		w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
-		if err == nil {
+		if w := stdoutTerminalWidth(); w > 0 {
 			if len(status) > w {
 				max := w - len(status) - 4
 				status = status[:max] + "... "
@@ -75,13 +72,17 @@ func runPrune(gopts GlobalOptions) error {
 		return err
 	}
 
-	err = repo.LoadIndex()
+	return pruneRepository(gopts, repo)
+}
+
+func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
+	err := repo.LoadIndex()
 	if err != nil {
 		return err
 	}
 
-	done := make(chan struct{})
-	defer close(done)
+	ctx, cancel := context.WithCancel(gopts.ctx)
+	defer cancel()
 
 	var stats struct {
 		blobs     int
@@ -91,7 +92,7 @@ func runPrune(gopts GlobalOptions) error {
 	}
 
 	Verbosef("counting files in repo\n")
-	for _ = range repo.List(restic.DataFile, done) {
+	for _ = range repo.List(restic.DataFile, ctx.Done()) {
 		stats.packs++
 	}
 
@@ -103,11 +104,13 @@ func runPrune(gopts GlobalOptions) error {
 		return err
 	}
 
+	blobs := 0
 	for _, pack := range idx.Packs {
 		stats.bytes += pack.Size
+		blobs += len(pack.Entries)
 	}
 	Verbosef("repository contains %v packs (%v blobs) with %v bytes\n",
-		len(idx.Packs), len(idx.Blobs), formatBytes(uint64(stats.bytes)))
+		len(idx.Packs), blobs, formatBytes(uint64(stats.bytes)))
 
 	blobCount := make(map[restic.BlobHandle]int)
 	duplicateBlobs := 0
@@ -164,18 +167,21 @@ func runPrune(gopts GlobalOptions) error {
 
 	// find packs that need a rewrite
 	rewritePacks := restic.NewIDSet()
-	for h, blob := range idx.Blobs {
-		if !usedBlobs.Has(h) {
-			rewritePacks.Merge(blob.Packs)
-			continue
-		}
+	for _, pack := range idx.Packs {
+		for _, blob := range pack.Entries {
+			h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
+			if !usedBlobs.Has(h) {
+				rewritePacks.Insert(pack.ID)
+				continue
+			}
 
-		if blobCount[h] > 1 {
-			rewritePacks.Merge(blob.Packs)
+			if blobCount[h] > 1 {
+				rewritePacks.Insert(pack.ID)
+			}
 		}
 	}
 
-	removeBytes := 0
+	removeBytes := duplicateBytes
 
 	// find packs that are unneeded
 	removePacks := restic.NewIDSet()
@@ -208,46 +214,34 @@ func runPrune(gopts GlobalOptions) error {
 	Verbosef("will delete %d packs and rewrite %d packs, this frees %s\n",
 		len(removePacks), len(rewritePacks), formatBytes(uint64(removeBytes)))
 
-	err = repository.Repack(repo, rewritePacks, usedBlobs)
-	if err != nil {
-		return err
-	}
-
-	for packID := range removePacks {
-		err = repo.Backend().Remove(restic.DataFile, packID.String())
+	if len(rewritePacks) != 0 {
+		bar = newProgressMax(!gopts.Quiet, uint64(len(rewritePacks)), "packs rewritten")
+		bar.Start()
+		err = repository.Repack(repo, rewritePacks, usedBlobs, bar)
 		if err != nil {
-			Warnf("unable to remove file %v from the repository\n", packID.Str())
+			return err
 		}
+		bar.Done()
 	}
 
-	Verbosef("creating new index\n")
-
-	stats.packs = 0
-	for _ = range repo.List(restic.DataFile, done) {
-		stats.packs++
+	if len(removePacks) != 0 {
+		bar = newProgressMax(!gopts.Quiet, uint64(len(removePacks)), "packs deleted")
+		bar.Start()
+		for packID := range removePacks {
+			h := restic.Handle{Type: restic.DataFile, Name: packID.String()}
+			err = repo.Backend().Remove(h)
+			if err != nil {
+				Warnf("unable to remove file %v from the repository\n", packID.Str())
+			}
+			bar.Report(restic.Stat{Blobs: 1})
+		}
+		bar.Done()
 	}
-	bar = newProgressMax(!gopts.Quiet, uint64(stats.packs), "packs")
-	idx, err = index.New(repo, bar)
-	if err != nil {
+
+	if err = rebuildIndex(ctx, repo); err != nil {
 		return err
 	}
 
-	var supersedes restic.IDs
-	for idxID := range repo.List(restic.IndexFile, done) {
-		err := repo.Backend().Remove(restic.IndexFile, idxID.String())
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "unable to remove index %v: %v\n", idxID.Str(), err)
-		}
-
-		supersedes = append(supersedes, idxID)
-	}
-
-	id, err := idx.Save(repo, supersedes)
-	if err != nil {
-		return err
-	}
-	Verbosef("saved new index as %v\n", id.Str())
-
 	Verbosef("done\n")
 	return nil
 }

src/cmds/restic/cmd_rebuild_index.go

@@ -1,7 +1,9 @@
 package main
 
 import (
-	"restic/repository"
+	"context"
+	"restic"
+	"restic/index"
 
 	"github.com/spf13/cobra"
 )
@@ -10,8 +12,8 @@ var cmdRebuildIndex = &cobra.Command{
 	Use:   "rebuild-index [flags]",
 	Short: "build a new index file",
 	Long: `
-The "rebuild-index" command creates a new index by combining the index files
-into a new one.
+The "rebuild-index" command creates a new index based on the pack files in the
+repository.
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runRebuildIndex(globalOptions)
@@ -34,5 +36,49 @@ func runRebuildIndex(gopts GlobalOptions) error {
 		return err
 	}
 
-	return repository.RebuildIndex(repo)
+	ctx, cancel := context.WithCancel(gopts.ctx)
+	defer cancel()
+	return rebuildIndex(ctx, repo)
+}
+
+func rebuildIndex(ctx context.Context, repo restic.Repository) error {
+	Verbosef("counting files in repo\n")
+
+	var packs uint64
+	for _ = range repo.List(restic.DataFile, ctx.Done()) {
+		packs++
+	}
+
+	bar := newProgressMax(!globalOptions.Quiet, packs, "packs")
+	idx, err := index.New(repo, bar)
+	if err != nil {
+		return err
+	}
+
+	Verbosef("finding old index files\n")
+
+	var supersedes restic.IDs
+	for id := range repo.List(restic.IndexFile, ctx.Done()) {
+		supersedes = append(supersedes, id)
+	}
+
+	id, err := idx.Save(repo, supersedes)
+	if err != nil {
+		return err
+	}
+
+	Verbosef("saved new index as %v\n", id.Str())
+
+	Verbosef("remove %d old index files\n", len(supersedes))
+
+	for _, id := range supersedes {
+		if err := repo.Backend().Remove(restic.Handle{
+			Type: restic.IndexFile,
+			Name: id.String(),
+		}); err != nil {
+			Warnf("error removing old index %v: %v\n", id.Str(), err)
+		}
+	}
+
+	return nil
 }

src/cmds/restic/cmd_restore.go

@@ -31,6 +31,7 @@ type RestoreOptions struct {
 	Target  string
 	Host    string
 	Paths   []string
+	Tags    []string
 }
 
 var restoreOptions RestoreOptions
@@ -44,12 +45,13 @@ func init() {
 	flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to")
 
 	flags.StringVarP(&restoreOptions.Host, "host", "H", "", `only consider snapshots for this host when the snapshot ID is "latest"`)
+	flags.StringSliceVar(&restoreOptions.Tags, "tag", nil, "only consider snapshots which include this `tag` for snapshot ID \"latest\"")
 	flags.StringSliceVar(&restoreOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
 }
 
 func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 	if len(args) != 1 {
-		return errors.Fatalf("no snapshot ID specified")
+		return errors.Fatal("no snapshot ID specified")
 	}
 
 	if opts.Target == "" {
@@ -85,7 +87,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 	var id restic.ID
 
 	if snapshotIDString == "latest" {
-		id, err = restic.FindLatestSnapshot(repo, opts.Paths, opts.Host)
+		id, err = restic.FindLatestSnapshot(repo, opts.Paths, opts.Tags, opts.Host)
 		if err != nil {
 			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Host:%v", err, opts.Paths, opts.Host)
 		}

src/cmds/restic/cmd_snapshots.go

@@ -1,9 +1,10 @@
 package main
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
-	"os"
-	"restic/errors"
+	"io"
 	"sort"
 
 	"github.com/spf13/cobra"
@@ -12,19 +13,20 @@ import (
 )
 
 var cmdSnapshots = &cobra.Command{
-	Use:   "snapshots",
+	Use:   "snapshots [snapshotID ...]",
 	Short: "list all snapshots",
 	Long: `
-The "snapshots" command lists all snapshots stored in a repository.
+The "snapshots" command lists all snapshots stored in the repository.
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runSnapshots(snapshotOptions, globalOptions, args)
 	},
 }
 
-// SnapshotOptions bundle all options for the snapshots command.
+// SnapshotOptions bundles all options for the snapshots command.
 type SnapshotOptions struct {
 	Host  string
+	Tags  []string
 	Paths []string
 }
 
@@ -34,15 +36,12 @@ func init() {
 	cmdRoot.AddCommand(cmdSnapshots)
 
 	f := cmdSnapshots.Flags()
-	f.StringVar(&snapshotOptions.Host, "host", "", "only print snapshots for this host")
-	f.StringSliceVar(&snapshotOptions.Paths, "path", []string{}, "only print snapshots for this `path` (can be specified multiple times)")
+	f.StringVarP(&snapshotOptions.Host, "host", "H", "", "only consider snapshots for this `host`")
+	f.StringSliceVar(&snapshotOptions.Tags, "tag", nil, "only consider snapshots which include this `tag` (can be specified multiple times)")
+	f.StringSliceVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)")
 }
 
 func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error {
-	if len(args) != 0 {
-		return errors.Fatalf("wrong number of arguments")
-	}
-
 	repo, err := OpenRepository(gopts)
 	if err != nil {
 		return err
@@ -56,37 +55,47 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 		}
 	}
 
-	tab := NewTable()
-	tab.Header = fmt.Sprintf("%-8s  %-19s  %-10s  %-10s  %s", "ID", "Date", "Host", "Tags", "Directory")
-	tab.RowFormat = "%-8s  %-19s  %-10s  %-10s  %s"
+	ctx, cancel := context.WithCancel(gopts.ctx)
+	defer cancel()
 
-	done := make(chan struct{})
-	defer close(done)
+	var list restic.Snapshots
+	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
+		list = append(list, sn)
+	}
+	sort.Sort(sort.Reverse(list))
 
-	list := []*restic.Snapshot{}
-	for id := range repo.List(restic.SnapshotFile, done) {
-		sn, err := restic.LoadSnapshot(repo, id)
+	if gopts.JSON {
+		err := printSnapshotsJSON(gopts.stdout, list)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "error loading snapshot %s: %v\n", id, err)
-			continue
+			Warnf("error printing snapshot: %v\n", err)
 		}
+		return nil
+	}
+	PrintSnapshots(gopts.stdout, list)
 
-		if restic.SamePaths(sn.Paths, opts.Paths) && (opts.Host == "" || opts.Host == sn.Hostname) {
-			pos := sort.Search(len(list), func(i int) bool {
-				return list[i].Time.After(sn.Time)
-			})
+	return nil
+}
 
-			if pos < len(list) {
-				list = append(list, nil)
-				copy(list[pos+1:], list[pos:])
-				list[pos] = sn
-			} else {
-				list = append(list, sn)
+// PrintSnapshots prints a text table of the snapshots in list to stdout.
+func PrintSnapshots(stdout io.Writer, list restic.Snapshots) {
+
+	// Determine the max widths for host and tag.
+	maxHost, maxTag := 10, 6
+	for _, sn := range list {
+		if len(sn.Hostname) > maxHost {
+			maxHost = len(sn.Hostname)
+		}
+		for _, tag := range sn.Tags {
+			if len(tag) > maxTag {
+				maxTag = len(tag)
 			}
 		}
-
 	}
 
+	tab := NewTable()
+	tab.Header = fmt.Sprintf("%-8s  %-19s  %-*s  %-*s  %-3s %s", "ID", "Date", -maxHost, "Host", -maxTag, "Tags", "", "Directory")
+	tab.RowFormat = fmt.Sprintf("%%-8s  %%-19s  %%%ds  %%%ds  %%-3s %%s", -maxHost, -maxTag)
+
 	for _, sn := range list {
 		if len(sn.Paths) == 0 {
 			continue
@@ -97,9 +106,18 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 			firstTag = sn.Tags[0]
 		}
 
-		tab.Rows = append(tab.Rows, []interface{}{sn.ID().Str(), sn.Time.Format(TimeFormat), sn.Hostname, firstTag, sn.Paths[0]})
-
 		rows := len(sn.Paths)
+		if rows < len(sn.Tags) {
+			rows = len(sn.Tags)
+		}
+
+		treeElement := "   "
+		if rows != 1 {
+			treeElement = "┌──"
+		}
+
+		tab.Rows = append(tab.Rows, []interface{}{sn.ID().Str(), sn.Time.Format(TimeFormat), sn.Hostname, firstTag, treeElement, sn.Paths[0]})
+
 		if len(sn.Tags) > rows {
 			rows = len(sn.Tags)
 		}
@@ -115,11 +133,38 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 				tag = sn.Tags[i]
 			}
 
-			tab.Rows = append(tab.Rows, []interface{}{"", "", "", tag, path})
+			treeElement := "│"
+			if i == (rows - 1) {
+				treeElement = "└──"
+			}
+
+			tab.Rows = append(tab.Rows, []interface{}{"", "", "", tag, treeElement, path})
 		}
 	}
 
-	tab.Write(os.Stdout)
-
-	return nil
+	tab.Write(stdout)
+}
+
+// Snapshot helps to print Snaphots as JSON with their ID included.
+type Snapshot struct {
+	*restic.Snapshot
+
+	ID *restic.ID `json:"id"`
+}
+
+// printSnapshotsJSON writes the JSON representation of list to stdout.
+func printSnapshotsJSON(stdout io.Writer, list restic.Snapshots) error {
+
+	var snapshots []Snapshot
+
+	for _, sn := range list {
+
+		k := Snapshot{
+			Snapshot: sn,
+			ID:       sn.ID(),
+		}
+		snapshots = append(snapshots, k)
+	}
+
+	return json.NewEncoder(stdout).Encode(snapshots)
 }

src/cmds/restic/cmd_tag.go

@@ -0,0 +1,142 @@
+package main
+
+import (
+	"context"
+
+	"github.com/spf13/cobra"
+
+	"restic"
+	"restic/debug"
+	"restic/errors"
+	"restic/repository"
+)
+
+var cmdTag = &cobra.Command{
+	Use:   "tag [flags] [snapshot-ID ...]",
+	Short: "modifies tags on snapshots",
+	Long: `
+The "tag" command allows you to modify tags on exiting snapshots.
+
+You can either set/replace the entire set of tags on a snapshot, or
+add tags to/remove tags from the existing set.
+
+When no snapshot-ID is given, all snapshots matching the host, tag and path filter criteria are modified.
+`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runTag(tagOptions, globalOptions, args)
+	},
+}
+
+// TagOptions bundles all options for the 'tag' command.
+type TagOptions struct {
+	Host       string
+	Paths      []string
+	Tags       []string
+	SetTags    []string
+	AddTags    []string
+	RemoveTags []string
+}
+
+var tagOptions TagOptions
+
+func init() {
+	cmdRoot.AddCommand(cmdTag)
+
+	tagFlags := cmdTag.Flags()
+	tagFlags.StringSliceVar(&tagOptions.SetTags, "set", nil, "`tag` which will replace the existing tags (can be given multiple times)")
+	tagFlags.StringSliceVar(&tagOptions.AddTags, "add", nil, "`tag` which will be added to the existing tags (can be given multiple times)")
+	tagFlags.StringSliceVar(&tagOptions.RemoveTags, "remove", nil, "`tag` which will be removed from the existing tags (can be given multiple times)")
+
+	tagFlags.StringVarP(&tagOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
+	tagFlags.StringSliceVar(&tagOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot-ID is given")
+	tagFlags.StringSliceVar(&tagOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
+}
+
+func changeTags(repo *repository.Repository, sn *restic.Snapshot, setTags, addTags, removeTags []string) (bool, error) {
+	var changed bool
+
+	if len(setTags) != 0 {
+		// Setting the tag to an empty string really means no tags.
+		if len(setTags) == 1 && setTags[0] == "" {
+			setTags = nil
+		}
+		sn.Tags = setTags
+		changed = true
+	} else {
+		changed = sn.AddTags(addTags)
+		if sn.RemoveTags(removeTags) {
+			changed = true
+		}
+	}
+
+	if changed {
+		// Retain the original snapshot id over all tag changes.
+		if sn.Original == nil {
+			sn.Original = sn.ID()
+		}
+
+		// Save the new snapshot.
+		id, err := repo.SaveJSONUnpacked(restic.SnapshotFile, sn)
+		if err != nil {
+			return false, err
+		}
+
+		debug.Log("new snapshot saved as %v", id.Str())
+
+		if err = repo.Flush(); err != nil {
+			return false, err
+		}
+
+		// Remove the old snapshot.
+		h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
+		if err = repo.Backend().Remove(h); err != nil {
+			return false, err
+		}
+
+		debug.Log("old snapshot %v removed", sn.ID())
+	}
+	return changed, nil
+}
+
+func runTag(opts TagOptions, gopts GlobalOptions, args []string) error {
+	if len(opts.SetTags) == 0 && len(opts.AddTags) == 0 && len(opts.RemoveTags) == 0 {
+		return errors.Fatal("nothing to do!")
+	}
+	if len(opts.SetTags) != 0 && (len(opts.AddTags) != 0 || len(opts.RemoveTags) != 0) {
+		return errors.Fatal("--set and --add/--remove cannot be given at the same time")
+	}
+
+	repo, err := OpenRepository(gopts)
+	if err != nil {
+		return err
+	}
+
+	if !gopts.NoLock {
+		Verbosef("Create exclusive lock for repository\n")
+		lock, err := lockRepoExclusive(repo)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+	}
+
+	changeCnt := 0
+	ctx, cancel := context.WithCancel(gopts.ctx)
+	defer cancel()
+	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
+		changed, err := changeTags(repo, sn, opts.SetTags, opts.AddTags, opts.RemoveTags)
+		if err != nil {
+			Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", sn.ID(), err)
+			continue
+		}
+		if changed {
+			changeCnt++
+		}
+	}
+	if changeCnt == 0 {
+		Verbosef("No snapshots were modified\n")
+	} else {
+		Verbosef("Modified tags on %v snapshots\n", changeCnt)
+	}
+	return nil
+}

src/cmds/restic/cmd_unlock.go

@@ -27,7 +27,7 @@ var unlockOptions UnlockOptions
 func init() {
 	cmdRoot.AddCommand(unlockCmd)
 
-	unlockCmd.Flags().BoolVar(&unlockOptions.RemoveAll, "remove-all", false, "Remove all locks, even non-stale ones")
+	unlockCmd.Flags().BoolVar(&unlockOptions.RemoveAll, "remove-all", false, "remove all locks, even non-stale ones")
 }
 
 func runUnlock(opts UnlockOptions, gopts GlobalOptions) error {

src/cmds/restic/cmd_version.go

@@ -9,14 +9,14 @@ import (
 
 var versionCmd = &cobra.Command{
 	Use:   "version",
-	Short: "Print version information",
+	Short: "print version information",
 	Long: `
 The "version" command prints detailed information about the build environment
 and the version of this software.
 `,
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Printf("restic %s\ncompiled at %s with %v on %v/%v\n",
-			version, compiledAt, runtime.Version(), runtime.GOOS, runtime.GOARCH)
+		fmt.Printf("restic %s\ncompiled with %v on %v/%v\n",
+			version, runtime.Version(), runtime.GOOS, runtime.GOARCH)
 	},
 }
 

src/cmds/restic/find.go

@@ -0,0 +1,78 @@
+package main
+
+import (
+	"context"
+
+	"restic"
+	"restic/repository"
+)
+
+// FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots.
+func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, host string, tags []string, paths []string, snapshotIDs []string) <-chan *restic.Snapshot {
+	out := make(chan *restic.Snapshot)
+	go func() {
+		defer close(out)
+		if len(snapshotIDs) != 0 {
+			var (
+				id         restic.ID
+				usedFilter bool
+				err        error
+			)
+			ids := make(restic.IDs, 0, len(snapshotIDs))
+			// Process all snapshot IDs given as arguments.
+			for _, s := range snapshotIDs {
+				if s == "latest" {
+					id, err = restic.FindLatestSnapshot(repo, paths, tags, host)
+					if err != nil {
+						Warnf("Ignoring %q, no snapshot matched given filter (Paths:%v Tags:%v Host:%v)\n", s, paths, tags, host)
+						usedFilter = true
+						continue
+					}
+				} else {
+					id, err = restic.FindSnapshot(repo, s)
+					if err != nil {
+						Warnf("Ignoring %q, it is not a snapshot id\n", s)
+						continue
+					}
+				}
+				ids = append(ids, id)
+			}
+
+			// Give the user some indication their filters are not used.
+			if !usedFilter && (host != "" || len(tags) != 0 || len(paths) != 0) {
+				Warnf("Ignoring filters as there are explicit snapshot ids given\n")
+			}
+
+			for _, id := range ids.Uniq() {
+				sn, err := restic.LoadSnapshot(repo, id)
+				if err != nil {
+					Warnf("Ignoring %q, could not load snapshot: %v\n", id, err)
+					continue
+				}
+				select {
+				case <-ctx.Done():
+					return
+				case out <- sn:
+				}
+			}
+			return
+		}
+
+		for id := range repo.List(restic.SnapshotFile, ctx.Done()) {
+			sn, err := restic.LoadSnapshot(repo, id)
+			if err != nil {
+				Warnf("Ignoring %q, could not load snapshot: %v\n", id, err)
+				continue
+			}
+			if (host != "" && host != sn.Hostname) || !sn.HasTags(tags) || !sn.HasPaths(paths) {
+				continue
+			}
+			select {
+			case <-ctx.Done():
+				return
+			case out <- sn:
+			}
+		}
+	}()
+	return out
+}

src/cmds/restic/format.go

@@ -2,7 +2,11 @@ package main
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
 	"time"
+
+	"restic"
 )
 
 func formatBytes(c uint64) string {
@@ -58,3 +62,23 @@ func formatDuration(d time.Duration) string {
 	sec := uint64(d / time.Second)
 	return formatSeconds(sec)
 }
+
+func formatNode(prefix string, n *restic.Node, long bool) string {
+	if !long {
+		return filepath.Join(prefix, n.Name)
+	}
+
+	switch n.Type {
+	case "file":
+		return fmt.Sprintf("%s %5d %5d %6d %s %s",
+			n.Mode, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name))
+	case "dir":
+		return fmt.Sprintf("%s %5d %5d %6d %s %s",
+			n.Mode|os.ModeDir, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name))
+	case "symlink":
+		return fmt.Sprintf("%s %5d %5d %6d %s %s -> %s",
+			n.Mode|os.ModeSymlink, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name), n.LinkTarget)
+	default:
+		return fmt.Sprintf("<Node(%s) %s>", n.Type, n.Name)
+	}
+}

src/cmds/restic/global.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -10,8 +11,6 @@ import (
 	"strings"
 	"syscall"
 
-	"github.com/spf13/cobra"
-
 	"restic/backend/local"
 	"restic/backend/rest"
 	"restic/backend/s3"
@@ -26,19 +25,6 @@ import (
 )
 
 var version = "compiled manually"
-var compiledAt = "unknown time"
-
-func parseEnvironment(cmd *cobra.Command, args []string) {
-	repo := os.Getenv("RESTIC_REPOSITORY")
-	if repo != "" {
-		globalOptions.Repo = repo
-	}
-
-	pw := os.Getenv("RESTIC_PASSWORD")
-	if pw != "" {
-		globalOptions.password = pw
-	}
-}
 
 // GlobalOptions hold all global options for restic.
 type GlobalOptions struct {
@@ -46,7 +32,9 @@ type GlobalOptions struct {
 	PasswordFile string
 	Quiet        bool
 	NoLock       bool
+	JSON         bool
 
+	ctx      context.Context
 	password string
 	stdout   io.Writer
 	stderr   io.Writer
@@ -58,11 +46,24 @@ var globalOptions = GlobalOptions{
 }
 
 func init() {
+	pw := os.Getenv("RESTIC_PASSWORD")
+	if pw != "" {
+		globalOptions.password = pw
+	}
+
+	var cancel context.CancelFunc
+	globalOptions.ctx, cancel = context.WithCancel(context.Background())
+	AddCleanupHandler(func() error {
+		cancel()
+		return nil
+	})
+
 	f := cmdRoot.PersistentFlags()
-	f.StringVarP(&globalOptions.Repo, "repo", "r", "", "repository to backup to or restore from (default: $RESTIC_REPOSITORY)")
+	f.StringVarP(&globalOptions.Repo, "repo", "r", os.Getenv("RESTIC_REPOSITORY"), "repository to backup to or restore from (default: $RESTIC_REPOSITORY)")
 	f.StringVarP(&globalOptions.PasswordFile, "password-file", "p", "", "read the repository password from a file")
-	f.BoolVarP(&globalOptions.Quiet, "quiet", "q", false, "do not outputcomprehensive progress report")
+	f.BoolVarP(&globalOptions.Quiet, "quiet", "q", false, "do not output comprehensive progress report")
 	f.BoolVar(&globalOptions.NoLock, "no-lock", false, "do not lock the repo, this allows some operations on read-only repos")
+	f.BoolVarP(&globalOptions.JSON, "json", "", false, "set output mode to JSON for commands that support it")
 
 	restoreTerminal()
 }
@@ -90,6 +91,14 @@ func stdoutIsTerminal() bool {
 	return terminal.IsTerminal(int(os.Stdout.Fd()))
 }
 
+func stdoutTerminalWidth() int {
+	w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
+	if err != nil {
+		return 0
+	}
+	return w
+}
+
 // restoreTerminal installs a cleanup handler that restores the previous
 // terminal state on exit.
 func restoreTerminal() {
@@ -118,8 +127,7 @@ func restoreTerminal() {
 // current windows cmd shell.
 func ClearLine() string {
 	if runtime.GOOS == "windows" {
-		w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
-		if err == nil {
+		if w := stdoutTerminalWidth(); w > 0 {
 			return strings.Repeat(" ", w-1) + "\r"
 		}
 		return ""
@@ -132,7 +140,7 @@ func Printf(format string, args ...interface{}) {
 	_, err := fmt.Fprintf(globalOptions.stdout, format, args...)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to write to stdout: %v\n", err)
-		os.Exit(100)
+		Exit(100)
 	}
 }
 
@@ -175,18 +183,19 @@ func Warnf(format string, args ...interface{}) {
 	_, err := fmt.Fprintf(globalOptions.stderr, format, args...)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to write to stderr: %v\n", err)
-		os.Exit(100)
+		Exit(100)
 	}
 }
 
-// Exitf uses Warnf to write the message and then calls os.Exit(exitcode).
+// Exitf uses Warnf to write the message and then terminates the process with
+// the given exit code.
 func Exitf(exitcode int, format string, args ...interface{}) {
 	if format[len(format)-1] != '\n' {
 		format += "\n"
 	}
 
 	Warnf(format, args...)
-	os.Exit(exitcode)
+	Exit(exitcode)
 }
 
 // readPassword reads the password from the given reader directly.

src/cmds/restic/global_debug.go

@@ -0,0 +1,60 @@
+// +build debug
+
+package main
+
+import (
+	"fmt"
+	"net/http"
+	_ "net/http/pprof"
+	"os"
+	"restic/errors"
+
+	"github.com/pkg/profile"
+)
+
+var (
+	listenMemoryProfile string
+	memProfilePath      string
+	cpuProfilePath      string
+
+	prof interface {
+		Stop()
+	}
+)
+
+func init() {
+	f := cmdRoot.PersistentFlags()
+	f.StringVar(&listenMemoryProfile, "listen-profile", "", "listen on this `address:port` for memory profiling")
+	f.StringVar(&memProfilePath, "mem-profile", "", "write memory profile to `dir`")
+	f.StringVar(&cpuProfilePath, "cpu-profile", "", "write cpu profile to `dir`")
+}
+
+func runDebug() error {
+	if listenMemoryProfile != "" {
+		fmt.Fprintf(os.Stderr, "running memory profile HTTP server on %v\n", listenMemoryProfile)
+		go func() {
+			err := http.ListenAndServe(listenMemoryProfile, nil)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "memory profile listen failed: %v\n", err)
+			}
+		}()
+	}
+
+	if memProfilePath != "" && cpuProfilePath != "" {
+		return errors.Fatal("only one profile (memory or CPU) may be activated at the same time")
+	}
+
+	if memProfilePath != "" {
+		prof = profile.Start(profile.Quiet, profile.MemProfile, profile.ProfilePath(memProfilePath))
+	} else if memProfilePath != "" {
+		prof = profile.Start(profile.Quiet, profile.CPUProfile, profile.ProfilePath(memProfilePath))
+	}
+
+	return nil
+}
+
+func shutdownDebug() {
+	if prof != nil {
+		prof.Stop()
+	}
+}

src/cmds/restic/global_release.go

@@ -0,0 +1,9 @@
+// +build !debug
+
+package main
+
+// runDebug is a noop without the debug tag.
+func runDebug() error { return nil }
+
+// shutdownDebug is a noop without the debug tag.
+func shutdownDebug() {}

src/cmds/restic/integration_helpers_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -15,6 +16,7 @@ import (
 type dirEntry struct {
 	path string
 	fi   os.FileInfo
+	link uint64
 }
 
 func walkDir(dir string) <-chan *dirEntry {
@@ -36,6 +38,7 @@ func walkDir(dir string) <-chan *dirEntry {
 			ch <- &dirEntry{
 				path: name,
 				fi:   info,
+				link: nlink(info),
 			}
 
 			return nil
@@ -192,6 +195,7 @@ func withTestEnvironment(t testing.TB, f func(*testEnvironment, GlobalOptions))
 	gopts := GlobalOptions{
 		Repo:     env.repo,
 		Quiet:    true,
+		ctx:      context.Background(),
 		password: TestPassword,
 		stdout:   os.Stdout,
 		stderr:   os.Stderr,

src/cmds/restic/integration_helpers_unix_test.go

@@ -4,7 +4,9 @@ package main
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
+	"path/filepath"
 	"syscall"
 )
 
@@ -37,5 +39,37 @@ func (e *dirEntry) equals(other *dirEntry) bool {
 		return false
 	}
 
+	if stat.Nlink != stat2.Nlink {
+		fmt.Fprintf(os.Stderr, "%v: Number of links do not match (%v != %v)\n", e.path, stat.Nlink, stat2.Nlink)
+		return false
+	}
+
 	return true
 }
+
+func nlink(info os.FileInfo) uint64 {
+	stat, _ := info.Sys().(*syscall.Stat_t)
+	return uint64(stat.Nlink)
+}
+
+func inode(info os.FileInfo) uint64 {
+	stat, _ := info.Sys().(*syscall.Stat_t)
+	return uint64(stat.Ino)
+}
+
+func createFileSetPerHardlink(dir string) map[uint64][]string {
+	var stat syscall.Stat_t
+	linkTests := make(map[uint64][]string)
+	files, err := ioutil.ReadDir(dir)
+	if err != nil {
+		return nil
+	}
+	for _, f := range files {
+
+		if err := syscall.Stat(filepath.Join(dir, f.Name()), &stat); err != nil {
+			return nil
+		}
+		linkTests[uint64(stat.Ino)] = append(linkTests[uint64(stat.Ino)], f.Name())
+	}
+	return linkTests
+}

src/cmds/restic/integration_helpers_windows_test.go

@@ -4,6 +4,7 @@ package main
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
 )
 
@@ -25,3 +26,24 @@ func (e *dirEntry) equals(other *dirEntry) bool {
 
 	return true
 }
+
+func nlink(info os.FileInfo) uint64 {
+	return 1
+}
+
+func inode(info os.FileInfo) uint64 {
+	return uint64(0)
+}
+
+func createFileSetPerHardlink(dir string) map[uint64][]string {
+	linkTests := make(map[uint64][]string)
+	files, err := ioutil.ReadDir(dir)
+	if err != nil {
+		return nil
+	}
+	for i, f := range files {
+		linkTests[uint64(i)] = append(linkTests[uint64(i)], f.Name())
+		i++
+	}
+	return linkTests
+}

src/cmds/restic/integration_test.go

@@ -4,9 +4,11 @@ import (
 	"bufio"
 	"bytes"
 	"crypto/rand"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
+	mrand "math/rand"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -140,7 +142,9 @@ func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
 		globalOptions.Quiet = quiet
 	}()
 
-	OK(t, runLs(gopts, []string{snapshotID}))
+	opts := LsOptions{}
+
+	OK(t, runLs(opts, gopts, []string{snapshotID}))
 
 	return strings.Split(string(buf.Bytes()), "\n")
 }
@@ -159,6 +163,41 @@ func testRunFind(t testing.TB, gopts GlobalOptions, pattern string) []string {
 	return strings.Split(string(buf.Bytes()), "\n")
 }
 
+func testRunSnapshots(t testing.TB, gopts GlobalOptions) (newest *Snapshot, snapmap map[restic.ID]Snapshot) {
+	buf := bytes.NewBuffer(nil)
+	globalOptions.stdout = buf
+	globalOptions.JSON = true
+	defer func() {
+		globalOptions.stdout = os.Stdout
+		globalOptions.JSON = gopts.JSON
+	}()
+
+	opts := SnapshotOptions{}
+
+	OK(t, runSnapshots(opts, globalOptions, []string{}))
+
+	snapshots := []Snapshot{}
+	OK(t, json.Unmarshal(buf.Bytes(), &snapshots))
+
+	snapmap = make(map[restic.ID]Snapshot, len(snapshots))
+	for _, sn := range snapshots {
+		snapmap[*sn.ID] = sn
+		if newest == nil || sn.Time.After(newest.Time) {
+			newest = &sn
+		}
+	}
+	return
+}
+
+func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) {
+	opts := ForgetOptions{}
+	OK(t, runForget(opts, gopts, args))
+}
+
+func testRunPrune(t testing.TB, gopts GlobalOptions) {
+	OK(t, runPrune(gopts))
+}
+
 func TestBackup(t *testing.T) {
 	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
 		datafile := filepath.Join("testdata", "backup-data.tar.gz")
@@ -343,6 +382,52 @@ func TestBackupMissingFile2(t *testing.T) {
 	})
 }
 
+func TestBackupChangedFile(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		fd, err := os.Open(datafile)
+		if os.IsNotExist(errors.Cause(err)) {
+			t.Skipf("unable to find data file %q, skipping", datafile)
+			return
+		}
+		OK(t, err)
+		OK(t, fd.Close())
+
+		SetupTarTestFixture(t, env.testdata, datafile)
+
+		testRunInit(t, gopts)
+
+		globalOptions.stderr = ioutil.Discard
+		defer func() {
+			globalOptions.stderr = os.Stderr
+		}()
+
+		modFile := filepath.Join(env.testdata, "0", "0", "6", "18")
+
+		ranHook := false
+		debug.Hook("archiver.SaveFile", func(context interface{}) {
+			pathname := context.(string)
+
+			if pathname != modFile {
+				return
+			}
+
+			t.Logf("in hook, modifying test file %v", modFile)
+			ranHook = true
+
+			OK(t, ioutil.WriteFile(modFile, []byte("modified"), 0600))
+		})
+
+		opts := BackupOptions{}
+
+		testRunBackup(t, []string{env.testdata}, opts, gopts)
+		testRunCheck(t, gopts)
+
+		Assert(t, ranHook, "hook did not run")
+		debug.RemoveHook("archiver.SaveFile")
+	})
+}
+
 func TestBackupDirectoryError(t *testing.T) {
 	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
 		datafile := filepath.Join("testdata", "backup-data.tar.gz")
@@ -460,23 +545,23 @@ func TestBackupExclude(t *testing.T) {
 		testRunBackup(t, []string{datadir}, opts, gopts)
 		snapshots, snapshotID := lastSnapshot(snapshots, loadSnapshotMap(t, gopts))
 		files := testRunLs(t, gopts, snapshotID)
-		Assert(t, includes(files, filepath.Join("testdata", "foo.tar.gz")),
+		Assert(t, includes(files, filepath.Join(string(filepath.Separator), "testdata", "foo.tar.gz")),
 			"expected file %q in first snapshot, but it's not included", "foo.tar.gz")
 
 		opts.Excludes = []string{"*.tar.gz"}
 		testRunBackup(t, []string{datadir}, opts, gopts)
 		snapshots, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, gopts))
 		files = testRunLs(t, gopts, snapshotID)
-		Assert(t, !includes(files, filepath.Join("testdata", "foo.tar.gz")),
+		Assert(t, !includes(files, filepath.Join(string(filepath.Separator), "testdata", "foo.tar.gz")),
 			"expected file %q not in first snapshot, but it's included", "foo.tar.gz")
 
 		opts.Excludes = []string{"*.tar.gz", "private/secret"}
 		testRunBackup(t, []string{datadir}, opts, gopts)
 		snapshots, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, gopts))
 		files = testRunLs(t, gopts, snapshotID)
-		Assert(t, !includes(files, filepath.Join("testdata", "foo.tar.gz")),
+		Assert(t, !includes(files, filepath.Join(string(filepath.Separator), "testdata", "foo.tar.gz")),
 			"expected file %q not in first snapshot, but it's included", "foo.tar.gz")
-		Assert(t, !includes(files, filepath.Join("testdata", "private", "secret", "passwords.txt")),
+		Assert(t, !includes(files, filepath.Join(string(filepath.Separator), "testdata", "private", "secret", "passwords.txt")),
 			"expected file %q not in first snapshot, but it's included", "passwords.txt")
 	})
 }
@@ -546,6 +631,105 @@ func TestIncrementalBackup(t *testing.T) {
 	})
 }
 
+func TestBackupTags(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		testRunInit(t, gopts)
+		SetupTarTestFixture(t, env.testdata, datafile)
+
+		opts := BackupOptions{}
+
+		testRunBackup(t, []string{env.testdata}, opts, gopts)
+		testRunCheck(t, gopts)
+		newest, _ := testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 0,
+			"expected no tags, got %v", newest.Tags)
+
+		opts.Tags = []string{"NL"}
+		testRunBackup(t, []string{env.testdata}, opts, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 1 && newest.Tags[0] == "NL",
+			"expected one NL tag, got %v", newest.Tags)
+	})
+}
+
+func testRunTag(t testing.TB, opts TagOptions, gopts GlobalOptions) {
+	OK(t, runTag(opts, gopts, []string{}))
+}
+
+func TestTag(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		testRunInit(t, gopts)
+		SetupTarTestFixture(t, env.testdata, datafile)
+
+		testRunBackup(t, []string{env.testdata}, BackupOptions{}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ := testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 0,
+			"expected no tags, got %v", newest.Tags)
+		Assert(t, newest.Original == nil,
+			"expected original ID to be nil, got %v", newest.Original)
+		originalID := *newest.ID
+
+		testRunTag(t, TagOptions{SetTags: []string{"NL"}}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 1 && newest.Tags[0] == "NL",
+			"set failed, expected one NL tag, got %v", newest.Tags)
+		Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+		Assert(t, *newest.Original == originalID,
+			"expected original ID to be set to the first snapshot id")
+
+		testRunTag(t, TagOptions{AddTags: []string{"CH"}}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 2 && newest.Tags[0] == "NL" && newest.Tags[1] == "CH",
+			"add failed, expected CH,NL tags, got %v", newest.Tags)
+		Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+		Assert(t, *newest.Original == originalID,
+			"expected original ID to be set to the first snapshot id")
+
+		testRunTag(t, TagOptions{RemoveTags: []string{"NL"}}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 1 && newest.Tags[0] == "CH",
+			"remove failed, expected one CH tag, got %v", newest.Tags)
+		Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+		Assert(t, *newest.Original == originalID,
+			"expected original ID to be set to the first snapshot id")
+
+		testRunTag(t, TagOptions{AddTags: []string{"US", "RU"}}, gopts)
+		testRunTag(t, TagOptions{RemoveTags: []string{"CH", "US", "RU"}}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 0,
+			"expected no tags, got %v", newest.Tags)
+		Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+		Assert(t, *newest.Original == originalID,
+			"expected original ID to be set to the first snapshot id")
+
+		// Check special case of removing all tags.
+		testRunTag(t, TagOptions{SetTags: []string{""}}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 0,
+			"expected no tags, got %v", newest.Tags)
+		Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+		Assert(t, *newest.Original == originalID,
+			"expected original ID to be set to the first snapshot id")
+	})
+}
+
 func testRunKeyListOtherIDs(t testing.TB, gopts GlobalOptions) []string {
 	buf := bytes.NewBuffer(nil)
 
@@ -684,6 +868,30 @@ func TestRestoreFilter(t *testing.T) {
 	})
 }
 
+func TestRestore(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+		testRunInit(t, gopts)
+
+		for i := 0; i < 10; i++ {
+			p := filepath.Join(env.testdata, fmt.Sprintf("foo/bar/testfile%v", i))
+			OK(t, os.MkdirAll(filepath.Dir(p), 0755))
+			OK(t, appendRandomData(p, uint(mrand.Intn(5<<21))))
+		}
+
+		opts := BackupOptions{}
+
+		testRunBackup(t, []string{env.testdata}, opts, gopts)
+		testRunCheck(t, gopts)
+
+		// Restore latest without any filters
+		restoredir := filepath.Join(env.base, "restore")
+		testRunRestoreLatest(t, gopts, restoredir, nil, "")
+
+		Assert(t, directoriesEqualContents(env.testdata, filepath.Join(restoredir, filepath.Base(env.testdata))),
+			"directories are not equal")
+	})
+}
+
 func TestRestoreLatest(t *testing.T) {
 
 	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
@@ -756,7 +964,7 @@ func TestRestoreWithPermissionFailure(t *testing.T) {
 
 		testRunRestore(t, gopts, filepath.Join(env.base, "restore"), snapshots[0])
 
-		// make sure that all files have been restored, regardeless of any
+		// make sure that all files have been restored, regardless of any
 		// permission errors
 		files := testRunLs(t, gopts, snapshots[0].String())
 		for _, filename := range files {
@@ -855,7 +1063,7 @@ func TestRebuildIndex(t *testing.T) {
 		}
 
 		if !strings.Contains(out, "restic rebuild-index") {
-			t.Fatalf("did not find hint for rebuild-index comman")
+			t.Fatalf("did not find hint for rebuild-index command")
 		}
 
 		testRunRebuildIndex(t, gopts)
@@ -901,3 +1109,130 @@ func TestCheckRestoreNoLock(t *testing.T) {
 		testRunRestore(t, gopts, filepath.Join(env.base, "restore"), snapshotIDs[0])
 	})
 }
+
+func TestPrune(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		fd, err := os.Open(datafile)
+		if os.IsNotExist(errors.Cause(err)) {
+			t.Skipf("unable to find data file %q, skipping", datafile)
+			return
+		}
+		OK(t, err)
+		OK(t, fd.Close())
+
+		testRunInit(t, gopts)
+
+		SetupTarTestFixture(t, env.testdata, datafile)
+		opts := BackupOptions{}
+
+		testRunBackup(t, []string{filepath.Join(env.testdata, "0", "0", "1")}, opts, gopts)
+		testRunBackup(t, []string{filepath.Join(env.testdata, "0", "0", "2")}, opts, gopts)
+		testRunBackup(t, []string{filepath.Join(env.testdata, "0", "0", "3")}, opts, gopts)
+
+		snapshotIDs := testRunList(t, "snapshots", gopts)
+		Assert(t, len(snapshotIDs) == 3,
+			"expected one snapshot, got %v", snapshotIDs)
+
+		testRunForget(t, gopts, snapshotIDs[0].String())
+		testRunPrune(t, gopts)
+		testRunCheck(t, gopts)
+	})
+}
+
+func TestHardLink(t *testing.T) {
+	// this test assumes a test set with a single directory containing hard linked files
+	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+		datafile := filepath.Join("testdata", "test.hl.tar.gz")
+		fd, err := os.Open(datafile)
+		if os.IsNotExist(errors.Cause(err)) {
+			t.Skipf("unable to find data file %q, skipping", datafile)
+			return
+		}
+		OK(t, err)
+		OK(t, fd.Close())
+
+		testRunInit(t, gopts)
+
+		SetupTarTestFixture(t, env.testdata, datafile)
+
+		linkTests := createFileSetPerHardlink(env.testdata)
+
+		opts := BackupOptions{}
+
+		// first backup
+		testRunBackup(t, []string{env.testdata}, opts, gopts)
+		snapshotIDs := testRunList(t, "snapshots", gopts)
+		Assert(t, len(snapshotIDs) == 1,
+			"expected one snapshot, got %v", snapshotIDs)
+
+		testRunCheck(t, gopts)
+
+		// restore all backups and compare
+		for i, snapshotID := range snapshotIDs {
+			restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i))
+			t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir)
+			testRunRestore(t, gopts, restoredir, snapshotIDs[0])
+			Assert(t, directoriesEqualContents(env.testdata, filepath.Join(restoredir, "testdata")),
+				"directories are not equal")
+
+			linkResults := createFileSetPerHardlink(filepath.Join(restoredir, "testdata"))
+			Assert(t, linksEqual(linkTests, linkResults),
+				"links are not equal")
+		}
+
+		testRunCheck(t, gopts)
+	})
+}
+
+func linksEqual(source, dest map[uint64][]string) bool {
+	for _, vs := range source {
+		found := false
+		for kd, vd := range dest {
+			if linkEqual(vs, vd) {
+				delete(dest, kd)
+				found = true
+				break
+			}
+		}
+		if !found {
+			return false
+		}
+	}
+
+	if len(dest) != 0 {
+		return false
+	}
+
+	return true
+}
+
+func linkEqual(source, dest []string) bool {
+	// equal if sliced are equal without considering order
+	if source == nil && dest == nil {
+		return true
+	}
+
+	if source == nil || dest == nil {
+		return false
+	}
+
+	if len(source) != len(dest) {
+		return false
+	}
+
+	for i := range source {
+		found := false
+		for j := range dest {
+			if source[i] == dest[j] {
+				found = true
+				break
+			}
+		}
+		if !found {
+			return false
+		}
+	}
+
+	return true
+}

src/cmds/restic/lock.go

@@ -36,6 +36,7 @@ func lockRepository(repo *repository.Repository, exclusive bool) (*restic.Lock,
 	if err != nil {
 		return nil, err
 	}
+	debug.Log("create lock %p (exclusive %v)", lock, exclusive)
 
 	globalLocks.Lock()
 	if globalLocks.cancelRefresh == nil {
@@ -88,7 +89,7 @@ func unlockRepo(lock *restic.Lock) error {
 	globalLocks.Lock()
 	defer globalLocks.Unlock()
 
-	debug.Log("unlocking repository")
+	debug.Log("unlocking repository with lock %p", lock)
 	if err := lock.Unlock(); err != nil {
 		debug.Log("error while unlocking: %v", err)
 		return err

src/cmds/restic/main.go

@@ -5,7 +5,6 @@ import (
 	"os"
 	"restic"
 	"restic/debug"
-	"runtime"
 
 	"github.com/spf13/cobra"
 
@@ -20,20 +19,17 @@ var cmdRoot = &cobra.Command{
 restic is a backup program which allows saving multiple revisions of files and
 directories in an encrypted repository stored on different backends.
 `,
-	SilenceErrors:    true,
-	SilenceUsage:     true,
-	PersistentPreRun: parseEnvironment,
-}
+	SilenceErrors: true,
+	SilenceUsage:  true,
 
-func init() {
-	// set GOMAXPROCS to number of CPUs
-	if runtime.Version() < "go1.5" {
-		gomaxprocs := os.Getenv("GOMAXPROCS")
-		debug.Log("read GOMAXPROCS from env variable, value: %s", gomaxprocs)
-		if gomaxprocs == "" {
-			runtime.GOMAXPROCS(runtime.NumCPU())
-		}
-	}
+	// run the debug functions for all subcommands (if build tag "debug" is
+	// enabled)
+	PersistentPreRunE: func(*cobra.Command, []string) error {
+		return runDebug()
+	},
+	PersistentPostRun: func(*cobra.Command, []string) {
+		shutdownDebug()
+	},
 }
 
 func main() {
@@ -49,9 +45,10 @@ func main() {
 		fmt.Fprintf(os.Stderr, "%+v\n", err)
 	}
 
-	RunCleanupHandlers()
-
+	var exitCode int
 	if err != nil {
-		os.Exit(1)
+		exitCode = 1
 	}
+
+	Exit(exitCode)
 }

src/restic/archiver/archive_reader.go

@@ -11,11 +11,22 @@ import (
 	"github.com/restic/chunker"
 )
 
-// ArchiveReader reads from the reader and archives the data. Returned is the
-// resulting snapshot and its ID.
-func ArchiveReader(repo restic.Repository, p *restic.Progress, rd io.Reader, name string, tags []string) (*restic.Snapshot, restic.ID, error) {
+// Reader allows saving a stream of data to the repository.
+type Reader struct {
+	restic.Repository
+
+	Tags     []string
+	Hostname string
+}
+
+// Archive reads data from the reader and saves it to the repo.
+func (r *Reader) Archive(name string, rd io.Reader, p *restic.Progress) (*restic.Snapshot, restic.ID, error) {
+	if name == "" {
+		return nil, restic.ID{}, errors.New("no filename given")
+	}
+
 	debug.Log("start archiving %s", name)
-	sn, err := restic.NewSnapshot([]string{name}, tags)
+	sn, err := restic.NewSnapshot([]string{name}, r.Tags, r.Hostname)
 	if err != nil {
 		return nil, restic.ID{}, err
 	}
@@ -23,9 +34,10 @@ func ArchiveReader(repo restic.Repository, p *restic.Progress, rd io.Reader, nam
 	p.Start()
 	defer p.Done()
 
+	repo := r.Repository
 	chnker := chunker.New(rd, repo.Config().ChunkerPolynomial)
 
-	var ids restic.IDs
+	ids := restic.IDs{}
 	var fileSize uint64
 
 	for {

src/restic/archiver/archive_reader_test.go

@@ -2,9 +2,11 @@ package archiver
 
 import (
 	"bytes"
+	"errors"
 	"io"
 	"math/rand"
 	"restic"
+	"restic/checker"
 	"restic/repository"
 	"testing"
 )
@@ -44,7 +46,7 @@ func checkSavedFile(t *testing.T, repo restic.Repository, treeID restic.ID, name
 			t.Fatal(err)
 		}
 
-		buf := make([]byte, int(size))
+		buf := restic.NewBlobBuffer(int(size))
 		n := loadBlob(t, repo, id, buf)
 		if n != len(buf) {
 			t.Errorf("wrong number of bytes read, want %d, got %d", len(buf), n)
@@ -77,7 +79,13 @@ func TestArchiveReader(t *testing.T) {
 
 	f := fakeFile(t, seed, size)
 
-	sn, id, err := ArchiveReader(repo, nil, f, "fakefile", []string{"test"})
+	r := &Reader{
+		Repository: repo,
+		Hostname:   "localhost",
+		Tags:       []string{"test"},
+	}
+
+	sn, id, err := r.Archive("fakefile", f, nil)
 	if err != nil {
 		t.Fatalf("ArchiveReader() returned error %v", err)
 	}
@@ -89,6 +97,80 @@ func TestArchiveReader(t *testing.T) {
 	t.Logf("snapshot saved as %v, tree is %v", id.Str(), sn.Tree.Str())
 
 	checkSavedFile(t, repo, *sn.Tree, "fakefile", fakeFile(t, seed, size))
+
+	checker.TestCheckRepo(t, repo)
+}
+
+func TestArchiveReaderNull(t *testing.T) {
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
+
+	r := &Reader{
+		Repository: repo,
+		Hostname:   "localhost",
+		Tags:       []string{"test"},
+	}
+
+	sn, id, err := r.Archive("fakefile", bytes.NewReader(nil), nil)
+	if err != nil {
+		t.Fatalf("ArchiveReader() returned error %v", err)
+	}
+
+	if id.IsNull() {
+		t.Fatalf("ArchiveReader() returned null ID")
+	}
+
+	t.Logf("snapshot saved as %v, tree is %v", id.Str(), sn.Tree.Str())
+
+	checker.TestCheckRepo(t, repo)
+}
+
+type errReader string
+
+func (e errReader) Read([]byte) (int, error) {
+	return 0, errors.New(string(e))
+}
+
+func countSnapshots(t testing.TB, repo restic.Repository) int {
+	done := make(chan struct{})
+	defer close(done)
+
+	snapshots := 0
+	for range repo.List(restic.SnapshotFile, done) {
+		snapshots++
+	}
+	return snapshots
+}
+
+func TestArchiveReaderError(t *testing.T) {
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
+
+	r := &Reader{
+		Repository: repo,
+		Hostname:   "localhost",
+		Tags:       []string{"test"},
+	}
+
+	sn, id, err := r.Archive("fakefile", errReader("error returned by reading stdin"), nil)
+	if err == nil {
+		t.Errorf("expected error not returned")
+	}
+
+	if sn != nil {
+		t.Errorf("Snapshot should be nil, but isn't")
+	}
+
+	if !id.IsNull() {
+		t.Errorf("id should be null, but %v returned", id.Str())
+	}
+
+	n := countSnapshots(t, repo)
+	if n > 0 {
+		t.Errorf("expected zero snapshots, but got %d", n)
+	}
+
+	checker.TestCheckRepo(t, repo)
 }
 
 func BenchmarkArchiveReader(t *testing.B) {
@@ -103,11 +185,17 @@ func BenchmarkArchiveReader(t *testing.B) {
 		t.Fatal(err)
 	}
 
+	r := &Reader{
+		Repository: repo,
+		Hostname:   "localhost",
+		Tags:       []string{"test"},
+	}
+
 	t.SetBytes(size)
 	t.ResetTimer()
 
 	for i := 0; i < t.N; i++ {
-		_, _, err := ArchiveReader(repo, nil, bytes.NewReader(buf), "fakefile", []string{"test"})
+		_, _, err := r.Archive("fakefile", bytes.NewReader(buf), nil)
 		if err != nil {
 			t.Fatal(err)
 		}

src/restic/archiver/archiver.go

@@ -26,7 +26,9 @@ const (
 	maxConcurrency     = 10
 )
 
-var archiverAbortOnAllErrors = func(str string, fi os.FileInfo, err error) error { return err }
+var archiverPrintWarnings = func(path string, fi os.FileInfo, err error) {
+	fmt.Fprintf(os.Stderr, "warning for %v: %v", path, err)
+}
 var archiverAllowAllFiles = func(string, os.FileInfo) bool { return true }
 
 // Archiver is used to backup a set of directories.
@@ -39,7 +41,7 @@ type Archiver struct {
 
 	blobToken chan struct{}
 
-	Error        func(dir string, fi os.FileInfo, err error) error
+	Warn         func(dir string, fi os.FileInfo, err error)
 	SelectFilter pipe.SelectFunc
 	Excludes     []string
 }
@@ -61,7 +63,7 @@ func New(repo restic.Repository) *Archiver {
 		arch.blobToken <- struct{}{}
 	}
 
-	arch.Error = archiverAbortOnAllErrors
+	arch.Warn = archiverPrintWarnings
 	arch.SelectFilter = archiverAllowAllFiles
 
 	return arch
@@ -135,15 +137,12 @@ func (arch *Archiver) reloadFileIfChanged(node *restic.Node, file fs.File) (*res
 		return node, nil
 	}
 
-	err = arch.Error(node.Path, fi, errors.New("file has changed"))
-	if err != nil {
-		return nil, err
-	}
+	arch.Warn(node.Path, fi, errors.New("file has changed"))
 
 	node, err = restic.NodeFromFileInfo(node.Path, fi)
 	if err != nil {
 		debug.Log("restic.NodeFromFileInfo returned error for %v: %v", node.Path, err)
-		return nil, err
+		arch.Warn(node.Path, fi, err)
 	}
 
 	return node, nil
@@ -197,7 +196,7 @@ func updateNodeContent(node *restic.Node, results []saveResult) error {
 	}
 
 	if bytes != node.Size {
-		return errors.Errorf("errors saving node %q: saved %d bytes, wanted %d bytes", node.Path, bytes, node.Size)
+		fmt.Fprintf(os.Stderr, "warning for %v: expected %d bytes, saved %d bytes\n", node.Path, node.Size, bytes)
 	}
 
 	debug.Log("SaveFile(%q): %v blobs\n", node.Path, len(results))
@@ -207,16 +206,18 @@ func updateNodeContent(node *restic.Node, results []saveResult) error {
 
 // SaveFile stores the content of the file on the backend as a Blob by calling
 // Save for each chunk.
-func (arch *Archiver) SaveFile(p *restic.Progress, node *restic.Node) error {
+func (arch *Archiver) SaveFile(p *restic.Progress, node *restic.Node) (*restic.Node, error) {
 	file, err := fs.Open(node.Path)
 	defer file.Close()
 	if err != nil {
-		return errors.Wrap(err, "Open")
+		return node, errors.Wrap(err, "Open")
 	}
 
+	debug.RunHook("archiver.SaveFile", node.Path)
+
 	node, err = arch.reloadFileIfChanged(node, file)
 	if err != nil {
-		return err
+		return node, err
 	}
 
 	chnker := chunker.New(file, arch.repo.Config().ChunkerPolynomial)
@@ -229,7 +230,7 @@ func (arch *Archiver) SaveFile(p *restic.Progress, node *restic.Node) error {
 		}
 
 		if err != nil {
-			return errors.Wrap(err, "chunker.Next")
+			return node, errors.Wrap(err, "chunker.Next")
 		}
 
 		resCh := make(chan saveResult, 1)
@@ -239,11 +240,11 @@ func (arch *Archiver) SaveFile(p *restic.Progress, node *restic.Node) error {
 
 	results, err := waitForResults(resultChannels)
 	if err != nil {
-		return err
+		return node, err
 	}
-
 	err = updateNodeContent(node, results)
-	return err
+
+	return node, err
 }
 
 func (arch *Archiver) fileWorker(wg *sync.WaitGroup, p *restic.Progress, done <-chan struct{}, entCh <-chan pipe.Entry) {
@@ -274,11 +275,8 @@ func (arch *Archiver) fileWorker(wg *sync.WaitGroup, p *restic.Progress, done <-
 
 			node, err := restic.NodeFromFileInfo(e.Fullpath(), e.Info())
 			if err != nil {
-				// TODO: integrate error reporting
 				debug.Log("restic.NodeFromFileInfo returned error for %v: %v", node.Path, err)
-				e.Result() <- nil
-				p.Report(restic.Stat{Errors: 1})
-				continue
+				arch.Warn(e.Fullpath(), e.Info(), err)
 			}
 
 			// try to use old node, if present
@@ -306,11 +304,11 @@ func (arch *Archiver) fileWorker(wg *sync.WaitGroup, p *restic.Progress, done <-
 
 			// otherwise read file normally
 			if node.Type == "file" && len(node.Content) == 0 {
-				debug.Log("   read and save %v, content: %v", e.Path(), node.Content)
-				err = arch.SaveFile(p, node)
+				debug.Log("   read and save %v", e.Path())
+				node, err = arch.SaveFile(p, node)
 				if err != nil {
-					// TODO: integrate error reporting
 					fmt.Fprintf(os.Stderr, "error for %v: %v\n", node.Path, err)
+					arch.Warn(e.Path(), nil, err)
 					// ignore this file
 					e.Result() <- nil
 					p.Report(restic.Stat{Errors: 1})
@@ -370,25 +368,28 @@ func (arch *Archiver) dirWorker(wg *sync.WaitGroup, p *restic.Progress, done <-c
 
 				// else insert node
 				node := res.(*restic.Node)
-				tree.Insert(node)
 
 				if node.Type == "dir" {
 					debug.Log("got tree node for %s: %v", node.Path, node.Subtree)
 
+					if node.Subtree == nil {
+						debug.Log("subtree is nil for node %v", node.Path)
+						continue
+					}
+
 					if node.Subtree.IsNull() {
 						panic("invalid null subtree restic.ID")
 					}
 				}
+				tree.Insert(node)
 			}
 
 			node := &restic.Node{}
 
 			if dir.Path() != "" && dir.Info() != nil {
-				n, err := restic.NodeFromFileInfo(dir.Path(), dir.Info())
+				n, err := restic.NodeFromFileInfo(dir.Fullpath(), dir.Info())
 				if err != nil {
-					n.Error = err.Error()
-					dir.Result() <- n
-					continue
+					arch.Warn(dir.Path(), dir.Info(), err)
 				}
 				node = n
 			}
@@ -633,7 +634,7 @@ func (p baseNameSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
 // Snapshot creates a snapshot of the given paths. If parentrestic.ID is set, this is
 // used to compare the files to the ones archived at the time this snapshot was
 // taken.
-func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, parentID *restic.ID) (*restic.Snapshot, restic.ID, error) {
+func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, hostname string, parentID *restic.ID) (*restic.Snapshot, restic.ID, error) {
 	paths = unique(paths)
 	sort.Sort(baseNameSlice(paths))
 
@@ -649,7 +650,7 @@ func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, parentI
 	defer p.Done()
 
 	// create new snapshot
-	sn, err := restic.NewSnapshot(paths, tags)
+	sn, err := restic.NewSnapshot(paths, tags, hostname)
 	if err != nil {
 		return nil, restic.ID{}, err
 	}
@@ -727,23 +728,25 @@ func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, parentI
 
 	debug.Log("workers terminated")
 
+	// flush repository
+	err = arch.repo.Flush()
+	if err != nil {
+		return nil, restic.ID{}, err
+	}
+
 	// receive the top-level tree
 	root := (<-resCh).(*restic.Node)
 	debug.Log("root node received: %v", root.Subtree.Str())
 	sn.Tree = root.Subtree
 
-	// save snapshot
-	id, err := arch.repo.SaveJSONUnpacked(restic.SnapshotFile, sn)
+	// load top-level tree again to see if it is empty
+	toptree, err := arch.repo.LoadTree(*root.Subtree)
 	if err != nil {
 		return nil, restic.ID{}, err
 	}
 
-	debug.Log("saved snapshot %v", id.Str())
-
-	// flush repository
-	err = arch.repo.Flush()
-	if err != nil {
-		return nil, restic.ID{}, err
+	if len(toptree.Nodes) == 0 {
+		return nil, restic.ID{}, errors.Fatal("no files/dirs saved, refusing to create empty snapshot")
 	}
 
 	// save index
@@ -755,6 +758,14 @@ func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, parentI
 
 	debug.Log("saved indexes")
 
+	// save snapshot
+	id, err := arch.repo.SaveJSONUnpacked(restic.SnapshotFile, sn)
+	if err != nil {
+		return nil, restic.ID{}, err
+	}
+
+	debug.Log("saved snapshot %v", id.Str())
+
 	return sn, id, nil
 }
 

src/restic/archiver/archiver_duplication_test.go

@@ -39,15 +39,15 @@ func randomID() restic.ID {
 func forgetfulBackend() restic.Backend {
 	be := &mock.Backend{}
 
-	be.TestFn = func(t restic.FileType, name string) (bool, error) {
+	be.TestFn = func(h restic.Handle) (bool, error) {
 		return false, nil
 	}
 
-	be.LoadFn = func(h restic.Handle, p []byte, off int64) (int, error) {
-		return 0, errors.New("not found")
+	be.LoadFn = func(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+		return nil, errors.New("not found")
 	}
 
-	be.SaveFn = func(h restic.Handle, p []byte) error {
+	be.SaveFn = func(h restic.Handle, rd io.Reader) error {
 		return nil
 	}
 
@@ -55,7 +55,7 @@ func forgetfulBackend() restic.Backend {
 		return restic.FileInfo{}, errors.New("not found")
 	}
 
-	be.RemoveFn = func(t restic.FileType, name string) error {
+	be.RemoveFn = func(h restic.Handle) error {
 		return nil
 	}
 
@@ -142,6 +142,11 @@ func testArchiverDuplication(t *testing.T) {
 	close(done)
 
 	wg.Wait()
+
+	err = repo.Flush()
+	if err != nil {
+		t.Fatal(err)
+	}
 }
 
 func TestArchiverDuplication(t *testing.T) {

src/restic/archiver/archiver_test.go

@@ -104,7 +104,7 @@ func archiveDirectory(b testing.TB) {
 
 	arch := archiver.New(repo)
 
-	_, id, err := arch.Snapshot(nil, []string{BenchArchiveDirectory}, nil, nil)
+	_, id, err := arch.Snapshot(nil, []string{BenchArchiveDirectory}, nil, "localhost", nil)
 	OK(b, err)
 
 	b.Logf("snapshot archived as %v", id)
@@ -220,7 +220,7 @@ func testParallelSaveWithDuplication(t *testing.T, seed int) {
 
 	errChannels := [](<-chan error){}
 
-	// interweaved processing of subsequent chunks
+	// interwoven processing of subsequent chunks
 	maxParallel := 2*duplication - 1
 	barrier := make(chan struct{}, maxParallel)
 
@@ -294,3 +294,23 @@ func assertNoUnreferencedPacks(t *testing.T, chkr *checker.Checker) {
 		OK(t, err)
 	}
 }
+
+func TestArchiveEmptySnapshot(t *testing.T) {
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
+
+	arch := archiver.New(repo)
+
+	sn, id, err := arch.Snapshot(nil, []string{"file-does-not-exist-123123213123", "file2-does-not-exist-too-123123123"}, nil, "localhost", nil)
+	if err == nil {
+		t.Errorf("expected error for empty snapshot, got nil")
+	}
+
+	if !id.IsNull() {
+		t.Errorf("expected null ID for empty snapshot, got %v", id.Str())
+	}
+
+	if sn != nil {
+		t.Errorf("expected null snapshot for empty snapshot, got %v", sn)
+	}
+}

src/restic/archiver/testing.go

@@ -8,7 +8,7 @@ import (
 // TestSnapshot creates a new snapshot of path.
 func TestSnapshot(t testing.TB, repo restic.Repository, path string, parent *restic.ID) *restic.Snapshot {
 	arch := New(repo)
-	sn, _, err := arch.Snapshot(nil, []string{path}, []string{"test"}, parent)
+	sn, _, err := arch.Snapshot(nil, []string{path}, []string{"test"}, "localhost", parent)
 	if err != nil {
 		t.Fatal(err)
 	}

src/restic/backend.go

@@ -1,5 +1,7 @@
 package restic
 
+import "io"
+
 // Backend is used to store and access data.
 type Backend interface {
 	// Location returns a string that describes the type and location of the
@@ -7,22 +9,22 @@ type Backend interface {
 	Location() string
 
 	// Test a boolean value whether a File with the name and type exists.
-	Test(t FileType, name string) (bool, error)
+	Test(h Handle) (bool, error)
 
 	// Remove removes a File with type t and name.
-	Remove(t FileType, name string) error
+	Remove(h Handle) error
 
 	// Close the backend
 	Close() error
 
-	// Load returns the data stored in the backend for h at the given offset
-	// and saves it in p. Load has the same semantics as io.ReaderAt, except
-	// that a negative offset is also allowed. In this case it references a
-	// position relative to the end of the file (similar to Seek()).
-	Load(h Handle, p []byte, off int64) (int, error)
-
 	// Save stores the data in the backend under the given handle.
-	Save(h Handle, p []byte) error
+	Save(h Handle, rd io.Reader) error
+
+	// Load returns a reader that yields the contents of the file at h at the
+	// given offset. If length is larger than zero, only a portion of the file
+	// is returned. rd must be closed after use. If an error is returned, the
+	// ReadCloser must be nil.
+	Load(h Handle, length int, offset int64) (io.ReadCloser, error)
 
 	// Stat returns information about the File identified by h.
 	Stat(h Handle) (FileInfo, error)

src/restic/backend/local/backend_test.go

@@ -51,13 +51,6 @@ func TestLocalBackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestLocalBackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestLocalBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/local/local.go

@@ -101,67 +101,22 @@ func dirname(base string, t restic.FileType, name string) string {
 	return filepath.Join(base, n)
 }
 
-// Load returns the data stored in the backend for h at the given offset and
-// saves it in p. Load has the same semantics as io.ReaderAt, with one
-// exception: when off is lower than zero, it is treated as an offset relative
-// to the end of the file.
-func (b *Local) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
-	debug.Log("Load %v, length %v at %v", h, len(p), off)
-	if err := h.Valid(); err != nil {
-		return 0, err
-	}
-
-	f, err := fs.Open(filename(b.p, h.Type, h.Name))
-	if err != nil {
-		return 0, errors.Wrap(err, "Open")
-	}
-
-	defer func() {
-		e := f.Close()
-		if err == nil && e != nil {
-			err = errors.Wrap(e, "Close")
-		}
-	}()
-
-	switch {
-	case off > 0:
-		_, err = f.Seek(off, 0)
-	case off < 0:
-		_, err = f.Seek(off, 2)
-	}
-
-	if err != nil {
-		return 0, errors.Wrap(err, "Seek")
-	}
-
-	return io.ReadFull(f, p)
-}
-
-// writeToTempfile saves p into a tempfile in tempdir.
-func writeToTempfile(tempdir string, p []byte) (filename string, err error) {
+// copyToTempfile saves p into a tempfile in tempdir.
+func copyToTempfile(tempdir string, rd io.Reader) (filename string, err error) {
 	tmpfile, err := ioutil.TempFile(tempdir, "temp-")
 	if err != nil {
 		return "", errors.Wrap(err, "TempFile")
 	}
 
-	n, err := tmpfile.Write(p)
+	_, err = io.Copy(tmpfile, rd)
 	if err != nil {
 		return "", errors.Wrap(err, "Write")
 	}
 
-	if n != len(p) {
-		return "", errors.New("not all bytes writen")
-	}
-
 	if err = tmpfile.Sync(); err != nil {
 		return "", errors.Wrap(err, "Syncn")
 	}
 
-	err = fs.ClearCache(tmpfile)
-	if err != nil {
-		return "", errors.Wrap(err, "ClearCache")
-	}
-
 	err = tmpfile.Close()
 	if err != nil {
 		return "", errors.Wrap(err, "Close")
@@ -171,14 +126,14 @@ func writeToTempfile(tempdir string, p []byte) (filename string, err error) {
 }
 
 // Save stores data in the backend at the handle.
-func (b *Local) Save(h restic.Handle, p []byte) (err error) {
-	debug.Log("Save %v, length %v", h, len(p))
+func (b *Local) Save(h restic.Handle, rd io.Reader) (err error) {
+	debug.Log("Save %v", h)
 	if err := h.Valid(); err != nil {
 		return err
 	}
 
-	tmpfile, err := writeToTempfile(filepath.Join(b.p, backend.Paths.Temp), p)
-	debug.Log("saved %v (%d bytes) to %v", h, len(p), tmpfile)
+	tmpfile, err := copyToTempfile(filepath.Join(b.p, backend.Paths.Temp), rd)
+	debug.Log("saved %v to %v", h, tmpfile)
 	if err != nil {
 		return err
 	}
@@ -215,6 +170,39 @@ func (b *Local) Save(h restic.Handle, p []byte) (err error) {
 	return setNewFileMode(filename, fi)
 }
 
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (b *Local) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	f, err := os.Open(filename(b.p, h.Type, h.Name))
+	if err != nil {
+		return nil, err
+	}
+
+	if offset > 0 {
+		_, err = f.Seek(offset, 0)
+		if err != nil {
+			f.Close()
+			return nil, err
+		}
+	}
+
+	if length > 0 {
+		return backend.LimitReadCloser(f, int64(length)), nil
+	}
+
+	return f, nil
+}
+
 // Stat returns information about a blob.
 func (b *Local) Stat(h restic.Handle) (restic.FileInfo, error) {
 	debug.Log("Stat %v", h)
@@ -231,9 +219,9 @@ func (b *Local) Stat(h restic.Handle) (restic.FileInfo, error) {
 }
 
 // Test returns true if a blob of the given type and name exists in the backend.
-func (b *Local) Test(t restic.FileType, name string) (bool, error) {
-	debug.Log("Test %v %v", t, name)
-	_, err := fs.Stat(filename(b.p, t, name))
+func (b *Local) Test(h restic.Handle) (bool, error) {
+	debug.Log("Test %v", h)
+	_, err := fs.Stat(filename(b.p, h.Type, h.Name))
 	if err != nil {
 		if os.IsNotExist(errors.Cause(err)) {
 			return false, nil
@@ -245,9 +233,9 @@ func (b *Local) Test(t restic.FileType, name string) (bool, error) {
 }
 
 // Remove removes the blob with the given name and type.
-func (b *Local) Remove(t restic.FileType, name string) error {
-	debug.Log("Remove %v %v", t, name)
-	fn := filename(b.p, t, name)
+func (b *Local) Remove(h restic.Handle) error {
+	debug.Log("Remove %v", h)
+	fn := filename(b.p, h.Type, h.Name)
 
 	// reset read-only flag
 	err := fs.Chmod(fn, 0666)

src/restic/backend/mem/backend_test.go

@@ -51,13 +51,6 @@ func TestMemBackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestMemBackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestMemBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/mem/mem_backend.go

@@ -1,21 +1,19 @@
 package mem
 
 import (
+	"bytes"
 	"io"
+	"io/ioutil"
 	"restic"
 	"sync"
 
+	"restic/backend"
 	"restic/errors"
 
 	"restic/debug"
 )
 
-type entry struct {
-	Type restic.FileType
-	Name string
-}
-
-type memMap map[entry][]byte
+type memMap map[restic.Handle][]byte
 
 // make sure that MemoryBackend implements backend.Backend
 var _ restic.Backend = &MemoryBackend{}
@@ -39,61 +37,21 @@ func New() *MemoryBackend {
 }
 
 // Test returns whether a file exists.
-func (be *MemoryBackend) Test(t restic.FileType, name string) (bool, error) {
+func (be *MemoryBackend) Test(h restic.Handle) (bool, error) {
 	be.m.Lock()
 	defer be.m.Unlock()
 
-	debug.Log("test %v %v", t, name)
+	debug.Log("Test %v", h)
 
-	if _, ok := be.data[entry{t, name}]; ok {
+	if _, ok := be.data[h]; ok {
 		return true, nil
 	}
 
 	return false, nil
 }
 
-// Load reads data from the backend.
-func (be *MemoryBackend) Load(h restic.Handle, p []byte, off int64) (int, error) {
-	if err := h.Valid(); err != nil {
-		return 0, err
-	}
-
-	be.m.Lock()
-	defer be.m.Unlock()
-
-	if h.Type == restic.ConfigFile {
-		h.Name = ""
-	}
-
-	debug.Log("get %v offset %v len %v", h, off, len(p))
-
-	if _, ok := be.data[entry{h.Type, h.Name}]; !ok {
-		return 0, errors.New("no such data")
-	}
-
-	buf := be.data[entry{h.Type, h.Name}]
-	switch {
-	case off > int64(len(buf)):
-		return 0, errors.New("offset beyond end of file")
-	case off < -int64(len(buf)):
-		off = 0
-	case off < 0:
-		off = int64(len(buf)) + off
-	}
-
-	buf = buf[off:]
-
-	n := copy(p, buf)
-
-	if len(p) > len(buf) {
-		return n, io.ErrUnexpectedEOF
-	}
-
-	return n, nil
-}
-
 // Save adds new Data to the backend.
-func (be *MemoryBackend) Save(h restic.Handle, p []byte) error {
+func (be *MemoryBackend) Save(h restic.Handle, rd io.Reader) error {
 	if err := h.Valid(); err != nil {
 		return err
 	}
@@ -105,18 +63,59 @@ func (be *MemoryBackend) Save(h restic.Handle, p []byte) error {
 		h.Name = ""
 	}
 
-	if _, ok := be.data[entry{h.Type, h.Name}]; ok {
+	if _, ok := be.data[h]; ok {
 		return errors.New("file already exists")
 	}
 
-	debug.Log("save %v bytes at %v", len(p), h)
-	buf := make([]byte, len(p))
-	copy(buf, p)
-	be.data[entry{h.Type, h.Name}] = buf
+	buf, err := ioutil.ReadAll(rd)
+	if err != nil {
+		return err
+	}
+
+	be.data[h] = buf
+	debug.Log("saved %v bytes at %v", len(buf), h)
 
 	return nil
 }
 
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (be *MemoryBackend) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	be.m.Lock()
+	defer be.m.Unlock()
+
+	if h.Type == restic.ConfigFile {
+		h.Name = ""
+	}
+
+	debug.Log("Load %v offset %v len %v", h, offset, length)
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	if _, ok := be.data[h]; !ok {
+		return nil, errors.New("no such data")
+	}
+
+	buf := be.data[h]
+	if offset > int64(len(buf)) {
+		return nil, errors.New("offset beyond end of file")
+	}
+
+	buf = buf[offset:]
+	if length > 0 && len(buf) > length {
+		buf = buf[:length]
+	}
+
+	return backend.Closer{Reader: bytes.NewReader(buf)}, nil
+}
+
 // Stat returns information about a file in the backend.
 func (be *MemoryBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 	be.m.Lock()
@@ -132,7 +131,7 @@ func (be *MemoryBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 
 	debug.Log("stat %v", h)
 
-	e, ok := be.data[entry{h.Type, h.Name}]
+	e, ok := be.data[h]
 	if !ok {
 		return restic.FileInfo{}, errors.New("no such data")
 	}
@@ -141,17 +140,17 @@ func (be *MemoryBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 }
 
 // Remove deletes a file from the backend.
-func (be *MemoryBackend) Remove(t restic.FileType, name string) error {
+func (be *MemoryBackend) Remove(h restic.Handle) error {
 	be.m.Lock()
 	defer be.m.Unlock()
 
-	debug.Log("get %v %v", t, name)
+	debug.Log("Remove %v", h)
 
-	if _, ok := be.data[entry{t, name}]; !ok {
+	if _, ok := be.data[h]; !ok {
 		return errors.New("no such data")
 	}
 
-	delete(be.data, entry{t, name})
+	delete(be.data, h)
 
 	return nil
 }

src/restic/backend/rest/backend_test.go

@@ -51,13 +51,6 @@ func TestRestBackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestRestBackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestRestBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/rest/rest.go

@@ -1,22 +1,26 @@
 package rest
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"path"
 	"restic"
 	"strings"
 
+	"restic/debug"
 	"restic/errors"
 
 	"restic/backend"
 )
 
-const connLimit = 10
+const connLimit = 40
+
+// make sure the rest backend implements restic.Backend
+var _ restic.Backend = &restBackend{}
 
 // restPath returns the path to the given resource.
 func restPath(url *url.URL, h restic.Handle) string {
@@ -59,7 +63,7 @@ func Open(cfg Config) (restic.Backend, error) {
 	for i := 0; i < connLimit; i++ {
 		connChan <- struct{}{}
 	}
-	tr := &http.Transport{}
+	tr := &http.Transport{MaxIdleConnsPerHost: connLimit}
 	client := http.Client{Transport: tr}
 
 	return &restBackend{url: cfg.URL, connChan: connChan, client: client}, nil
@@ -70,68 +74,23 @@ func (b *restBackend) Location() string {
 	return b.url.String()
 }
 
-// Load returns the data stored in the backend for h at the given offset
-// and saves it in p. Load has the same semantics as io.ReaderAt.
-func (b *restBackend) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
-	if err := h.Valid(); err != nil {
-		return 0, err
-	}
-
-	// invert offset
-	if off < 0 {
-		info, err := b.Stat(h)
-		if err != nil {
-			return 0, errors.Wrap(err, "Stat")
-		}
-
-		if -off > info.Size {
-			off = 0
-		} else {
-			off = info.Size + off
-		}
-	}
-
-	req, err := http.NewRequest("GET", restPath(b.url, h), nil)
-	if err != nil {
-		return 0, errors.Wrap(err, "http.NewRequest")
-	}
-	req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", off, off+int64(len(p))))
-	<-b.connChan
-	resp, err := b.client.Do(req)
-	b.connChan <- struct{}{}
-
-	if resp != nil {
-		defer func() {
-			e := resp.Body.Close()
-
-			if err == nil {
-				err = errors.Wrap(e, "Close")
-			}
-		}()
-	}
-
-	if err != nil {
-		return 0, errors.Wrap(err, "client.Do")
-	}
-	if resp.StatusCode != 200 && resp.StatusCode != 206 {
-		return 0, errors.Errorf("unexpected HTTP response code %v", resp.StatusCode)
-	}
-
-	return io.ReadFull(resp.Body, p)
-}
-
 // Save stores data in the backend at the handle.
-func (b *restBackend) Save(h restic.Handle, p []byte) (err error) {
+func (b *restBackend) Save(h restic.Handle, rd io.Reader) (err error) {
 	if err := h.Valid(); err != nil {
 		return err
 	}
 
+	// make sure that client.Post() cannot close the reader by wrapping it in
+	// backend.Closer, which has a noop method.
+	rd = backend.Closer{Reader: rd}
+
 	<-b.connChan
-	resp, err := b.client.Post(restPath(b.url, h), "binary/octet-stream", bytes.NewReader(p))
+	resp, err := b.client.Post(restPath(b.url, h), "binary/octet-stream", rd)
 	b.connChan <- struct{}{}
 
 	if resp != nil {
 		defer func() {
+			io.Copy(ioutil.Discard, resp.Body)
 			e := resp.Body.Close()
 
 			if err == nil {
@@ -151,6 +110,56 @@ func (b *restBackend) Save(h restic.Handle, p []byte) (err error) {
 	return nil
 }
 
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (b *restBackend) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	if length < 0 {
+		return nil, errors.Errorf("invalid length %d", length)
+	}
+
+	req, err := http.NewRequest("GET", restPath(b.url, h), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "http.NewRequest")
+	}
+
+	byteRange := fmt.Sprintf("bytes=%d-", offset)
+	if length > 0 {
+		byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1)
+	}
+	req.Header.Add("Range", byteRange)
+	debug.Log("Load(%v) send range %v", h, byteRange)
+
+	<-b.connChan
+	resp, err := b.client.Do(req)
+	b.connChan <- struct{}{}
+
+	if err != nil {
+		if resp != nil {
+			io.Copy(ioutil.Discard, resp.Body)
+			resp.Body.Close()
+		}
+		return nil, errors.Wrap(err, "client.Do")
+	}
+
+	if resp.StatusCode != 200 && resp.StatusCode != 206 {
+		io.Copy(ioutil.Discard, resp.Body)
+		resp.Body.Close()
+		return nil, errors.Errorf("unexpected HTTP response code %v", resp.StatusCode)
+	}
+
+	return resp.Body, nil
+}
+
 // Stat returns information about a blob.
 func (b *restBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 	if err := h.Valid(); err != nil {
@@ -164,6 +173,7 @@ func (b *restBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 		return restic.FileInfo{}, errors.Wrap(err, "client.Head")
 	}
 
+	io.Copy(ioutil.Discard, resp.Body)
 	if err = resp.Body.Close(); err != nil {
 		return restic.FileInfo{}, errors.Wrap(err, "Close")
 	}
@@ -184,8 +194,8 @@ func (b *restBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 }
 
 // Test returns true if a blob of the given type and name exists in the backend.
-func (b *restBackend) Test(t restic.FileType, name string) (bool, error) {
-	_, err := b.Stat(restic.Handle{Type: t, Name: name})
+func (b *restBackend) Test(h restic.Handle) (bool, error) {
+	_, err := b.Stat(h)
 	if err != nil {
 		return false, nil
 	}
@@ -194,8 +204,7 @@ func (b *restBackend) Test(t restic.FileType, name string) (bool, error) {
 }
 
 // Remove removes the blob with the given name and type.
-func (b *restBackend) Remove(t restic.FileType, name string) error {
-	h := restic.Handle{Type: t, Name: name}
+func (b *restBackend) Remove(h restic.Handle) error {
 	if err := h.Valid(); err != nil {
 		return err
 	}
@@ -216,6 +225,7 @@ func (b *restBackend) Remove(t restic.FileType, name string) error {
 		return errors.New("blob not removed")
 	}
 
+	io.Copy(ioutil.Discard, resp.Body)
 	return resp.Body.Close()
 }
 
@@ -235,7 +245,14 @@ func (b *restBackend) List(t restic.FileType, done <-chan struct{}) <-chan strin
 	b.connChan <- struct{}{}
 
 	if resp != nil {
-		defer resp.Body.Close()
+		defer func() {
+			io.Copy(ioutil.Discard, resp.Body)
+			e := resp.Body.Close()
+
+			if err == nil {
+				err = errors.Wrap(e, "Close")
+			}
+		}()
 	}
 
 	if err != nil {

src/restic/backend/rest/rest_test.go

@@ -37,7 +37,7 @@ func init() {
 			return nil, err
 		}
 
-		exists, err := be.Test(restic.ConfigFile, "")
+		exists, err := be.Test(restic.Handle{Type: restic.ConfigFile, Name: ""})
 		if err != nil {
 			return nil, err
 		}

src/restic/backend/s3/backend_test.go

@@ -51,13 +51,6 @@ func TestS3BackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestS3BackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestS3BackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/s3/s3.go

@@ -3,9 +3,12 @@ package s3
 import (
 	"bytes"
 	"io"
+	"net/http"
+	"path"
 	"restic"
 	"strings"
 
+	"restic/backend"
 	"restic/errors"
 
 	"github.com/minio/minio-go"
@@ -13,7 +16,7 @@ import (
 	"restic/debug"
 )
 
-const connLimit = 10
+const connLimit = 40
 
 // s3 is a backend which stores the data on an S3 endpoint.
 type s3 struct {
@@ -34,15 +37,19 @@ func Open(cfg Config) (restic.Backend, error) {
 	}
 
 	be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix}
+
+	tr := &http.Transport{MaxIdleConnsPerHost: connLimit}
+	client.SetCustomTransport(tr)
+
 	be.createConnections()
 
-	ok, err := client.BucketExists(cfg.Bucket)
+	found, err := client.BucketExists(cfg.Bucket)
 	if err != nil {
-		debug.Log("BucketExists(%v) returned err %v, trying to create the bucket", cfg.Bucket, err)
+		debug.Log("BucketExists(%v) returned err %v", cfg.Bucket, err)
 		return nil, errors.Wrap(err, "client.BucketExists")
 	}
 
-	if !ok {
+	if !found {
 		// create new bucket with default ACL in default region
 		err = client.MakeBucket(cfg.Bucket, "")
 		if err != nil {
@@ -53,18 +60,11 @@ func Open(cfg Config) (restic.Backend, error) {
 	return be, nil
 }
 
-func (be *s3) s3path(t restic.FileType, name string) string {
-	var path string
-
-	if be.prefix != "" {
-		path = be.prefix + "/"
+func (be *s3) s3path(h restic.Handle) string {
+	if h.Type == restic.ConfigFile {
+		return path.Join(be.prefix, string(h.Type))
 	}
-	path += string(t)
-
-	if t == restic.ConfigFile {
-		return path
-	}
-	return path + "/" + name
+	return path.Join(be.prefix, string(h.Type), h.Name)
 }
 
 func (be *s3) createConnections() {
@@ -79,91 +79,18 @@ func (be *s3) Location() string {
 	return be.bucketname
 }
 
-// Load returns the data stored in the backend for h at the given offset
-// and saves it in p. Load has the same semantics as io.ReaderAt.
-func (be s3) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
-	var obj *minio.Object
-
-	debug.Log("%v, offset %v, len %v", h, off, len(p))
-	path := be.s3path(h.Type, h.Name)
-
-	<-be.connChan
-	defer func() {
-		be.connChan <- struct{}{}
-	}()
-
-	obj, err = be.client.GetObject(be.bucketname, path)
-	if err != nil {
-		debug.Log("  err %v", err)
-		return 0, errors.Wrap(err, "client.GetObject")
-	}
-
-	// make sure that the object is closed properly.
-	defer func() {
-		e := obj.Close()
-		if err == nil {
-			err = errors.Wrap(e, "Close")
-		}
-	}()
-
-	info, err := obj.Stat()
-	if err != nil {
-		return 0, errors.Wrap(err, "obj.Stat")
-	}
-
-	// handle negative offsets
-	if off < 0 {
-		// if the negative offset is larger than the object itself, read from
-		// the beginning.
-		if -off > info.Size {
-			off = 0
-		} else {
-			// otherwise compute the offset from the end of the file.
-			off = info.Size + off
-		}
-	}
-
-	// return an error if the offset is beyond the end of the file
-	if off > info.Size {
-		return 0, errors.Wrap(io.EOF, "")
-	}
-
-	var nextError error
-
-	// manually create an io.ErrUnexpectedEOF
-	if off+int64(len(p)) > info.Size {
-		newlen := info.Size - off
-		p = p[:newlen]
-
-		nextError = io.ErrUnexpectedEOF
-
-		debug.Log("    capped buffer to %v byte", len(p))
-	}
-
-	n, err = obj.ReadAt(p, off)
-	if int64(n) == info.Size-off && errors.Cause(err) == io.EOF {
-		err = nil
-	}
-
-	if err == nil {
-		err = nextError
-	}
-
-	return n, err
-}
-
 // Save stores data in the backend at the handle.
-func (be s3) Save(h restic.Handle, p []byte) (err error) {
+func (be *s3) Save(h restic.Handle, rd io.Reader) (err error) {
 	if err := h.Valid(); err != nil {
 		return err
 	}
 
-	debug.Log("%v with %d bytes", h, len(p))
+	debug.Log("Save %v", h)
 
-	path := be.s3path(h.Type, h.Name)
+	objName := be.s3path(h)
 
 	// Check key does not already exist
-	_, err = be.client.StatObject(be.bucketname, path)
+	_, err = be.client.StatObject(be.bucketname, objName)
 	if err == nil {
 		debug.Log("%v already exists", h)
 		return errors.New("key already exists")
@@ -174,22 +101,132 @@ func (be s3) Save(h restic.Handle, p []byte) (err error) {
 		be.connChan <- struct{}{}
 	}()
 
-	debug.Log("PutObject(%v, %v, %v, %v)",
-		be.bucketname, path, int64(len(p)), "binary/octet-stream")
-	n, err := be.client.PutObject(be.bucketname, path, bytes.NewReader(p), "binary/octet-stream")
-	debug.Log("%v -> %v bytes, err %#v", path, n, err)
+	debug.Log("PutObject(%v, %v)",
+		be.bucketname, objName)
+	n, err := be.client.PutObject(be.bucketname, objName, rd, "binary/octet-stream")
+	debug.Log("%v -> %v bytes, err %#v", objName, n, err)
 
 	return errors.Wrap(err, "client.PutObject")
 }
 
-// Stat returns information about a blob.
-func (be s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) {
-	debug.Log("%v", h)
+// wrapReader wraps an io.ReadCloser to run an additional function on Close.
+type wrapReader struct {
+	io.ReadCloser
+	f func()
+}
+
+func (wr wrapReader) Close() error {
+	err := wr.ReadCloser.Close()
+	wr.f()
+	return err
+}
+
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	if length < 0 {
+		return nil, errors.Errorf("invalid length %d", length)
+	}
 
-	path := be.s3path(h.Type, h.Name)
 	var obj *minio.Object
 
-	obj, err = be.client.GetObject(be.bucketname, path)
+	objName := be.s3path(h)
+
+	// get token for connection
+	<-be.connChan
+
+	obj, err := be.client.GetObject(be.bucketname, objName)
+	if err != nil {
+		debug.Log("  err %v", err)
+
+		// return token
+		be.connChan <- struct{}{}
+
+		return nil, errors.Wrap(err, "client.GetObject")
+	}
+
+	// if we're going to read the whole object, just pass it on.
+	if length == 0 {
+		debug.Log("Load %v: pass on object", h)
+
+		_, err = obj.Seek(offset, 0)
+		if err != nil {
+			_ = obj.Close()
+
+			// return token
+			be.connChan <- struct{}{}
+
+			return nil, errors.Wrap(err, "obj.Seek")
+		}
+
+		rd := wrapReader{
+			ReadCloser: obj,
+			f: func() {
+				debug.Log("Close()")
+				// return token
+				be.connChan <- struct{}{}
+			},
+		}
+		return rd, nil
+	}
+
+	defer func() {
+		// return token
+		be.connChan <- struct{}{}
+	}()
+
+	// otherwise use a buffer with ReadAt
+	info, err := obj.Stat()
+	if err != nil {
+		_ = obj.Close()
+		return nil, errors.Wrap(err, "obj.Stat")
+	}
+
+	if offset > info.Size {
+		_ = obj.Close()
+		return nil, errors.New("offset larger than file size")
+	}
+
+	l := int64(length)
+	if offset+l > info.Size {
+		l = info.Size - offset
+	}
+
+	buf := make([]byte, l)
+	n, err := obj.ReadAt(buf, offset)
+	debug.Log("Load %v: use buffer with ReadAt: %v, %v", h, n, err)
+	if err == io.EOF {
+		debug.Log("Load %v: shorten buffer %v -> %v", h, len(buf), n)
+		buf = buf[:n]
+		err = nil
+	}
+
+	if err != nil {
+		_ = obj.Close()
+		return nil, errors.Wrap(err, "obj.ReadAt")
+	}
+
+	return backend.Closer{Reader: bytes.NewReader(buf)}, nil
+}
+
+// Stat returns information about a blob.
+func (be *s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) {
+	debug.Log("%v", h)
+
+	objName := be.s3path(h)
+	var obj *minio.Object
+
+	obj, err = be.client.GetObject(be.bucketname, objName)
 	if err != nil {
 		debug.Log("GetObject() err %v", err)
 		return restic.FileInfo{}, errors.Wrap(err, "client.GetObject")
@@ -213,10 +250,10 @@ func (be s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) {
 }
 
 // Test returns true if a blob of the given type and name exists in the backend.
-func (be *s3) Test(t restic.FileType, name string) (bool, error) {
+func (be *s3) Test(h restic.Handle) (bool, error) {
 	found := false
-	path := be.s3path(t, name)
-	_, err := be.client.StatObject(be.bucketname, path)
+	objName := be.s3path(h)
+	_, err := be.client.StatObject(be.bucketname, objName)
 	if err == nil {
 		found = true
 	}
@@ -226,10 +263,10 @@ func (be *s3) Test(t restic.FileType, name string) (bool, error) {
 }
 
 // Remove removes the blob with the given name and type.
-func (be *s3) Remove(t restic.FileType, name string) error {
-	path := be.s3path(t, name)
-	err := be.client.RemoveObject(be.bucketname, path)
-	debug.Log("%v %v -> err %v", t, name, err)
+func (be *s3) Remove(h restic.Handle) error {
+	objName := be.s3path(h)
+	err := be.client.RemoveObject(be.bucketname, objName)
+	debug.Log("Remove(%v) -> err %v", h, err)
 	return errors.Wrap(err, "client.RemoveObject")
 }
 
@@ -240,7 +277,7 @@ func (be *s3) List(t restic.FileType, done <-chan struct{}) <-chan string {
 	debug.Log("listing %v", t)
 	ch := make(chan string)
 
-	prefix := be.s3path(t, "")
+	prefix := be.s3path(restic.Handle{Type: t}) + "/"
 
 	listresp := be.client.ListObjects(be.bucketname, prefix, true, done)
 
@@ -268,7 +305,7 @@ func (be *s3) removeKeys(t restic.FileType) error {
 	done := make(chan struct{})
 	defer close(done)
 	for key := range be.List(restic.DataFile, done) {
-		err := be.Remove(restic.DataFile, key)
+		err := be.Remove(restic.Handle{Type: restic.DataFile, Name: key})
 		if err != nil {
 			return err
 		}
@@ -293,7 +330,7 @@ func (be *s3) Delete() error {
 		}
 	}
 
-	return be.Remove(restic.ConfigFile, "")
+	return be.Remove(restic.Handle{Type: restic.ConfigFile})
 }
 
 // Close does nothing

src/restic/backend/s3/s3_test.go

@@ -44,7 +44,7 @@ func init() {
 			return nil, err
 		}
 
-		exists, err := be.Test(restic.ConfigFile, "")
+		exists, err := be.Test(restic.Handle{Type: restic.ConfigFile})
 		if err != nil {
 			return nil, err
 		}

src/restic/backend/sftp/backend_test.go

@@ -51,13 +51,6 @@ func TestSftpBackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestSftpBackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestSftpBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/sftp/sftp.go

@@ -153,7 +153,7 @@ func buildSSHCommand(cfg Config) []string {
 }
 
 // OpenWithConfig opens an sftp backend as described by the config by running
-// "ssh" with the appropiate arguments.
+// "ssh" with the appropriate arguments.
 func OpenWithConfig(cfg Config) (*SFTP, error) {
 	debug.Log("open with config %v", cfg)
 	return Open(cfg.Dir, "ssh", buildSSHCommand(cfg)...)
@@ -193,7 +193,7 @@ func Create(dir string, program string, args ...string) (*SFTP, error) {
 }
 
 // CreateWithConfig creates an sftp backend as described by the config by running
-// "ssh" with the appropiate arguments.
+// "ssh" with the appropriate arguments.
 func CreateWithConfig(cfg Config) (*SFTP, error) {
 	debug.Log("config %v", cfg)
 	return Create(cfg.Dir, "ssh", buildSSHCommand(cfg)...)
@@ -259,11 +259,11 @@ func (r *SFTP) mkdirAll(dir string, mode os.FileMode) error {
 }
 
 // Rename temp file to final name according to type and name.
-func (r *SFTP) renameFile(oldname string, t restic.FileType, name string) error {
-	filename := r.filename(t, name)
+func (r *SFTP) renameFile(oldname string, h restic.Handle) error {
+	filename := r.filename(h)
 
 	// create directories if necessary
-	if t == restic.DataFile {
+	if h.Type == restic.DataFile {
 		err := r.mkdirAll(path.Dir(filename), backend.Modes.Dir)
 		if err != nil {
 			return err
@@ -297,22 +297,22 @@ func Join(parts ...string) string {
 }
 
 // Construct path for given restic.Type and name.
-func (r *SFTP) filename(t restic.FileType, name string) string {
-	if t == restic.ConfigFile {
+func (r *SFTP) filename(h restic.Handle) string {
+	if h.Type == restic.ConfigFile {
 		return Join(r.p, "config")
 	}
 
-	return Join(r.dirname(t, name), name)
+	return Join(r.dirname(h), h.Name)
 }
 
 // Construct directory for given backend.Type.
-func (r *SFTP) dirname(t restic.FileType, name string) string {
+func (r *SFTP) dirname(h restic.Handle) string {
 	var n string
-	switch t {
+	switch h.Type {
 	case restic.DataFile:
 		n = backend.Paths.Data
-		if len(name) > 2 {
-			n = Join(n, name[:2])
+		if len(h.Name) > 2 {
+			n = Join(n, h.Name[:2])
 		}
 	case restic.SnapshotFile:
 		n = backend.Paths.Snapshots
@@ -326,47 +326,9 @@ func (r *SFTP) dirname(t restic.FileType, name string) string {
 	return Join(r.p, n)
 }
 
-// Load returns the data stored in the backend for h at the given offset
-// and saves it in p. Load has the same semantics as io.ReaderAt.
-func (r *SFTP) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
-	debug.Log("load %v, %d bytes, offset %v", h, len(p), off)
-	if err := r.clientError(); err != nil {
-		return 0, err
-	}
-
-	if err := h.Valid(); err != nil {
-		return 0, err
-	}
-
-	f, err := r.c.Open(r.filename(h.Type, h.Name))
-	if err != nil {
-		return 0, errors.Wrap(err, "Open")
-	}
-
-	defer func() {
-		e := f.Close()
-		if err == nil && e != nil {
-			err = errors.Wrap(e, "Close")
-		}
-	}()
-
-	switch {
-	case off > 0:
-		_, err = f.Seek(off, 0)
-	case off < 0:
-		_, err = f.Seek(off, 2)
-	}
-
-	if err != nil {
-		return 0, errors.Wrap(err, "Seek")
-	}
-
-	return io.ReadFull(f, p)
-}
-
 // Save stores data in the backend at the handle.
-func (r *SFTP) Save(h restic.Handle, p []byte) (err error) {
-	debug.Log("save %v bytes to %v", h, len(p))
+func (r *SFTP) Save(h restic.Handle, rd io.Reader) (err error) {
+	debug.Log("save to %v", h)
 	if err := r.clientError(); err != nil {
 		return err
 	}
@@ -380,31 +342,60 @@ func (r *SFTP) Save(h restic.Handle, p []byte) (err error) {
 		return err
 	}
 
-	debug.Log("save %v (%d bytes) to %v", h, len(p), filename)
-
-	n, err := tmpfile.Write(p)
+	n, err := io.Copy(tmpfile, rd)
 	if err != nil {
 		return errors.Wrap(err, "Write")
 	}
 
-	if n != len(p) {
-		return errors.New("not all bytes writen")
-	}
+	debug.Log("saved %v (%d bytes) to %v", h, n, filename)
 
 	err = tmpfile.Close()
 	if err != nil {
 		return errors.Wrap(err, "Close")
 	}
 
-	err = r.renameFile(filename, h.Type, h.Name)
+	err = r.renameFile(filename, h)
 	debug.Log("save %v: rename %v: %v",
 		h, path.Base(filename), err)
 	return err
 }
 
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (r *SFTP) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	f, err := r.c.Open(r.filename(h))
+	if err != nil {
+		return nil, err
+	}
+
+	if offset > 0 {
+		_, err = f.Seek(offset, 0)
+		if err != nil {
+			_ = f.Close()
+			return nil, err
+		}
+	}
+
+	if length > 0 {
+		return backend.LimitReadCloser(f, int64(length)), nil
+	}
+
+	return f, nil
+}
+
 // Stat returns information about a blob.
 func (r *SFTP) Stat(h restic.Handle) (restic.FileInfo, error) {
-	debug.Log("stat %v", h)
+	debug.Log("Stat(%v)", h)
 	if err := r.clientError(); err != nil {
 		return restic.FileInfo{}, err
 	}
@@ -413,7 +404,7 @@ func (r *SFTP) Stat(h restic.Handle) (restic.FileInfo, error) {
 		return restic.FileInfo{}, err
 	}
 
-	fi, err := r.c.Lstat(r.filename(h.Type, h.Name))
+	fi, err := r.c.Lstat(r.filename(h))
 	if err != nil {
 		return restic.FileInfo{}, errors.Wrap(err, "Lstat")
 	}
@@ -422,13 +413,13 @@ func (r *SFTP) Stat(h restic.Handle) (restic.FileInfo, error) {
 }
 
 // Test returns true if a blob of the given type and name exists in the backend.
-func (r *SFTP) Test(t restic.FileType, name string) (bool, error) {
-	debug.Log("type %v, name %v", t, name)
+func (r *SFTP) Test(h restic.Handle) (bool, error) {
+	debug.Log("Test(%v)", h)
 	if err := r.clientError(); err != nil {
 		return false, err
 	}
 
-	_, err := r.c.Lstat(r.filename(t, name))
+	_, err := r.c.Lstat(r.filename(h))
 	if os.IsNotExist(errors.Cause(err)) {
 		return false, nil
 	}
@@ -441,13 +432,13 @@ func (r *SFTP) Test(t restic.FileType, name string) (bool, error) {
 }
 
 // Remove removes the content stored at name.
-func (r *SFTP) Remove(t restic.FileType, name string) error {
-	debug.Log("type %v, name %v", t, name)
+func (r *SFTP) Remove(h restic.Handle) error {
+	debug.Log("Remove(%v)", h)
 	if err := r.clientError(); err != nil {
 		return err
 	}
 
-	return r.c.Remove(r.filename(t, name))
+	return r.c.Remove(r.filename(h))
 }
 
 // List returns a channel that yields all names of blobs of type t. A
@@ -462,7 +453,7 @@ func (r *SFTP) List(t restic.FileType, done <-chan struct{}) <-chan string {
 
 		if t == restic.DataFile {
 			// read first level
-			basedir := r.dirname(t, "")
+			basedir := r.dirname(restic.Handle{Type: t})
 
 			list1, err := r.c.ReadDir(basedir)
 			if err != nil {
@@ -495,7 +486,7 @@ func (r *SFTP) List(t restic.FileType, done <-chan struct{}) <-chan string {
 				}
 			}
 		} else {
-			entries, err := r.c.ReadDir(r.dirname(t, ""))
+			entries, err := r.c.ReadDir(r.dirname(restic.Handle{Type: t}))
 			if err != nil {
 				return
 			}

src/restic/backend/test/backend_test.go

@@ -51,13 +51,6 @@ func TestTestBackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestTestBackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestTestBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/test/tests.go

@@ -4,13 +4,16 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"math/rand"
+	"os"
 	"reflect"
 	"restic"
+	"restic/errors"
 	"sort"
+	"strings"
 	"testing"
 
-	"restic/errors"
 	"restic/test"
 
 	"restic/backend"
@@ -127,7 +130,7 @@ func TestCreateWithConfig(t testing.TB) {
 	}
 
 	// remove config
-	err = b.Remove(restic.ConfigFile, "")
+	err = b.Remove(restic.Handle{Type: restic.ConfigFile, Name: ""})
 	if err != nil {
 		t.Fatalf("unexpected error removing config: %v", err)
 	}
@@ -152,12 +155,12 @@ func TestConfig(t testing.TB) {
 	var testString = "Config"
 
 	// create config and read it back
-	_, err := backend.LoadAll(b, restic.Handle{Type: restic.ConfigFile}, nil)
+	_, err := backend.LoadAll(b, restic.Handle{Type: restic.ConfigFile})
 	if err == nil {
 		t.Fatalf("did not get expected error for non-existing config")
 	}
 
-	err = b.Save(restic.Handle{Type: restic.ConfigFile}, []byte(testString))
+	err = b.Save(restic.Handle{Type: restic.ConfigFile}, strings.NewReader(testString))
 	if err != nil {
 		t.Fatalf("Save() error: %v", err)
 	}
@@ -166,7 +169,7 @@ func TestConfig(t testing.TB) {
 	// same config
 	for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} {
 		h := restic.Handle{Type: restic.ConfigFile, Name: name}
-		buf, err := backend.LoadAll(b, h, nil)
+		buf, err := backend.LoadAll(b, h)
 		if err != nil {
 			t.Fatalf("unable to read config with name %q: %v", name, err)
 		}
@@ -182,12 +185,12 @@ func TestLoad(t testing.TB) {
 	b := open(t)
 	defer close(t)
 
-	_, err := b.Load(restic.Handle{}, nil, 0)
+	_, err := b.Load(restic.Handle{}, 0, 0)
 	if err == nil {
 		t.Fatalf("Load() did not return an error for invalid handle")
 	}
 
-	_, err = b.Load(restic.Handle{Type: restic.DataFile, Name: "foobar"}, nil, 0)
+	_, err = b.Load(restic.Handle{Type: restic.DataFile, Name: "foobar"}, 0, 0)
 	if err == nil {
 		t.Fatalf("Load() did not return an error for non-existing blob")
 	}
@@ -198,11 +201,20 @@ func TestLoad(t testing.TB) {
 	id := restic.Hash(data)
 
 	handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
-	err = b.Save(handle, data)
+	err = b.Save(handle, bytes.NewReader(data))
 	if err != nil {
 		t.Fatalf("Save() error: %v", err)
 	}
 
+	rd, err := b.Load(handle, 100, -1)
+	if err == nil {
+		t.Fatalf("Load() returned no error for negative offset!")
+	}
+
+	if rd != nil {
+		t.Fatalf("Load() returned a non-nil reader for negative offset!")
+	}
+
 	for i := 0; i < 50; i++ {
 		l := rand.Intn(length + 2000)
 		o := rand.Intn(length + 2000)
@@ -215,157 +227,64 @@ func TestLoad(t testing.TB) {
 			d = d[:0]
 		}
 
-		if l > 0 && l < len(d) {
-			d = d[:l]
-		}
-
-		buf := make([]byte, l)
-		n, err := b.Load(handle, buf, int64(o))
-
-		// if we requested data beyond the end of the file, require
-		// ErrUnexpectedEOF error
-		if l > len(d) {
-			if errors.Cause(err) != io.ErrUnexpectedEOF {
-				t.Errorf("Load(%d, %d) did not return io.ErrUnexpectedEOF", len(buf), int64(o))
-			}
-			err = nil
-			buf = buf[:len(d)]
-		}
-
-		if err != nil {
-			t.Errorf("Load(%d, %d): unexpected error: %v", len(buf), int64(o), err)
-			continue
-		}
-
-		if n != len(buf) {
-			t.Errorf("Load(%d, %d): wrong length returned, want %d, got %d",
-				len(buf), int64(o), len(buf), n)
-			continue
-		}
-
-		buf = buf[:n]
-		if !bytes.Equal(buf, d) {
-			t.Errorf("Load(%d, %d) returned wrong bytes", len(buf), int64(o))
-			continue
-		}
-	}
-
-	// test with negative offset
-	for i := 0; i < 50; i++ {
-		l := rand.Intn(length + 2000)
-		o := rand.Intn(length + 2000)
-
-		d := data
-		if o < len(d) {
-			d = d[len(d)-o:]
-		} else {
-			o = 0
+		getlen := l
+		if l >= len(d) && rand.Float32() >= 0.5 {
+			getlen = 0
 		}
 
 		if l > 0 && l < len(d) {
 			d = d[:l]
 		}
 
-		buf := make([]byte, l)
-		n, err := b.Load(handle, buf, -int64(o))
-
-		// if we requested data beyond the end of the file, require
-		// ErrUnexpectedEOF error
-		if l > len(d) {
-			if errors.Cause(err) != io.ErrUnexpectedEOF {
-				t.Errorf("Load(%d, %d) did not return io.ErrUnexpectedEOF", len(buf), int64(o))
-				continue
-			}
-			err = nil
-			buf = buf[:len(d)]
-		}
-
+		rd, err := b.Load(handle, getlen, int64(o))
 		if err != nil {
-			t.Errorf("Load(%d, %d): unexpected error: %v", len(buf), int64(o), err)
+			t.Errorf("Load(%d, %d) returned unexpected error: %v", l, o, err)
 			continue
 		}
 
-		if n != len(buf) {
-			t.Errorf("Load(%d, %d): wrong length returned, want %d, got %d",
-				len(buf), int64(o), len(buf), n)
+		buf, err := ioutil.ReadAll(rd)
+		if err != nil {
+			t.Errorf("Load(%d, %d) ReadAll() returned unexpected error: %v", l, o, err)
+			rd.Close()
+			continue
+		}
+
+		if l <= len(d) && len(buf) != l {
+			t.Errorf("Load(%d, %d) wrong number of bytes read: want %d, got %d", l, o, l, len(buf))
+			rd.Close()
+			continue
+		}
+
+		if l > len(d) && len(buf) != len(d) {
+			t.Errorf("Load(%d, %d) wrong number of bytes read for overlong read: want %d, got %d", l, o, l, len(buf))
+			rd.Close()
 			continue
 		}
 
-		buf = buf[:n]
 		if !bytes.Equal(buf, d) {
-			t.Errorf("Load(%d, %d) returned wrong bytes", len(buf), int64(o))
+			t.Errorf("Load(%d, %d) returned wrong bytes", l, o)
+			rd.Close()
+			continue
+		}
+
+		err = rd.Close()
+		if err != nil {
+			t.Errorf("Load(%d, %d) rd.Close() returned unexpected error: %v", l, o, err)
 			continue
 		}
 	}
 
-	// load with a too-large buffer, this should return io.ErrUnexpectedEOF
-	buf := make([]byte, length+100)
-	n, err := b.Load(handle, buf, 0)
-	if n != length {
-		t.Errorf("wrong length for larger buffer returned, want %d, got %d", length, n)
-	}
-
-	if errors.Cause(err) != io.ErrUnexpectedEOF {
-		t.Errorf("wrong error returned for larger buffer: want io.ErrUnexpectedEOF, got %#v", err)
-	}
-
-	test.OK(t, b.Remove(restic.DataFile, id.String()))
+	test.OK(t, b.Remove(handle))
 }
 
-// TestLoadNegativeOffset tests the backend's Load function with negative offsets.
-func TestLoadNegativeOffset(t testing.TB) {
-	b := open(t)
-	defer close(t)
+type errorCloser struct {
+	io.Reader
+	t testing.TB
+}
 
-	length := rand.Intn(1<<24) + 2000
-
-	data := test.Random(23, length)
-	id := restic.Hash(data)
-
-	handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
-	err := b.Save(handle, data)
-	if err != nil {
-		t.Fatalf("Save() error: %v", err)
-	}
-
-	// test normal reads
-	for i := 0; i < 50; i++ {
-		l := rand.Intn(length + 2000)
-		o := -rand.Intn(length + 2000)
-
-		buf := make([]byte, l)
-		n, err := b.Load(handle, buf, int64(o))
-
-		// if we requested data beyond the end of the file, require
-		// ErrUnexpectedEOF error
-		if len(buf) > -o {
-			if errors.Cause(err) != io.ErrUnexpectedEOF {
-				t.Errorf("Load(%d, %d) did not return io.ErrUnexpectedEOF", len(buf), o)
-				continue
-			}
-			err = nil
-			buf = buf[:-o]
-		}
-
-		if err != nil {
-			t.Errorf("Load(%d, %d) returned error: %v", len(buf), o, err)
-			continue
-		}
-
-		if n != len(buf) {
-			t.Errorf("Load(%d, %d) returned short read, only got %d bytes", len(buf), o, n)
-			continue
-		}
-
-		p := len(data) + o
-		if !bytes.Equal(buf, data[p:p+len(buf)]) {
-			t.Errorf("Load(%d, %d) returned wrong bytes", len(buf), o)
-			continue
-		}
-
-	}
-
-	test.OK(t, b.Remove(restic.DataFile, id.String()))
+func (ec errorCloser) Close() error {
+	ec.t.Error("forbidden method close was called")
+	return errors.New("forbidden method close was called")
 }
 
 // TestSave tests saving data in the backend.
@@ -384,10 +303,10 @@ func TestSave(t testing.TB) {
 			Type: restic.DataFile,
 			Name: fmt.Sprintf("%s-%d", id, i),
 		}
-		err := b.Save(h, data)
+		err := b.Save(h, bytes.NewReader(data))
 		test.OK(t, err)
 
-		buf, err := backend.LoadAll(b, h, nil)
+		buf, err := backend.LoadAll(b, h)
 		test.OK(t, err)
 		if len(buf) != len(data) {
 			t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf))
@@ -404,11 +323,51 @@ func TestSave(t testing.TB) {
 			t.Fatalf("Stat() returned different size, want %q, got %d", len(data), fi.Size)
 		}
 
-		err = b.Remove(h.Type, h.Name)
+		err = b.Remove(h)
 		if err != nil {
 			t.Fatalf("error removing item: %v", err)
 		}
 	}
+
+	// test saving from a tempfile
+	tmpfile, err := ioutil.TempFile("", "restic-backend-save-test-")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	length := rand.Intn(1<<23) + 200000
+	data := test.Random(23, length)
+	copy(id[:], data)
+
+	if _, err = tmpfile.Write(data); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err = tmpfile.Seek(0, 0); err != nil {
+		t.Fatal(err)
+	}
+
+	h := restic.Handle{Type: restic.DataFile, Name: id.String()}
+
+	// wrap the tempfile in an errorCloser, so we can detect if the backend
+	// closes the reader
+	err = b.Save(h, errorCloser{t: t, Reader: tmpfile})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err = tmpfile.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	if err = os.Remove(tmpfile.Name()); err != nil {
+		t.Fatal(err)
+	}
+
+	err = b.Remove(h)
+	if err != nil {
+		t.Fatalf("error removing item: %v", err)
+	}
 }
 
 var filenameTests = []struct {
@@ -430,13 +389,13 @@ func TestSaveFilenames(t testing.TB) {
 
 	for i, test := range filenameTests {
 		h := restic.Handle{Name: test.name, Type: restic.DataFile}
-		err := b.Save(h, []byte(test.data))
+		err := b.Save(h, strings.NewReader(test.data))
 		if err != nil {
 			t.Errorf("test %d failed: Save() returned %v", i, err)
 			continue
 		}
 
-		buf, err := backend.LoadAll(b, h, nil)
+		buf, err := backend.LoadAll(b, h)
 		if err != nil {
 			t.Errorf("test %d failed: Load() returned %v", i, err)
 			continue
@@ -446,7 +405,7 @@ func TestSaveFilenames(t testing.TB) {
 			t.Errorf("test %d: returned wrong bytes", i)
 		}
 
-		err = b.Remove(h.Type, h.Name)
+		err = b.Remove(h)
 		if err != nil {
 			t.Errorf("test %d failed: Remove() returned %v", i, err)
 			continue
@@ -464,10 +423,12 @@ var testStrings = []struct {
 	{"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"},
 }
 
-func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) {
+func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) restic.Handle {
 	id := restic.Hash(data)
-	err := b.Save(restic.Handle{Name: id.String(), Type: tpe}, data)
+	h := restic.Handle{Name: id.String(), Type: tpe}
+	err := b.Save(h, bytes.NewReader(data))
 	test.OK(t, err)
+	return h
 }
 
 // TestBackend tests all functions of the backend.
@@ -485,21 +446,21 @@ func TestBackend(t testing.TB) {
 			test.OK(t, err)
 
 			// test if blob is already in repository
-			ret, err := b.Test(tpe, id.String())
+			h := restic.Handle{Type: tpe, Name: id.String()}
+			ret, err := b.Test(h)
 			test.OK(t, err)
 			test.Assert(t, !ret, "blob was found to exist before creating")
 
 			// try to stat a not existing blob
-			h := restic.Handle{Type: tpe, Name: id.String()}
 			_, err = b.Stat(h)
 			test.Assert(t, err != nil, "blob data could be extracted before creation")
 
 			// try to read not existing blob
-			_, err = b.Load(h, nil, 0)
+			_, err = b.Load(h, 0, 0)
 			test.Assert(t, err != nil, "blob reader could be obtained before creation")
 
 			// try to get string out, should fail
-			ret, err = b.Test(tpe, id.String())
+			ret, err = b.Test(h)
 			test.OK(t, err)
 			test.Assert(t, !ret, "id %q was found (but should not have)", ts.id)
 		}
@@ -510,7 +471,7 @@ func TestBackend(t testing.TB) {
 
 			// test Load()
 			h := restic.Handle{Type: tpe, Name: ts.id}
-			buf, err := backend.LoadAll(b, h, nil)
+			buf, err := backend.LoadAll(b, h)
 			test.OK(t, err)
 			test.Equals(t, ts.data, string(buf))
 
@@ -520,9 +481,18 @@ func TestBackend(t testing.TB) {
 			length := end - start
 
 			buf2 := make([]byte, length)
-			n, err := b.Load(h, buf2, int64(start))
+			rd, err := b.Load(h, len(buf2), int64(start))
 			test.OK(t, err)
-			test.Equals(t, length, n)
+			n, err := io.ReadFull(rd, buf2)
+			test.OK(t, err)
+			test.Equals(t, len(buf2), n)
+
+			remaining, err := io.Copy(ioutil.Discard, rd)
+			test.OK(t, err)
+			test.Equals(t, int64(0), remaining)
+
+			test.OK(t, rd.Close())
+
 			test.Equals(t, ts.data[start:end], string(buf2))
 		}
 
@@ -530,20 +500,21 @@ func TestBackend(t testing.TB) {
 		ts := testStrings[0]
 
 		// create blob
-		err := b.Save(restic.Handle{Type: tpe, Name: ts.id}, []byte(ts.data))
+		err := b.Save(restic.Handle{Type: tpe, Name: ts.id}, strings.NewReader(ts.data))
 		test.Assert(t, err != nil, "expected error, got %v", err)
 
 		// remove and recreate
-		err = b.Remove(tpe, ts.id)
+		h := restic.Handle{Type: tpe, Name: ts.id}
+		err = b.Remove(h)
 		test.OK(t, err)
 
 		// test that the blob is gone
-		ok, err := b.Test(tpe, ts.id)
+		ok, err := b.Test(h)
 		test.OK(t, err)
 		test.Assert(t, ok == false, "removed blob still present")
 
 		// create blob
-		err = b.Save(restic.Handle{Type: tpe, Name: ts.id}, []byte(ts.data))
+		err = b.Save(h, strings.NewReader(ts.data))
 		test.OK(t, err)
 
 		// list items
@@ -578,12 +549,14 @@ func TestBackend(t testing.TB) {
 				id, err := restic.ParseID(ts.id)
 				test.OK(t, err)
 
-				found, err := b.Test(tpe, id.String())
+				h := restic.Handle{Type: tpe, Name: id.String()}
+
+				found, err := b.Test(h)
 				test.OK(t, err)
 
-				test.OK(t, b.Remove(tpe, id.String()))
+				test.OK(t, b.Remove(h))
 
-				found, err = b.Test(tpe, id.String())
+				found, err = b.Test(h)
 				test.OK(t, err)
 				test.Assert(t, !found, fmt.Sprintf("id %q not found after removal", id))
 			}

src/restic/backend/utils.go

@@ -2,29 +2,51 @@ package backend
 
 import (
 	"io"
+	"io/ioutil"
 	"restic"
-
-	"restic/errors"
 )
 
-// LoadAll reads all data stored in the backend for the handle. The buffer buf
-// is resized to accomodate all data in the blob. Errors returned by be.Load()
-// are passed on, except io.ErrUnexpectedEOF is silenced and nil returned
-// instead, since it means this function is working properly.
-func LoadAll(be restic.Backend, h restic.Handle, buf []byte) ([]byte, error) {
-	fi, err := be.Stat(h)
+// LoadAll reads all data stored in the backend for the handle.
+func LoadAll(be restic.Backend, h restic.Handle) (buf []byte, err error) {
+	rd, err := be.Load(h, 0, 0)
 	if err != nil {
-		return nil, errors.Wrap(err, "Stat")
+		return nil, err
 	}
 
-	if fi.Size > int64(len(buf)) {
-		buf = make([]byte, int(fi.Size))
-	}
+	defer func() {
+		io.Copy(ioutil.Discard, rd)
+		e := rd.Close()
+		if err == nil {
+			err = e
+		}
+	}()
 
-	n, err := be.Load(h, buf, 0)
-	if errors.Cause(err) == io.ErrUnexpectedEOF {
-		err = nil
-	}
-	buf = buf[:n]
-	return buf, err
+	return ioutil.ReadAll(rd)
+}
+
+// Closer wraps an io.Reader and adds a Close() method that does nothing.
+type Closer struct {
+	io.Reader
+}
+
+// Close is a no-op.
+func (c Closer) Close() error {
+	return nil
+}
+
+// LimitedReadCloser wraps io.LimitedReader and exposes the Close() method.
+type LimitedReadCloser struct {
+	io.ReadCloser
+	io.Reader
+}
+
+// Read reads data from the limited reader.
+func (l *LimitedReadCloser) Read(p []byte) (int, error) {
+	return l.Reader.Read(p)
+}
+
+// LimitReadCloser returns a new reader wraps r in an io.LimitReader, but also
+// exposes the Close() method.
+func LimitReadCloser(r io.ReadCloser, n int64) *LimitedReadCloser {
+	return &LimitedReadCloser{ReadCloser: r, Reader: io.LimitReader(r, n)}
 }

src/restic/backend/utils_test.go

@@ -21,10 +21,10 @@ func TestLoadAll(t *testing.T) {
 		data := Random(23+i, rand.Intn(MiB)+500*KiB)
 
 		id := restic.Hash(data)
-		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, data)
+		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data))
 		OK(t, err)
 
-		buf, err := backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()}, nil)
+		buf, err := backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()})
 		OK(t, err)
 
 		if len(buf) != len(data) {
@@ -46,11 +46,11 @@ func TestLoadSmallBuffer(t *testing.T) {
 		data := Random(23+i, rand.Intn(MiB)+500*KiB)
 
 		id := restic.Hash(data)
-		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, data)
+		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data))
 		OK(t, err)
 
 		buf := make([]byte, len(data)-23)
-		buf, err = backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()}, buf)
+		buf, err = backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()})
 		OK(t, err)
 
 		if len(buf) != len(data) {
@@ -72,11 +72,11 @@ func TestLoadLargeBuffer(t *testing.T) {
 		data := Random(23+i, rand.Intn(MiB)+500*KiB)
 
 		id := restic.Hash(data)
-		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, data)
+		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data))
 		OK(t, err)
 
 		buf := make([]byte, len(data)+100)
-		buf, err = backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()}, buf)
+		buf, err = backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()})
 		OK(t, err)
 
 		if len(buf) != len(data) {

src/restic/blob.go

@@ -14,6 +14,11 @@ type Blob struct {
 	Offset uint
 }
 
+func (b Blob) String() string {
+	return fmt.Sprintf("<Blob (%v) %v, offset %v, length %v>",
+		b.Type, b.ID.Str(), b.Offset, b.Length)
+}
+
 // PackedBlob is a blob stored within a file.
 type PackedBlob struct {
 	Blob

src/restic/buffer.go

@@ -0,0 +1,21 @@
+package restic
+
+import "restic/crypto"
+
+// NewBlobBuffer returns a buffer that is large enough to hold a blob of size
+// plaintext bytes, including the crypto overhead.
+func NewBlobBuffer(size int) []byte {
+	return make([]byte, size, size+crypto.Extension)
+}
+
+// PlaintextLength returns the plaintext length of a blob with ciphertextSize
+// bytes.
+func PlaintextLength(ciphertextSize int) int {
+	return ciphertextSize - crypto.Extension
+}
+
+// CiphertextLength returns the encrypted length of a blob with plaintextSize
+// bytes.
+func CiphertextLength(plaintextSize int) int {
+	return plaintextSize + crypto.Extension
+}

src/restic/checker/checker.go

@@ -1,14 +1,17 @@
 package checker
 
 import (
-	"bytes"
+	"crypto/sha256"
 	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
 	"sync"
 
 	"restic/errors"
+	"restic/hashing"
 
 	"restic"
-	"restic/backend"
 	"restic/crypto"
 	"restic/debug"
 	"restic/pack"
@@ -77,6 +80,7 @@ func (c *Checker) LoadIndex() (hints []error, errs []error) {
 	debug.Log("Start")
 	type indexRes struct {
 		Index *repository.Index
+		err   error
 		ID    string
 	}
 
@@ -92,39 +96,40 @@ func (c *Checker) LoadIndex() (hints []error, errs []error) {
 			idx, err = repository.LoadIndexWithDecoder(c.repo, id, repository.DecodeOldIndex)
 		}
 
-		if err != nil {
-			return err
-		}
+		err = errors.Wrapf(err, "error loading index %v", id.Str())
 
 		select {
-		case indexCh <- indexRes{Index: idx, ID: id.String()}:
+		case indexCh <- indexRes{Index: idx, ID: id.String(), err: err}:
 		case <-done:
 		}
 
 		return nil
 	}
 
-	var perr error
 	go func() {
 		defer close(indexCh)
 		debug.Log("start loading indexes in parallel")
-		perr = repository.FilesInParallel(c.repo.Backend(), restic.IndexFile, defaultParallelism,
+		err := repository.FilesInParallel(c.repo.Backend(), restic.IndexFile, defaultParallelism,
 			repository.ParallelWorkFuncParseID(worker))
-		debug.Log("loading indexes finished, error: %v", perr)
+		debug.Log("loading indexes finished, error: %v", err)
+		if err != nil {
+			panic(err)
+		}
 	}()
 
 	done := make(chan struct{})
 	defer close(done)
 
-	if perr != nil {
-		errs = append(errs, perr)
-		return hints, errs
-	}
-
 	packToIndex := make(map[restic.ID]restic.IDSet)
 
 	for res := range indexCh {
-		debug.Log("process index %v", res.ID)
+		debug.Log("process index %v, err %v", res.ID, res.err)
+
+		if res.err != nil {
+			errs = append(errs, res.err)
+			continue
+		}
+
 		idxID, err := restic.ParseID(res.ID)
 		if err != nil {
 			errs = append(errs, errors.Errorf("unable to parse as index ID: %v", res.ID))
@@ -151,8 +156,6 @@ func (c *Checker) LoadIndex() (hints []error, errs []error) {
 		debug.Log("%d blobs processed", cnt)
 	}
 
-	debug.Log("done, error %v", perr)
-
 	debug.Log("checking for duplicate packs")
 	for packID := range c.packs {
 		debug.Log("  check pack %v: contained in %d indexes", packID.Str(), len(packToIndex[packID]))
@@ -187,7 +190,8 @@ func packIDTester(repo restic.Repository, inChan <-chan restic.ID, errChan chan<
 	defer wg.Done()
 
 	for id := range inChan {
-		ok, err := repo.Backend().Test(restic.DataFile, id.String())
+		h := restic.Handle{Type: restic.DataFile, Name: id.String()}
+		ok, err := repo.Backend().Test(h)
 		if err != nil {
 			err = PackError{ID: id, Err: err}
 		} else {
@@ -658,36 +662,77 @@ func (c *Checker) CountPacks() uint64 {
 func checkPack(r restic.Repository, id restic.ID) error {
 	debug.Log("checking pack %v", id.Str())
 	h := restic.Handle{Type: restic.DataFile, Name: id.String()}
-	buf, err := backend.LoadAll(r.Backend(), h, nil)
+
+	rd, err := r.Backend().Load(h, 0, 0)
 	if err != nil {
 		return err
 	}
 
-	hash := restic.Hash(buf)
+	packfile, err := ioutil.TempFile("", "restic-temp-check-")
+	if err != nil {
+		return errors.Wrap(err, "TempFile")
+	}
+
+	defer func() {
+		packfile.Close()
+		os.Remove(packfile.Name())
+	}()
+
+	hrd := hashing.NewReader(rd, sha256.New())
+	size, err := io.Copy(packfile, hrd)
+	if err != nil {
+		return errors.Wrap(err, "Copy")
+	}
+
+	if err = rd.Close(); err != nil {
+		return err
+	}
+
+	hash := restic.IDFromHash(hrd.Sum(nil))
+	debug.Log("hash for pack %v is %v", id.Str(), hash.Str())
+
 	if !hash.Equal(id) {
 		debug.Log("Pack ID does not match, want %v, got %v", id.Str(), hash.Str())
 		return errors.Errorf("Pack ID does not match, want %v, got %v", id.Str(), hash.Str())
 	}
 
-	blobs, err := pack.List(r.Key(), bytes.NewReader(buf), int64(len(buf)))
+	blobs, err := pack.List(r.Key(), packfile, size)
 	if err != nil {
 		return err
 	}
 
 	var errs []error
+	var buf []byte
 	for i, blob := range blobs {
-		debug.Log("  check blob %d: %v", i, blob.ID.Str())
+		debug.Log("  check blob %d: %v", i, blob)
 
-		plainBuf := make([]byte, blob.Length)
-		n, err := crypto.Decrypt(r.Key(), plainBuf, buf[blob.Offset:blob.Offset+blob.Length])
+		buf = buf[:cap(buf)]
+		if uint(len(buf)) < blob.Length {
+			buf = make([]byte, blob.Length)
+		}
+		buf = buf[:blob.Length]
+
+		_, err := packfile.Seek(int64(blob.Offset), 0)
+		if err != nil {
+			return errors.Errorf("Seek(%v): %v", blob.Offset, err)
+		}
+
+		_, err = io.ReadFull(packfile, buf)
+		if err != nil {
+			debug.Log("  error loading blob %v: %v", blob.ID.Str(), err)
+			errs = append(errs, errors.Errorf("blob %v: %v", i, err))
+			continue
+		}
+
+		n, err := crypto.Decrypt(r.Key(), buf, buf)
 		if err != nil {
 			debug.Log("  error decrypting blob %v: %v", blob.ID.Str(), err)
 			errs = append(errs, errors.Errorf("blob %v: %v", i, err))
 			continue
 		}
-		plainBuf = plainBuf[:n]
+		buf = buf[:n]
 
-		hash := restic.Hash(plainBuf)
+		hash := restic.Hash(buf)
 		if !hash.Equal(blob.ID) {
 			debug.Log("  Blob ID does not match, want %v, got %v", blob.ID.Str(), hash.Str())
 			errs = append(errs, errors.Errorf("Blob ID does not match, want %v, got %v", blob.ID.Str(), hash.Str()))

src/restic/checker/checker_test.go

@@ -1,6 +1,7 @@
 package checker_test
 
 import (
+	"io"
 	"math/rand"
 	"path/filepath"
 	"sort"
@@ -8,7 +9,6 @@ import (
 
 	"restic"
 	"restic/archiver"
-	"restic/backend/mem"
 	"restic/checker"
 	"restic/repository"
 	"restic/test"
@@ -73,8 +73,11 @@ func TestMissingPack(t *testing.T) {
 
 	repo := repository.TestOpenLocal(t, repodir)
 
-	packID := "657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6"
-	test.OK(t, repo.Backend().Remove(restic.DataFile, packID))
+	packHandle := restic.Handle{
+		Type: restic.DataFile,
+		Name: "657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6",
+	}
+	test.OK(t, repo.Backend().Remove(packHandle))
 
 	chkr := checker.New(repo)
 	hints, errs := chkr.LoadIndex()
@@ -92,7 +95,7 @@ func TestMissingPack(t *testing.T) {
 		"expected exactly one error, got %v", len(errs))
 
 	if err, ok := errs[0].(checker.PackError); ok {
-		test.Equals(t, packID, err.ID.String())
+		test.Equals(t, packHandle.Name, err.ID.String())
 	} else {
 		t.Errorf("expected error returned by checker.Packs() to be PackError, got %v", err)
 	}
@@ -105,9 +108,12 @@ func TestUnreferencedPack(t *testing.T) {
 	repo := repository.TestOpenLocal(t, repodir)
 
 	// index 3f1a only references pack 60e0
-	indexID := "3f1abfcb79c6f7d0a3be517d2c83c8562fba64ef2c8e9a3544b4edaf8b5e3b44"
 	packID := "60e0438dcb978ec6860cc1f8c43da648170ee9129af8f650f876bad19f8f788e"
-	test.OK(t, repo.Backend().Remove(restic.IndexFile, indexID))
+	indexHandle := restic.Handle{
+		Type: restic.IndexFile,
+		Name: "3f1abfcb79c6f7d0a3be517d2c83c8562fba64ef2c8e9a3544b4edaf8b5e3b44",
+	}
+	test.OK(t, repo.Backend().Remove(indexHandle))
 
 	chkr := checker.New(repo)
 	hints, errs := chkr.LoadIndex()
@@ -137,8 +143,11 @@ func TestUnreferencedBlobs(t *testing.T) {
 
 	repo := repository.TestOpenLocal(t, repodir)
 
-	snID := "51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02"
-	test.OK(t, repo.Backend().Remove(restic.SnapshotFile, snID))
+	snapshotHandle := restic.Handle{
+		Type: restic.SnapshotFile,
+		Name: "51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02",
+	}
+	test.OK(t, repo.Backend().Remove(snapshotHandle))
 
 	unusedBlobsBySnapshot := restic.IDs{
 		restic.TestParseID("58c748bbe2929fdf30c73262bd8313fe828f8925b05d1d4a87fe109082acb849"),
@@ -170,6 +179,48 @@ func TestUnreferencedBlobs(t *testing.T) {
 	test.Equals(t, unusedBlobsBySnapshot, blobs)
 }
 
+func TestModifiedIndex(t *testing.T) {
+	repodir, cleanup := test.Env(t, checkerTestData)
+	defer cleanup()
+
+	repo := repository.TestOpenLocal(t, repodir)
+
+	done := make(chan struct{})
+	defer close(done)
+
+	h := restic.Handle{
+		Type: restic.IndexFile,
+		Name: "90f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
+	}
+	f, err := repo.Backend().Load(h, 0, 0)
+	test.OK(t, err)
+
+	// save the index again with a modified name so that the hash doesn't match
+	// the content any more
+	h2 := restic.Handle{
+		Type: restic.IndexFile,
+		Name: "80f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
+	}
+	err = repo.Backend().Save(h2, f)
+	test.OK(t, err)
+
+	test.OK(t, f.Close())
+
+	chkr := checker.New(repo)
+	hints, errs := chkr.LoadIndex()
+	if len(errs) == 0 {
+		t.Fatalf("expected errors not found")
+	}
+
+	for _, err := range errs {
+		t.Logf("found expected error %v", err)
+	}
+
+	if len(hints) > 0 {
+		t.Errorf("expected no hints, got %v: %v", len(hints), hints)
+	}
+}
+
 var checkerDuplicateIndexTestData = filepath.Join("testdata", "duplicate-packs-in-index-test-repo.tar.gz")
 
 func TestDuplicatePacksInIndex(t *testing.T) {
@@ -208,15 +259,35 @@ type errorBackend struct {
 	ProduceErrors bool
 }
 
-func (b errorBackend) Load(h restic.Handle, p []byte, off int64) (int, error) {
-	n, err := b.Backend.Load(h, p, off)
+func (b errorBackend) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	rd, err := b.Backend.Load(h, length, offset)
+	if err != nil {
+		return rd, err
+	}
 
 	if b.ProduceErrors {
-		induceError(p)
+		return errorReadCloser{rd}, err
+	}
+
+	return rd, nil
+}
+
+type errorReadCloser struct {
+	io.ReadCloser
+}
+
+func (erd errorReadCloser) Read(p []byte) (int, error) {
+	n, err := erd.ReadCloser.Read(p)
+	if n > 0 {
+		induceError(p[:n])
 	}
 	return n, err
 }
 
+func (erd errorReadCloser) Close() error {
+	return erd.ReadCloser.Close()
+}
+
 // induceError flips a bit in the slice.
 func induceError(data []byte) {
 	if rand.Float32() < 0.2 {
@@ -228,19 +299,15 @@ func induceError(data []byte) {
 }
 
 func TestCheckerModifiedData(t *testing.T) {
-	be := mem.New()
-
-	repository.TestUseLowSecurityKDFParameters(t)
-
-	repo := repository.New(be)
-	test.OK(t, repo.Init(test.TestPassword))
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
 
 	arch := archiver.New(repo)
-	_, id, err := arch.Snapshot(nil, []string{"."}, nil, nil)
+	_, id, err := arch.Snapshot(nil, []string{"."}, nil, "localhost", nil)
 	test.OK(t, err)
 	t.Logf("archived as %v", id.Str())
 
-	beError := &errorBackend{Backend: be}
+	beError := &errorBackend{Backend: repo.Backend()}
 	checkRepo := repository.New(beError)
 	test.OK(t, checkRepo.SearchKey(test.TestPassword, 5))
 
@@ -266,7 +333,7 @@ func TestCheckerModifiedData(t *testing.T) {
 	}
 
 	for _, err := range checkData(chkr) {
-		t.Logf("struct error: %v", err)
+		t.Logf("data error: %v", err)
 		errFound = true
 	}
 
@@ -274,3 +341,28 @@ func TestCheckerModifiedData(t *testing.T) {
 		t.Fatal("no error found, checker is broken")
 	}
 }
+
+func BenchmarkChecker(t *testing.B) {
+	repodir, cleanup := test.Env(t, checkerTestData)
+	defer cleanup()
+
+	repo := repository.TestOpenLocal(t, repodir)
+
+	chkr := checker.New(repo)
+	hints, errs := chkr.LoadIndex()
+	if len(errs) > 0 {
+		t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
+	}
+
+	if len(hints) > 0 {
+		t.Errorf("expected no hints, got %v: %v", len(hints), hints)
+	}
+
+	t.ResetTimer()
+
+	for i := 0; i < t.N; i++ {
+		test.OKs(t, checkPacks(chkr))
+		test.OKs(t, checkStruct(chkr))
+		test.OKs(t, checkData(chkr))
+	}
+}

src/restic/errors/wrap.go

@@ -18,3 +18,7 @@ var Errorf = errors.Errorf
 // Wrap wraps an error retrieved from outside of restic. Wrapped so that this
 // package does not appear in the stack trace.
 var Wrap = errors.Wrap
+
+// Wrapf returns an error annotating err with the format specifier. If err is
+// nil, Wrapf returns nil.
+var Wrapf = errors.Wrapf

src/restic/fs/deviceid_unix.go

@@ -12,10 +12,19 @@ import (
 // DeviceID extracts the device ID from an os.FileInfo object by casting it
 // to syscall.Stat_t
 func DeviceID(fi os.FileInfo) (deviceID uint64, err error) {
+	if fi == nil {
+		return 0, errors.New("unable to determine device: fi is nil")
+	}
+
+	if fi.Sys() == nil {
+		return 0, errors.New("unable to determine device: fi.Sys() is nil")
+	}
+
 	if st, ok := fi.Sys().(*syscall.Stat_t); ok {
 		// st.Dev is uint32 on Darwin and uint64 on Linux. Just cast
 		// everything to uint64.
 		return uint64(st.Dev), nil
 	}
+
 	return 0, errors.New("Could not cast to syscall.Stat_t")
 }

src/restic/fs/doc.go

@@ -1,3 +1,3 @@
-// Package fs implements an OS independend abstraction of a file system
+// Package fs implements an OS independent abstraction of a file system
 // suitable for backup purposes.
 package fs

src/restic/fs/file.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 )
 
 // File is an open file on a file system.
@@ -26,7 +27,20 @@ func fixpath(name string) string {
 	if runtime.GOOS == "windows" {
 		abspath, err := filepath.Abs(name)
 		if err == nil {
-			return "\\\\?\\" + abspath
+			// Check if \\?\UNC\ already exist
+			if strings.HasPrefix(abspath, `\\?\UNC\`) {
+				return abspath
+			}
+			// Check if \\?\ already exist
+			if strings.HasPrefix(abspath, `\\?\`) {
+				return abspath
+			}
+			// Check if path starts with \\
+			if strings.HasPrefix(abspath, `\\`) {
+				return strings.Replace(abspath, `\\`, `\\?\UNC\`, 1)
+			}
+			// Normal path
+			return `\\?\` + abspath
 		}
 	}
 	return name
@@ -88,6 +102,12 @@ func Symlink(oldname, newname string) error {
 	return os.Symlink(fixpath(oldname), fixpath(newname))
 }
 
+// Link creates newname as a hard link to oldname.
+// If there is an error, it will be of type *LinkError.
+func Link(oldname, newname string) error {
+	return os.Link(fixpath(oldname), fixpath(newname))
+}
+
 // Stat returns a FileInfo structure describing the named file.
 // If there is an error, it will be of type *PathError.
 func Stat(name string) (os.FileInfo, error) {
@@ -111,6 +131,11 @@ func Create(name string) (*os.File, error) {
 	return os.Create(fixpath(name))
 }
 
+// Open opens a file for reading.
+func Open(name string) (File, error) {
+	return os.Open(fixpath(name))
+}
+
 // OpenFile is the generalized open call; most users will use Open
 // or Create instead.  It opens the named file with specified flag
 // (O_RDONLY etc.) and perm, (0666 etc.) if applicable.  If successful,

src/restic/fs/file_linux.go

@@ -1,59 +0,0 @@
-// +build linux,go1.4
-
-package fs
-
-import (
-	"os"
-	"syscall"
-
-	"restic/errors"
-
-	"golang.org/x/sys/unix"
-)
-
-// Open opens a file for reading, without updating the atime and without caching data on read.
-func Open(name string) (File, error) {
-	file, err := os.OpenFile(name, os.O_RDONLY|syscall.O_NOATIME, 0)
-	if os.IsPermission(errors.Cause(err)) {
-		file, err = os.OpenFile(name, os.O_RDONLY, 0)
-	}
-	return &nonCachingFile{File: file}, err
-}
-
-// nonCachingFile wraps an *os.File and calls fadvise() to instantly forget
-// data that has been read or written.
-type nonCachingFile struct {
-	*os.File
-	readOffset int64
-}
-
-func (f *nonCachingFile) Read(p []byte) (int, error) {
-	n, err := f.File.Read(p)
-
-	if n > 0 {
-		ferr := unix.Fadvise(int(f.File.Fd()), f.readOffset, int64(n), unix.FADV_DONTNEED)
-
-		f.readOffset += int64(n)
-
-		if err == nil {
-			err = ferr
-		}
-	}
-
-	return n, err
-}
-
-// ClearCache syncs and then removes the file's content from the OS cache.
-func ClearCache(file File) error {
-	f, ok := file.(*os.File)
-	if !ok {
-		panic("ClearCache called for file not *os.File")
-	}
-
-	err := f.Sync()
-	if err != nil {
-		return err
-	}
-
-	return unix.Fadvise(int(f.Fd()), 0, 0, unix.FADV_DONTNEED)
-}

src/restic/fs/file_nonlinux.go

@@ -1,15 +0,0 @@
-// +build !linux !go1.4
-
-package fs
-
-import "os"
-
-// Open opens a file for reading.
-func Open(name string) (File, error) {
-	return os.OpenFile(fixpath(name), os.O_RDONLY, 0)
-}
-
-// ClearCache syncs and then removes the file's content from the OS cache.
-func ClearCache(f File) error {
-	return nil
-}

src/restic/fuse/dir.go

@@ -114,9 +114,25 @@ func (d *dir) Attr(ctx context.Context, a *fuse.Attr) error {
 	a.Atime = d.node.AccessTime
 	a.Ctime = d.node.ChangeTime
 	a.Mtime = d.node.ModTime
+
+	a.Nlink = d.calcNumberOfLinks()
+
 	return nil
 }
 
+func (d *dir) calcNumberOfLinks() uint32 {
+	// a directory d has 2 hardlinks + the number
+	// of directories contained by d
+	var count uint32
+	count = 2
+	for _, node := range d.items {
+		if node.Type == "dir" {
+			count++
+		}
+	}
+	return count
+}
+
 func (d *dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
 	debug.Log("called")
 	ret := make([]fuse.Dirent, 0, len(d.items))
@@ -161,3 +177,21 @@ func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
 		return nil, fuse.ENOENT
 	}
 }
+
+func (d *dir) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error {
+	debug.Log("Listxattr(%v, %v)", d.node.Name, req.Size)
+	for _, attr := range d.node.ExtendedAttributes {
+		resp.Append(attr.Name)
+	}
+	return nil
+}
+
+func (d *dir) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
+	debug.Log("Getxattr(%v, %v, %v)", d.node.Name, req.Name, req.Size)
+	attrval := d.node.GetExtendedAttribute(req.Name)
+	if attrval != nil {
+		resp.Xattr = attrval
+		return nil
+	}
+	return fuse.ErrNoXattr
+}

src/restic/fuse/file.go

@@ -4,8 +4,6 @@
 package fuse
 
 import (
-	"sync"
-
 	"restic/errors"
 
 	"restic"
@@ -35,29 +33,23 @@ type file struct {
 	node        *restic.Node
 	ownerIsRoot bool
 
-	sizes []uint
+	sizes []int
 	blobs [][]byte
 }
 
 const defaultBlobSize = 128 * 1024
 
-var blobPool = sync.Pool{
-	New: func() interface{} {
-		return make([]byte, defaultBlobSize)
-	},
-}
-
 func newFile(repo BlobLoader, node *restic.Node, ownerIsRoot bool) (*file, error) {
 	debug.Log("create new file for %v with %d blobs", node.Name, len(node.Content))
 	var bytes uint64
-	sizes := make([]uint, len(node.Content))
+	sizes := make([]int, len(node.Content))
 	for i, id := range node.Content {
 		size, err := repo.LookupBlobSize(id, restic.DataBlob)
 		if err != nil {
 			return nil, err
 		}
 
-		sizes[i] = size
+		sizes[i] = int(size)
 		bytes += uint64(size)
 	}
 
@@ -82,6 +74,7 @@ func (f *file) Attr(ctx context.Context, a *fuse.Attr) error {
 	a.Size = f.node.Size
 	a.Blocks = (f.node.Size / blockSize) + 1
 	a.BlockSize = blockSize
+	a.Nlink = uint32(f.node.Links)
 
 	if !f.ownerIsRoot {
 		a.Uid = f.node.UID
@@ -90,7 +83,9 @@ func (f *file) Attr(ctx context.Context, a *fuse.Attr) error {
 	a.Atime = f.node.AccessTime
 	a.Ctime = f.node.ChangeTime
 	a.Mtime = f.node.ModTime
+
 	return nil
+
 }
 
 func (f *file) getBlobAt(i int) (blob []byte, err error) {
@@ -99,16 +94,12 @@ func (f *file) getBlobAt(i int) (blob []byte, err error) {
 		return f.blobs[i], nil
 	}
 
-	buf := blobPool.Get().([]byte)
-	buf = buf[:cap(buf)]
-
-	if uint(len(buf)) < f.sizes[i] {
-		if len(buf) > defaultBlobSize {
-			blobPool.Put(buf)
-		}
-		buf = make([]byte, f.sizes[i])
+	// release earlier blobs
+	for j := 0; j < i; j++ {
+		f.blobs[j] = nil
 	}
 
+	buf := restic.NewBlobBuffer(f.sizes[i])
 	n, err := f.repo.LoadBlob(restic.DataBlob, f.node.Content[i], buf)
 	if err != nil {
 		debug.Log("LoadBlob(%v, %v) failed: %v", f.node.Name, f.node.Content[i], err)
@@ -169,10 +160,25 @@ func (f *file) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadR
 
 func (f *file) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
 	for i := range f.blobs {
-		if f.blobs[i] != nil {
-			blobPool.Put(f.blobs[i])
-			f.blobs[i] = nil
-		}
+		f.blobs[i] = nil
 	}
 	return nil
 }
+
+func (f *file) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error {
+	debug.Log("Listxattr(%v, %v)", f.node.Name, req.Size)
+	for _, attr := range f.node.ExtendedAttributes {
+		resp.Append(attr.Name)
+	}
+	return nil
+}
+
+func (f *file) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
+	debug.Log("Getxattr(%v, %v, %v)", f.node.Name, req.Name, req.Size)
+	attrval := f.node.GetExtendedAttribute(req.Name)
+	if attrval != nil {
+		resp.Xattr = attrval
+		return nil
+	}
+	return fuse.ErrNoXattr
+}

src/restic/fuse/file_test.go

@@ -9,7 +9,9 @@ import (
 	"testing"
 	"time"
 
-	"restic/errors"
+	"golang.org/x/net/context"
+
+	"restic/repository"
 
 	"bazil.org/fuse"
 
@@ -17,108 +19,96 @@ import (
 	. "restic/test"
 )
 
-type MockRepo struct {
-	blobs map[restic.ID][]byte
-}
-
-func NewMockRepo(content map[restic.ID][]byte) *MockRepo {
-	return &MockRepo{blobs: content}
-}
-
-func (m *MockRepo) LookupBlobSize(id restic.ID, t restic.BlobType) (uint, error) {
-	buf, ok := m.blobs[id]
-	if !ok {
-		return 0, errors.New("blob not found")
-	}
-
-	return uint(len(buf)), nil
-}
-
-func (m *MockRepo) LoadBlob(t restic.BlobType, id restic.ID, buf []byte) (int, error) {
-	size, err := m.LookupBlobSize(id, t)
-	if err != nil {
-		return 0, err
-	}
-
-	if uint(len(buf)) < size {
-		return 0, errors.New("buffer too small")
-	}
-
-	buf = buf[:size]
-	copy(buf, m.blobs[id])
-	return int(size), nil
-}
-
-type MockContext struct{}
-
-func (m MockContext) Deadline() (time.Time, bool)       { return time.Now(), false }
-func (m MockContext) Done() <-chan struct{}             { return nil }
-func (m MockContext) Err() error                        { return nil }
-func (m MockContext) Value(key interface{}) interface{} { return nil }
-
-var testContent = genTestContent()
-var testContentLengths = []uint{
-	4646 * 1024,
-	655 * 1024,
-	378 * 1024,
-	8108 * 1024,
-	558 * 1024,
-}
-var testMaxFileSize uint
-
-func genTestContent() map[restic.ID][]byte {
-	m := make(map[restic.ID][]byte)
-
-	for _, length := range testContentLengths {
-		buf := Random(int(length), int(length))
-		id := restic.Hash(buf)
-		m[id] = buf
-		testMaxFileSize += length
-	}
-
-	return m
-}
-
-const maxBufSize = 20 * 1024 * 1024
-
-func testRead(t *testing.T, f *file, offset, length int, data []byte) {
-	ctx := MockContext{}
+func testRead(t testing.TB, f *file, offset, length int, data []byte) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
 
 	req := &fuse.ReadRequest{
 		Offset: int64(offset),
 		Size:   length,
 	}
 	resp := &fuse.ReadResponse{
-		Data: make([]byte, length),
+		Data: data,
 	}
 	OK(t, f.Read(ctx, req, resp))
 }
 
-var offsetReadsTests = []struct {
-	offset, length int
-}{
-	{0, 5 * 1024 * 1024},
-	{4000 * 1024, 1000 * 1024},
+func firstSnapshotID(t testing.TB, repo restic.Repository) (first restic.ID) {
+	done := make(chan struct{})
+	defer close(done)
+	for id := range repo.List(restic.SnapshotFile, done) {
+		if first.IsNull() {
+			first = id
+		}
+	}
+	return first
+}
+
+func loadFirstSnapshot(t testing.TB, repo restic.Repository) *restic.Snapshot {
+	id := firstSnapshotID(t, repo)
+	sn, err := restic.LoadSnapshot(repo, id)
+	OK(t, err)
+	return sn
+}
+
+func loadTree(t testing.TB, repo restic.Repository, id restic.ID) *restic.Tree {
+	tree, err := repo.LoadTree(id)
+	OK(t, err)
+	return tree
 }
 
 func TestFuseFile(t *testing.T) {
-	repo := NewMockRepo(testContent)
-	ctx := MockContext{}
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
 
-	memfile := make([]byte, 0, maxBufSize)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	timestamp, err := time.Parse(time.RFC3339, "2017-01-24T10:42:56+01:00")
+	OK(t, err)
+	restic.TestCreateSnapshot(t, repo, timestamp, 2, 0.1)
+
+	sn := loadFirstSnapshot(t, repo)
+	tree := loadTree(t, repo, *sn.Tree)
+
+	var content restic.IDs
+	for _, node := range tree.Nodes {
+		content = append(content, node.Content...)
+	}
+	t.Logf("tree loaded, content: %v", content)
+
+	var (
+		filesize uint64
+		memfile  []byte
+	)
+	for _, id := range content {
+		size, err := repo.LookupBlobSize(id, restic.DataBlob)
+		OK(t, err)
+		filesize += uint64(size)
+
+		buf := restic.NewBlobBuffer(int(size))
+		n, err := repo.LoadBlob(restic.DataBlob, id, buf)
+		OK(t, err)
+
+		if uint(n) != size {
+			t.Fatalf("not enough bytes read for id %v: want %v, got %v", id.Str(), size, n)
+		}
+
+		if uint(len(buf)) != size {
+			t.Fatalf("buffer has wrong length for id %v: want %v, got %v", id.Str(), size, len(buf))
+		}
 
-	var ids restic.IDs
-	for id, buf := range repo.blobs {
-		ids = append(ids, id)
 		memfile = append(memfile, buf...)
 	}
 
+	t.Logf("filesize is %v, memfile has size %v", filesize, len(memfile))
+
 	node := &restic.Node{
 		Name:    "foo",
 		Inode:   23,
 		Mode:    0742,
-		Size:    42,
-		Content: ids,
+		Size:    filesize,
+		Content: content,
 	}
 	f, err := newFile(repo, node, false)
 	OK(t, err)
@@ -131,28 +121,19 @@ func TestFuseFile(t *testing.T) {
 	Equals(t, node.Size, attr.Size)
 	Equals(t, (node.Size/uint64(attr.BlockSize))+1, attr.Blocks)
 
-	for i, test := range offsetReadsTests {
-		b := memfile[test.offset : test.offset+test.length]
-		buf := make([]byte, test.length)
-		testRead(t, f, test.offset, test.length, buf)
-		if !bytes.Equal(b, buf) {
-			t.Errorf("test %d failed, wrong data returned", i)
-		}
-	}
-
 	for i := 0; i < 200; i++ {
-		length := rand.Intn(int(testMaxFileSize) / 2)
-		offset := rand.Intn(int(testMaxFileSize))
-		if length+offset > int(testMaxFileSize) {
-			diff := length + offset - int(testMaxFileSize)
-			length -= diff
-		}
+		offset := rand.Intn(int(filesize))
+		length := rand.Intn(int(filesize)-offset) + 100
 
 		b := memfile[offset : offset+length]
+
 		buf := make([]byte, length)
+
 		testRead(t, f, offset, length, buf)
 		if !bytes.Equal(b, buf) {
-			t.Errorf("test %d failed (offset %d, length %d), wrong data returned", i, offset, length)
+			t.Errorf("test %d failed, wrong data returned (offset %v, length %v)", i, offset, length)
 		}
 	}
+
+	OK(t, f.Release(ctx, nil))
 }

src/restic/fuse/link.go

@@ -38,5 +38,8 @@ func (l *link) Attr(ctx context.Context, a *fuse.Attr) error {
 	a.Atime = l.node.AccessTime
 	a.Ctime = l.node.ChangeTime
 	a.Mtime = l.node.ModTime
+
+	a.Nlink = uint32(l.node.Links)
+
 	return nil
 }

src/restic/fuse/snapshot.go

@@ -32,6 +32,9 @@ var _ = fs.NodeStringLookuper(&SnapshotsDir{})
 type SnapshotsDir struct {
 	repo        restic.Repository
 	ownerIsRoot bool
+	paths       []string
+	tags        []string
+	host        string
 
 	// knownSnapshots maps snapshot timestamp to the snapshot
 	sync.RWMutex
@@ -40,12 +43,15 @@ type SnapshotsDir struct {
 }
 
 // NewSnapshotsDir returns a new dir object for the snapshots.
-func NewSnapshotsDir(repo restic.Repository, ownerIsRoot bool) *SnapshotsDir {
+func NewSnapshotsDir(repo restic.Repository, ownerIsRoot bool, paths []string, tags []string, host string) *SnapshotsDir {
 	debug.Log("fuse mount initiated")
 	return &SnapshotsDir{
 		repo:           repo,
-		knownSnapshots: make(map[string]SnapshotWithId),
 		ownerIsRoot:    ownerIsRoot,
+		paths:          paths,
+		tags:           tags,
+		host:           host,
+		knownSnapshots: make(map[string]SnapshotWithId),
 		processed:      restic.NewIDSet(),
 	}
 }
@@ -79,6 +85,13 @@ func (sn *SnapshotsDir) updateCache(ctx context.Context) error {
 			return err
 		}
 
+		// Filter snapshots we don't care for.
+		if (sn.host != "" && sn.host != snapshot.Hostname) ||
+			!snapshot.HasTags(sn.tags) ||
+			!snapshot.HasPaths(sn.paths) {
+			continue
+		}
+
 		timestamp := snapshot.Time.Format(time.RFC3339)
 		for i := 1; ; i++ {
 			if _, ok := sn.knownSnapshots[timestamp]; !ok {

src/restic/hardlinks_index.go

@@ -0,0 +1,57 @@
+package restic
+
+import (
+	"sync"
+)
+
+// HardlinkKey is a composed key for finding inodes on a specific device.
+type HardlinkKey struct {
+	Inode, Device uint64
+}
+
+// HardlinkIndex contains a list of inodes, devices these inodes are one, and associated file names.
+type HardlinkIndex struct {
+	m     sync.Mutex
+	Index map[HardlinkKey]string
+}
+
+// NewHardlinkIndex create a new index for hard links
+func NewHardlinkIndex() *HardlinkIndex {
+	return &HardlinkIndex{
+		Index: make(map[HardlinkKey]string),
+	}
+}
+
+// Has checks wether the link already exist in the index.
+func (idx *HardlinkIndex) Has(inode uint64, device uint64) bool {
+	idx.m.Lock()
+	defer idx.m.Unlock()
+	_, ok := idx.Index[HardlinkKey{inode, device}]
+
+	return ok
+}
+
+// Add adds a link to the index.
+func (idx *HardlinkIndex) Add(inode uint64, device uint64, name string) {
+	idx.m.Lock()
+	defer idx.m.Unlock()
+	_, ok := idx.Index[HardlinkKey{inode, device}]
+
+	if !ok {
+		idx.Index[HardlinkKey{inode, device}] = name
+	}
+}
+
+// GetFilename obtains the filename from the index.
+func (idx *HardlinkIndex) GetFilename(inode uint64, device uint64) string {
+	idx.m.Lock()
+	defer idx.m.Unlock()
+	return idx.Index[HardlinkKey{inode, device}]
+}
+
+// Remove removes a link from the index.
+func (idx *HardlinkIndex) Remove(inode uint64, device uint64) {
+	idx.m.Lock()
+	defer idx.m.Unlock()
+	delete(idx.Index, HardlinkKey{inode, device})
+}

src/restic/hardlinks_index_test.go

@@ -0,0 +1,35 @@
+package restic_test
+
+import (
+	"testing"
+
+	"restic"
+	. "restic/test"
+)
+
+// TestHardLinks contains various tests for HardlinkIndex.
+func TestHardLinks(t *testing.T) {
+
+	idx := restic.NewHardlinkIndex()
+
+	idx.Add(1, 2, "inode1-file1-on-device2")
+	idx.Add(2, 3, "inode2-file2-on-device3")
+
+	var sresult string
+	sresult = idx.GetFilename(1, 2)
+	Equals(t, sresult, "inode1-file1-on-device2")
+
+	sresult = idx.GetFilename(2, 3)
+	Equals(t, sresult, "inode2-file2-on-device3")
+
+	var bresult bool
+	bresult = idx.Has(1, 2)
+	Equals(t, bresult, true)
+
+	bresult = idx.Has(1, 3)
+	Equals(t, bresult, false)
+
+	idx.Remove(1, 2)
+	bresult = idx.Has(1, 2)
+	Equals(t, bresult, false)
+}

src/restic/hashing/reader.go

@@ -0,0 +1,29 @@
+package hashing
+
+import (
+	"hash"
+	"io"
+)
+
+// Reader hashes all data read from the underlying reader.
+type Reader struct {
+	r io.Reader
+	h hash.Hash
+}
+
+// NewReader returns a new Reader that uses the hash h.
+func NewReader(r io.Reader, h hash.Hash) *Reader {
+	return &Reader{
+		h: h,
+		r: io.TeeReader(r, h),
+	}
+}
+
+func (h *Reader) Read(p []byte) (int, error) {
+	return h.r.Read(p)
+}
+
+// Sum returns the hash of the data read so far.
+func (h *Reader) Sum(d []byte) []byte {
+	return h.h.Sum(d)
+}

src/restic/hashing/reader_test.go

@@ -0,0 +1,73 @@
+package hashing
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/sha256"
+	"io"
+	"io/ioutil"
+	"testing"
+)
+
+func TestReader(t *testing.T) {
+	tests := []int{5, 23, 2<<18 + 23, 1 << 20}
+
+	for _, size := range tests {
+		data := make([]byte, size)
+		_, err := io.ReadFull(rand.Reader, data)
+		if err != nil {
+			t.Fatalf("ReadFull: %v", err)
+		}
+
+		expectedHash := sha256.Sum256(data)
+
+		rd := NewReader(bytes.NewReader(data), sha256.New())
+		n, err := io.Copy(ioutil.Discard, rd)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if n != int64(size) {
+			t.Errorf("Reader: invalid number of bytes written: got %d, expected %d",
+				n, size)
+		}
+
+		resultingHash := rd.Sum(nil)
+
+		if !bytes.Equal(expectedHash[:], resultingHash) {
+			t.Errorf("Reader: hashes do not match: expected %02x, got %02x",
+				expectedHash, resultingHash)
+		}
+	}
+}
+
+func BenchmarkReader(b *testing.B) {
+	buf := make([]byte, 1<<22)
+	_, err := io.ReadFull(rand.Reader, buf)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	expectedHash := sha256.Sum256(buf)
+
+	b.SetBytes(int64(len(buf)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		rd := NewReader(bytes.NewReader(buf), sha256.New())
+		n, err := io.Copy(ioutil.Discard, rd)
+		if err != nil {
+			b.Fatal(err)
+		}
+
+		if n != int64(len(buf)) {
+			b.Errorf("Reader: invalid number of bytes written: got %d, expected %d",
+				n, len(buf))
+		}
+
+		resultingHash := rd.Sum(nil)
+		if !bytes.Equal(expectedHash[:], resultingHash) {
+			b.Errorf("Reader: hashes do not match: expected %02x, got %02x",
+				expectedHash, resultingHash)
+		}
+	}
+}