Add VERSION file for 0.4.0

This was a temporary fix for Go earlier than 1.5 to run code on all
avaialble cores. We don't need that any more since we require at least
Go 1.6.

.envrc

@@ -0,0 +1 @@
+GOPATH=$PWD:$PWD/vendor

.travis.yml

@@ -2,8 +2,9 @@ language: go
 sudo: false
 
 go:
-  - 1.6.3
-  - 1.7.1
+  - 1.6.4
+  - 1.7.4
+  - tip
 
 os:
   - linux
@@ -16,16 +17,23 @@ env:
 matrix:
   exclude:
     - os: osx
-      go: 1.6.3
+      go: 1.6.4
+    - os: osx
+      go: tip
     - os: linux
-      go: 1.7.1
+      go: 1.7.4
   include:
     - os: linux
-      go: 1.7.1
+      go: 1.7.4
       sudo: true
       env:
         RESTIC_TEST_FUSE=1
+  allow_failures:
+    - go: tip
 
+branches:
+  only:
+    - master
 
 notifications:
   irc:
@@ -46,5 +54,4 @@ script:
   - go run run_integration_tests.go
 
 after_success:
-  - GOPATH=$PWD:$PWD/vendor goveralls -coverprofile=all.cov -service=travis-ci -repotoken "$COVERALLS_TOKEN"
   - bash <(curl -s https://codecov.io/bash) -f all.cov

CONTRIBUTING.md

@@ -3,7 +3,10 @@ This document describes the way you can contribute to the restic project.
 Ways to Help Out
 ================
 
-Thank you for your contribution!
+Thank you for your contribution! Please **open an issue first** (or add a
+comment to an existing issue) if you plan to work on any code or add a new
+feature. This way, duplicate work is prevented and we can discuss your ideas
+and design first.
 
 There are several ways you can help us out. First of all code contributions and
 bug fixes are most welcome. However even "minor" details as fixing spelling
@@ -83,7 +86,7 @@ The following commands can be used to run all the tests:
     [...]
 
 If you want to run your tests on Linux, OpenBSD or FreeBSD, you can use
-[vagrant](https://www.vagrantup.com/) with the proveded `Vagrantfile` to
+[vagrant](https://www.vagrantup.com/) with the provided `Vagrantfile` to
 quickly set up VMs and run the tests, e.g.:
 
     $ vagrant up freebsd
@@ -92,6 +95,16 @@ quickly set up VMs and run the tests, e.g.:
     $ vagrant ssh freebsd -c 'cd restic/restic; go test -v ./...'
     [...]
 
+The default `go` tool can also be used by setting the environment variable
+`GOPATH` to the following value while being in the top level directory in the
+git repository:
+
+    $ export GOPATH=$PWD:$PWD/vendor
+
+The file `.envrc` allows automatic `GOPATH` configuration with
+[direnv](https://direnv.net/), inspect the file and then allow automatic
+configuration by running `direnv allow`.
+
 Providing Patches
 =================
 

README.md

@@ -11,7 +11,8 @@ restic is a backup program that is fast, efficient and secure. Detailed
 information can be found in [the documentation](doc/index.md) and [the user
 manual](doc/Manual.md). The [design document](doc/Design.md) lists the
 technical background and gives detailed information about the structure of the
-repository and the data saved therein.
+repository and the data saved therein. The file [FAQ.md](doc/FAQ.md) lists the
+most frequently asked questions.
 
 The latest documentation can be viewed online at
 <https://restic.readthedocs.io/en/latest>. On the bottom left corner there is
@@ -39,10 +40,15 @@ Building restic with gccgo may work, but is not supported.
 Contribute and Documentation
 ============================
 
-Contributions are welcome! More information and a description of the
-development environment can be found in [`CONTRIBUTING.md`](CONTRIBUTING.md). A
-document describing the design of restic and the data structures stored on the
-back end is contained in [`doc/Design.md`](doc/Design.md).
+Contributions are welcome! Please **open an issue first** (or add a comment to
+an existing issue) if you plan to work on any code or add a new feature. This
+way, duplicate work is prevented and we can discuss your ideas and design
+first.
+
+More information and a description of the development environment can be found
+in [`CONTRIBUTING.md`](CONTRIBUTING.md). A document describing the design of
+restic and the data structures stored on the back end is contained in
+[`doc/Design.md`](doc/Design.md).
 
 If you'd like to start contributing to restic, but don't know exactly what do
 to, have a look at this great article by Dave Cheney:
@@ -67,7 +73,6 @@ alexander@bumpern.de. If possible, please encrypt your email using the following
 pub   4096R/91A6868BD3F7A907 2014-11-01
       Key fingerprint = CF8F 18F2 8445 7597 3F79  D4E1 91A6 868B D3F7 A907
       uid                          Alexander Neumann <alexander@bumpern.de>
-      uid                          Alexander Neumann <alexander@debian.org>
       sub   4096R/D5FC2ACF4043FDF1 2014-11-01
 ```
 

VERSION

@@ -1 +1 @@
-0.3.0
+0.4.0

appveyor.yml

@@ -3,6 +3,10 @@ clone_folder: c:\restic
 environment:
   GOPATH: c:\gopath
 
+branches:
+  only:
+    - master
+
 init:
   - ps: >-
       $app = Get-WmiObject -Class Win32_Product -Filter "Vendor = 'http://golang.org'"
@@ -13,8 +17,8 @@ init:
 
 install:
   - rmdir c:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.7.windows-amd64.msi
-  - msiexec /i go1.7.windows-amd64.msi /q
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.7.4.windows-amd64.msi
+  - msiexec /i go1.7.4.windows-amd64.msi /q
   - go version
   - go env
   - appveyor DownloadFile http://sourceforge.netcologne.de/project/gnuwin32/tar/1.13-1/tar-1.13-1-bin.zip -FileName tar.zip

build.go

@@ -12,7 +12,6 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
-	"time"
 )
 
 var (
@@ -21,7 +20,17 @@ var (
 	runTests   bool
 )
 
-const timeFormat = "2006-01-02 15:04:05"
+var config = struct {
+	Name      string
+	Namespace string
+	Main      string
+	Tests     []string
+}{
+	Name:      "restic",                           // name of the program executable and directory
+	Namespace: "",                                 // subdir of GOPATH, e.g. "github.com/foo/bar"
+	Main:      "cmds/restic",                      // package name for the main package
+	Tests:     []string{"restic/...", "cmds/..."}, // tests to run
+}
 
 // specialDir returns true if the file begins with a special character ('.' or '_').
 func specialDir(name string) bool {
@@ -96,6 +105,15 @@ func updateGopath(dst, src, prefix string) error {
 	})
 }
 
+func directoryExists(dirname string) bool {
+	stat, err := os.Stat(dirname)
+	if err != nil && os.IsNotExist(err) {
+		return false
+	}
+
+	return stat.IsDir()
+}
+
 // copyFile creates dst from src, preserving file attributes and timestamps.
 func copyFile(dst, src string) error {
 	fi, err := os.Stat(src)
@@ -156,6 +174,7 @@ func showUsage(output io.Writer) {
 	fmt.Fprintf(output, "  -t     --tags          specify additional build tags\n")
 	fmt.Fprintf(output, "  -k     --keep-gopath   do not remove the GOPATH after build\n")
 	fmt.Fprintf(output, "  -T     --test          run tests\n")
+	fmt.Fprintf(output, "  -o     --output        set output file name\n")
 	fmt.Fprintf(output, "         --goos value    set GOOS for cross-compilation\n")
 	fmt.Fprintf(output, "         --goarch value  set GOARCH for cross-compilation\n")
 }
@@ -186,7 +205,7 @@ func cleanEnv() (env []string) {
 func build(cwd, goos, goarch, gopath string, args ...string) error {
 	args = append([]string{"build"}, args...)
 	cmd := exec.Command("go", args...)
-	cmd.Env = append(cleanEnv(), "GOPATH="+gopath, "GOARCH="+goarch, "GOOS="+goos)
+	cmd.Env = append(cleanEnv(), "GOPATH="+gopath, "GOARCH="+goarch, "GOOS="+goos, "CGO_ENABLED=0")
 	cmd.Dir = cwd
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
@@ -264,24 +283,14 @@ type Constants map[string]string
 func (cs Constants) LDFlags() string {
 	l := make([]string, 0, len(cs))
 
-	if runtime.Version() < "go1.5" {
-		for k, v := range cs {
-			l = append(l, fmt.Sprintf(`-X %q %q`, k, v))
-		}
-	} else {
-		for k, v := range cs {
-			l = append(l, fmt.Sprintf(`-X "%s=%s"`, k, v))
-		}
+	for k, v := range cs {
+		l = append(l, fmt.Sprintf(`-X "%s=%s"`, k, v))
 	}
 
 	return strings.Join(l, " ")
 }
 
 func main() {
-	if runtime.Version() < "go1.6" {
-		fmt.Fprintf(os.Stderr, "old version of Go detected (%v), I'll try but no guarantees\n", runtime.Version())
-	}
-
 	buildTags := []string{}
 
 	skipNext := false
@@ -290,6 +299,8 @@ func main() {
 	targetGOOS := runtime.GOOS
 	targetGOARCH := runtime.GOARCH
 
+	var outputFilename string
+
 	for i, arg := range params {
 		if skipNext {
 			skipNext = false
@@ -302,8 +313,14 @@ func main() {
 		case "-k", "--keep-gopath":
 			keepGopath = true
 		case "-t", "-tags", "--tags":
+			if i+1 >= len(params) {
+				die("-t given but no tag specified")
+			}
 			skipNext = true
 			buildTags = strings.Split(params[i+1], " ")
+		case "-o", "--output":
+			skipNext = true
+			outputFilename = params[i+1]
 		case "-T", "--test":
 			runTests = true
 		case "--goos":
@@ -338,23 +355,21 @@ func main() {
 		die("Getwd(): %v\n", err)
 	}
 
-	gopath, err := ioutil.TempDir("", "restic-build-")
+	gopath, err := ioutil.TempDir("", fmt.Sprintf("%v-build-", config.Name))
 	if err != nil {
 		die("TempDir(): %v\n", err)
 	}
 
 	verbosePrintf("create GOPATH at %v\n", gopath)
-	if err = updateGopath(gopath, filepath.Join(root, "src", "restic"), "restic"); err != nil {
-		die("copying files from %v/src/restic to %v/src/restic failed: %v\n", root, gopath, err)
-	}
-
-	if err = updateGopath(gopath, filepath.Join(root, "src", "cmds"), "cmds"); err != nil {
-		die("copying files from %v/src/cmds to %v/src/restic/cmds failed: %v\n", root, gopath, err)
+	if err = updateGopath(gopath, filepath.Join(root, "src"), config.Namespace); err != nil {
+		die("copying files from %v/src to %v/src failed: %v\n", root, gopath, err)
 	}
 
 	vendor := filepath.Join(root, "vendor", "src")
-	if err = updateGopath(gopath, vendor, ""); err != nil {
-		die("copying files from %v to %v/src failed: %v\n", vendor, gopath, err)
+	if directoryExists(vendor) {
+		if err = updateGopath(gopath, vendor, ""); err != nil {
+			die("copying files from %v to %v failed: %v\n", root, gopath, err)
+		}
 	}
 
 	defer func() {
@@ -368,9 +383,11 @@ func main() {
 		}
 	}()
 
-	outputFilename := "restic"
-	if targetGOOS == "windows" {
-		outputFilename = "restic.exe"
+	if outputFilename == "" {
+		outputFilename = config.Name
+		if targetGOOS == "windows" {
+			outputFilename += ".exe"
+		}
 	}
 
 	cwd, err := os.Getwd()
@@ -380,8 +397,7 @@ func main() {
 	output := filepath.Join(cwd, outputFilename)
 
 	version := getVersion()
-	compileTime := time.Now().Format(timeFormat)
-	constants := Constants{`main.compiledAt`: compileTime}
+	constants := Constants{}
 	if version != "" {
 		constants["main.version"] = version
 	}
@@ -391,7 +407,7 @@ func main() {
 	args := []string{
 		"-tags", strings.Join(buildTags, " "),
 		"-ldflags", ldflags,
-		"-o", output, "cmds/restic",
+		"-o", output, config.Main,
 	}
 
 	err = build(filepath.Join(gopath, "src"), targetGOOS, targetGOARCH, gopath, args...)
@@ -402,7 +418,7 @@ func main() {
 	if runTests {
 		verbosePrintf("running tests\n")
 
-		err = test(filepath.Join(gopath, "src"), gopath, "restic/...")
+		err = test(cwd, gopath, config.Tests...)
 		if err != nil {
 			die("running tests failed: %v\n", err)
 		}

build_release_binaries.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+set -e
+
+if [[ -z "$VERSION" ]]; then
+    echo '$VERSION unset'
+    exit 1
+fi
+
+dir=$(mktemp -d --tmpdir restic-release-XXXXXX)
+echo "path is ${dir}"
+
+for R in       \
+    darwin/386     \
+    darwin/amd64   \
+    freebsd/386    \
+    freebsd/amd64  \
+    freebsd/arm    \
+    linux/386      \
+    linux/amd64    \
+    linux/arm      \
+    linux/arm64    \
+    openbsd/386   \
+    openbsd/amd64 \
+    windows/386    \
+    windows/amd64  \
+    ; do \
+
+    OS=$(dirname $R)
+    ARCH=$(basename $R)
+    filename=restic_${VERSION}_${OS}_${ARCH}
+
+    if [[ "$OS" == "windows" ]]; then
+        filename="${filename}.exe"
+    fi
+
+    echo $filename
+
+    go run ../build.go --goos $OS --goarch $ARCH --output ${filename}
+    if [[ "$OS" == "windows" ]]; then
+        zip ${filename%.exe}.zip ${filename}
+        rm ${filename}
+        mv ${filename%.exe}.zip ${dir}
+    else
+        bzip2 ${filename}
+        mv ${filename}.bz2 ${dir}
+    fi
+done
+
+echo "packing sources"
+git archive --format=tar --prefix=restic-$VERSION/ v$VERSION | gzip -n > restic-$VERSION.tar.gz
+mv restic-$VERSION.tar.gz ${dir}
+
+echo "creating checksums"
+pushd ${dir}
+sha256sum restic_*.{zip,bz2} > SHA256SUMS
+gpg --armor --detach-sign SHA256SUMS
+popd
+
+echo "creating source signature file"
+gpg --armor --detach-sign ${dir}/restic-$VERSION.tar.gz
+
+echo
+echo "done, path is ${dir}"

doc/Design.md

@@ -62,11 +62,13 @@ overhead is 32 bytes. For each file, a new random IV is selected.
 The file `config` is encrypted this way and contains a JSON document like the
 following:
 
-    {
-      "version": 1,
-      "id": "5956a3f67a6230d4a92cefb29529f10196c7d92582ec305fd71ff6d331d6271b",
-      "chunker_polynomial": "25b468838dcb75"
-    }
+```json
+{
+  "version": 1,
+  "id": "5956a3f67a6230d4a92cefb29529f10196c7d92582ec305fd71ff6d331d6271b",
+  "chunker_polynomial": "25b468838dcb75"
+}
+```
 
 After decryption, restic first checks that the version field contains a version
 number that it understands, otherwise it aborts. At the moment, the version is
@@ -102,7 +104,9 @@ The basic layout of a sample restic repository is shown here:
 
 A repository can be initialized with the `restic init` command, e.g.:
 
-    $ restic -r /tmp/restic-repo init
+```console
+$ restic -r /tmp/restic-repo init
+```
 
 Pack Format
 -----------
@@ -163,35 +167,37 @@ used to reconstruct the index. The files are encrypted and authenticated like
 Data and Tree Blobs, so the outer structure is `IV || Ciphertext || MAC` again.
 The plaintext consists of a JSON document like the following:
 
+```json
+{
+  "supersedes": [
+    "ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452"
+  ],
+  "packs": [
     {
-      "supersedes": [
-        "ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452"
-      ],
-      "packs": [
+      "id": "73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c",
+      "blobs": [
         {
-          "id": "73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c",
-          "blobs": [
-            {
-              "id": "3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce",
-              "type": "data",
-              "offset": 0,
-              "length": 25
-            },{
-              "id": "9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae",
-              "type": "tree",
-              "offset": 38,
-              "length": 100
-            },
-            {
-              "id": "d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66",
-              "type": "data",
-              "offset": 150,
-              "length": 123
-            }
-          ]
-        }, [...]
+          "id": "3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce",
+          "type": "data",
+          "offset": 0,
+          "length": 25
+        },{
+          "id": "9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae",
+          "type": "tree",
+          "offset": 38,
+          "length": 100
+        },
+        {
+          "id": "d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66",
+          "type": "data",
+          "offset": 150,
+          "length": 123
+        }
       ]
-    }
+    }, [...]
+  ]
+}
+```
 
 This JSON document lists Packs and the blobs contained therein. In this
 example, the Pack `73d04e61` contains two data Blobs and one Tree blob, the
@@ -258,14 +264,16 @@ document which contains the master encryption and message authentication keys
 for this repository (encoded in Base64). The command `restic cat masterkey` can
 be used as follows to decrypt and pretty-print the master key:
 
-    $ restic -r /tmp/restic-repo cat masterkey
-    {
-        "mac": {
-          "k": "evFWd9wWlndL9jc501268g==",
-          "r": "E9eEDnSJZgqwTOkDtOp+Dw=="
-        },
-        "encrypt": "UQCqa0lKZ94PygPxMRqkePTZnHRYh1k1pX2k2lM2v3Q=",
-    }
+```console
+$ restic -r /tmp/restic-repo cat masterkey
+{
+    "mac": {
+      "k": "evFWd9wWlndL9jc501268g==",
+      "r": "E9eEDnSJZgqwTOkDtOp+Dw=="
+    },
+    "encrypt": "UQCqa0lKZ94PygPxMRqkePTZnHRYh1k1pX2k2lM2v3Q=",
+}
+```
 
 All data in the repository is encrypted and authenticated with these master keys.
 For encryption, the AES-256 algorithm in Counter mode is used. For message
@@ -286,17 +294,19 @@ string is unique and used within restic to uniquely identify a snapshot.
 The command `restic cat snapshot` can be used as follows to decrypt and
 pretty-print the contents of a snapshot file:
 
-    $ restic -r /tmp/restic-repo cat snapshot 22a5af1b
-    enter password for repository:
-    {
-      "time": "2015-01-02T18:10:50.895208559+01:00",
-      "tree": "2da81727b6585232894cfbb8f8bdab8d1eccd3d8f7c92bc934d62e62e618ffdf",
-      "dir": "/tmp/testdata",
-      "hostname": "kasimir",
-      "username": "fd0",
-      "uid": 1000,
-      "gid": 100
-    }
+```console
+$ restic -r /tmp/restic-repo cat snapshot 22a5af1b
+enter password for repository:
+{
+  "time": "2015-01-02T18:10:50.895208559+01:00",
+  "tree": "2da81727b6585232894cfbb8f8bdab8d1eccd3d8f7c92bc934d62e62e618ffdf",
+  "dir": "/tmp/testdata",
+  "hostname": "kasimir",
+  "username": "fd0",
+  "uid": 1000,
+  "gid": 100
+}
+```
 
 Here it can be seen that this snapshot represents the contents of the directory
 `/tmp/testdata`. The most important field is `tree`.
@@ -317,70 +327,77 @@ A snapshot references a tree by the SHA-256 hash of the JSON string
 representation of its contents. Trees and data are saved in pack files in a
 subdirectory of the directory `data`.
 
-The command `restic cat tree` can be used to inspect the tree referenced above:
+The command `restic cat blob` can be used to inspect the tree referenced above
+(piping the output of the command to `jq .` so that the JSON is indented):
 
-    $ restic -r /tmp/restic-repo cat tree b8138ab08a4722596ac89c917827358da4672eac68e3c03a8115b88dbf4bfb59
-    enter password for repository:
+```console
+$ restic -r /tmp/restic-repo cat blob b8138ab08a4722596ac89c917827358da4672eac68e3c03a8115b88dbf4bfb59 | jq .
+enter password for repository:
+{
+  "nodes": [
     {
-      "nodes": [
-        {
-          "name": "testdata",
-          "type": "dir",
-          "mode": 493,
-          "mtime": "2014-12-22T14:47:59.912418701+01:00",
-          "atime": "2014-12-06T17:49:21.748468803+01:00",
-          "ctime": "2014-12-22T14:47:59.912418701+01:00",
-          "uid": 1000,
-          "gid": 100,
-          "user": "fd0",
-          "inode": 409704562,
-          "content": null,
-          "subtree": "b26e315b0988ddcd1cee64c351d13a100fedbc9fdbb144a67d1b765ab280b4dc"
-        }
-      ]
+      "name": "testdata",
+      "type": "dir",
+      "mode": 493,
+      "mtime": "2014-12-22T14:47:59.912418701+01:00",
+      "atime": "2014-12-06T17:49:21.748468803+01:00",
+      "ctime": "2014-12-22T14:47:59.912418701+01:00",
+      "uid": 1000,
+      "gid": 100,
+      "user": "fd0",
+      "inode": 409704562,
+      "content": null,
+      "subtree": "b26e315b0988ddcd1cee64c351d13a100fedbc9fdbb144a67d1b765ab280b4dc"
     }
+  ]
+}
+```
 
 A tree contains a list of entries (in the field `nodes`) which contain meta
 data like a name and timestamps. When the entry references a directory, the
 field `subtree` contains the plain text ID of another tree object.
 
-When the command `restic cat tree` is used, the storage hash is needed to print
+When the command `restic cat blob` is used, the plaintext ID is needed to print
 a tree. The tree referenced above can be dumped as follows:
 
-    $ restic -r /tmp/restic-repo cat tree 8b238c8811cc362693e91a857460c78d3acf7d9edb2f111048691976803cf16e
-    enter password for repository:
+```console
+$ restic -r /tmp/restic-repo cat blob 8b238c8811cc362693e91a857460c78d3acf7d9edb2f111048691976803cf16e
+enter password for repository:
+{
+  "nodes": [
     {
-      "nodes": [
-        {
-          "name": "testfile",
-          "type": "file",
-          "mode": 420,
-          "mtime": "2014-12-06T17:50:23.34513538+01:00",
-          "atime": "2014-12-06T17:50:23.338468713+01:00",
-          "ctime": "2014-12-06T17:50:23.34513538+01:00",
-          "uid": 1000,
-          "gid": 100,
-          "user": "fd0",
-          "inode": 416863351,
-          "size": 1234,
-          "links": 1,
-          "content": [
-            "50f77b3b4291e8411a027b9f9b9e64658181cc676ce6ba9958b95f268cb1109d"
-          ]
-        },
-        [...]
+      "name": "testfile",
+      "type": "file",
+      "mode": 420,
+      "mtime": "2014-12-06T17:50:23.34513538+01:00",
+      "atime": "2014-12-06T17:50:23.338468713+01:00",
+      "ctime": "2014-12-06T17:50:23.34513538+01:00",
+      "uid": 1000,
+      "gid": 100,
+      "user": "fd0",
+      "inode": 416863351,
+      "size": 1234,
+      "links": 1,
+      "content": [
+        "50f77b3b4291e8411a027b9f9b9e64658181cc676ce6ba9958b95f268cb1109d"
       ]
-    }
+    },
+    [...]
+  ]
+}
+```
 
 This tree contains a file entry. This time, the `subtree` field is not present
 and the `content` field contains a list with one plain text SHA-256 hash.
 
-The command `restic cat data` can be used to extract and decrypt data given a
-plaintext ID, e.g. for the data mentioned above:
+The command `restic cat blob` can also be used to extract and decrypt data
+given a plaintext ID, e.g. for the data mentioned above:
 
-    $ restic -r /tmp/restic-repo cat blob 50f77b3b4291e8411a027b9f9b9e64658181cc676ce6ba9958b95f268cb1109d | sha256sum
-    enter password for repository:
-    50f77b3b4291e8411a027b9f9b9e64658181cc676ce6ba9958b95f268cb1109d  -
+```console
+$ restic -r /tmp/restic-repo cat blob 50f77b3b4291e8411a027b9f9b9e64658181cc676ce6ba9958b95f268cb1109d | sha256sum
+enter password for repository:
+50f77b3b4291e8411a027b9f9b9e64658181cc676ce6ba9958b95f268cb1109d  -
+```
 
 As can be seen from the output of the program `sha256sum`, the hash matches the
 plaintext hash from the map included in the tree above, so the correct data has
@@ -404,15 +421,17 @@ A lock is a file in the subdir `locks` whose filename is the storage ID of
 the contents. It is encrypted and authenticated the same way as other files
 in the repository and contains the following JSON structure:
 
-    {
-      "time": "2015-06-27T12:18:51.759239612+02:00",
-      "exclusive": false,
-      "hostname": "kasimir",
-      "username": "fd0",
-      "pid": 13607,
-      "uid": 1000,
-      "gid": 100
-    }
+```json
+{
+  "time": "2015-06-27T12:18:51.759239612+02:00",
+  "exclusive": false,
+  "hostname": "kasimir",
+  "username": "fd0",
+  "pid": 13607,
+  "uid": 1000,
+  "gid": 100
+}
+```
 
 The field `exclusive` defines the type of lock. When a new lock is to be
 created, restic checks all locks in the repository. When a lock is found, it

doc/FAQ.md

@@ -0,0 +1,20 @@
+FAQ
+===
+
+This is the list of Frequently Asked Questions for restic.
+
+`restic check` reports packs that aren't referenced in any index, is my repository broken?
+------------------------------------------------------------------------------------------
+
+When `restic check` reports that there are pack files in the repository that are not referenced in any index, that's (in contrast to what restic reports at the moment) not a source for concern. The output looks like this:
+
+    $ restic check
+    Create exclusive lock for repository
+    Load indexes
+    Check all packs
+    pack 819a9a52e4f51230afa89aefbf90df37fb70996337ae57e6f7a822959206a85e: not referenced in any index
+    pack de299e69fb075354a3775b6b045d152387201f1cdc229c31d1caa34c3b340141: not referenced in any index
+    Check snapshots, trees and blobs
+    Fatal: repository contains errors
+
+The message means that there is more data stored in the repo than strictly necessary. With high probability this is duplicate data. In order to clean it up, the command `restic prune` can be used. The cause of this bug is not yet known.

doc/Manual.md

@@ -6,16 +6,22 @@ functionality provided by restic.
 If you are using Mac OS X, you can install restic using the
 [homebrew](http://brew.sh/) packet manager:
 
-    $ brew tap restic/restic
-    $ brew install restic
+```console
+$ brew tap restic/restic
+$ brew install restic
+```
 
 On archlinux, there is a package called `restic-git` which can be installed from AUR, e.g. with `pacaur`:
 
-    $ pacaur -S restic-git
+```console
+$ pacaur -S restic-git
+```
 
 At debian stable you can install 'go' directly from the repositories (as root):
 
-     $ apt-get install golang-go
+```console
+$ apt-get install golang-go
+```
 
 after installation of 'go' go straight forward to 'git clone [...]'
 
@@ -28,80 +34,85 @@ instructions how to install Go.
 
 In order to build restic from source, execute the following steps:
 
-    $ git clone https://github.com/restic/restic
-    [...]
+```console
+$ git clone https://github.com/restic/restic
+[...]
 
-    $ cd restic
+$ cd restic
 
-    $ go run build.go
+$ go run build.go
+```
 
 At the moment, the only tested compiler for restic is the official Go compiler.
 Building restic with gccgo may work, but is not supported.
 
 Usage help is available:
 
-    $ ./restic --help
-    Usage:
-      restic [OPTIONS] <command>
+```console
+$ ./restic --help
+restic is a backup program which allows saving multiple revisions of files and
+directories in an encrypted repository stored on different backends.
 
-    Application Options:
-      -r, --repo=      Repository directory to backup to/restore from
-          --cache-dir= Directory to use as a local cache
-      -q, --quiet      Do not output comprehensive progress report (false)
-          --no-lock    Do not lock the repo, this allows some operations on read-only repos. (false)
-      -o, --option=    Specify options in the form 'foo.key=value'
+Usage:
+  restic [command]
 
-    Help Options:
-      -h, --help       Show this help message
+Available Commands:
+  backup        create a new backup of files and/or directories
+  cat           print internal objects to stdout
+  check         check the repository for errors
+  find          find a file or directory
+  forget        forget removes snapshots from the repository
+  init          initialize a new repository
+  key           manage keys (passwords)
+  list          list items in the repository
+  ls            list files in a snapshot
+  mount         mount the repository
+  prune         remove unneeded data from the repository
+  rebuild-index build a new index file
+  restore       extract the data from a snapshot
+  snapshots     list all snapshots
+  unlock        remove locks other processes created
+  version       Print version information
 
-    Available commands:
-      backup         save file/directory
-      cat            dump something
-      check          check the repository
-      find           find a file/directory
-      forget         removes snapshots from a repository
-      init           create repository
-      key            manage keys
-      list           lists data
-      ls             list files
-      mount          mount a repository
-      prune          removes content from a repository
-      rebuild-index  rebuild the index
-      restore        restore a snapshot
-      snapshots      show snapshots
-      unlock         remove locks
-      version        display version
+Flags:
+      --no-lock                do not lock the repo, this allows some operations on read-only repos
+  -p, --password-file string   read the repository password from a file
+  -q, --quiet                  do not output comprehensive progress report
+  -r, --repo string            repository to backup to or restore from (default: $RESTIC_REPOSITORY)
+
+Use "restic [command] --help" for more information about a command.
+```
 
 Similar to programs such as `git`, restic has a number of sub-commands. You can
 see these commands in the listing above. Each sub-command may have own
 command-line options, and there is a help option for each command which lists
 them, e.g. for the `backup` command:
 
-    $ ./restic backup --help
-    Usage:
-      restic [OPTIONS] backup DIR/FILE [DIR/FILE] [...]
+```console
+$ ./restic backup --help
+The "backup" command creates a new snapshot and saves the files and directories
+given as the arguments.
 
-    The backup command creates a snapshot of a file or directory
+Usage:
+  restic backup [flags] FILE/DIR [FILE/DIR] ...
 
-    Application Options:
-      -r, --repo=               Repository directory to backup to/restore from (/tmp/repo)
-      -p, --password-file=      Read the repository password from a file
-          --cache-dir=          Directory to use as a local cache
-      -q, --quiet               Do not output comprehensive progress report (false)
-          --no-lock             Do not lock the repo, this allows some operations on read-only repos. (false)
-      -o, --option=             Specify options in the form 'foo.key=value'
+Flags:
+  -e, --exclude pattern         exclude a pattern (can be specified multiple times)
+      --exclude-file string     read exclude patterns from a file
+      --files-from string       read the files to backup from file (can be combined with file args)
+  -f, --force                   force re-reading the target files/directories. Overrides the "parent" flag
+  -x, --one-file-system         Exclude other file systems
+      --parent string           use this parent snapshot (default: last snapshot in the repo that has the same target files/directories)
+      --stdin                   read backup from stdin
+      --stdin-filename string   file name to use when reading from stdin
+      --tag tag                 add a tag for the new snapshot (can be specified multiple times)
 
-    Help Options:
-      -h, --help                Show this help message
-
-    [backup command options]
-          -p, --parent=         use this parent snapshot (default: last snapshot in repo that has the same target)
-          -f, --force           Force re-reading the target. Overrides the "parent" flag
-          -e, --exclude=        Exclude a pattern (can be specified multiple times)
-              --exclude-file=   Read exclude-patterns from file
-              --stdin           read backup data from stdin
-              --stdin-filename= file name to use when reading from stdin (stdin)
-              --tag=            Add a tag (can be specified multiple times)
+Global Flags:
+      --no-lock                do not lock the repo, this allows some operations on read-only repos
+  -p, --password-file string   read the repository password from a file
+  -q, --quiet                  do not output comprehensive progress report
+  -r, --repo string            repository to backup to or restore from (default: $RESTIC_REPOSITORY)
+```
 
 Subcommand that support showing progress information such as `backup`, `check` and `prune` will do so unless
 the quiet flag `-q` or `--quiet` is set. When running from a non-interactive console progress reporting will
@@ -119,12 +130,14 @@ will be saved at.
 In order to create a repository at `/tmp/backup`, run the following command and
 enter the same password twice:
 
-    $ restic init --repo /tmp/backup
-    enter password for new backend:
-    enter password again:
-    created restic backend 085b3c76b9 at /tmp/backup
-    Please note that knowledge of your password is required to access the repository.
-    Losing your password means that your data is irrecoverably lost.
+```console
+$ restic init --repo /tmp/backup
+enter password for new backend:
+enter password again:
+created restic backend 085b3c76b9 at /tmp/backup
+Please note that knowledge of your password is required to access the repository.
+Losing your password means that your data is irrecoverably lost.
+```
 
 Remembering your password is important! If you lose it, you won't be able to
 access data stored in the repository.
@@ -146,8 +159,10 @@ You can workaround this by using a special tool called `winpty` (look
 [here](https://github.com/rprichard/winpty) for detail information). On MSYS2,
 you can install `winpty` as follows:
 
-    $ pacman -S winpty
-    $ winpty restic -r /tmp/backup init
+```console
+$ pacman -S winpty
+$ winpty restic -r /tmp/backup init
+```
 
 # Create a snapshot
 
@@ -155,13 +170,15 @@ Now we're ready to backup some data. The contents of a directory at a specific
 point in time is called a "snapshot" in restic. Run the following command and
 enter the repository password you chose above again:
 
-    $ restic -r /tmp/backup backup ~/work
-    enter password for repository:
-    scan [/home/user/work]
-    scanned 764 directories, 1816 files in 0:00
-    [0:29] 100.00%  54.732 MiB/s  1.582 GiB / 1.582 GiB  2580 / 2580 items  0 errors  ETA 0:00
-    duration: 0:29, 54.47MiB/s
-    snapshot 40dc1520 saved
+```console
+$ restic -r /tmp/backup backup ~/work
+enter password for repository:
+scan [/home/user/work]
+scanned 764 directories, 1816 files in 0:00
+[0:29] 100.00%  54.732 MiB/s  1.582 GiB / 1.582 GiB  2580 / 2580 items  0 errors  ETA 0:00
+duration: 0:29, 54.47MiB/s
+snapshot 40dc1520 saved
+```
 
 As you can see, restic created a backup of the directory and was pretty fast!
 The specific snapshot just created is identified by a sequence of hexadecimal
@@ -170,40 +187,46 @@ characters, `40dc1520` in this case.
 If you run the command again, restic will create another snapshot of your data,
 but this time it's even faster. This is de-duplication at work!
 
-    $ restic -r /tmp/backup backup ~/shared/work/web
-    enter password for repository:
-    using parent snapshot 40dc1520aa6a07b7b3ae561786770a01951245d2367241e71e9485f18ae8228c
-    scan [/home/user/work]
-    scanned 764 directories, 1816 files in 0:00
-    [0:00] 100.00%  0B/s  1.582 GiB / 1.582 GiB  2580 / 2580 items  0 errors  ETA 0:00
-    duration: 0:00, 6572.38MiB/s
-    snapshot 79766175 saved
+```console
+$ restic -r /tmp/backup backup ~/shared/work/web
+enter password for repository:
+using parent snapshot 40dc1520aa6a07b7b3ae561786770a01951245d2367241e71e9485f18ae8228c
+scan [/home/user/work]
+scanned 764 directories, 1816 files in 0:00
+[0:00] 100.00%  0B/s  1.582 GiB / 1.582 GiB  2580 / 2580 items  0 errors  ETA 0:00
+duration: 0:00, 6572.38MiB/s
+snapshot 79766175 saved
+```
 
 You can even backup individual files in the same repository.
 
-    $ restic -r /tmp/backup backup ~/work.txt
-    scan [~/work.txt]
-    scanned 0 directories, 1 files in 0:00
-    [0:00] 100.00%  0B/s  220B / 220B  1 / 1 items  0 errors  ETA 0:00
-    duration: 0:00, 0.03MiB/s
-    snapshot 31f7bd63 saved
+```console
+$ restic -r /tmp/backup backup ~/work.txt
+scan [~/work.txt]
+scanned 0 directories, 1 files in 0:00
+[0:00] 100.00%  0B/s  220B / 220B  1 / 1 items  0 errors  ETA 0:00
+duration: 0:00, 0.03MiB/s
+snapshot 31f7bd63 saved
+```
 
 In fact several hosts may use the same repository to backup directories and
 files leading to a greater de-duplication.
 
-You can exclude folders and files by specifying exclude-patterns.  
+You can exclude folders and files by specifying exclude-patterns.
 Either specify them with multiple `--exclude`'s or one `--exclude-file`
 
-    $ cat exclude
-    # exclude go-files
-    *.go
-    # exclude foo/x/y/z/bar foo/x/bar foo/bar
-    foo/**/bar
-    $ restic -r /tmp/backup backup ~/work --exclude=*.c --exclude-file=exclude
+```console
+$ cat exclude
+# exclude go-files
+*.go
+# exclude foo/x/y/z/bar foo/x/bar foo/bar
+foo/**/bar
+$ restic -r /tmp/backup backup ~/work --exclude=*.c --exclude-file=exclude
+```
 
 Patterns use [`filepath.Glob`](https://golang.org/pkg/path/filepath/#Glob) internally,
 see [`filepath.Match`](https://golang.org/pkg/path/filepath/#Match) for syntax.
-Additionally `**` exludes arbitrary subdirectories.  
+Additionally `**` exludes arbitrary subdirectories.
 Environment-variables in exclude-files are expanded with [`os.ExpandEnv`](https://golang.org/pkg/os/#ExpandEnv).
 
 By specifying the option `--one-file-system` you can instruct restic to only
@@ -211,7 +234,30 @@ backup files from the file systems the initially specified files or directories
 reside on. For example, calling restic like this won't backup `/sys` or
 `/dev` on a Linux system:
 
-    $ restic -r /tmp/backup backup --one-file-system /
+```console
+$ restic -r /tmp/backup backup --one-file-system /
+```
+By using the `--files-from` option you can read the files you want to backup
+from a file. This is especially useful if a lot of files have to be backed up
+that are not in the same folder or are maybe pre-filtered by other software.
+
+For example maybe you want to backup files that have a certain filename in them:
+
+```console
+$ find /tmp/somefiles | grep 'PATTERN' > /tmp/files_to_backup
+```
+
+You can then use restic to backup the filtered files:
+
+```console
+$ restic -r /tmp/backup backup --files-from /tmp/files_to_backup
+```
+
+Incidentally you can also combine `--files-from` with the normal files args:
+
+```console
+$ restic -r /tmp/backup backup --files-from /tmp/files_to_backup /tmp/some_additional_file
+```
 
 ## Reading data from stdin
 
@@ -219,7 +265,9 @@ Sometimes it can be nice to directly save the output of a program, e.g.
 `mysqldump` so that the SQL can later be restored. Restic supports this mode of
 operation, just supply the option `--stdin` to the `backup` command like this:
 
-    $ mysqldump [...] | restic -r /tmp/backup backup --stdin
+```console
+$ mysqldump [...] | restic -r /tmp/backup backup --stdin
+```
 
 This creates a new snapshot of the output of `mysqldump`. You can then use e.g.
 the fuse mounting option (see below) to mount the repository and read the file.
@@ -227,15 +275,19 @@ the fuse mounting option (see below) to mount the repository and read the file.
 By default, the file name `stdin` is used, a different name can be specified
 with `--stdin-filename`, e.g. like this:
 
-    $ mysqldump [...] | restic -r /tmp/backup backup --stdin --stdin-filename production.sql
+```console
+$ mysqldump [...] | restic -r /tmp/backup backup --stdin --stdin-filename production.sql
+```
 
 ## Tags
 
 Snapshots can have one or more tags, short strings which add identifying
 information. Just specify the tags for a snapshot with `--tag`:
 
-    $ restic -r /tmp/backup backup --tag projectX ~/shared/work/web
-    [...]
+```console
+$ restic -r /tmp/backup backup --tag projectX ~/shared/work/web
+[...]
+```
 
 The tags can later be used to keep (or forget) snapshots.
 
@@ -243,52 +295,62 @@ The tags can later be used to keep (or forget) snapshots.
 
 Now, you can list all the snapshots stored in the repository:
 
-    $ restic -r /tmp/backup snapshots
-    enter password for repository:
-    ID        Date                 Host    Tags   Directory
-    ----------------------------------------------------------------------
-    40dc1520  2015-05-08 21:38:30  kasimir        /home/user/work
-    79766175  2015-05-08 21:40:19  kasimir        /home/user/work
-    bdbd3439  2015-05-08 21:45:17  luigi          /home/art
-    590c8fc8  2015-05-08 21:47:38  kazik          /srv
-    9f0bc19e  2015-05-08 21:46:11  luigi          /srv
+```console
+$ restic -r /tmp/backup snapshots
+enter password for repository:
+ID        Date                 Host    Tags   Directory
+----------------------------------------------------------------------
+40dc1520  2015-05-08 21:38:30  kasimir        /home/user/work
+79766175  2015-05-08 21:40:19  kasimir        /home/user/work
+bdbd3439  2015-05-08 21:45:17  luigi          /home/art
+590c8fc8  2015-05-08 21:47:38  kazik          /srv
+9f0bc19e  2015-05-08 21:46:11  luigi          /srv
+```
 
 You can filter the listing by directory path:
 
-    $ restic -r /tmp/backup snapshots --path="/srv"
-    enter password for repository:
-    ID        Date                 Host    Tags   Directory
-    ----------------------------------------------------------------------
-    590c8fc8  2015-05-08 21:47:38  kazik          /srv
-    9f0bc19e  2015-05-08 21:46:11  luigi          /srv
+```console
+$ restic -r /tmp/backup snapshots --path="/srv"
+enter password for repository:
+ID        Date                 Host    Tags   Directory
+----------------------------------------------------------------------
+590c8fc8  2015-05-08 21:47:38  kazik          /srv
+9f0bc19e  2015-05-08 21:46:11  luigi          /srv
+```
 
 Or filter by host:
 
-    $ restic -r /tmp/backup snapshots --host luigi
-    enter password for repository:
-    ID        Date                 Host    Tags   Directory
-    ----------------------------------------------------------------------
-    bdbd3439  2015-05-08 21:45:17  luigi          /home/art
-    9f0bc19e  2015-05-08 21:46:11  luigi          /srv
+```console
+$ restic -r /tmp/backup snapshots --host luigi
+enter password for repository:
+ID        Date                 Host    Tags   Directory
+----------------------------------------------------------------------
+bdbd3439  2015-05-08 21:45:17  luigi          /home/art
+9f0bc19e  2015-05-08 21:46:11  luigi          /srv
+```
 
-Combining filters is also possible.    
+Combining filters is also possible.
 
 # Restore a snapshot
 
 Restoring a snapshot is as easy as it sounds, just use the following command to
 restore the contents of the latest snapshot to `/tmp/restore-work`:
 
-    $ restic -r /tmp/backup restore 79766175 --target ~/tmp/restore-work
-    enter password for repository:
-    restoring <Snapshot of [/home/user/work] at 2015-05-08 21:40:19.884408621 +0200 CEST> to /tmp/restore-work
+```console
+$ restic -r /tmp/backup restore 79766175 --target ~/tmp/restore-work
+enter password for repository:
+restoring <Snapshot of [/home/user/work] at 2015-05-08 21:40:19.884408621 +0200 CEST> to /tmp/restore-work
+```
 
 Use the word `latest` to restore the last backup. You can also combine `latest`
 with the `--host` and `--path` filters to choose the last backup for a specific
 host, path or both.
 
-    $ restic -r /tmp/backup restore latest --target ~/tmp/restore-work --path "/home/art" --host luigi
-    enter password for repository:
-    restoring <Snapshot of [/home/art] at 2015-05-08 21:45:17.884408621 +0200 CEST> to /tmp/restore-work
+```console
+$ restic -r /tmp/backup restore latest --target ~/tmp/restore-work --path "/home/art" --host luigi
+enter password for repository:
+restoring <Snapshot of [/home/art] at 2015-05-08 21:45:17.884408621 +0200 CEST> to /tmp/restore-work
+```
 
 
 # Manage repository keys
@@ -297,24 +359,26 @@ The `key` command allows you to set multiple access keys or passwords per
 repository. In fact, you can use the `list`, `add`, `remove` and `passwd`
 sub-commands to manage these keys very precisely:
 
-    $ restic -r /tmp/backup key list
-    enter password for repository:
-     ID          User        Host        Created
-    ----------------------------------------------------------------------
-    *eb78040b    username    kasimir   2015-08-12 13:29:57
+```console
+$ restic -r /tmp/backup key list
+enter password for repository:
+ ID          User        Host        Created
+----------------------------------------------------------------------
+*eb78040b    username    kasimir   2015-08-12 13:29:57
 
-    $ restic -r /tmp/backup key add
-    enter password for repository:
-    enter password for new key:
-    enter password again:
-    saved new key as <Key of username@kasimir, created on 2015-08-12 13:35:05.316831933 +0200 CEST>
+$ restic -r /tmp/backup key add
+enter password for repository:
+enter password for new key:
+enter password again:
+saved new key as <Key of username@kasimir, created on 2015-08-12 13:35:05.316831933 +0200 CEST>
 
-    $ restic -r backup key list
-    enter password for repository:
-     ID          User        Host        Created
-    ----------------------------------------------------------------------
-     5c657874    username    kasimir   2015-08-12 13:35:05
-    *eb78040b    username    kasimir   2015-08-12 13:29:57
+$ restic -r backup key list
+enter password for repository:
+ ID          User        Host        Created
+----------------------------------------------------------------------
+ 5c657874    username    kasimir   2015-08-12 13:35:05
+*eb78040b    username    kasimir   2015-08-12 13:29:57
+```
 
 # Check integrity and consistency
 
@@ -322,22 +386,28 @@ Imagine your repository is saved on a server that has a faulty hard drive, or
 even worse, attackers get privileged access and modify your backup with the
 intention to make you restore malicious data:
 
-    $ sudo echo "boom" >> backup/index/d795ffa99a8ab8f8e42cec1f814df4e48b8f49129360fb57613df93739faee97
+```console
+$ sudo echo "boom" >> backup/index/d795ffa99a8ab8f8e42cec1f814df4e48b8f49129360fb57613df93739faee97
+```
 
 In order to detect these things, it is a good idea to regularly use the `check`
 command to test whether everything is alright, your precious backup data is
 consistent and the integrity is unharmed:
 
-    $ restic -r /tmp/backup check
-    Load indexes
-    ciphertext verification failed
+```console
+$ restic -r /tmp/backup check
+Load indexes
+ciphertext verification failed
+```
 
 Trying to restore a snapshot which has been modified as shown above will yield
 the same error:
 
-    $ restic -r /tmp/backup restore 79766175 --target ~/tmp/restore-work
-    Load indexes
-    ciphertext verification failed
+```console
+$ restic -r /tmp/backup restore 79766175 --target ~/tmp/restore-work
+Load indexes
+ciphertext verification failed
+```
 
 # Mount a repository
 
@@ -345,11 +415,13 @@ Browsing your backup as a regular file system is also very easy. First, create
 a mount point such as `/mnt/restic` and then use the following command to serve
 the repository with FUSE:
 
-    $ mkdir /mnt/restic
-    $ restic -r /tmp/backup mount /mnt/restic
-    enter password for repository:
-    Now serving /tmp/backup at /tmp/restic
-    Don't forget to umount after quitting!
+```console
+$ mkdir /mnt/restic
+$ restic -r /tmp/backup mount /mnt/restic
+enter password for repository:
+Now serving /tmp/backup at /tmp/restic
+Don't forget to umount after quitting!
+```
 
 Mounting repositories via FUSE is not possible on Windows and OpenBSD.
 
@@ -363,34 +435,71 @@ credentials.
 Once the server is configured, the setup of the SFTP repository can simply be
 achieved by changing the URL scheme in the `init` command:
 
-    $ restic -r sftp:user@host:/tmp/backup init
-    enter password for new backend:
-    enter password again:
-    created restic backend f1c6108821 at sftp:user@host:/tmp/backup
-    Please note that knowledge of your password is required to access the repository.
-    Losing your password means that your data is irrecoverably lost.
+```console
+$ restic -r sftp:user@host:/tmp/backup init
+enter password for new backend:
+enter password again:
+created restic backend f1c6108821 at sftp:user@host:/tmp/backup
+Please note that knowledge of your password is required to access the repository.
+Losing your password means that your data is irrecoverably lost.
+```
 
 You can also specify a relative (read: no slash (`/`) character at the
 beginning) directory, in this case the dir is relative to the remote user's
 home directory.
 
+# Create a REST server repository
+
+In order to backup data to the remote server via HTTP or HTTPS protocol,
+you must first set up a remote [REST server](https://github.com/restic/rest-server)
+instance.  Once the server is configured, accessing it is achieved by changing the
+URL scheme like this:
+
+```console
+$ restic -r rest:http://host:8000/
+```
+
+Depending on your REST server setup, you can use HTTPS protocol, password
+protection, or multiple repositories.  Or any combination of those features, as
+you see fit. TCP/IP port is also configurable. Here are some more examples:
+
+```console
+$ restic -r rest:https://host:8000/
+$ restic -r rest:https://user:pass@host:8000/
+$ restic -r rest:https://user:pass@host:8000/my_backup_repo/
+```
+
+If you use TLS, make sure your certificates are signed, 'cause restic client
+will refuse to communicate otherwise.  It's easy to obtain such certificates
+today, thanks to free certificate authorities like [Let’s
+Encrypt](https://letsencrypt.org/).
+
+REST server uses exactly the same directory structure as local backend, so you
+should be able to access it both locally and via HTTP, even simultaneously.
+
 # Create an Amazon S3 repository
 
 Restic can backup data to any Amazon S3 bucket. However, in this case, changing the URL scheme is not enough since Amazon uses special security credentials to sign HTTP requests. By consequence, you must first setup the following environment variables with the credentials you obtained while creating the bucket.
 
-    $ export AWS_ACCESS_KEY_ID=<MY_ACCESS_KEY>
-    $ export AWS_SECRET_ACCESS_KEY=<MY_SECRET_ACCESS_KEY>
+```console
+$ export AWS_ACCESS_KEY_ID=<MY_ACCESS_KEY>
+$ export AWS_SECRET_ACCESS_KEY=<MY_SECRET_ACCESS_KEY>
+```
 
-You can then easily initialize a repository that uses your Amazon S3 as a backend.
+You can then easily initialize a repository that uses your Amazon S3 as a backend, if the bucket does not exist yet it will be created in the default location:
 
-    $ restic -r s3:eu-central-1/bucket_name init
-    enter password for new backend:
-    enter password again:
-    created restic backend eefee03bbd at s3:eu-central-1/bucket_name
-    Please note that knowledge of your password is required to access the repository.
-    Losing your password means that your data is irrecoverably lost.
+```console
+$ restic -r s3:s3.amazonaws.com/bucket_name init
+enter password for new backend:
+enter password again:
+created restic backend eefee03bbd at s3:s3.amazonaws.com/bucket_name
+Please note that knowledge of your password is required to access the repository.
+Losing your password means that your data is irrecoverably lost.
+```
 
-Fro an s3-compatible server that is not Amazon (like Minio, see below), or is
+It is not possible at the moment to have restic create a new bucket in a different location, so you need to create it using a different program. Afterwards, the S3 server (`s3.amazonaws.com`) will redirect restic to the correct endpoint.
+
+For an S3-compatible server that is not Amazon (like Minio, see below), or is
 only available via HTTP, you can specify the URL to the server like this:
 `s3:http://server:port/bucket_name`.
 
@@ -400,22 +509,26 @@ only available via HTTP, you can specify the URL to the server like this:
 
 ### Pre-Requisites
 
-* Download and Install [Minio Server](https://minio.io/download/). 
+* Download and Install [Minio Server](https://minio.io/download/).
 * You can also refer to [https://docs.minio.io](https://docs.minio.io) for step by step guidance on installation and getting started on Minio CLient and Minio Server.
 
 You must first setup the following environment variables with the credentials of your running Minio Server.
 
-    $ export AWS_ACCESS_KEY_ID=<YOUR-MINIO-ACCESS-KEY-ID>
-    $ export AWS_SECRET_ACCESS_KEY= <YOUR-MINIO-SECRET-ACCESS-KEY>
+```console
+$ export AWS_ACCESS_KEY_ID=<YOUR-MINIO-ACCESS-KEY-ID>
+$ export AWS_SECRET_ACCESS_KEY= <YOUR-MINIO-SECRET-ACCESS-KEY>
+```
 
 Now you can easily initialize restic to use Minio server as backend with this command.
 
-    $ ./restic -r s3:http://localhost:9000/restic init
-    enter password for new backend: 
-    enter password again: 
-    created restic backend 6ad29560f5 at s3:http://localhost:9000/restic1
-    Please note that knowledge of your password is required to access
-    the repository. Losing your password means that your data is irrecoverably lost. 
+```console
+$ ./restic -r s3:http://localhost:9000/restic init
+enter password for new backend:
+enter password again:
+created restic backend 6ad29560f5 at s3:http://localhost:9000/restic1
+Please note that knowledge of your password is required to access
+the repository. Losing your password means that your data is irrecoverably lost.
+```
 
 # Removing old snapshots
 
@@ -430,54 +543,62 @@ the repository.
 
 The command `snapshots` can be used to list all snapshots in a repository like this:
 
-    $ restic -r /tmp/backup snapshots
-    enter password for repository:
-    ID        Date                 Host      Tags  Directory
-    ----------------------------------------------------------------------
-    40dc1520  2015-05-08 21:38:30  kasimir         /home/user/work
-    79766175  2015-05-08 21:40:19  kasimir         /home/user/work
-    bdbd3439  2015-05-08 21:45:17  luigi           /home/art
-    590c8fc8  2015-05-08 21:47:38  kazik           /srv
-    9f0bc19e  2015-05-08 21:46:11  luigi           /srv
+```console
+$ restic -r /tmp/backup snapshots
+enter password for repository:
+ID        Date                 Host      Tags  Directory
+----------------------------------------------------------------------
+40dc1520  2015-05-08 21:38:30  kasimir         /home/user/work
+79766175  2015-05-08 21:40:19  kasimir         /home/user/work
+bdbd3439  2015-05-08 21:45:17  luigi           /home/art
+590c8fc8  2015-05-08 21:47:38  kazik           /srv
+9f0bc19e  2015-05-08 21:46:11  luigi           /srv
+```
 
 In order to remove the snapshot of `/home/art`, use the `forget` command and
 specify the snapshot ID on the command line:
 
-    $ restic -r /tmp/backup forget bdbd3439
-    enter password for repository:
-    removed snapshot d3f01f63
+```console
+$ restic -r /tmp/backup forget bdbd3439
+enter password for repository:
+removed snapshot d3f01f63
+```
 
 Afterwards this snapshot is removed:
 
-    $ restic -r /tmp/backup snapshots
-    enter password for repository:
-    ID        Date                 Host     Tags  Directory
-    ----------------------------------------------------------------------
-    40dc1520  2015-05-08 21:38:30  kasimir        /home/user/work
-    79766175  2015-05-08 21:40:19  kasimir        /home/user/work
-    590c8fc8  2015-05-08 21:47:38  kazik          /srv
-    9f0bc19e  2015-05-08 21:46:11  luigi          /srv
+```console
+$ restic -r /tmp/backup snapshots
+enter password for repository:
+ID        Date                 Host     Tags  Directory
+----------------------------------------------------------------------
+40dc1520  2015-05-08 21:38:30  kasimir        /home/user/work
+79766175  2015-05-08 21:40:19  kasimir        /home/user/work
+590c8fc8  2015-05-08 21:47:38  kazik          /srv
+9f0bc19e  2015-05-08 21:46:11  luigi          /srv
+```
 
 But the data that was referenced by files in this snapshot is still stored in
 the repository. To cleanup unreferenced data, the `prune` command must be run:
 
-    $ restic -r /tmp/backup prune
-    enter password for repository:
+```console
+$ restic -r /tmp/backup prune
+enter password for repository:
 
-    counting files in repo
-    building new index for repo
-    [0:00] 100.00%  22 / 22 files
-    repository contains 22 packs (8512 blobs) with 100.092 MiB bytes
-    processed 8512 blobs: 0 duplicate blobs, 0B duplicate
-    load all snapshots
-    find data that is still in use for 1 snapshots
-    [0:00] 100.00%  1 / 1 snapshots
-    found 8433 of 8512 data blobs still in use
-    will rewrite 3 packs
-    creating new index
-    [0:00] 86.36%  19 / 22 files
-    saved new index as 544a5084
-    done
+counting files in repo
+building new index for repo
+[0:00] 100.00%  22 / 22 files
+repository contains 22 packs (8512 blobs) with 100.092 MiB bytes
+processed 8512 blobs: 0 duplicate blobs, 0B duplicate
+load all snapshots
+find data that is still in use for 1 snapshots
+[0:00] 100.00%  1 / 1 snapshots
+found 8433 of 8512 data blobs still in use
+will rewrite 3 packs
+creating new index
+[0:00] 86.36%  19 / 22 files
+saved new index as 544a5084
+done
+```
 
 Afterwards the repository is smaller.
 
@@ -530,12 +651,16 @@ all snapshots!
 
 The program can be built with debug support like this:
 
-    $ go run build.go -tags debug
+```console
+$ go run build.go -tags debug
+```
 
 Afterwards, extensive debug messages are written to the file in environment
 variable `DEBUG_LOG`, e.g.:
 
-    $ DEBUG_LOG=/tmp/restic-debug.log restic backup ~/work
+```console
+$ DEBUG_LOG=/tmp/restic-debug.log restic backup ~/work
+```
 
 If you suspect that there is a bug, you can have a look at the debug log.
 Please be aware that the debug log might contain sensitive information such as
@@ -551,23 +676,29 @@ separated by commas. Patterns are case sensitive.
 Printing all log messages to the console can be achieved by setting the file
 filter to `*`:
 
-    $ DEBUG_FILES=* restic check
+```console
+$ DEBUG_FILES=* restic check
+```
 
 If you want restic to just print all debug log messages from the files
-`main.go` and `lock.go`, set the environment variable `DEBUG_FILES` like this: 
+`main.go` and `lock.go`, set the environment variable `DEBUG_FILES` like this:
 
-    $ DEBUG_FILES=main.go,lock.go restic check
+```console
+$ DEBUG_FILES=main.go,lock.go restic check
+```
 
 The following command line instructs restic to only print debug statements
 originating in functions that match the pattern `*unlock*` (case sensitive):
 
-    $ DEBUG_FUNCS=*unlock* restic check
+```console
+$ DEBUG_FUNCS=*unlock* restic check
+```
 
 # Under the hood: Browse repository objects
 
 Internally, a repository stores data of several different types described in the [design documentation](https://github.com/restic/restic/blob/master/doc/Design.md). You can `list` objects such as blobs, packs, index, snapshots, keys or locks with the following command:
 
-```shell
+```console
 $ restic -r /tmp/backup list snapshots
 d369ccc7d126594950bf74f0a348d5d98d9e99f3215082eb69bf02dc9b3e464c
 ```
@@ -575,26 +706,30 @@ d369ccc7d126594950bf74f0a348d5d98d9e99f3215082eb69bf02dc9b3e464c
 The `find` command searches for a given
 [pattern](http://golang.org/pkg/path/filepath/#Match) in the repository.
 
-    $ restic -r backup find test.txt
-    debug log file restic.log
-    debug enabled
-    enter password for repository:
-    found 1 matching entries in snapshot 196bc5760c909a7681647949e80e5448e276521489558525680acf1bd428af36
-      -rw-r--r--   501    20      5 2015-08-26 14:09:57 +0200 CEST path/to/test.txt
+```console
+$ restic -r backup find test.txt
+debug log file restic.log
+debug enabled
+enter password for repository:
+found 1 matching entries in snapshot 196bc5760c909a7681647949e80e5448e276521489558525680acf1bd428af36
+  -rw-r--r--   501    20      5 2015-08-26 14:09:57 +0200 CEST path/to/test.txt
+```
 
 The `cat` command allows you to display the JSON representation of the objects
 or its raw content.
 
-    $ restic -r /tmp/backup cat snapshot d369ccc7d126594950bf74f0a348d5d98d9e99f3215082eb69bf02dc9b3e464c
-    enter password for repository:
-    {
-      "time": "2015-08-12T12:52:44.091448856+02:00",
-      "tree": "05cec17e8d3349f402576d02576a2971fc0d9f9776ce2f441c7010849c4ff5af",
-      "paths": [
-        "/home/user/work"
-      ],
-      "hostname": "kasimir",
-      "username": "username",
-      "uid": 501,
-      "gid": 20
-    }
+```console
+$ restic -r /tmp/backup cat snapshot d369ccc7d126594950bf74f0a348d5d98d9e99f3215082eb69bf02dc9b3e464c
+enter password for repository:
+{
+  "time": "2015-08-12T12:52:44.091448856+02:00",
+  "tree": "05cec17e8d3349f402576d02576a2971fc0d9f9776ce2f441c7010849c4ff5af",
+  "paths": [
+    "/home/user/work"
+  ],
+  "hostname": "kasimir",
+  "username": "username",
+  "uid": 501,
+  "gid": 20
+}
+```

doc/REST_backend.md

@@ -6,6 +6,19 @@ following values are valid for `{type}`: `data`, `keys`, `locks`, `snapshots`,
 `index`, `config`. `{path}` is a path to the repository, so that multiple
 different repositories can be accessed. The default path is `/`.
 
+## POST {path}?create=true
+
+This request is used to initially create a new repository. The server responds
+with "200 OK" if the repository structure was created successfully or already
+exists, otherwise an error is returned.
+
+## DELETE {path}
+
+Deletes the repository on the server side. The server responds with "200 OK" if
+the repository was successfully removed. If this function is not implemented
+the server returns "501 Not Implemented", if this it is denied by the server it
+returns "403 Forbidden".
+
 ## HEAD {path}/config
 
 Returns "200 OK" if the repository has a configuration,

doc/code.css

@@ -0,0 +1,124 @@
+code {
+    font-size: 90%;
+}
+
+/* based on https://github.com/mkdocs/mkdocs/issues/1019 */
+
+.codehilite code, .codehilite pre {
+    color:#3F3F3F;background-color:#F7F7F7;
+    overflow: auto;
+    box-sizing: border-box;
+
+    padding: 0.01em 4px;
+    padding-top: 0.01em;
+    padding-right-value: 4px;
+    padding-bottom: 0.01em;
+    padding-left-value: 4px;
+    padding-left-ltr-source: physical;
+    padding-left-rtl-source: physical;
+    padding-right-ltr-source: physical;
+    padding-right-rtl-source: physical;
+
+    border-radius: 4px !important;
+    border-top-left-radius: 4px;
+    border-top-right-radius: 4px;
+    border-bottom-right-radius: 4px;
+    border-bottom-left-radius: 4px;
+
+    border: 1px solid #CCC !important;
+    border-top-width: 1px;
+    border-right-width-value: 1px;
+    border-right-width-ltr-source: physical;
+    border-right-width-rtl-source: physical;
+    border-bottom-width: 1px;
+    border-left-width-value: 1px;
+    border-left-width-ltr-source: physical;
+    border-left-width-rtl-source: physical;
+    border-top-style: solid;
+    border-right-style-value: solid;
+    border-right-style-ltr-source: physical;
+    border-right-style-rtl-source: physical;
+    border-bottom-style: solid;
+    border-left-style-value: solid;
+    border-left-style-ltr-source: physical;
+    border-left-style-rtl-source: physical;
+    border-top-color: #CCC;
+    border-right-color-value: #CCC;
+    border-right-color-ltr-source: physical;
+    border-right-color-rtl-source: physical;
+    border-bottom-color: #CCC;
+    border-left-color-value: #CCC;
+    border-left-color-ltr-source: physical;
+    border-left-color-rtl-source: physical;
+    -moz-border-top-colors: none;
+    -moz-border-right-colors: none;
+    -moz-border-bottom-colors: none;
+    -moz-border-left-colors: none;
+    border-image-source: none;
+    border-image-slice: 100% 100% 100% 100%;
+    border-image-width: 1 1 1 1;
+    border-image-outset: 0 0 0 0;
+    border-image-repeat: stretch stretch;
+}
+
+.codehilite .hll { background-color: #ffffcc  }
+.codehilite .c { color: #999988; font-style: italic  } /* Comment */
+.codehilite .err { color: #a61717; background-color: #e3d2d2  } /* Error */
+.codehilite .k { color: #000000; font-weight: bold  } /* Keyword */
+.codehilite .o { color: #000000; font-weight: bold  } /* Operator */
+.codehilite .cm { color: #999988; font-style: italic  } /* Comment.Multiline */
+.codehilite .cp { color: #999999; font-weight: bold; font-style: italic  } /* Comment.Preproc */
+.codehilite .c1 { color: #999988; font-style: italic  } /* Comment.Single */
+.codehilite .cs { color: #999999; font-weight: bold; font-style: italic  } /* Comment.Special */
+.codehilite .gd { color: #000000; background-color: #ffdddd  } /* Generic.Deleted */
+.codehilite .ge { color: #000000; font-style: italic  } /* Generic.Emph */
+.codehilite .gr { color: #aa0000  } /* Generic.Error */
+.codehilite .gh { color: #999999  } /* Generic.Heading */
+.codehilite .gi { color: #000000; background-color: #ddffdd  } /* Generic.Inserted */
+.codehilite .go { color: #888888  } /* Generic.Output */
+.codehilite .gp { color: #555555  } /* Generic.Prompt */
+.codehilite .gs { font-weight: bold  } /* Generic.Strong */
+.codehilite .gu { color: #aaaaaa  } /* Generic.Subheading */
+.codehilite .gt { color: #aa0000  } /* Generic.Traceback */
+.codehilite .kc { color: #000000; font-weight: bold  } /* Keyword.Constant */
+.codehilite .kd { color: #000000; font-weight: bold  } /* Keyword.Declaration */
+.codehilite .kn { color: #000000; font-weight: bold  } /* Keyword.Namespace */
+.codehilite .kp { color: #000000; font-weight: bold  } /* Keyword.Pseudo */
+.codehilite .kr { color: #000000; font-weight: bold  } /* Keyword.Reserved */
+.codehilite .kt { color: #445588; font-weight: bold  } /* Keyword.Type */
+.codehilite .m { color: #009999  } /* Literal.Number */
+.codehilite .s { color: #d01040  } /* Literal.String */
+.codehilite .na { color: #008080  } /* Name.Attribute */
+.codehilite .nb { color: #0086B3  } /* Name.Builtin */
+.codehilite .nc { color: #445588; font-weight: bold  } /* Name.Class */
+.codehilite .no { color: #008080  } /* Name.Constant */
+.codehilite .nd { color: #3c5d5d; font-weight: bold  } /* Name.Decorator */
+.codehilite .ni { color: #800080  } /* Name.Entity */
+.codehilite .ne { color: #990000; font-weight: bold  } /* Name.Exception */
+.codehilite .nf { color: #990000; font-weight: bold  } /* Name.Function */
+.codehilite .nl { color: #990000; font-weight: bold  } /* Name.Label */
+.codehilite .nn { color: #555555  } /* Name.Namespace */
+.codehilite .nt { color: #000080  } /* Name.Tag */
+.codehilite .nv { color: #008080  } /* Name.Variable */
+.codehilite .ow { color: #000000; font-weight: bold  } /* Operator.Word */
+.codehilite .w { color: #bbbbbb  } /* Text.Whitespace */
+.codehilite .mf { color: #009999  } /* Literal.Number.Float */
+.codehilite .mh { color: #009999  } /* Literal.Number.Hex */
+.codehilite .mi { color: #009999  } /* Literal.Number.Integer */
+.codehilite .mo { color: #009999  } /* Literal.Number.Oct */
+.codehilite .sb { color: #d01040  } /* Literal.String.Backtick */
+.codehilite .sc { color: #d01040  } /* Literal.String.Char */
+.codehilite .sd { color: #d01040  } /* Literal.String.Doc */
+.codehilite .s2 { color: #d01040  } /* Literal.String.Double */
+.codehilite .se { color: #d01040  } /* Literal.String.Escape */
+.codehilite .sh { color: #d01040  } /* Literal.String.Heredoc */
+.codehilite .si { color: #d01040  } /* Literal.String.Interpol */
+.codehilite .sx { color: #d01040  } /* Literal.String.Other */
+.codehilite .sr { color: #009926  } /* Literal.String.Regex */
+.codehilite .s1 { color: #d01040  } /* Literal.String.Single */
+.codehilite .ss { color: #990073  } /* Literal.String.Symbol */
+.codehilite .bp { color: #999999  } /* Name.Builtin.Pseudo */
+.codehilite .vc { color: #008080  } /* Name.Variable.Class */
+.codehilite .vg { color: #008080  } /* Name.Variable.Global */
+.codehilite .vi { color: #008080  } /* Name.Variable.Instance */
+.codehilite .il { color: #009999  } /* Literal.Number.Integer.Long */

doc/index.md

@@ -20,10 +20,12 @@ this page, where you can select the version.
 The restic documentation is built with [MkDocs](http://www.mkdocs.org). After
 installing it, you can edit and view the documentation locally by running:
 
-    $ mkdocs serve
-    INFO    -  Building documentation...
-    INFO    -  Cleaning site directory
-    [I 160221 12:33:57 server:271] Serving on http://127.0.0.1:8000
+```console
+$ mkdocs serve
+INFO    -  Building documentation...
+INFO    -  Cleaning site directory
+[I 160221 12:33:57 server:271] Serving on http://127.0.0.1:8000
+```
 
 Afterwards visit the URL with a browser.
 

mkdocs.yml

@@ -1,5 +1,9 @@
 site_name: Documentation for restic
 theme: readthedocs
+markdown_extensions:
+  - codehilite:
+extra_css:
+  - code.css
 docs_dir: doc
 pages:
   - Getting Started: index.md

run_integration_tests.go

@@ -176,7 +176,7 @@ func (env *TravisEnvironment) Prepare() error {
 				"windows/386", "windows/amd64",
 				"darwin/386", "darwin/amd64",
 				"freebsd/386", "freebsd/amd64",
-				"opendbsd/386", "opendbsd/amd64",
+				"openbsd/386", "openbsd/amd64",
 			}
 			if !strings.HasPrefix(runtime.Version(), "go1.3") {
 				env.goxOSArch = append(env.goxOSArch,

src/cmds/restic-server/.gitignore

@@ -1,24 +0,0 @@
-# Compiled Object files, Static and Dynamic libs (Shared Objects)
-*.o
-*.a
-*.so
-
-# Folders
-_obj
-_test
-
-# Architecture specific extensions/prefixes
-*.[568vq]
-[568vq].out
-
-*.cgo1.go
-*.cgo2.c
-_cgo_defun.c
-_cgo_gotypes.go
-_cgo_export.*
-
-_testmain.go
-
-*.exe
-*.test
-*.prof

src/cmds/restic-server/LICENSE

@@ -1,24 +0,0 @@
-Copyright (c) 2015, Bertil Chapuis
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-* Redistributions of source code must retain the above copyright notice, this
-  list of conditions and the following disclaimer.
-
-* Redistributions in binary form must reproduce the above copyright notice,
-  this list of conditions and the following disclaimer in the documentation
-  and/or other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-

src/cmds/restic-server/README.md

@@ -1,29 +0,0 @@
-# Restic Server
-
-Restic Server is a sample server that implement restic's rest backend api.
-It has been developed for demonstration purpose and is not intented to be used in production.
-
-## Getting started
-
-By default the server persists backup data in `/tmp/restic`. 
-Build and start the server with a custom persistence directory:
-
-```
-go build
-./restic-server -path /user/home/backup
-```
-
-The server use an `.htpasswd` file to specify users. You can create such a file at the root of the persistence directory by executing the following command. In order to append new user to the file, just omit the `-c` argument.
-
-```
-htpasswd -s -c .htpasswd username
-```
-
-By default the server uses http. This is not very secure since with Basic Authentication, username and passwords will be present in every request. In order to enable TLS support just add the `-tls` argument and add a private and public key at the root of your persistence directory. 
-
-Signed certificate are required by the restic backend but if you just want to test the feature you can generate unsigned keys with the following commands:
-
-```
-openssl genrsa -out private_key 2048
-openssl req -new -x509 -key private_key -out public_key -days 365
-```

src/cmds/restic-server/handlers.go

@@ -1,194 +0,0 @@
-// +build go1.4
-
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
-
-	"restic/fs"
-)
-
-// Context contains repository meta-data.
-type Context struct {
-	path string
-}
-
-// AuthHandler wraps h with a http.HandlerFunc that performs basic
-// authentication against the user/passwords pairs stored in f and returns the
-// http.HandlerFunc.
-func AuthHandler(f *HtpasswdFile, h http.Handler) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		username, password, ok := r.BasicAuth()
-		if !ok {
-			http.Error(w, "401 unauthorized", 401)
-			return
-		}
-		if !f.Validate(username, password) {
-			http.Error(w, "401 unauthorized", 401)
-			return
-		}
-		h.ServeHTTP(w, r)
-	}
-}
-
-// CheckConfig returns a http.HandlerFunc that checks whether
-// a configuration exists.
-func CheckConfig(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		config := filepath.Join(c.path, "config")
-		st, err := os.Stat(config)
-		if err != nil {
-			http.Error(w, "404 not found", 404)
-			return
-		}
-		w.Header().Add("Content-Length", fmt.Sprint(st.Size()))
-	}
-}
-
-// GetConfig returns a http.HandlerFunc that allows for a
-// config to be retrieved.
-func GetConfig(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		config := filepath.Join(c.path, "config")
-		bytes, err := ioutil.ReadFile(config)
-		if err != nil {
-			http.Error(w, "404 not found", 404)
-			return
-		}
-		w.Write(bytes)
-	}
-}
-
-// SaveConfig returns a http.HandlerFunc that allows for a
-// config to be saved.
-func SaveConfig(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		config := filepath.Join(c.path, "config")
-		bytes, err := ioutil.ReadAll(r.Body)
-		if err != nil {
-			http.Error(w, "400 bad request", 400)
-			return
-		}
-		errw := ioutil.WriteFile(config, bytes, 0600)
-		if errw != nil {
-			http.Error(w, "500 internal server error", 500)
-			return
-		}
-		w.Write([]byte("200 ok"))
-	}
-}
-
-// ListBlobs returns a http.HandlerFunc that lists
-// all blobs of a given type in an arbitrary order.
-func ListBlobs(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		vars := strings.Split(r.RequestURI, "/")
-		dir := vars[1]
-		path := filepath.Join(c.path, dir)
-		files, err := ioutil.ReadDir(path)
-		if err != nil {
-			http.Error(w, "404 not found", 404)
-			return
-		}
-		names := make([]string, len(files))
-		for i, f := range files {
-			names[i] = f.Name()
-		}
-		data, err := json.Marshal(names)
-		if err != nil {
-			http.Error(w, "500 internal server error", 500)
-			return
-		}
-		w.Write(data)
-	}
-}
-
-// CheckBlob reutrns a http.HandlerFunc that tests whether a blob exists
-// and returns 200, if it does, or 404 otherwise.
-func CheckBlob(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		vars := strings.Split(r.RequestURI, "/")
-		dir := vars[1]
-		name := vars[2]
-		path := filepath.Join(c.path, dir, name)
-		st, err := os.Stat(path)
-		if err != nil {
-			http.Error(w, "404 not found", 404)
-			return
-		}
-		w.Header().Add("Content-Length", fmt.Sprint(st.Size()))
-	}
-}
-
-// GetBlob returns a http.HandlerFunc that retrieves a blob
-// from the repository.
-func GetBlob(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		vars := strings.Split(r.RequestURI, "/")
-		dir := vars[1]
-		name := vars[2]
-		path := filepath.Join(c.path, dir, name)
-		file, err := fs.Open(path)
-		if err != nil {
-			http.Error(w, "404 not found", 404)
-			return
-		}
-		defer file.Close()
-		http.ServeContent(w, r, "", time.Unix(0, 0), file)
-	}
-}
-
-// SaveBlob returns a http.HandlerFunc that saves a blob to the repository.
-func SaveBlob(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		vars := strings.Split(r.RequestURI, "/")
-		dir := vars[1]
-		name := vars[2]
-		path := filepath.Join(c.path, dir, name)
-		tmp := path + "_tmp"
-		tf, err := fs.OpenFile(tmp, os.O_CREATE|os.O_WRONLY, 0600)
-		if err != nil {
-			http.Error(w, "500 internal server error", 500)
-			return
-		}
-		if _, err := io.Copy(tf, r.Body); err != nil {
-			http.Error(w, "400 bad request", 400)
-			tf.Close()
-			os.Remove(tmp)
-			return
-		}
-		if err := tf.Close(); err != nil {
-			http.Error(w, "500 internal server error", 500)
-		}
-		if err := os.Rename(tmp, path); err != nil {
-			http.Error(w, "500 internal server error", 500)
-			return
-		}
-		w.Write([]byte("200 ok"))
-	}
-}
-
-// DeleteBlob returns a http.HandlerFunc that deletes a blob from the
-// repository.
-func DeleteBlob(c *Context) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		vars := strings.Split(r.RequestURI, "/")
-		dir := vars[1]
-		name := vars[2]
-		path := filepath.Join(c.path, dir, name)
-		err := os.Remove(path)
-		if err != nil {
-			http.Error(w, "500 internal server error", 500)
-			return
-		}
-		w.Write([]byte("200 ok"))
-	}
-}

src/cmds/restic-server/htpasswd.go

@@ -1,97 +0,0 @@
-// +build go1.4
-
-package main
-
-/*
-Copied from: github.com/bitly/oauth2_proxy
-
-MIT License
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-*/
-
-import (
-	"crypto/sha1"
-	"encoding/base64"
-	"encoding/csv"
-	"io"
-	"log"
-
-	"restic/fs"
-)
-
-// lookup passwords in a htpasswd file
-// The entries must have been created with -s for SHA encryption
-
-// HtpasswdFile is a map for usernames to passwords.
-type HtpasswdFile struct {
-	Users map[string]string
-}
-
-// NewHtpasswdFromFile reads the users and passwords from a htpasswd
-// file and returns them. If an error is encountered, it is returned, together
-// with a nil-Pointer for the HtpasswdFile.
-func NewHtpasswdFromFile(path string) (*HtpasswdFile, error) {
-	r, err := fs.Open(path)
-	if err != nil {
-		return nil, err
-	}
-	defer r.Close()
-	return NewHtpasswd(r)
-}
-
-// NewHtpasswd reads the users and passwords from a htpasswd
-// datastream in file and returns them. If an error is encountered,
-// it is returned, together with a nil-Pointer for the HtpasswdFile.
-func NewHtpasswd(file io.Reader) (*HtpasswdFile, error) {
-	cr := csv.NewReader(file)
-	cr.Comma = ':'
-	cr.Comment = '#'
-	cr.TrimLeadingSpace = true
-
-	records, err := cr.ReadAll()
-	if err != nil {
-		return nil, err
-	}
-	h := &HtpasswdFile{Users: make(map[string]string)}
-	for _, record := range records {
-		h.Users[record[0]] = record[1]
-	}
-	return h, nil
-}
-
-// Validate returns true if password matches the stored password
-// for user. If no password for user is stored, or the password
-// is wrong, false is returned.
-func (h *HtpasswdFile) Validate(user string, password string) bool {
-	realPassword, exists := h.Users[user]
-	if !exists {
-		return false
-	}
-	if realPassword[:5] == "{SHA}" {
-		d := sha1.New()
-		d.Write([]byte(password))
-		if realPassword[5:] == base64.StdEncoding.EncodeToString(d.Sum(nil)) {
-			return true
-		}
-	} else {
-		log.Printf("Invalid htpasswd entry for %s. Must be a SHA entry.", user)
-	}
-	return false
-}

src/cmds/restic-server/router.go

@@ -1,137 +0,0 @@
-// +build go1.4
-
-package main
-
-import (
-	"log"
-	"net/http"
-	"strings"
-)
-
-// Route is a handler for a path that was already split.
-type Route struct {
-	path    []string
-	handler http.Handler
-}
-
-// Router maps HTTP methods to a slice of Route handlers.
-type Router struct {
-	routes map[string][]Route
-}
-
-// NewRouter creates a new Router and returns a pointer to it.
-func NewRouter() *Router {
-	return &Router{make(map[string][]Route)}
-}
-
-// Options registers handler for path with method "OPTIONS".
-func (router *Router) Options(path string, handler http.Handler) {
-	router.Handle("OPTIONS", path, handler)
-}
-
-// OptionsFunc registers handler for path with method "OPTIONS".
-func (router *Router) OptionsFunc(path string, handler http.HandlerFunc) {
-	router.Handle("OPTIONS", path, handler)
-}
-
-// Get registers handler for path with method "GET".
-func (router *Router) Get(path string, handler http.Handler) {
-	router.Handle("GET", path, handler)
-}
-
-// GetFunc registers handler for path with method "GET".
-func (router *Router) GetFunc(path string, handler http.HandlerFunc) {
-	router.Handle("GET", path, handler)
-}
-
-// Head registers handler for path with method "HEAD".
-func (router *Router) Head(path string, handler http.Handler) {
-	router.Handle("HEAD", path, handler)
-}
-
-// HeadFunc registers handler for path with method "HEAD".
-func (router *Router) HeadFunc(path string, handler http.HandlerFunc) {
-	router.Handle("HEAD", path, handler)
-}
-
-// Post registers handler for path with method "POST".
-func (router *Router) Post(path string, handler http.Handler) {
-	router.Handle("POST", path, handler)
-}
-
-// PostFunc registers handler for path with method "POST".
-func (router *Router) PostFunc(path string, handler http.HandlerFunc) {
-	router.Handle("POST", path, handler)
-}
-
-// Put registers handler for path with method "PUT".
-func (router *Router) Put(path string, handler http.Handler) {
-	router.Handle("PUT", path, handler)
-}
-
-// PutFunc registers handler for path with method "PUT".
-func (router *Router) PutFunc(path string, handler http.HandlerFunc) {
-	router.Handle("PUT", path, handler)
-}
-
-// Delete registers handler for path with method "DELETE".
-func (router *Router) Delete(path string, handler http.Handler) {
-	router.Handle("DELETE", path, handler)
-}
-
-// DeleteFunc registers handler for path with method "DELETE".
-func (router *Router) DeleteFunc(path string, handler http.HandlerFunc) {
-	router.Handle("DELETE", path, handler)
-}
-
-// Trace registers handler for path with method "TRACE".
-func (router *Router) Trace(path string, handler http.Handler) {
-	router.Handle("TRACE", path, handler)
-}
-
-// TraceFunc registers handler for path with method "TRACE".
-func (router *Router) TraceFunc(path string, handler http.HandlerFunc) {
-	router.Handle("TRACE", path, handler)
-}
-
-// Connect registers handler for path with method "Connect".
-func (router *Router) Connect(path string, handler http.Handler) {
-	router.Handle("Connect", path, handler)
-}
-
-// ConnectFunc registers handler for path with method "Connect".
-func (router *Router) ConnectFunc(path string, handler http.HandlerFunc) {
-	router.Handle("Connect", path, handler)
-}
-
-// Handle registers a http.Handler for method and uri
-func (router *Router) Handle(method string, uri string, handler http.Handler) {
-	routes := router.routes[method]
-	path := strings.Split(uri, "/")
-	routes = append(routes, Route{path, handler})
-	router.routes[method] = routes
-}
-
-func (router *Router) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	method := r.Method
-	uri := r.RequestURI
-	path := strings.Split(uri, "/")
-
-	log.Printf("%s %s", method, uri)
-
-ROUTE:
-	for _, route := range router.routes[method] {
-		if len(route.path) != len(path) {
-			continue
-		}
-		for i := 0; i < len(route.path); i++ {
-			if !strings.HasPrefix(route.path[i], ":") && route.path[i] != path[i] {
-				continue ROUTE
-			}
-		}
-		route.handler.ServeHTTP(w, r)
-		return
-	}
-
-	http.Error(w, "404 not found", 404)
-}

src/cmds/restic-server/router_test.go

@@ -1,74 +0,0 @@
-// +build go1.4
-
-package main
-
-import (
-	"io/ioutil"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-)
-
-func TestRouter(t *testing.T) {
-	router := NewRouter()
-
-	getConfig := []byte("GET /config")
-	router.GetFunc("/config", func(w http.ResponseWriter, r *http.Request) {
-		w.Write(getConfig)
-	})
-
-	postConfig := []byte("POST /config")
-	router.PostFunc("/config", func(w http.ResponseWriter, r *http.Request) {
-		w.Write(postConfig)
-	})
-
-	getBlobs := []byte("GET /blobs/")
-	router.GetFunc("/blobs/", func(w http.ResponseWriter, r *http.Request) {
-		w.Write(getBlobs)
-	})
-
-	getBlob := []byte("GET /blobs/:sha")
-	router.GetFunc("/blobs/:sha", func(w http.ResponseWriter, r *http.Request) {
-		w.Write(getBlob)
-	})
-
-	server := httptest.NewServer(router)
-	defer server.Close()
-
-	getConfigResp, _ := http.Get(server.URL + "/config")
-	getConfigBody, _ := ioutil.ReadAll(getConfigResp.Body)
-	if getConfigResp.StatusCode != 200 {
-		t.Fatalf("Wanted HTTP Status 200, got %d", getConfigResp.StatusCode)
-	}
-	if string(getConfig) != string(getConfigBody) {
-		t.Fatalf("Config wrong:\nWanted '%s'\nGot: '%s'", string(getConfig), string(getConfigBody))
-	}
-
-	postConfigResp, _ := http.Post(server.URL+"/config", "binary/octet-stream", strings.NewReader("post test"))
-	postConfigBody, _ := ioutil.ReadAll(postConfigResp.Body)
-	if postConfigResp.StatusCode != 200 {
-		t.Fatalf("Wanted HTTP Status 200, got %d", postConfigResp.StatusCode)
-	}
-	if string(postConfig) != string(postConfigBody) {
-		t.Fatalf("Config wrong:\nWanted '%s'\nGot: '%s'", string(postConfig), string(postConfigBody))
-	}
-
-	getBlobsResp, _ := http.Get(server.URL + "/blobs/")
-	getBlobsBody, _ := ioutil.ReadAll(getBlobsResp.Body)
-	if getBlobsResp.StatusCode != 200 {
-		t.Fatalf("Wanted HTTP Status 200, got %d", getBlobsResp.StatusCode)
-	}
-	if string(getBlobs) != string(getBlobsBody) {
-		t.Fatalf("Config wrong:\nWanted '%s'\nGot: '%s'", string(getBlobs), string(getBlobsBody))
-	}
-
-	getBlobResp, _ := http.Get(server.URL + "/blobs/test")
-	getBlobBody, _ := ioutil.ReadAll(getBlobResp.Body)
-	if getBlobResp.StatusCode != 200 {
-		t.Fatalf("Wanted HTTP Status 200, got %d", getBlobResp.StatusCode)
-	}
-	if string(getBlob) != string(getBlobBody) {
-		t.Fatalf("Config wrong:\nWanted '%s'\nGot: '%s'", string(getBlob), string(getBlobBody))
-	}
-}

src/cmds/restic-server/server.go

@@ -1,73 +0,0 @@
-// +build go1.4
-
-package main
-
-import (
-	"flag"
-	"log"
-	"net/http"
-	"os"
-	"path/filepath"
-)
-
-const (
-	defaultHTTPPort  = ":8000"
-	defaultHTTPSPort = ":8443"
-)
-
-func main() {
-	// Parse command-line args
-	var path = flag.String("path", "/tmp/restic", "specifies the path of the data directory")
-	var tls = flag.Bool("tls", false, "turns on tls support")
-	flag.Parse()
-
-	// Create the missing directories
-	dirs := []string{
-		"data",
-		"snapshots",
-		"index",
-		"locks",
-		"keys",
-		"tmp",
-	}
-	for _, d := range dirs {
-		os.MkdirAll(filepath.Join(*path, d), 0700)
-	}
-
-	// Define the routes
-	context := &Context{*path}
-	router := NewRouter()
-	router.HeadFunc("/config", CheckConfig(context))
-	router.GetFunc("/config", GetConfig(context))
-	router.PostFunc("/config", SaveConfig(context))
-	router.GetFunc("/:dir/", ListBlobs(context))
-	router.HeadFunc("/:dir/:name", CheckBlob(context))
-	router.GetFunc("/:type/:name", GetBlob(context))
-	router.PostFunc("/:type/:name", SaveBlob(context))
-	router.DeleteFunc("/:type/:name", DeleteBlob(context))
-
-	// Check for a password file
-	var handler http.Handler
-	htpasswdFile, err := NewHtpasswdFromFile(filepath.Join(*path, ".htpasswd"))
-	if err != nil {
-		log.Println("Authentication disabled")
-		handler = router
-	} else {
-		log.Println("Authentication enabled")
-		handler = AuthHandler(htpasswdFile, router)
-	}
-
-	// start the server
-	if !*tls {
-		log.Printf("start server on port %s\n", defaultHTTPPort)
-		http.ListenAndServe(defaultHTTPPort, handler)
-	} else {
-		privateKey := filepath.Join(*path, "private_key")
-		publicKey := filepath.Join(*path, "public_key")
-		log.Println("TLS enabled")
-		log.Printf("private key: %s", privateKey)
-		log.Printf("public key: %s", publicKey)
-		log.Printf("start server on port %s\n", defaultHTTPSPort)
-		http.ListenAndServeTLS(defaultHTTPSPort, publicKey, privateKey, handler)
-	}
-}

src/cmds/restic/background.go

@@ -0,0 +1,9 @@
+// +build !linux
+
+package main
+
+// IsProcessBackground should return true if it is running in the background or false if not
+func IsProcessBackground() bool {
+	//TODO: Check if the process are running in the background in other OS than linux
+	return false
+}

src/cmds/restic/background_linux.go

@@ -0,0 +1,21 @@
+package main
+
+import (
+	"syscall"
+	"unsafe"
+
+	"restic/debug"
+)
+
+// IsProcessBackground returns true if it is running in the background or false if not
+func IsProcessBackground() bool {
+	var pid int
+	_, _, err := syscall.Syscall(syscall.SYS_IOCTL, uintptr(syscall.Stdin), syscall.TIOCGPGRP, uintptr(unsafe.Pointer(&pid)))
+
+	if err != 0 {
+		debug.Log("Can't check if we are in the background. Using default behaviour. Error: %s\n", err.Error())
+		return false
+	}
+
+	return pid != syscall.Getpgrp()
+}

src/cmds/restic/cleanup.go

@@ -62,8 +62,13 @@ func CleanupHandler(c <-chan os.Signal) {
 	for s := range c {
 		debug.Log("signal %v received, cleaning up", s)
 		fmt.Printf("%sInterrupt received, cleaning up\n", ClearLine())
-		RunCleanupHandlers()
-		fmt.Println("exiting")
-		os.Exit(0)
+		Exit(0)
 	}
 }
+
+// Exit runs the cleanup handlers and then terminates the process with the
+// given exit code.
+func Exit(code int) {
+	RunCleanupHandlers()
+	os.Exit(code)
+}

src/cmds/restic/cmd_backup.go

@@ -46,6 +46,7 @@ type BackupOptions struct {
 	Stdin          bool
 	StdinFilename  string
 	Tags           []string
+	FilesFrom      string
 }
 
 var backupOptions BackupOptions
@@ -62,6 +63,7 @@ func init() {
 	f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
 	f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "", "file name to use when reading from stdin")
 	f.StringSliceVar(&backupOptions.Tags, "tag", []string{}, "add a `tag` for the new snapshot (can be specified multiple times)")
+	f.StringVar(&backupOptions.FilesFrom, "files-from", "", "read the files to backup from file (can be combined with file args)")
 }
 
 func newScanProgress(gopts GlobalOptions) *restic.Progress {
@@ -71,8 +73,13 @@ func newScanProgress(gopts GlobalOptions) *restic.Progress {
 
 	p := restic.NewProgress()
 	p.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
+		if IsProcessBackground() {
+			return
+		}
+
 		PrintProgress("[%s] %d directories, %d files, %s", formatDuration(d), s.Dirs, s.Files, formatBytes(s.Bytes))
 	}
+
 	p.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
 		PrintProgress("scanned %d directories, %d files in %s\n", s.Dirs, s.Files, formatDuration(d))
 	}
@@ -91,6 +98,10 @@ func newArchiveProgress(gopts GlobalOptions, todo restic.Stat) *restic.Progress
 	itemsTodo := todo.Files + todo.Dirs
 
 	archiveProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
+		if IsProcessBackground() {
+			return
+		}
+
 		sec := uint64(d / time.Second)
 		if todo.Bytes > 0 && sec > 0 && ticker {
 			bps = s.Bytes / sec
@@ -144,6 +155,10 @@ func newArchiveStdinProgress(gopts GlobalOptions) *restic.Progress {
 	var bps uint64
 
 	archiveProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
+		if IsProcessBackground() {
+			return
+		}
+
 		sec := uint64(d / time.Second)
 		if s.Bytes > 0 && sec > 0 && ticker {
 			bps = s.Bytes / sec
@@ -241,11 +256,48 @@ func readBackupFromStdin(opts BackupOptions, gopts GlobalOptions, args []string)
 		return err
 	}
 
-	fmt.Printf("archived as %v\n", id.Str())
+	Verbosef("archived as %v\n", id.Str())
 	return nil
 }
 
+// readFromFile will read all lines from the given filename and write them to a
+// string array, if filename is empty readFromFile returns and empty string
+// array
+func readLinesFromFile(filename string) ([]string, error) {
+	if filename == "" {
+		return nil, nil
+	}
+
+	file, err := os.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	var lines []string
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		lines = append(lines, scanner.Text())
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+
+	return lines, nil
+}
+
 func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
+	fromfile, err := readLinesFromFile(opts.FilesFrom)
+	if err != nil {
+		return err
+	}
+
+	// merge files from files-from into normal args so we can reuse the normal
+	// args checks and have the ability to use both files-from and args at the
+	// same time
+	args = append(args, fromfile...)
 	if len(args) == 0 {
 		return errors.Fatalf("wrong number of parameters")
 	}
@@ -258,7 +310,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
 		target = append(target, d)
 	}
 
-	target, err := filterExisting(target)
+	target, err = filterExisting(target)
 	if err != nil {
 		return err
 	}
@@ -303,7 +355,13 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
 
 	// Find last snapshot to set it as parent, if not already set
 	if !opts.Force && parentSnapshotID == nil {
-		id, err := restic.FindLatestSnapshot(repo, target, "")
+		hostname, err := os.Hostname()
+		if err != nil {
+			debug.Log("os.Hostname() returned err: %v", err)
+			hostname = ""
+		}
+
+		id, err := restic.FindLatestSnapshot(repo, target, hostname)
 		if err == nil {
 			parentSnapshotID = &id
 		} else if err != restic.ErrNoSnapshotFound {
@@ -346,7 +404,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
 			return false
 		}
 
-		if !opts.ExcludeOtherFS {
+		if !opts.ExcludeOtherFS || fi == nil {
 			return true
 		}
 
@@ -374,10 +432,9 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
 	arch.Excludes = opts.Excludes
 	arch.SelectFilter = selectFilter
 
-	arch.Error = func(dir string, fi os.FileInfo, err error) error {
+	arch.Warn = func(dir string, fi os.FileInfo, err error) {
 		// TODO: make ignoring errors configurable
-		Warnf("%s\rerror for %s: %v\n", ClearLine(), dir, err)
-		return nil
+		Warnf("%s\rwarning for %s: %v\n", ClearLine(), dir, err)
 	}
 
 	_, id, err := arch.Snapshot(newArchiveProgress(gopts, stat), target, opts.Tags, parentSnapshotID)

src/cmds/restic/cmd_cat.go

@@ -9,13 +9,12 @@ import (
 
 	"restic"
 	"restic/backend"
-	"restic/debug"
 	"restic/errors"
 	"restic/repository"
 )
 
 var cmdCat = &cobra.Command{
-	Use:   "cat [flags] [pack|blob|tree|snapshot|key|masterkey|config|lock] ID",
+	Use:   "cat [flags] [pack|blob|snapshot|index|key|masterkey|config|lock] ID",
 	Short: "print internal objects to stdout",
 	Long: `
 The "cat" command is used to print internal objects to stdout.
@@ -99,7 +98,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 		return nil
 	case "key":
 		h := restic.Handle{Type: restic.KeyFile, Name: id.String()}
-		buf, err := backend.LoadAll(repo.Backend(), h, nil)
+		buf, err := backend.LoadAll(repo.Backend(), h)
 		if err != nil {
 			return err
 		}
@@ -150,7 +149,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 	switch tpe {
 	case "pack":
 		h := restic.Handle{Type: restic.DataFile, Name: id.String()}
-		buf, err := backend.LoadAll(repo.Backend(), h, nil)
+		buf, err := backend.LoadAll(repo.Backend(), h)
 		if err != nil {
 			return err
 		}
@@ -172,7 +171,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 			blob := list[0]
 
 			buf := make([]byte, blob.Length)
-			n, err := repo.LoadBlob(restic.DataBlob, id, buf)
+			n, err := repo.LoadBlob(t, id, buf)
 			if err != nil {
 				return err
 			}
@@ -184,23 +183,6 @@ func runCat(gopts GlobalOptions, args []string) error {
 
 		return errors.Fatal("blob not found")
 
-	case "tree":
-		debug.Log("cat tree %v", id.Str())
-		tree, err := repo.LoadTree(id)
-		if err != nil {
-			debug.Log("unable to load tree %v: %v", id.Str(), err)
-			return err
-		}
-
-		buf, err := json.MarshalIndent(&tree, "", "  ")
-		if err != nil {
-			debug.Log("error json.MarshalIndent(): %v", err)
-			return err
-		}
-
-		_, err = os.Stdout.Write(append(buf, '\n'))
-		return nil
-
 	default:
 		return errors.Fatal("invalid type")
 	}

src/cmds/restic/cmd_forget.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"encoding/hex"
 	"fmt"
 	"io"
 	"restic"
@@ -111,27 +112,36 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	err = repo.LoadIndex()
-	if err != nil {
-		return err
+	// parse arguments as hex strings
+	var ids []string
+	for _, s := range args {
+		_, err := hex.DecodeString(s)
+		if err != nil {
+			Warnf("argument %q is not a snapshot ID, ignoring\n", s)
+			continue
+		}
+
+		ids = append(ids, s)
 	}
 
-	// first, process all snapshot IDs given as arguments
-	for _, s := range args {
+	// process all snapshot IDs given as arguments
+	for _, s := range ids {
 		id, err := restic.FindSnapshot(repo, s)
 		if err != nil {
-			return err
+			Warnf("cound not find a snapshot for ID %q, ignoring\n", s)
+			continue
 		}
 
 		if !opts.DryRun {
-			err = repo.Backend().Remove(restic.SnapshotFile, id.String())
+			h := restic.Handle{Type: restic.SnapshotFile, Name: id.String()}
+			err = repo.Backend().Remove(h)
 			if err != nil {
 				return err
 			}
 
 			Verbosef("removed snapshot %v\n", id.Str())
 		} else {
-			Verbosef("would removed snapshot %v\n", id.Str())
+			Verbosef("would remove snapshot %v\n", id.Str())
 		}
 	}
 
@@ -192,7 +202,8 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 
 		if !opts.DryRun {
 			for _, sn := range remove {
-				err = repo.Backend().Remove(restic.SnapshotFile, sn.ID().String())
+				h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
+				err = repo.Backend().Remove(h)
 				if err != nil {
 					return err
 				}

src/cmds/restic/cmd_key.go

@@ -87,7 +87,8 @@ func deleteKey(repo *repository.Repository, name string) error {
 		return errors.Fatal("refusing to remove key currently used to access repository")
 	}
 
-	err := repo.Backend().Remove(restic.KeyFile, name)
+	h := restic.Handle{Type: restic.KeyFile, Name: name}
+	err := repo.Backend().Remove(h)
 	if err != nil {
 		return err
 	}
@@ -107,7 +108,8 @@ func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
 		return errors.Fatalf("creating new key failed: %v\n", err)
 	}
 
-	err = repo.Backend().Remove(restic.KeyFile, repo.KeyName())
+	h := restic.Handle{Type: restic.KeyFile, Name: repo.KeyName()}
+	err = repo.Backend().Remove(h)
 	if err != nil {
 		return err
 	}

src/cmds/restic/cmd_list.go

@@ -1,8 +1,10 @@
 package main
 
 import (
+	"fmt"
 	"restic"
 	"restic/errors"
+	"restic/index"
 
 	"github.com/spf13/cobra"
 )
@@ -52,6 +54,19 @@ func runList(opts GlobalOptions, args []string) error {
 		t = restic.KeyFile
 	case "locks":
 		t = restic.LockFile
+	case "blobs":
+		idx, err := index.Load(repo, nil)
+		if err != nil {
+			return err
+		}
+
+		for _, pack := range idx.Packs {
+			for _, entry := range pack.Entries {
+				fmt.Printf("%v %v\n", entry.Type, entry.ID)
+			}
+		}
+
+		return nil
 	default:
 		return errors.Fatal("invalid type")
 	}

src/cmds/restic/cmd_ls.go

@@ -17,35 +17,48 @@ var cmdLs = &cobra.Command{
 	Short: "list files in a snapshot",
 	Long: `
 The "ls" command allows listing files and directories in a snapshot.
+
+The special snapshot-ID "latest" can be used to list files and directories of the latest snapshot in the repository.
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runLs(globalOptions, args)
 	},
 }
 
-var listLong bool
+// LsOptions collects all options for the ls command.
+type LsOptions struct {
+	ListLong bool
+	Host     string
+	Paths    []string
+}
+
+var lsOptions LsOptions
 
 func init() {
 	cmdRoot.AddCommand(cmdLs)
 
-	cmdLs.Flags().BoolVarP(&listLong, "long", "l", false, "use a long listing format showing size and mode")
+	flags := cmdLs.Flags()
+	flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
+
+	flags.StringVarP(&lsOptions.Host, "host", "H", "", `only consider snapshots for this host when the snapshot ID is "latest"`)
+	flags.StringSliceVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
 }
 
 func printNode(prefix string, n *restic.Node) string {
-	if !listLong {
+	if !lsOptions.ListLong {
 		return filepath.Join(prefix, n.Name)
 	}
 
 	switch n.Type {
 	case "file":
 		return fmt.Sprintf("%s %5d %5d %6d %s %s",
-			n.Mode, n.UID, n.GID, n.Size, n.ModTime, filepath.Join(prefix, n.Name))
+			n.Mode, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name))
 	case "dir":
 		return fmt.Sprintf("%s %5d %5d %6d %s %s",
-			n.Mode|os.ModeDir, n.UID, n.GID, n.Size, n.ModTime, filepath.Join(prefix, n.Name))
+			n.Mode|os.ModeDir, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name))
 	case "symlink":
 		return fmt.Sprintf("%s %5d %5d %6d %s %s -> %s",
-			n.Mode|os.ModeSymlink, n.UID, n.GID, n.Size, n.ModTime, filepath.Join(prefix, n.Name), n.LinkTarget)
+			n.Mode|os.ModeSymlink, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name), n.LinkTarget)
 	default:
 		return fmt.Sprintf("<Node(%s) %s>", n.Type, n.Name)
 	}
@@ -86,9 +99,19 @@ func runLs(gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	id, err := restic.FindSnapshot(repo, args[0])
-	if err != nil {
-		return err
+	snapshotIDString := args[0]
+	var id restic.ID
+
+	if snapshotIDString == "latest" {
+		id, err = restic.FindLatestSnapshot(repo, lsOptions.Paths, lsOptions.Host)
+		if err != nil {
+			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Host:%v", err, lsOptions.Paths, lsOptions.Host)
+		}
+	} else {
+		id, err = restic.FindSnapshot(repo, snapshotIDString)
+		if err != nil {
+			Exitf(1, "invalid id %q: %v", snapshotIDString, err)
+		}
 	}
 
 	sn, err := restic.LoadSnapshot(repo, id)

src/cmds/restic/cmd_prune.go

@@ -103,11 +103,13 @@ func runPrune(gopts GlobalOptions) error {
 		return err
 	}
 
+	blobs := 0
 	for _, pack := range idx.Packs {
 		stats.bytes += pack.Size
+		blobs += len(pack.Entries)
 	}
 	Verbosef("repository contains %v packs (%v blobs) with %v bytes\n",
-		len(idx.Packs), len(idx.Blobs), formatBytes(uint64(stats.bytes)))
+		len(idx.Packs), blobs, formatBytes(uint64(stats.bytes)))
 
 	blobCount := make(map[restic.BlobHandle]int)
 	duplicateBlobs := 0
@@ -164,14 +166,17 @@ func runPrune(gopts GlobalOptions) error {
 
 	// find packs that need a rewrite
 	rewritePacks := restic.NewIDSet()
-	for h, blob := range idx.Blobs {
-		if !usedBlobs.Has(h) {
-			rewritePacks.Merge(blob.Packs)
-			continue
-		}
+	for _, pack := range idx.Packs {
+		for _, blob := range pack.Entries {
+			h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
+			if !usedBlobs.Has(h) {
+				rewritePacks.Insert(pack.ID)
+				continue
+			}
 
-		if blobCount[h] > 1 {
-			rewritePacks.Merge(blob.Packs)
+			if blobCount[h] > 1 {
+				rewritePacks.Insert(pack.ID)
+			}
 		}
 	}
 
@@ -214,7 +219,8 @@ func runPrune(gopts GlobalOptions) error {
 	}
 
 	for packID := range removePacks {
-		err = repo.Backend().Remove(restic.DataFile, packID.String())
+		h := restic.Handle{Type: restic.DataFile, Name: packID.String()}
+		err = repo.Backend().Remove(h)
 		if err != nil {
 			Warnf("unable to remove file %v from the repository\n", packID.Str())
 		}
@@ -234,7 +240,8 @@ func runPrune(gopts GlobalOptions) error {
 
 	var supersedes restic.IDs
 	for idxID := range repo.List(restic.IndexFile, done) {
-		err := repo.Backend().Remove(restic.IndexFile, idxID.String())
+		h := restic.Handle{Type: restic.IndexFile, Name: idxID.String()}
+		err := repo.Backend().Remove(h)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "unable to remove index %v: %v\n", idxID.Str(), err)
 		}

src/cmds/restic/cmd_snapshots.go

@@ -57,8 +57,8 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 	}
 
 	tab := NewTable()
-	tab.Header = fmt.Sprintf("%-8s  %-19s  %-10s  %-10s  %s", "ID", "Date", "Host", "Tags", "Directory")
-	tab.RowFormat = "%-8s  %-19s  %-10s  %-10s  %s"
+	tab.Header = fmt.Sprintf("%-8s  %-19s  %-10s  %-10s  %-3s %s", "ID", "Date", "Host", "Tags", "", "Directory")
+	tab.RowFormat = "%-8s  %-19s  %-10s  %-10s  %-3s %s"
 
 	done := make(chan struct{})
 	defer close(done)
@@ -97,9 +97,15 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 			firstTag = sn.Tags[0]
 		}
 
-		tab.Rows = append(tab.Rows, []interface{}{sn.ID().Str(), sn.Time.Format(TimeFormat), sn.Hostname, firstTag, sn.Paths[0]})
-
 		rows := len(sn.Paths)
+
+		treeElement := "   "
+		if rows != 1 {
+			treeElement = "┌──"
+		}
+
+		tab.Rows = append(tab.Rows, []interface{}{sn.ID().Str(), sn.Time.Format(TimeFormat), sn.Hostname, firstTag, treeElement, sn.Paths[0]})
+
 		if len(sn.Tags) > rows {
 			rows = len(sn.Tags)
 		}
@@ -115,7 +121,12 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 				tag = sn.Tags[i]
 			}
 
-			tab.Rows = append(tab.Rows, []interface{}{"", "", "", tag, path})
+			treeElement := "│"
+			if i == (rows - 1) {
+				treeElement = "└──"
+			}
+
+			tab.Rows = append(tab.Rows, []interface{}{"", "", "", tag, treeElement, path})
 		}
 	}
 

src/cmds/restic/cmd_version.go

@@ -15,8 +15,8 @@ The "version" command prints detailed information about the build environment
 and the version of this software.
 `,
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Printf("restic %s\ncompiled at %s with %v on %v/%v\n",
-			version, compiledAt, runtime.Version(), runtime.GOOS, runtime.GOARCH)
+		fmt.Printf("restic %s\ncompiled with %v on %v/%v\n",
+			version, runtime.Version(), runtime.GOOS, runtime.GOARCH)
 	},
 }
 

src/cmds/restic/global.go

@@ -10,8 +10,6 @@ import (
 	"strings"
 	"syscall"
 
-	"github.com/spf13/cobra"
-
 	"restic/backend/local"
 	"restic/backend/rest"
 	"restic/backend/s3"
@@ -26,19 +24,6 @@ import (
 )
 
 var version = "compiled manually"
-var compiledAt = "unknown time"
-
-func parseEnvironment(cmd *cobra.Command, args []string) {
-	repo := os.Getenv("RESTIC_REPOSITORY")
-	if repo != "" {
-		globalOptions.Repo = repo
-	}
-
-	pw := os.Getenv("RESTIC_PASSWORD")
-	if pw != "" {
-		globalOptions.password = pw
-	}
-}
 
 // GlobalOptions hold all global options for restic.
 type GlobalOptions struct {
@@ -58,10 +43,15 @@ var globalOptions = GlobalOptions{
 }
 
 func init() {
+	pw := os.Getenv("RESTIC_PASSWORD")
+	if pw != "" {
+		globalOptions.password = pw
+	}
+
 	f := cmdRoot.PersistentFlags()
-	f.StringVarP(&globalOptions.Repo, "repo", "r", "", "repository to backup to or restore from (default: $RESTIC_REPOSITORY)")
+	f.StringVarP(&globalOptions.Repo, "repo", "r", os.Getenv("RESTIC_REPOSITORY"), "repository to backup to or restore from (default: $RESTIC_REPOSITORY)")
 	f.StringVarP(&globalOptions.PasswordFile, "password-file", "p", "", "read the repository password from a file")
-	f.BoolVarP(&globalOptions.Quiet, "quiet", "q", false, "do not outputcomprehensive progress report")
+	f.BoolVarP(&globalOptions.Quiet, "quiet", "q", false, "do not output comprehensive progress report")
 	f.BoolVar(&globalOptions.NoLock, "no-lock", false, "do not lock the repo, this allows some operations on read-only repos")
 
 	restoreTerminal()
@@ -132,7 +122,7 @@ func Printf(format string, args ...interface{}) {
 	_, err := fmt.Fprintf(globalOptions.stdout, format, args...)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to write to stdout: %v\n", err)
-		os.Exit(100)
+		Exit(100)
 	}
 }
 
@@ -175,18 +165,19 @@ func Warnf(format string, args ...interface{}) {
 	_, err := fmt.Fprintf(globalOptions.stderr, format, args...)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "unable to write to stderr: %v\n", err)
-		os.Exit(100)
+		Exit(100)
 	}
 }
 
-// Exitf uses Warnf to write the message and then calls os.Exit(exitcode).
+// Exitf uses Warnf to write the message and then terminates the process with
+// the given exit code.
 func Exitf(exitcode int, format string, args ...interface{}) {
 	if format[len(format)-1] != '\n' {
 		format += "\n"
 	}
 
 	Warnf(format, args...)
-	os.Exit(exitcode)
+	Exit(exitcode)
 }
 
 // readPassword reads the password from the given reader directly.

src/cmds/restic/global_debug.go

@@ -0,0 +1,60 @@
+// +build debug
+
+package main
+
+import (
+	"fmt"
+	"net/http"
+	_ "net/http/pprof"
+	"os"
+	"restic/errors"
+
+	"github.com/pkg/profile"
+)
+
+var (
+	listenMemoryProfile string
+	memProfilePath      string
+	cpuProfilePath      string
+
+	prof interface {
+		Stop()
+	}
+)
+
+func init() {
+	f := cmdRoot.PersistentFlags()
+	f.StringVar(&listenMemoryProfile, "listen-profile", "", "listen on this `address:port` for memory profiling")
+	f.StringVar(&memProfilePath, "mem-profile", "", "write memory profile to `dir`")
+	f.StringVar(&cpuProfilePath, "cpu-profile", "", "write cpu profile to `dir`")
+}
+
+func runDebug() error {
+	if listenMemoryProfile != "" {
+		fmt.Fprintf(os.Stderr, "running memory profile HTTP server on %v\n", listenMemoryProfile)
+		go func() {
+			err := http.ListenAndServe(listenMemoryProfile, nil)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "memory profile listen failed: %v\n", err)
+			}
+		}()
+	}
+
+	if memProfilePath != "" && cpuProfilePath != "" {
+		return errors.Fatal("only one profile (memory or CPU) may be activated at the same time")
+	}
+
+	if memProfilePath != "" {
+		prof = profile.Start(profile.Quiet, profile.MemProfile, profile.ProfilePath(memProfilePath))
+	} else if memProfilePath != "" {
+		prof = profile.Start(profile.Quiet, profile.CPUProfile, profile.ProfilePath(memProfilePath))
+	}
+
+	return nil
+}
+
+func shutdownDebug() {
+	if prof != nil {
+		prof.Stop()
+	}
+}

src/cmds/restic/global_release.go

@@ -0,0 +1,9 @@
+// +build !debug
+
+package main
+
+// runDebug is a noop without the debug tag.
+func runDebug() error { return nil }
+
+// shutdownDebug is a noop without the debug tag.
+func shutdownDebug() {}

src/cmds/restic/integration_test.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	mrand "math/rand"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -159,6 +160,15 @@ func testRunFind(t testing.TB, gopts GlobalOptions, pattern string) []string {
 	return strings.Split(string(buf.Bytes()), "\n")
 }
 
+func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) {
+	opts := ForgetOptions{}
+	OK(t, runForget(opts, gopts, args))
+}
+
+func testRunPrune(t testing.TB, gopts GlobalOptions) {
+	OK(t, runPrune(gopts))
+}
+
 func TestBackup(t *testing.T) {
 	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
 		datafile := filepath.Join("testdata", "backup-data.tar.gz")
@@ -343,6 +353,52 @@ func TestBackupMissingFile2(t *testing.T) {
 	})
 }
 
+func TestBackupChangedFile(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		fd, err := os.Open(datafile)
+		if os.IsNotExist(errors.Cause(err)) {
+			t.Skipf("unable to find data file %q, skipping", datafile)
+			return
+		}
+		OK(t, err)
+		OK(t, fd.Close())
+
+		SetupTarTestFixture(t, env.testdata, datafile)
+
+		testRunInit(t, gopts)
+
+		globalOptions.stderr = ioutil.Discard
+		defer func() {
+			globalOptions.stderr = os.Stderr
+		}()
+
+		modFile := filepath.Join(env.testdata, "0", "0", "6", "18")
+
+		ranHook := false
+		debug.Hook("archiver.SaveFile", func(context interface{}) {
+			pathname := context.(string)
+
+			if pathname != modFile {
+				return
+			}
+
+			t.Logf("in hook, modifying test file %v", modFile)
+			ranHook = true
+
+			OK(t, ioutil.WriteFile(modFile, []byte("modified"), 0600))
+		})
+
+		opts := BackupOptions{}
+
+		testRunBackup(t, []string{env.testdata}, opts, gopts)
+		testRunCheck(t, gopts)
+
+		Assert(t, ranHook, "hook did not run")
+		debug.RemoveHook("archiver.SaveFile")
+	})
+}
+
 func TestBackupDirectoryError(t *testing.T) {
 	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
 		datafile := filepath.Join("testdata", "backup-data.tar.gz")
@@ -684,6 +740,30 @@ func TestRestoreFilter(t *testing.T) {
 	})
 }
 
+func TestRestore(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+		testRunInit(t, gopts)
+
+		for i := 0; i < 10; i++ {
+			p := filepath.Join(env.testdata, fmt.Sprintf("foo/bar/testfile%v", i))
+			OK(t, os.MkdirAll(filepath.Dir(p), 0755))
+			OK(t, appendRandomData(p, uint(mrand.Intn(5<<21))))
+		}
+
+		opts := BackupOptions{}
+
+		testRunBackup(t, []string{env.testdata}, opts, gopts)
+		testRunCheck(t, gopts)
+
+		// Restore latest without any filters
+		restoredir := filepath.Join(env.base, "restore")
+		testRunRestoreLatest(t, gopts, restoredir, nil, "")
+
+		Assert(t, directoriesEqualContents(env.testdata, filepath.Join(restoredir, filepath.Base(env.testdata))),
+			"directories are not equal")
+	})
+}
+
 func TestRestoreLatest(t *testing.T) {
 
 	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
@@ -901,3 +981,33 @@ func TestCheckRestoreNoLock(t *testing.T) {
 		testRunRestore(t, gopts, filepath.Join(env.base, "restore"), snapshotIDs[0])
 	})
 }
+
+func TestPrune(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		fd, err := os.Open(datafile)
+		if os.IsNotExist(errors.Cause(err)) {
+			t.Skipf("unable to find data file %q, skipping", datafile)
+			return
+		}
+		OK(t, err)
+		OK(t, fd.Close())
+
+		testRunInit(t, gopts)
+
+		SetupTarTestFixture(t, env.testdata, datafile)
+		opts := BackupOptions{}
+
+		testRunBackup(t, []string{filepath.Join(env.testdata, "0", "0", "1")}, opts, gopts)
+		testRunBackup(t, []string{filepath.Join(env.testdata, "0", "0", "2")}, opts, gopts)
+		testRunBackup(t, []string{filepath.Join(env.testdata, "0", "0", "3")}, opts, gopts)
+
+		snapshotIDs := testRunList(t, "snapshots", gopts)
+		Assert(t, len(snapshotIDs) == 3,
+			"expected one snapshot, got %v", snapshotIDs)
+
+		testRunForget(t, gopts, snapshotIDs[0].String())
+		testRunPrune(t, gopts)
+		testRunCheck(t, gopts)
+	})
+}

src/cmds/restic/lock.go

@@ -36,6 +36,7 @@ func lockRepository(repo *repository.Repository, exclusive bool) (*restic.Lock,
 	if err != nil {
 		return nil, err
 	}
+	debug.Log("create lock %p (exclusive %v)", lock, exclusive)
 
 	globalLocks.Lock()
 	if globalLocks.cancelRefresh == nil {
@@ -88,7 +89,7 @@ func unlockRepo(lock *restic.Lock) error {
 	globalLocks.Lock()
 	defer globalLocks.Unlock()
 
-	debug.Log("unlocking repository")
+	debug.Log("unlocking repository with lock %p", lock)
 	if err := lock.Unlock(); err != nil {
 		debug.Log("error while unlocking: %v", err)
 		return err

src/cmds/restic/main.go

@@ -5,7 +5,6 @@ import (
 	"os"
 	"restic"
 	"restic/debug"
-	"runtime"
 
 	"github.com/spf13/cobra"
 
@@ -20,20 +19,17 @@ var cmdRoot = &cobra.Command{
 restic is a backup program which allows saving multiple revisions of files and
 directories in an encrypted repository stored on different backends.
 `,
-	SilenceErrors:    true,
-	SilenceUsage:     true,
-	PersistentPreRun: parseEnvironment,
-}
+	SilenceErrors: true,
+	SilenceUsage:  true,
 
-func init() {
-	// set GOMAXPROCS to number of CPUs
-	if runtime.Version() < "go1.5" {
-		gomaxprocs := os.Getenv("GOMAXPROCS")
-		debug.Log("read GOMAXPROCS from env variable, value: %s", gomaxprocs)
-		if gomaxprocs == "" {
-			runtime.GOMAXPROCS(runtime.NumCPU())
-		}
-	}
+	// run the debug functions for all subcommands (if build tag "debug" is
+	// enabled)
+	PersistentPreRunE: func(*cobra.Command, []string) error {
+		return runDebug()
+	},
+	PersistentPostRun: func(*cobra.Command, []string) {
+		shutdownDebug()
+	},
 }
 
 func main() {
@@ -49,9 +45,10 @@ func main() {
 		fmt.Fprintf(os.Stderr, "%+v\n", err)
 	}
 
-	RunCleanupHandlers()
-
+	var exitCode int
 	if err != nil {
-		os.Exit(1)
+		exitCode = 1
 	}
+
+	Exit(exitCode)
 }

src/restic/archiver/archive_reader_test.go

@@ -44,7 +44,7 @@ func checkSavedFile(t *testing.T, repo restic.Repository, treeID restic.ID, name
 			t.Fatal(err)
 		}
 
-		buf := make([]byte, int(size))
+		buf := restic.NewBlobBuffer(int(size))
 		n := loadBlob(t, repo, id, buf)
 		if n != len(buf) {
 			t.Errorf("wrong number of bytes read, want %d, got %d", len(buf), n)

src/restic/archiver/archiver.go

@@ -26,7 +26,9 @@ const (
 	maxConcurrency     = 10
 )
 
-var archiverAbortOnAllErrors = func(str string, fi os.FileInfo, err error) error { return err }
+var archiverPrintWarnings = func(path string, fi os.FileInfo, err error) {
+	fmt.Fprintf(os.Stderr, "warning for %v: %v", path, err)
+}
 var archiverAllowAllFiles = func(string, os.FileInfo) bool { return true }
 
 // Archiver is used to backup a set of directories.
@@ -39,7 +41,7 @@ type Archiver struct {
 
 	blobToken chan struct{}
 
-	Error        func(dir string, fi os.FileInfo, err error) error
+	Warn         func(dir string, fi os.FileInfo, err error)
 	SelectFilter pipe.SelectFunc
 	Excludes     []string
 }
@@ -61,7 +63,7 @@ func New(repo restic.Repository) *Archiver {
 		arch.blobToken <- struct{}{}
 	}
 
-	arch.Error = archiverAbortOnAllErrors
+	arch.Warn = archiverPrintWarnings
 	arch.SelectFilter = archiverAllowAllFiles
 
 	return arch
@@ -135,10 +137,7 @@ func (arch *Archiver) reloadFileIfChanged(node *restic.Node, file fs.File) (*res
 		return node, nil
 	}
 
-	err = arch.Error(node.Path, fi, errors.New("file has changed"))
-	if err != nil {
-		return nil, err
-	}
+	arch.Warn(node.Path, fi, errors.New("file has changed"))
 
 	node, err = restic.NodeFromFileInfo(node.Path, fi)
 	if err != nil {
@@ -197,7 +196,7 @@ func updateNodeContent(node *restic.Node, results []saveResult) error {
 	}
 
 	if bytes != node.Size {
-		return errors.Errorf("errors saving node %q: saved %d bytes, wanted %d bytes", node.Path, bytes, node.Size)
+		fmt.Fprintf(os.Stderr, "warning for %v: expected %d bytes, saved %d bytes\n", node.Path, node.Size, bytes)
 	}
 
 	debug.Log("SaveFile(%q): %v blobs\n", node.Path, len(results))
@@ -207,16 +206,18 @@ func updateNodeContent(node *restic.Node, results []saveResult) error {
 
 // SaveFile stores the content of the file on the backend as a Blob by calling
 // Save for each chunk.
-func (arch *Archiver) SaveFile(p *restic.Progress, node *restic.Node) error {
+func (arch *Archiver) SaveFile(p *restic.Progress, node *restic.Node) (*restic.Node, error) {
 	file, err := fs.Open(node.Path)
 	defer file.Close()
 	if err != nil {
-		return errors.Wrap(err, "Open")
+		return node, errors.Wrap(err, "Open")
 	}
 
+	debug.RunHook("archiver.SaveFile", node.Path)
+
 	node, err = arch.reloadFileIfChanged(node, file)
 	if err != nil {
-		return err
+		return node, err
 	}
 
 	chnker := chunker.New(file, arch.repo.Config().ChunkerPolynomial)
@@ -229,7 +230,7 @@ func (arch *Archiver) SaveFile(p *restic.Progress, node *restic.Node) error {
 		}
 
 		if err != nil {
-			return errors.Wrap(err, "chunker.Next")
+			return node, errors.Wrap(err, "chunker.Next")
 		}
 
 		resCh := make(chan saveResult, 1)
@@ -239,11 +240,11 @@ func (arch *Archiver) SaveFile(p *restic.Progress, node *restic.Node) error {
 
 	results, err := waitForResults(resultChannels)
 	if err != nil {
-		return err
+		return node, err
 	}
-
 	err = updateNodeContent(node, results)
-	return err
+
+	return node, err
 }
 
 func (arch *Archiver) fileWorker(wg *sync.WaitGroup, p *restic.Progress, done <-chan struct{}, entCh <-chan pipe.Entry) {
@@ -307,7 +308,7 @@ func (arch *Archiver) fileWorker(wg *sync.WaitGroup, p *restic.Progress, done <-
 			// otherwise read file normally
 			if node.Type == "file" && len(node.Content) == 0 {
 				debug.Log("   read and save %v, content: %v", e.Path(), node.Content)
-				err = arch.SaveFile(p, node)
+				node, err = arch.SaveFile(p, node)
 				if err != nil {
 					// TODO: integrate error reporting
 					fmt.Fprintf(os.Stderr, "error for %v: %v\n", node.Path, err)
@@ -727,19 +728,6 @@ func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, parentI
 
 	debug.Log("workers terminated")
 
-	// receive the top-level tree
-	root := (<-resCh).(*restic.Node)
-	debug.Log("root node received: %v", root.Subtree.Str())
-	sn.Tree = root.Subtree
-
-	// save snapshot
-	id, err := arch.repo.SaveJSONUnpacked(restic.SnapshotFile, sn)
-	if err != nil {
-		return nil, restic.ID{}, err
-	}
-
-	debug.Log("saved snapshot %v", id.Str())
-
 	// flush repository
 	err = arch.repo.Flush()
 	if err != nil {
@@ -755,6 +743,19 @@ func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, parentI
 
 	debug.Log("saved indexes")
 
+	// receive the top-level tree
+	root := (<-resCh).(*restic.Node)
+	debug.Log("root node received: %v", root.Subtree.Str())
+	sn.Tree = root.Subtree
+
+	// save snapshot
+	id, err := arch.repo.SaveJSONUnpacked(restic.SnapshotFile, sn)
+	if err != nil {
+		return nil, restic.ID{}, err
+	}
+
+	debug.Log("saved snapshot %v", id.Str())
+
 	return sn, id, nil
 }
 

src/restic/archiver/archiver_duplication_test.go

@@ -39,15 +39,15 @@ func randomID() restic.ID {
 func forgetfulBackend() restic.Backend {
 	be := &mock.Backend{}
 
-	be.TestFn = func(t restic.FileType, name string) (bool, error) {
+	be.TestFn = func(h restic.Handle) (bool, error) {
 		return false, nil
 	}
 
-	be.LoadFn = func(h restic.Handle, p []byte, off int64) (int, error) {
-		return 0, errors.New("not found")
+	be.LoadFn = func(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+		return nil, errors.New("not found")
 	}
 
-	be.SaveFn = func(h restic.Handle, p []byte) error {
+	be.SaveFn = func(h restic.Handle, rd io.Reader) error {
 		return nil
 	}
 
@@ -55,7 +55,7 @@ func forgetfulBackend() restic.Backend {
 		return restic.FileInfo{}, errors.New("not found")
 	}
 
-	be.RemoveFn = func(t restic.FileType, name string) error {
+	be.RemoveFn = func(h restic.Handle) error {
 		return nil
 	}
 
@@ -142,6 +142,11 @@ func testArchiverDuplication(t *testing.T) {
 	close(done)
 
 	wg.Wait()
+
+	err = repo.Flush()
+	if err != nil {
+		t.Fatal(err)
+	}
 }
 
 func TestArchiverDuplication(t *testing.T) {

src/restic/backend.go

@@ -1,5 +1,7 @@
 package restic
 
+import "io"
+
 // Backend is used to store and access data.
 type Backend interface {
 	// Location returns a string that describes the type and location of the
@@ -7,22 +9,22 @@ type Backend interface {
 	Location() string
 
 	// Test a boolean value whether a File with the name and type exists.
-	Test(t FileType, name string) (bool, error)
+	Test(h Handle) (bool, error)
 
 	// Remove removes a File with type t and name.
-	Remove(t FileType, name string) error
+	Remove(h Handle) error
 
 	// Close the backend
 	Close() error
 
-	// Load returns the data stored in the backend for h at the given offset
-	// and saves it in p. Load has the same semantics as io.ReaderAt, except
-	// that a negative offset is also allowed. In this case it references a
-	// position relative to the end of the file (similar to Seek()).
-	Load(h Handle, p []byte, off int64) (int, error)
-
 	// Save stores the data in the backend under the given handle.
-	Save(h Handle, p []byte) error
+	Save(h Handle, rd io.Reader) error
+
+	// Load returns a reader that yields the contents of the file at h at the
+	// given offset. If length is larger than zero, only a portion of the file
+	// is returned. rd must be closed after use. If an error is returned, the
+	// ReadCloser must be nil.
+	Load(h Handle, length int, offset int64) (io.ReadCloser, error)
 
 	// Stat returns information about the File identified by h.
 	Stat(h Handle) (FileInfo, error)

src/restic/backend/local/backend_test.go

@@ -51,13 +51,6 @@ func TestLocalBackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestLocalBackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestLocalBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/local/local.go

@@ -101,67 +101,22 @@ func dirname(base string, t restic.FileType, name string) string {
 	return filepath.Join(base, n)
 }
 
-// Load returns the data stored in the backend for h at the given offset and
-// saves it in p. Load has the same semantics as io.ReaderAt, with one
-// exception: when off is lower than zero, it is treated as an offset relative
-// to the end of the file.
-func (b *Local) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
-	debug.Log("Load %v, length %v at %v", h, len(p), off)
-	if err := h.Valid(); err != nil {
-		return 0, err
-	}
-
-	f, err := fs.Open(filename(b.p, h.Type, h.Name))
-	if err != nil {
-		return 0, errors.Wrap(err, "Open")
-	}
-
-	defer func() {
-		e := f.Close()
-		if err == nil && e != nil {
-			err = errors.Wrap(e, "Close")
-		}
-	}()
-
-	switch {
-	case off > 0:
-		_, err = f.Seek(off, 0)
-	case off < 0:
-		_, err = f.Seek(off, 2)
-	}
-
-	if err != nil {
-		return 0, errors.Wrap(err, "Seek")
-	}
-
-	return io.ReadFull(f, p)
-}
-
-// writeToTempfile saves p into a tempfile in tempdir.
-func writeToTempfile(tempdir string, p []byte) (filename string, err error) {
+// copyToTempfile saves p into a tempfile in tempdir.
+func copyToTempfile(tempdir string, rd io.Reader) (filename string, err error) {
 	tmpfile, err := ioutil.TempFile(tempdir, "temp-")
 	if err != nil {
 		return "", errors.Wrap(err, "TempFile")
 	}
 
-	n, err := tmpfile.Write(p)
+	_, err = io.Copy(tmpfile, rd)
 	if err != nil {
 		return "", errors.Wrap(err, "Write")
 	}
 
-	if n != len(p) {
-		return "", errors.New("not all bytes writen")
-	}
-
 	if err = tmpfile.Sync(); err != nil {
 		return "", errors.Wrap(err, "Syncn")
 	}
 
-	err = fs.ClearCache(tmpfile)
-	if err != nil {
-		return "", errors.Wrap(err, "ClearCache")
-	}
-
 	err = tmpfile.Close()
 	if err != nil {
 		return "", errors.Wrap(err, "Close")
@@ -171,14 +126,14 @@ func writeToTempfile(tempdir string, p []byte) (filename string, err error) {
 }
 
 // Save stores data in the backend at the handle.
-func (b *Local) Save(h restic.Handle, p []byte) (err error) {
-	debug.Log("Save %v, length %v", h, len(p))
+func (b *Local) Save(h restic.Handle, rd io.Reader) (err error) {
+	debug.Log("Save %v", h)
 	if err := h.Valid(); err != nil {
 		return err
 	}
 
-	tmpfile, err := writeToTempfile(filepath.Join(b.p, backend.Paths.Temp), p)
-	debug.Log("saved %v (%d bytes) to %v", h, len(p), tmpfile)
+	tmpfile, err := copyToTempfile(filepath.Join(b.p, backend.Paths.Temp), rd)
+	debug.Log("saved %v to %v", h, tmpfile)
 	if err != nil {
 		return err
 	}
@@ -215,6 +170,39 @@ func (b *Local) Save(h restic.Handle, p []byte) (err error) {
 	return setNewFileMode(filename, fi)
 }
 
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (b *Local) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	f, err := os.Open(filename(b.p, h.Type, h.Name))
+	if err != nil {
+		return nil, err
+	}
+
+	if offset > 0 {
+		_, err = f.Seek(offset, 0)
+		if err != nil {
+			f.Close()
+			return nil, err
+		}
+	}
+
+	if length > 0 {
+		return backend.LimitReadCloser(f, int64(length)), nil
+	}
+
+	return f, nil
+}
+
 // Stat returns information about a blob.
 func (b *Local) Stat(h restic.Handle) (restic.FileInfo, error) {
 	debug.Log("Stat %v", h)
@@ -231,9 +219,9 @@ func (b *Local) Stat(h restic.Handle) (restic.FileInfo, error) {
 }
 
 // Test returns true if a blob of the given type and name exists in the backend.
-func (b *Local) Test(t restic.FileType, name string) (bool, error) {
-	debug.Log("Test %v %v", t, name)
-	_, err := fs.Stat(filename(b.p, t, name))
+func (b *Local) Test(h restic.Handle) (bool, error) {
+	debug.Log("Test %v", h)
+	_, err := fs.Stat(filename(b.p, h.Type, h.Name))
 	if err != nil {
 		if os.IsNotExist(errors.Cause(err)) {
 			return false, nil
@@ -245,9 +233,9 @@ func (b *Local) Test(t restic.FileType, name string) (bool, error) {
 }
 
 // Remove removes the blob with the given name and type.
-func (b *Local) Remove(t restic.FileType, name string) error {
-	debug.Log("Remove %v %v", t, name)
-	fn := filename(b.p, t, name)
+func (b *Local) Remove(h restic.Handle) error {
+	debug.Log("Remove %v", h)
+	fn := filename(b.p, h.Type, h.Name)
 
 	// reset read-only flag
 	err := fs.Chmod(fn, 0666)

src/restic/backend/mem/backend_test.go

@@ -51,13 +51,6 @@ func TestMemBackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestMemBackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestMemBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/mem/mem_backend.go

@@ -1,21 +1,19 @@
 package mem
 
 import (
+	"bytes"
 	"io"
+	"io/ioutil"
 	"restic"
 	"sync"
 
+	"restic/backend"
 	"restic/errors"
 
 	"restic/debug"
 )
 
-type entry struct {
-	Type restic.FileType
-	Name string
-}
-
-type memMap map[entry][]byte
+type memMap map[restic.Handle][]byte
 
 // make sure that MemoryBackend implements backend.Backend
 var _ restic.Backend = &MemoryBackend{}
@@ -39,61 +37,21 @@ func New() *MemoryBackend {
 }
 
 // Test returns whether a file exists.
-func (be *MemoryBackend) Test(t restic.FileType, name string) (bool, error) {
+func (be *MemoryBackend) Test(h restic.Handle) (bool, error) {
 	be.m.Lock()
 	defer be.m.Unlock()
 
-	debug.Log("test %v %v", t, name)
+	debug.Log("Test %v", h)
 
-	if _, ok := be.data[entry{t, name}]; ok {
+	if _, ok := be.data[h]; ok {
 		return true, nil
 	}
 
 	return false, nil
 }
 
-// Load reads data from the backend.
-func (be *MemoryBackend) Load(h restic.Handle, p []byte, off int64) (int, error) {
-	if err := h.Valid(); err != nil {
-		return 0, err
-	}
-
-	be.m.Lock()
-	defer be.m.Unlock()
-
-	if h.Type == restic.ConfigFile {
-		h.Name = ""
-	}
-
-	debug.Log("get %v offset %v len %v", h, off, len(p))
-
-	if _, ok := be.data[entry{h.Type, h.Name}]; !ok {
-		return 0, errors.New("no such data")
-	}
-
-	buf := be.data[entry{h.Type, h.Name}]
-	switch {
-	case off > int64(len(buf)):
-		return 0, errors.New("offset beyond end of file")
-	case off < -int64(len(buf)):
-		off = 0
-	case off < 0:
-		off = int64(len(buf)) + off
-	}
-
-	buf = buf[off:]
-
-	n := copy(p, buf)
-
-	if len(p) > len(buf) {
-		return n, io.ErrUnexpectedEOF
-	}
-
-	return n, nil
-}
-
 // Save adds new Data to the backend.
-func (be *MemoryBackend) Save(h restic.Handle, p []byte) error {
+func (be *MemoryBackend) Save(h restic.Handle, rd io.Reader) error {
 	if err := h.Valid(); err != nil {
 		return err
 	}
@@ -105,18 +63,59 @@ func (be *MemoryBackend) Save(h restic.Handle, p []byte) error {
 		h.Name = ""
 	}
 
-	if _, ok := be.data[entry{h.Type, h.Name}]; ok {
+	if _, ok := be.data[h]; ok {
 		return errors.New("file already exists")
 	}
 
-	debug.Log("save %v bytes at %v", len(p), h)
-	buf := make([]byte, len(p))
-	copy(buf, p)
-	be.data[entry{h.Type, h.Name}] = buf
+	buf, err := ioutil.ReadAll(rd)
+	if err != nil {
+		return err
+	}
+
+	be.data[h] = buf
+	debug.Log("saved %v bytes at %v", len(buf), h)
 
 	return nil
 }
 
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (be *MemoryBackend) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	be.m.Lock()
+	defer be.m.Unlock()
+
+	if h.Type == restic.ConfigFile {
+		h.Name = ""
+	}
+
+	debug.Log("Load %v offset %v len %v", h, offset, length)
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	if _, ok := be.data[h]; !ok {
+		return nil, errors.New("no such data")
+	}
+
+	buf := be.data[h]
+	if offset > int64(len(buf)) {
+		return nil, errors.New("offset beyond end of file")
+	}
+
+	buf = buf[offset:]
+	if length > 0 && len(buf) > length {
+		buf = buf[:length]
+	}
+
+	return backend.Closer{Reader: bytes.NewReader(buf)}, nil
+}
+
 // Stat returns information about a file in the backend.
 func (be *MemoryBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 	be.m.Lock()
@@ -132,7 +131,7 @@ func (be *MemoryBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 
 	debug.Log("stat %v", h)
 
-	e, ok := be.data[entry{h.Type, h.Name}]
+	e, ok := be.data[h]
 	if !ok {
 		return restic.FileInfo{}, errors.New("no such data")
 	}
@@ -141,17 +140,17 @@ func (be *MemoryBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 }
 
 // Remove deletes a file from the backend.
-func (be *MemoryBackend) Remove(t restic.FileType, name string) error {
+func (be *MemoryBackend) Remove(h restic.Handle) error {
 	be.m.Lock()
 	defer be.m.Unlock()
 
-	debug.Log("get %v %v", t, name)
+	debug.Log("Remove %v", h)
 
-	if _, ok := be.data[entry{t, name}]; !ok {
+	if _, ok := be.data[h]; !ok {
 		return errors.New("no such data")
 	}
 
-	delete(be.data, entry{t, name})
+	delete(be.data, h)
 
 	return nil
 }

src/restic/backend/rest/backend_test.go

@@ -51,13 +51,6 @@ func TestRestBackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestRestBackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestRestBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/rest/rest.go

@@ -1,16 +1,17 @@
 package rest
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"path"
 	"restic"
 	"strings"
 
+	"restic/debug"
 	"restic/errors"
 
 	"restic/backend"
@@ -18,6 +19,9 @@ import (
 
 const connLimit = 10
 
+// make sure the rest backend implements restic.Backend
+var _ restic.Backend = &restBackend{}
+
 // restPath returns the path to the given resource.
 func restPath(url *url.URL, h restic.Handle) string {
 	u := *url
@@ -59,7 +63,7 @@ func Open(cfg Config) (restic.Backend, error) {
 	for i := 0; i < connLimit; i++ {
 		connChan <- struct{}{}
 	}
-	tr := &http.Transport{}
+	tr := &http.Transport{MaxIdleConnsPerHost: connLimit}
 	client := http.Client{Transport: tr}
 
 	return &restBackend{url: cfg.URL, connChan: connChan, client: client}, nil
@@ -70,68 +74,23 @@ func (b *restBackend) Location() string {
 	return b.url.String()
 }
 
-// Load returns the data stored in the backend for h at the given offset
-// and saves it in p. Load has the same semantics as io.ReaderAt.
-func (b *restBackend) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
-	if err := h.Valid(); err != nil {
-		return 0, err
-	}
-
-	// invert offset
-	if off < 0 {
-		info, err := b.Stat(h)
-		if err != nil {
-			return 0, errors.Wrap(err, "Stat")
-		}
-
-		if -off > info.Size {
-			off = 0
-		} else {
-			off = info.Size + off
-		}
-	}
-
-	req, err := http.NewRequest("GET", restPath(b.url, h), nil)
-	if err != nil {
-		return 0, errors.Wrap(err, "http.NewRequest")
-	}
-	req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", off, off+int64(len(p))))
-	<-b.connChan
-	resp, err := b.client.Do(req)
-	b.connChan <- struct{}{}
-
-	if resp != nil {
-		defer func() {
-			e := resp.Body.Close()
-
-			if err == nil {
-				err = errors.Wrap(e, "Close")
-			}
-		}()
-	}
-
-	if err != nil {
-		return 0, errors.Wrap(err, "client.Do")
-	}
-	if resp.StatusCode != 200 && resp.StatusCode != 206 {
-		return 0, errors.Errorf("unexpected HTTP response code %v", resp.StatusCode)
-	}
-
-	return io.ReadFull(resp.Body, p)
-}
-
 // Save stores data in the backend at the handle.
-func (b *restBackend) Save(h restic.Handle, p []byte) (err error) {
+func (b *restBackend) Save(h restic.Handle, rd io.Reader) (err error) {
 	if err := h.Valid(); err != nil {
 		return err
 	}
 
+	// make sure that client.Post() cannot close the reader by wrapping it in
+	// backend.Closer, which has a noop method.
+	rd = backend.Closer{Reader: rd}
+
 	<-b.connChan
-	resp, err := b.client.Post(restPath(b.url, h), "binary/octet-stream", bytes.NewReader(p))
+	resp, err := b.client.Post(restPath(b.url, h), "binary/octet-stream", rd)
 	b.connChan <- struct{}{}
 
 	if resp != nil {
 		defer func() {
+			io.Copy(ioutil.Discard, resp.Body)
 			e := resp.Body.Close()
 
 			if err == nil {
@@ -151,6 +110,56 @@ func (b *restBackend) Save(h restic.Handle, p []byte) (err error) {
 	return nil
 }
 
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (b *restBackend) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	if length < 0 {
+		return nil, errors.Errorf("invalid length %d", length)
+	}
+
+	req, err := http.NewRequest("GET", restPath(b.url, h), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "http.NewRequest")
+	}
+
+	byteRange := fmt.Sprintf("bytes=%d-", offset)
+	if length > 0 {
+		byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1)
+	}
+	req.Header.Add("Range", byteRange)
+	debug.Log("Load(%v) send range %v", h, byteRange)
+
+	<-b.connChan
+	resp, err := b.client.Do(req)
+	b.connChan <- struct{}{}
+
+	if err != nil {
+		if resp != nil {
+			io.Copy(ioutil.Discard, resp.Body)
+			resp.Body.Close()
+		}
+		return nil, errors.Wrap(err, "client.Do")
+	}
+
+	if resp.StatusCode != 200 && resp.StatusCode != 206 {
+		io.Copy(ioutil.Discard, resp.Body)
+		resp.Body.Close()
+		return nil, errors.Errorf("unexpected HTTP response code %v", resp.StatusCode)
+	}
+
+	return resp.Body, nil
+}
+
 // Stat returns information about a blob.
 func (b *restBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 	if err := h.Valid(); err != nil {
@@ -164,6 +173,7 @@ func (b *restBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 		return restic.FileInfo{}, errors.Wrap(err, "client.Head")
 	}
 
+	io.Copy(ioutil.Discard, resp.Body)
 	if err = resp.Body.Close(); err != nil {
 		return restic.FileInfo{}, errors.Wrap(err, "Close")
 	}
@@ -184,8 +194,8 @@ func (b *restBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 }
 
 // Test returns true if a blob of the given type and name exists in the backend.
-func (b *restBackend) Test(t restic.FileType, name string) (bool, error) {
-	_, err := b.Stat(restic.Handle{Type: t, Name: name})
+func (b *restBackend) Test(h restic.Handle) (bool, error) {
+	_, err := b.Stat(h)
 	if err != nil {
 		return false, nil
 	}
@@ -194,8 +204,7 @@ func (b *restBackend) Test(t restic.FileType, name string) (bool, error) {
 }
 
 // Remove removes the blob with the given name and type.
-func (b *restBackend) Remove(t restic.FileType, name string) error {
-	h := restic.Handle{Type: t, Name: name}
+func (b *restBackend) Remove(h restic.Handle) error {
 	if err := h.Valid(); err != nil {
 		return err
 	}
@@ -216,6 +225,7 @@ func (b *restBackend) Remove(t restic.FileType, name string) error {
 		return errors.New("blob not removed")
 	}
 
+	io.Copy(ioutil.Discard, resp.Body)
 	return resp.Body.Close()
 }
 
@@ -235,7 +245,14 @@ func (b *restBackend) List(t restic.FileType, done <-chan struct{}) <-chan strin
 	b.connChan <- struct{}{}
 
 	if resp != nil {
-		defer resp.Body.Close()
+		defer func() {
+			io.Copy(ioutil.Discard, resp.Body)
+			e := resp.Body.Close()
+
+			if err == nil {
+				err = errors.Wrap(e, "Close")
+			}
+		}()
 	}
 
 	if err != nil {

src/restic/backend/rest/rest_test.go

@@ -37,7 +37,7 @@ func init() {
 			return nil, err
 		}
 
-		exists, err := be.Test(restic.ConfigFile, "")
+		exists, err := be.Test(restic.Handle{Type: restic.ConfigFile, Name: ""})
 		if err != nil {
 			return nil, err
 		}

src/restic/backend/s3/backend_test.go

@@ -51,13 +51,6 @@ func TestS3BackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestS3BackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestS3BackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/s3/s3.go

@@ -3,9 +3,11 @@ package s3
 import (
 	"bytes"
 	"io"
+	"path"
 	"restic"
 	"strings"
 
+	"restic/backend"
 	"restic/errors"
 
 	"github.com/minio/minio-go"
@@ -36,13 +38,13 @@ func Open(cfg Config) (restic.Backend, error) {
 	be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix}
 	be.createConnections()
 
-	ok, err := client.BucketExists(cfg.Bucket)
+	found, err := client.BucketExists(cfg.Bucket)
 	if err != nil {
-		debug.Log("BucketExists(%v) returned err %v, trying to create the bucket", cfg.Bucket, err)
+		debug.Log("BucketExists(%v) returned err %v", cfg.Bucket, err)
 		return nil, errors.Wrap(err, "client.BucketExists")
 	}
 
-	if !ok {
+	if !found {
 		// create new bucket with default ACL in default region
 		err = client.MakeBucket(cfg.Bucket, "")
 		if err != nil {
@@ -53,18 +55,11 @@ func Open(cfg Config) (restic.Backend, error) {
 	return be, nil
 }
 
-func (be *s3) s3path(t restic.FileType, name string) string {
-	var path string
-
-	if be.prefix != "" {
-		path = be.prefix + "/"
+func (be *s3) s3path(h restic.Handle) string {
+	if h.Type == restic.ConfigFile {
+		return path.Join(be.prefix, string(h.Type))
 	}
-	path += string(t)
-
-	if t == restic.ConfigFile {
-		return path
-	}
-	return path + "/" + name
+	return path.Join(be.prefix, string(h.Type), h.Name)
 }
 
 func (be *s3) createConnections() {
@@ -79,91 +74,18 @@ func (be *s3) Location() string {
 	return be.bucketname
 }
 
-// Load returns the data stored in the backend for h at the given offset
-// and saves it in p. Load has the same semantics as io.ReaderAt.
-func (be s3) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
-	var obj *minio.Object
-
-	debug.Log("%v, offset %v, len %v", h, off, len(p))
-	path := be.s3path(h.Type, h.Name)
-
-	<-be.connChan
-	defer func() {
-		be.connChan <- struct{}{}
-	}()
-
-	obj, err = be.client.GetObject(be.bucketname, path)
-	if err != nil {
-		debug.Log("  err %v", err)
-		return 0, errors.Wrap(err, "client.GetObject")
-	}
-
-	// make sure that the object is closed properly.
-	defer func() {
-		e := obj.Close()
-		if err == nil {
-			err = errors.Wrap(e, "Close")
-		}
-	}()
-
-	info, err := obj.Stat()
-	if err != nil {
-		return 0, errors.Wrap(err, "obj.Stat")
-	}
-
-	// handle negative offsets
-	if off < 0 {
-		// if the negative offset is larger than the object itself, read from
-		// the beginning.
-		if -off > info.Size {
-			off = 0
-		} else {
-			// otherwise compute the offset from the end of the file.
-			off = info.Size + off
-		}
-	}
-
-	// return an error if the offset is beyond the end of the file
-	if off > info.Size {
-		return 0, errors.Wrap(io.EOF, "")
-	}
-
-	var nextError error
-
-	// manually create an io.ErrUnexpectedEOF
-	if off+int64(len(p)) > info.Size {
-		newlen := info.Size - off
-		p = p[:newlen]
-
-		nextError = io.ErrUnexpectedEOF
-
-		debug.Log("    capped buffer to %v byte", len(p))
-	}
-
-	n, err = obj.ReadAt(p, off)
-	if int64(n) == info.Size-off && errors.Cause(err) == io.EOF {
-		err = nil
-	}
-
-	if err == nil {
-		err = nextError
-	}
-
-	return n, err
-}
-
 // Save stores data in the backend at the handle.
-func (be s3) Save(h restic.Handle, p []byte) (err error) {
+func (be *s3) Save(h restic.Handle, rd io.Reader) (err error) {
 	if err := h.Valid(); err != nil {
 		return err
 	}
 
-	debug.Log("%v with %d bytes", h, len(p))
+	debug.Log("Save %v", h)
 
-	path := be.s3path(h.Type, h.Name)
+	objName := be.s3path(h)
 
 	// Check key does not already exist
-	_, err = be.client.StatObject(be.bucketname, path)
+	_, err = be.client.StatObject(be.bucketname, objName)
 	if err == nil {
 		debug.Log("%v already exists", h)
 		return errors.New("key already exists")
@@ -174,22 +96,100 @@ func (be s3) Save(h restic.Handle, p []byte) (err error) {
 		be.connChan <- struct{}{}
 	}()
 
-	debug.Log("PutObject(%v, %v, %v, %v)",
-		be.bucketname, path, int64(len(p)), "binary/octet-stream")
-	n, err := be.client.PutObject(be.bucketname, path, bytes.NewReader(p), "binary/octet-stream")
-	debug.Log("%v -> %v bytes, err %#v", path, n, err)
+	debug.Log("PutObject(%v, %v)",
+		be.bucketname, objName)
+	n, err := be.client.PutObject(be.bucketname, objName, rd, "binary/octet-stream")
+	debug.Log("%v -> %v bytes, err %#v", objName, n, err)
 
 	return errors.Wrap(err, "client.PutObject")
 }
 
-// Stat returns information about a blob.
-func (be s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) {
-	debug.Log("%v", h)
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	if length < 0 {
+		return nil, errors.Errorf("invalid length %d", length)
+	}
 
-	path := be.s3path(h.Type, h.Name)
 	var obj *minio.Object
 
-	obj, err = be.client.GetObject(be.bucketname, path)
+	objName := be.s3path(h)
+
+	<-be.connChan
+	defer func() {
+		be.connChan <- struct{}{}
+	}()
+
+	obj, err := be.client.GetObject(be.bucketname, objName)
+	if err != nil {
+		debug.Log("  err %v", err)
+		return nil, errors.Wrap(err, "client.GetObject")
+	}
+
+	// if we're going to read the whole object, just pass it on.
+	if length == 0 {
+		debug.Log("Load %v: pass on object", h)
+		_, err = obj.Seek(offset, 0)
+		if err != nil {
+			_ = obj.Close()
+			return nil, errors.Wrap(err, "obj.Seek")
+		}
+
+		return obj, nil
+	}
+
+	// otherwise use a buffer with ReadAt
+	info, err := obj.Stat()
+	if err != nil {
+		_ = obj.Close()
+		return nil, errors.Wrap(err, "obj.Stat")
+	}
+
+	if offset > info.Size {
+		_ = obj.Close()
+		return nil, errors.Errorf("offset larger than file size")
+	}
+
+	l := int64(length)
+	if offset+l > info.Size {
+		l = info.Size - offset
+	}
+
+	buf := make([]byte, l)
+	n, err := obj.ReadAt(buf, offset)
+	debug.Log("Load %v: use buffer with ReadAt: %v, %v", h, n, err)
+	if err == io.EOF {
+		debug.Log("Load %v: shorten buffer %v -> %v", h, len(buf), n)
+		buf = buf[:n]
+		err = nil
+	}
+
+	if err != nil {
+		_ = obj.Close()
+		return nil, errors.Wrap(err, "obj.ReadAt")
+	}
+
+	return backend.Closer{Reader: bytes.NewReader(buf)}, nil
+}
+
+// Stat returns information about a blob.
+func (be *s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) {
+	debug.Log("%v", h)
+
+	objName := be.s3path(h)
+	var obj *minio.Object
+
+	obj, err = be.client.GetObject(be.bucketname, objName)
 	if err != nil {
 		debug.Log("GetObject() err %v", err)
 		return restic.FileInfo{}, errors.Wrap(err, "client.GetObject")
@@ -213,10 +213,10 @@ func (be s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) {
 }
 
 // Test returns true if a blob of the given type and name exists in the backend.
-func (be *s3) Test(t restic.FileType, name string) (bool, error) {
+func (be *s3) Test(h restic.Handle) (bool, error) {
 	found := false
-	path := be.s3path(t, name)
-	_, err := be.client.StatObject(be.bucketname, path)
+	objName := be.s3path(h)
+	_, err := be.client.StatObject(be.bucketname, objName)
 	if err == nil {
 		found = true
 	}
@@ -226,10 +226,10 @@ func (be *s3) Test(t restic.FileType, name string) (bool, error) {
 }
 
 // Remove removes the blob with the given name and type.
-func (be *s3) Remove(t restic.FileType, name string) error {
-	path := be.s3path(t, name)
-	err := be.client.RemoveObject(be.bucketname, path)
-	debug.Log("%v %v -> err %v", t, name, err)
+func (be *s3) Remove(h restic.Handle) error {
+	objName := be.s3path(h)
+	err := be.client.RemoveObject(be.bucketname, objName)
+	debug.Log("Remove(%v) -> err %v", h, err)
 	return errors.Wrap(err, "client.RemoveObject")
 }
 
@@ -240,7 +240,7 @@ func (be *s3) List(t restic.FileType, done <-chan struct{}) <-chan string {
 	debug.Log("listing %v", t)
 	ch := make(chan string)
 
-	prefix := be.s3path(t, "")
+	prefix := be.s3path(restic.Handle{Type: t}) + "/"
 
 	listresp := be.client.ListObjects(be.bucketname, prefix, true, done)
 
@@ -268,7 +268,7 @@ func (be *s3) removeKeys(t restic.FileType) error {
 	done := make(chan struct{})
 	defer close(done)
 	for key := range be.List(restic.DataFile, done) {
-		err := be.Remove(restic.DataFile, key)
+		err := be.Remove(restic.Handle{Type: restic.DataFile, Name: key})
 		if err != nil {
 			return err
 		}
@@ -293,7 +293,7 @@ func (be *s3) Delete() error {
 		}
 	}
 
-	return be.Remove(restic.ConfigFile, "")
+	return be.Remove(restic.Handle{Type: restic.ConfigFile})
 }
 
 // Close does nothing

src/restic/backend/s3/s3_test.go

@@ -44,7 +44,7 @@ func init() {
 			return nil, err
 		}
 
-		exists, err := be.Test(restic.ConfigFile, "")
+		exists, err := be.Test(restic.Handle{Type: restic.ConfigFile})
 		if err != nil {
 			return nil, err
 		}

src/restic/backend/sftp/backend_test.go

@@ -51,13 +51,6 @@ func TestSftpBackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestSftpBackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestSftpBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/sftp/sftp.go

@@ -259,11 +259,11 @@ func (r *SFTP) mkdirAll(dir string, mode os.FileMode) error {
 }
 
 // Rename temp file to final name according to type and name.
-func (r *SFTP) renameFile(oldname string, t restic.FileType, name string) error {
-	filename := r.filename(t, name)
+func (r *SFTP) renameFile(oldname string, h restic.Handle) error {
+	filename := r.filename(h)
 
 	// create directories if necessary
-	if t == restic.DataFile {
+	if h.Type == restic.DataFile {
 		err := r.mkdirAll(path.Dir(filename), backend.Modes.Dir)
 		if err != nil {
 			return err
@@ -297,22 +297,22 @@ func Join(parts ...string) string {
 }
 
 // Construct path for given restic.Type and name.
-func (r *SFTP) filename(t restic.FileType, name string) string {
-	if t == restic.ConfigFile {
+func (r *SFTP) filename(h restic.Handle) string {
+	if h.Type == restic.ConfigFile {
 		return Join(r.p, "config")
 	}
 
-	return Join(r.dirname(t, name), name)
+	return Join(r.dirname(h), h.Name)
 }
 
 // Construct directory for given backend.Type.
-func (r *SFTP) dirname(t restic.FileType, name string) string {
+func (r *SFTP) dirname(h restic.Handle) string {
 	var n string
-	switch t {
+	switch h.Type {
 	case restic.DataFile:
 		n = backend.Paths.Data
-		if len(name) > 2 {
-			n = Join(n, name[:2])
+		if len(h.Name) > 2 {
+			n = Join(n, h.Name[:2])
 		}
 	case restic.SnapshotFile:
 		n = backend.Paths.Snapshots
@@ -326,47 +326,9 @@ func (r *SFTP) dirname(t restic.FileType, name string) string {
 	return Join(r.p, n)
 }
 
-// Load returns the data stored in the backend for h at the given offset
-// and saves it in p. Load has the same semantics as io.ReaderAt.
-func (r *SFTP) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
-	debug.Log("load %v, %d bytes, offset %v", h, len(p), off)
-	if err := r.clientError(); err != nil {
-		return 0, err
-	}
-
-	if err := h.Valid(); err != nil {
-		return 0, err
-	}
-
-	f, err := r.c.Open(r.filename(h.Type, h.Name))
-	if err != nil {
-		return 0, errors.Wrap(err, "Open")
-	}
-
-	defer func() {
-		e := f.Close()
-		if err == nil && e != nil {
-			err = errors.Wrap(e, "Close")
-		}
-	}()
-
-	switch {
-	case off > 0:
-		_, err = f.Seek(off, 0)
-	case off < 0:
-		_, err = f.Seek(off, 2)
-	}
-
-	if err != nil {
-		return 0, errors.Wrap(err, "Seek")
-	}
-
-	return io.ReadFull(f, p)
-}
-
 // Save stores data in the backend at the handle.
-func (r *SFTP) Save(h restic.Handle, p []byte) (err error) {
-	debug.Log("save %v bytes to %v", h, len(p))
+func (r *SFTP) Save(h restic.Handle, rd io.Reader) (err error) {
+	debug.Log("save to %v", h)
 	if err := r.clientError(); err != nil {
 		return err
 	}
@@ -380,31 +342,60 @@ func (r *SFTP) Save(h restic.Handle, p []byte) (err error) {
 		return err
 	}
 
-	debug.Log("save %v (%d bytes) to %v", h, len(p), filename)
-
-	n, err := tmpfile.Write(p)
+	n, err := io.Copy(tmpfile, rd)
 	if err != nil {
 		return errors.Wrap(err, "Write")
 	}
 
-	if n != len(p) {
-		return errors.New("not all bytes writen")
-	}
+	debug.Log("saved %v (%d bytes) to %v", h, n, filename)
 
 	err = tmpfile.Close()
 	if err != nil {
 		return errors.Wrap(err, "Close")
 	}
 
-	err = r.renameFile(filename, h.Type, h.Name)
+	err = r.renameFile(filename, h)
 	debug.Log("save %v: rename %v: %v",
 		h, path.Base(filename), err)
 	return err
 }
 
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (r *SFTP) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	f, err := r.c.Open(r.filename(h))
+	if err != nil {
+		return nil, err
+	}
+
+	if offset > 0 {
+		_, err = f.Seek(offset, 0)
+		if err != nil {
+			_ = f.Close()
+			return nil, err
+		}
+	}
+
+	if length > 0 {
+		return backend.LimitReadCloser(f, int64(length)), nil
+	}
+
+	return f, nil
+}
+
 // Stat returns information about a blob.
 func (r *SFTP) Stat(h restic.Handle) (restic.FileInfo, error) {
-	debug.Log("stat %v", h)
+	debug.Log("Stat(%v)", h)
 	if err := r.clientError(); err != nil {
 		return restic.FileInfo{}, err
 	}
@@ -413,7 +404,7 @@ func (r *SFTP) Stat(h restic.Handle) (restic.FileInfo, error) {
 		return restic.FileInfo{}, err
 	}
 
-	fi, err := r.c.Lstat(r.filename(h.Type, h.Name))
+	fi, err := r.c.Lstat(r.filename(h))
 	if err != nil {
 		return restic.FileInfo{}, errors.Wrap(err, "Lstat")
 	}
@@ -422,13 +413,13 @@ func (r *SFTP) Stat(h restic.Handle) (restic.FileInfo, error) {
 }
 
 // Test returns true if a blob of the given type and name exists in the backend.
-func (r *SFTP) Test(t restic.FileType, name string) (bool, error) {
-	debug.Log("type %v, name %v", t, name)
+func (r *SFTP) Test(h restic.Handle) (bool, error) {
+	debug.Log("Test(%v)", h)
 	if err := r.clientError(); err != nil {
 		return false, err
 	}
 
-	_, err := r.c.Lstat(r.filename(t, name))
+	_, err := r.c.Lstat(r.filename(h))
 	if os.IsNotExist(errors.Cause(err)) {
 		return false, nil
 	}
@@ -441,13 +432,13 @@ func (r *SFTP) Test(t restic.FileType, name string) (bool, error) {
 }
 
 // Remove removes the content stored at name.
-func (r *SFTP) Remove(t restic.FileType, name string) error {
-	debug.Log("type %v, name %v", t, name)
+func (r *SFTP) Remove(h restic.Handle) error {
+	debug.Log("Remove(%v)", h)
 	if err := r.clientError(); err != nil {
 		return err
 	}
 
-	return r.c.Remove(r.filename(t, name))
+	return r.c.Remove(r.filename(h))
 }
 
 // List returns a channel that yields all names of blobs of type t. A
@@ -462,7 +453,7 @@ func (r *SFTP) List(t restic.FileType, done <-chan struct{}) <-chan string {
 
 		if t == restic.DataFile {
 			// read first level
-			basedir := r.dirname(t, "")
+			basedir := r.dirname(restic.Handle{Type: t})
 
 			list1, err := r.c.ReadDir(basedir)
 			if err != nil {
@@ -495,7 +486,7 @@ func (r *SFTP) List(t restic.FileType, done <-chan struct{}) <-chan string {
 				}
 			}
 		} else {
-			entries, err := r.c.ReadDir(r.dirname(t, ""))
+			entries, err := r.c.ReadDir(r.dirname(restic.Handle{Type: t}))
 			if err != nil {
 				return
 			}

src/restic/backend/test/backend_test.go

@@ -51,13 +51,6 @@ func TestTestBackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestTestBackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestTestBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/test/tests.go

@@ -4,13 +4,16 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"math/rand"
+	"os"
 	"reflect"
 	"restic"
+	"restic/errors"
 	"sort"
+	"strings"
 	"testing"
 
-	"restic/errors"
 	"restic/test"
 
 	"restic/backend"
@@ -127,7 +130,7 @@ func TestCreateWithConfig(t testing.TB) {
 	}
 
 	// remove config
-	err = b.Remove(restic.ConfigFile, "")
+	err = b.Remove(restic.Handle{Type: restic.ConfigFile, Name: ""})
 	if err != nil {
 		t.Fatalf("unexpected error removing config: %v", err)
 	}
@@ -152,12 +155,12 @@ func TestConfig(t testing.TB) {
 	var testString = "Config"
 
 	// create config and read it back
-	_, err := backend.LoadAll(b, restic.Handle{Type: restic.ConfigFile}, nil)
+	_, err := backend.LoadAll(b, restic.Handle{Type: restic.ConfigFile})
 	if err == nil {
 		t.Fatalf("did not get expected error for non-existing config")
 	}
 
-	err = b.Save(restic.Handle{Type: restic.ConfigFile}, []byte(testString))
+	err = b.Save(restic.Handle{Type: restic.ConfigFile}, strings.NewReader(testString))
 	if err != nil {
 		t.Fatalf("Save() error: %v", err)
 	}
@@ -166,7 +169,7 @@ func TestConfig(t testing.TB) {
 	// same config
 	for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} {
 		h := restic.Handle{Type: restic.ConfigFile, Name: name}
-		buf, err := backend.LoadAll(b, h, nil)
+		buf, err := backend.LoadAll(b, h)
 		if err != nil {
 			t.Fatalf("unable to read config with name %q: %v", name, err)
 		}
@@ -182,12 +185,12 @@ func TestLoad(t testing.TB) {
 	b := open(t)
 	defer close(t)
 
-	_, err := b.Load(restic.Handle{}, nil, 0)
+	_, err := b.Load(restic.Handle{}, 0, 0)
 	if err == nil {
 		t.Fatalf("Load() did not return an error for invalid handle")
 	}
 
-	_, err = b.Load(restic.Handle{Type: restic.DataFile, Name: "foobar"}, nil, 0)
+	_, err = b.Load(restic.Handle{Type: restic.DataFile, Name: "foobar"}, 0, 0)
 	if err == nil {
 		t.Fatalf("Load() did not return an error for non-existing blob")
 	}
@@ -198,11 +201,20 @@ func TestLoad(t testing.TB) {
 	id := restic.Hash(data)
 
 	handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
-	err = b.Save(handle, data)
+	err = b.Save(handle, bytes.NewReader(data))
 	if err != nil {
 		t.Fatalf("Save() error: %v", err)
 	}
 
+	rd, err := b.Load(handle, 100, -1)
+	if err == nil {
+		t.Fatalf("Load() returned no error for negative offset!")
+	}
+
+	if rd != nil {
+		t.Fatalf("Load() returned a non-nil reader for negative offset!")
+	}
+
 	for i := 0; i < 50; i++ {
 		l := rand.Intn(length + 2000)
 		o := rand.Intn(length + 2000)
@@ -215,157 +227,60 @@ func TestLoad(t testing.TB) {
 			d = d[:0]
 		}
 
-		if l > 0 && l < len(d) {
-			d = d[:l]
-		}
-
-		buf := make([]byte, l)
-		n, err := b.Load(handle, buf, int64(o))
-
-		// if we requested data beyond the end of the file, require
-		// ErrUnexpectedEOF error
-		if l > len(d) {
-			if errors.Cause(err) != io.ErrUnexpectedEOF {
-				t.Errorf("Load(%d, %d) did not return io.ErrUnexpectedEOF", len(buf), int64(o))
-			}
-			err = nil
-			buf = buf[:len(d)]
-		}
-
-		if err != nil {
-			t.Errorf("Load(%d, %d): unexpected error: %v", len(buf), int64(o), err)
-			continue
-		}
-
-		if n != len(buf) {
-			t.Errorf("Load(%d, %d): wrong length returned, want %d, got %d",
-				len(buf), int64(o), len(buf), n)
-			continue
-		}
-
-		buf = buf[:n]
-		if !bytes.Equal(buf, d) {
-			t.Errorf("Load(%d, %d) returned wrong bytes", len(buf), int64(o))
-			continue
-		}
-	}
-
-	// test with negative offset
-	for i := 0; i < 50; i++ {
-		l := rand.Intn(length + 2000)
-		o := rand.Intn(length + 2000)
-
-		d := data
-		if o < len(d) {
-			d = d[len(d)-o:]
-		} else {
-			o = 0
+		getlen := l
+		if l >= len(d) && rand.Float32() >= 0.5 {
+			getlen = 0
 		}
 
 		if l > 0 && l < len(d) {
 			d = d[:l]
 		}
 
-		buf := make([]byte, l)
-		n, err := b.Load(handle, buf, -int64(o))
-
-		// if we requested data beyond the end of the file, require
-		// ErrUnexpectedEOF error
-		if l > len(d) {
-			if errors.Cause(err) != io.ErrUnexpectedEOF {
-				t.Errorf("Load(%d, %d) did not return io.ErrUnexpectedEOF", len(buf), int64(o))
-				continue
-			}
-			err = nil
-			buf = buf[:len(d)]
-		}
-
+		rd, err := b.Load(handle, getlen, int64(o))
 		if err != nil {
-			t.Errorf("Load(%d, %d): unexpected error: %v", len(buf), int64(o), err)
+			t.Errorf("Load(%d, %d) returned unexpected error: %v", l, o, err)
 			continue
 		}
 
-		if n != len(buf) {
-			t.Errorf("Load(%d, %d): wrong length returned, want %d, got %d",
-				len(buf), int64(o), len(buf), n)
+		buf, err := ioutil.ReadAll(rd)
+		if err != nil {
+			t.Errorf("Load(%d, %d) ReadAll() returned unexpected error: %v", l, o, err)
+			continue
+		}
+
+		if l <= len(d) && len(buf) != l {
+			t.Errorf("Load(%d, %d) wrong number of bytes read: want %d, got %d", l, o, l, len(buf))
+			continue
+		}
+
+		if l > len(d) && len(buf) != len(d) {
+			t.Errorf("Load(%d, %d) wrong number of bytes read for overlong read: want %d, got %d", l, o, l, len(buf))
 			continue
 		}
 
-		buf = buf[:n]
 		if !bytes.Equal(buf, d) {
-			t.Errorf("Load(%d, %d) returned wrong bytes", len(buf), int64(o))
+			t.Errorf("Load(%d, %d) returned wrong bytes", l, o)
+			continue
+		}
+
+		err = rd.Close()
+		if err != nil {
+			t.Errorf("Load(%d, %d) rd.Close() returned unexpected error: %v", l, o, err)
 			continue
 		}
 	}
 
-	// load with a too-large buffer, this should return io.ErrUnexpectedEOF
-	buf := make([]byte, length+100)
-	n, err := b.Load(handle, buf, 0)
-	if n != length {
-		t.Errorf("wrong length for larger buffer returned, want %d, got %d", length, n)
-	}
-
-	if errors.Cause(err) != io.ErrUnexpectedEOF {
-		t.Errorf("wrong error returned for larger buffer: want io.ErrUnexpectedEOF, got %#v", err)
-	}
-
-	test.OK(t, b.Remove(restic.DataFile, id.String()))
+	test.OK(t, b.Remove(handle))
 }
 
-// TestLoadNegativeOffset tests the backend's Load function with negative offsets.
-func TestLoadNegativeOffset(t testing.TB) {
-	b := open(t)
-	defer close(t)
+type errorCloser struct {
+	io.Reader
+	t testing.TB
+}
 
-	length := rand.Intn(1<<24) + 2000
-
-	data := test.Random(23, length)
-	id := restic.Hash(data)
-
-	handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
-	err := b.Save(handle, data)
-	if err != nil {
-		t.Fatalf("Save() error: %v", err)
-	}
-
-	// test normal reads
-	for i := 0; i < 50; i++ {
-		l := rand.Intn(length + 2000)
-		o := -rand.Intn(length + 2000)
-
-		buf := make([]byte, l)
-		n, err := b.Load(handle, buf, int64(o))
-
-		// if we requested data beyond the end of the file, require
-		// ErrUnexpectedEOF error
-		if len(buf) > -o {
-			if errors.Cause(err) != io.ErrUnexpectedEOF {
-				t.Errorf("Load(%d, %d) did not return io.ErrUnexpectedEOF", len(buf), o)
-				continue
-			}
-			err = nil
-			buf = buf[:-o]
-		}
-
-		if err != nil {
-			t.Errorf("Load(%d, %d) returned error: %v", len(buf), o, err)
-			continue
-		}
-
-		if n != len(buf) {
-			t.Errorf("Load(%d, %d) returned short read, only got %d bytes", len(buf), o, n)
-			continue
-		}
-
-		p := len(data) + o
-		if !bytes.Equal(buf, data[p:p+len(buf)]) {
-			t.Errorf("Load(%d, %d) returned wrong bytes", len(buf), o)
-			continue
-		}
-
-	}
-
-	test.OK(t, b.Remove(restic.DataFile, id.String()))
+func (ec errorCloser) Close() error {
+	ec.t.Error("forbidden method close was called")
+	return errors.New("forbidden method close was called")
 }
 
 // TestSave tests saving data in the backend.
@@ -384,10 +299,10 @@ func TestSave(t testing.TB) {
 			Type: restic.DataFile,
 			Name: fmt.Sprintf("%s-%d", id, i),
 		}
-		err := b.Save(h, data)
+		err := b.Save(h, bytes.NewReader(data))
 		test.OK(t, err)
 
-		buf, err := backend.LoadAll(b, h, nil)
+		buf, err := backend.LoadAll(b, h)
 		test.OK(t, err)
 		if len(buf) != len(data) {
 			t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf))
@@ -404,11 +319,51 @@ func TestSave(t testing.TB) {
 			t.Fatalf("Stat() returned different size, want %q, got %d", len(data), fi.Size)
 		}
 
-		err = b.Remove(h.Type, h.Name)
+		err = b.Remove(h)
 		if err != nil {
 			t.Fatalf("error removing item: %v", err)
 		}
 	}
+
+	// test saving from a tempfile
+	tmpfile, err := ioutil.TempFile("", "restic-backend-save-test-")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	length := rand.Intn(1<<23) + 200000
+	data := test.Random(23, length)
+	copy(id[:], data)
+
+	if _, err = tmpfile.Write(data); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err = tmpfile.Seek(0, 0); err != nil {
+		t.Fatal(err)
+	}
+
+	h := restic.Handle{Type: restic.DataFile, Name: id.String()}
+
+	// wrap the tempfile in an errorCloser, so we can detect if the backend
+	// closes the reader
+	err = b.Save(h, errorCloser{t: t, Reader: tmpfile})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err = tmpfile.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	if err = os.Remove(tmpfile.Name()); err != nil {
+		t.Fatal(err)
+	}
+
+	err = b.Remove(h)
+	if err != nil {
+		t.Fatalf("error removing item: %v", err)
+	}
 }
 
 var filenameTests = []struct {
@@ -430,13 +385,13 @@ func TestSaveFilenames(t testing.TB) {
 
 	for i, test := range filenameTests {
 		h := restic.Handle{Name: test.name, Type: restic.DataFile}
-		err := b.Save(h, []byte(test.data))
+		err := b.Save(h, strings.NewReader(test.data))
 		if err != nil {
 			t.Errorf("test %d failed: Save() returned %v", i, err)
 			continue
 		}
 
-		buf, err := backend.LoadAll(b, h, nil)
+		buf, err := backend.LoadAll(b, h)
 		if err != nil {
 			t.Errorf("test %d failed: Load() returned %v", i, err)
 			continue
@@ -446,7 +401,7 @@ func TestSaveFilenames(t testing.TB) {
 			t.Errorf("test %d: returned wrong bytes", i)
 		}
 
-		err = b.Remove(h.Type, h.Name)
+		err = b.Remove(h)
 		if err != nil {
 			t.Errorf("test %d failed: Remove() returned %v", i, err)
 			continue
@@ -464,10 +419,12 @@ var testStrings = []struct {
 	{"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"},
 }
 
-func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) {
+func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) restic.Handle {
 	id := restic.Hash(data)
-	err := b.Save(restic.Handle{Name: id.String(), Type: tpe}, data)
+	h := restic.Handle{Name: id.String(), Type: tpe}
+	err := b.Save(h, bytes.NewReader(data))
 	test.OK(t, err)
+	return h
 }
 
 // TestBackend tests all functions of the backend.
@@ -485,21 +442,21 @@ func TestBackend(t testing.TB) {
 			test.OK(t, err)
 
 			// test if blob is already in repository
-			ret, err := b.Test(tpe, id.String())
+			h := restic.Handle{Type: tpe, Name: id.String()}
+			ret, err := b.Test(h)
 			test.OK(t, err)
 			test.Assert(t, !ret, "blob was found to exist before creating")
 
 			// try to stat a not existing blob
-			h := restic.Handle{Type: tpe, Name: id.String()}
 			_, err = b.Stat(h)
 			test.Assert(t, err != nil, "blob data could be extracted before creation")
 
 			// try to read not existing blob
-			_, err = b.Load(h, nil, 0)
+			_, err = b.Load(h, 0, 0)
 			test.Assert(t, err != nil, "blob reader could be obtained before creation")
 
 			// try to get string out, should fail
-			ret, err = b.Test(tpe, id.String())
+			ret, err = b.Test(h)
 			test.OK(t, err)
 			test.Assert(t, !ret, "id %q was found (but should not have)", ts.id)
 		}
@@ -510,7 +467,7 @@ func TestBackend(t testing.TB) {
 
 			// test Load()
 			h := restic.Handle{Type: tpe, Name: ts.id}
-			buf, err := backend.LoadAll(b, h, nil)
+			buf, err := backend.LoadAll(b, h)
 			test.OK(t, err)
 			test.Equals(t, ts.data, string(buf))
 
@@ -520,9 +477,18 @@ func TestBackend(t testing.TB) {
 			length := end - start
 
 			buf2 := make([]byte, length)
-			n, err := b.Load(h, buf2, int64(start))
+			rd, err := b.Load(h, len(buf2), int64(start))
 			test.OK(t, err)
-			test.Equals(t, length, n)
+			n, err := io.ReadFull(rd, buf2)
+			test.OK(t, err)
+			test.Equals(t, len(buf2), n)
+
+			remaining, err := io.Copy(ioutil.Discard, rd)
+			test.OK(t, err)
+			test.Equals(t, int64(0), remaining)
+
+			test.OK(t, rd.Close())
+
 			test.Equals(t, ts.data[start:end], string(buf2))
 		}
 
@@ -530,20 +496,21 @@ func TestBackend(t testing.TB) {
 		ts := testStrings[0]
 
 		// create blob
-		err := b.Save(restic.Handle{Type: tpe, Name: ts.id}, []byte(ts.data))
+		err := b.Save(restic.Handle{Type: tpe, Name: ts.id}, strings.NewReader(ts.data))
 		test.Assert(t, err != nil, "expected error, got %v", err)
 
 		// remove and recreate
-		err = b.Remove(tpe, ts.id)
+		h := restic.Handle{Type: tpe, Name: ts.id}
+		err = b.Remove(h)
 		test.OK(t, err)
 
 		// test that the blob is gone
-		ok, err := b.Test(tpe, ts.id)
+		ok, err := b.Test(h)
 		test.OK(t, err)
 		test.Assert(t, ok == false, "removed blob still present")
 
 		// create blob
-		err = b.Save(restic.Handle{Type: tpe, Name: ts.id}, []byte(ts.data))
+		err = b.Save(h, strings.NewReader(ts.data))
 		test.OK(t, err)
 
 		// list items
@@ -578,12 +545,14 @@ func TestBackend(t testing.TB) {
 				id, err := restic.ParseID(ts.id)
 				test.OK(t, err)
 
-				found, err := b.Test(tpe, id.String())
+				h := restic.Handle{Type: tpe, Name: id.String()}
+
+				found, err := b.Test(h)
 				test.OK(t, err)
 
-				test.OK(t, b.Remove(tpe, id.String()))
+				test.OK(t, b.Remove(h))
 
-				found, err = b.Test(tpe, id.String())
+				found, err = b.Test(h)
 				test.OK(t, err)
 				test.Assert(t, !found, fmt.Sprintf("id %q not found after removal", id))
 			}

src/restic/backend/utils.go

@@ -2,29 +2,51 @@ package backend
 
 import (
 	"io"
+	"io/ioutil"
 	"restic"
-
-	"restic/errors"
 )
 
-// LoadAll reads all data stored in the backend for the handle. The buffer buf
-// is resized to accomodate all data in the blob. Errors returned by be.Load()
-// are passed on, except io.ErrUnexpectedEOF is silenced and nil returned
-// instead, since it means this function is working properly.
-func LoadAll(be restic.Backend, h restic.Handle, buf []byte) ([]byte, error) {
-	fi, err := be.Stat(h)
+// LoadAll reads all data stored in the backend for the handle.
+func LoadAll(be restic.Backend, h restic.Handle) (buf []byte, err error) {
+	rd, err := be.Load(h, 0, 0)
 	if err != nil {
-		return nil, errors.Wrap(err, "Stat")
+		return nil, err
 	}
 
-	if fi.Size > int64(len(buf)) {
-		buf = make([]byte, int(fi.Size))
-	}
+	defer func() {
+		io.Copy(ioutil.Discard, rd)
+		e := rd.Close()
+		if err == nil {
+			err = e
+		}
+	}()
 
-	n, err := be.Load(h, buf, 0)
-	if errors.Cause(err) == io.ErrUnexpectedEOF {
-		err = nil
-	}
-	buf = buf[:n]
-	return buf, err
+	return ioutil.ReadAll(rd)
+}
+
+// Closer wraps an io.Reader and adds a Close() method that does nothing.
+type Closer struct {
+	io.Reader
+}
+
+// Close is a no-op.
+func (c Closer) Close() error {
+	return nil
+}
+
+// LimitedReadCloser wraps io.LimitedReader and exposes the Close() method.
+type LimitedReadCloser struct {
+	io.ReadCloser
+	io.Reader
+}
+
+// Read reads data from the limited reader.
+func (l *LimitedReadCloser) Read(p []byte) (int, error) {
+	return l.Reader.Read(p)
+}
+
+// LimitReadCloser returns a new reader wraps r in an io.LimitReader, but also
+// exposes the Close() method.
+func LimitReadCloser(r io.ReadCloser, n int64) *LimitedReadCloser {
+	return &LimitedReadCloser{ReadCloser: r, Reader: io.LimitReader(r, n)}
 }

src/restic/backend/utils_test.go

@@ -21,10 +21,10 @@ func TestLoadAll(t *testing.T) {
 		data := Random(23+i, rand.Intn(MiB)+500*KiB)
 
 		id := restic.Hash(data)
-		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, data)
+		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data))
 		OK(t, err)
 
-		buf, err := backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()}, nil)
+		buf, err := backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()})
 		OK(t, err)
 
 		if len(buf) != len(data) {
@@ -46,11 +46,11 @@ func TestLoadSmallBuffer(t *testing.T) {
 		data := Random(23+i, rand.Intn(MiB)+500*KiB)
 
 		id := restic.Hash(data)
-		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, data)
+		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data))
 		OK(t, err)
 
 		buf := make([]byte, len(data)-23)
-		buf, err = backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()}, buf)
+		buf, err = backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()})
 		OK(t, err)
 
 		if len(buf) != len(data) {
@@ -72,11 +72,11 @@ func TestLoadLargeBuffer(t *testing.T) {
 		data := Random(23+i, rand.Intn(MiB)+500*KiB)
 
 		id := restic.Hash(data)
-		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, data)
+		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data))
 		OK(t, err)
 
 		buf := make([]byte, len(data)+100)
-		buf, err = backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()}, buf)
+		buf, err = backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()})
 		OK(t, err)
 
 		if len(buf) != len(data) {

src/restic/buffer.go

@@ -0,0 +1,21 @@
+package restic
+
+import "restic/crypto"
+
+// NewBlobBuffer returns a buffer that is large enough to hold a blob of size
+// plaintext bytes, including the crypto overhead.
+func NewBlobBuffer(size int) []byte {
+	return make([]byte, size, size+crypto.Extension)
+}
+
+// PlaintextLength returns the plaintext length of a blob with ciphertextSize
+// bytes.
+func PlaintextLength(ciphertextSize int) int {
+	return ciphertextSize - crypto.Extension
+}
+
+// CiphertextLength returns the encrypted length of a blob with plaintextSize
+// bytes.
+func CiphertextLength(plaintextSize int) int {
+	return plaintextSize + crypto.Extension
+}

src/restic/checker/checker.go

@@ -187,7 +187,8 @@ func packIDTester(repo restic.Repository, inChan <-chan restic.ID, errChan chan<
 	defer wg.Done()
 
 	for id := range inChan {
-		ok, err := repo.Backend().Test(restic.DataFile, id.String())
+		h := restic.Handle{Type: restic.DataFile, Name: id.String()}
+		ok, err := repo.Backend().Test(h)
 		if err != nil {
 			err = PackError{ID: id, Err: err}
 		} else {
@@ -658,7 +659,7 @@ func (c *Checker) CountPacks() uint64 {
 func checkPack(r restic.Repository, id restic.ID) error {
 	debug.Log("checking pack %v", id.Str())
 	h := restic.Handle{Type: restic.DataFile, Name: id.String()}
-	buf, err := backend.LoadAll(r.Backend(), h, nil)
+	buf, err := backend.LoadAll(r.Backend(), h)
 	if err != nil {
 		return err
 	}

src/restic/checker/checker_test.go

@@ -1,6 +1,7 @@
 package checker_test
 
 import (
+	"io"
 	"math/rand"
 	"path/filepath"
 	"sort"
@@ -8,7 +9,6 @@ import (
 
 	"restic"
 	"restic/archiver"
-	"restic/backend/mem"
 	"restic/checker"
 	"restic/repository"
 	"restic/test"
@@ -73,8 +73,11 @@ func TestMissingPack(t *testing.T) {
 
 	repo := repository.TestOpenLocal(t, repodir)
 
-	packID := "657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6"
-	test.OK(t, repo.Backend().Remove(restic.DataFile, packID))
+	packHandle := restic.Handle{
+		Type: restic.DataFile,
+		Name: "657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6",
+	}
+	test.OK(t, repo.Backend().Remove(packHandle))
 
 	chkr := checker.New(repo)
 	hints, errs := chkr.LoadIndex()
@@ -92,7 +95,7 @@ func TestMissingPack(t *testing.T) {
 		"expected exactly one error, got %v", len(errs))
 
 	if err, ok := errs[0].(checker.PackError); ok {
-		test.Equals(t, packID, err.ID.String())
+		test.Equals(t, packHandle.Name, err.ID.String())
 	} else {
 		t.Errorf("expected error returned by checker.Packs() to be PackError, got %v", err)
 	}
@@ -105,9 +108,12 @@ func TestUnreferencedPack(t *testing.T) {
 	repo := repository.TestOpenLocal(t, repodir)
 
 	// index 3f1a only references pack 60e0
-	indexID := "3f1abfcb79c6f7d0a3be517d2c83c8562fba64ef2c8e9a3544b4edaf8b5e3b44"
 	packID := "60e0438dcb978ec6860cc1f8c43da648170ee9129af8f650f876bad19f8f788e"
-	test.OK(t, repo.Backend().Remove(restic.IndexFile, indexID))
+	indexHandle := restic.Handle{
+		Type: restic.IndexFile,
+		Name: "3f1abfcb79c6f7d0a3be517d2c83c8562fba64ef2c8e9a3544b4edaf8b5e3b44",
+	}
+	test.OK(t, repo.Backend().Remove(indexHandle))
 
 	chkr := checker.New(repo)
 	hints, errs := chkr.LoadIndex()
@@ -137,8 +143,11 @@ func TestUnreferencedBlobs(t *testing.T) {
 
 	repo := repository.TestOpenLocal(t, repodir)
 
-	snID := "51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02"
-	test.OK(t, repo.Backend().Remove(restic.SnapshotFile, snID))
+	snapshotHandle := restic.Handle{
+		Type: restic.SnapshotFile,
+		Name: "51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02",
+	}
+	test.OK(t, repo.Backend().Remove(snapshotHandle))
 
 	unusedBlobsBySnapshot := restic.IDs{
 		restic.TestParseID("58c748bbe2929fdf30c73262bd8313fe828f8925b05d1d4a87fe109082acb849"),
@@ -208,15 +217,35 @@ type errorBackend struct {
 	ProduceErrors bool
 }
 
-func (b errorBackend) Load(h restic.Handle, p []byte, off int64) (int, error) {
-	n, err := b.Backend.Load(h, p, off)
+func (b errorBackend) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	rd, err := b.Backend.Load(h, length, offset)
+	if err != nil {
+		return rd, err
+	}
 
 	if b.ProduceErrors {
-		induceError(p)
+		return errorReadCloser{rd}, err
+	}
+
+	return rd, nil
+}
+
+type errorReadCloser struct {
+	io.ReadCloser
+}
+
+func (erd errorReadCloser) Read(p []byte) (int, error) {
+	n, err := erd.ReadCloser.Read(p)
+	if n > 0 {
+		induceError(p[:n])
 	}
 	return n, err
 }
 
+func (erd errorReadCloser) Close() error {
+	return erd.ReadCloser.Close()
+}
+
 // induceError flips a bit in the slice.
 func induceError(data []byte) {
 	if rand.Float32() < 0.2 {
@@ -228,19 +257,15 @@ func induceError(data []byte) {
 }
 
 func TestCheckerModifiedData(t *testing.T) {
-	be := mem.New()
-
-	repository.TestUseLowSecurityKDFParameters(t)
-
-	repo := repository.New(be)
-	test.OK(t, repo.Init(test.TestPassword))
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
 
 	arch := archiver.New(repo)
 	_, id, err := arch.Snapshot(nil, []string{"."}, nil, nil)
 	test.OK(t, err)
 	t.Logf("archived as %v", id.Str())
 
-	beError := &errorBackend{Backend: be}
+	beError := &errorBackend{Backend: repo.Backend()}
 	checkRepo := repository.New(beError)
 	test.OK(t, checkRepo.SearchKey(test.TestPassword, 5))
 
@@ -266,7 +291,7 @@ func TestCheckerModifiedData(t *testing.T) {
 	}
 
 	for _, err := range checkData(chkr) {
-		t.Logf("struct error: %v", err)
+		t.Logf("data error: %v", err)
 		errFound = true
 	}
 

src/restic/fs/deviceid_unix.go

@@ -12,10 +12,19 @@ import (
 // DeviceID extracts the device ID from an os.FileInfo object by casting it
 // to syscall.Stat_t
 func DeviceID(fi os.FileInfo) (deviceID uint64, err error) {
+	if fi == nil {
+		return 0, errors.New("unable to determine device: fi is nil")
+	}
+
+	if fi.Sys() == nil {
+		return 0, errors.New("unable to determine device: fi.Sys() is nil")
+	}
+
 	if st, ok := fi.Sys().(*syscall.Stat_t); ok {
 		// st.Dev is uint32 on Darwin and uint64 on Linux. Just cast
 		// everything to uint64.
 		return uint64(st.Dev), nil
 	}
+
 	return 0, errors.New("Could not cast to syscall.Stat_t")
 }

src/restic/fs/file.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 )
 
 // File is an open file on a file system.
@@ -26,7 +27,20 @@ func fixpath(name string) string {
 	if runtime.GOOS == "windows" {
 		abspath, err := filepath.Abs(name)
 		if err == nil {
-			return "\\\\?\\" + abspath
+			// Check if \\?\UNC\ already exist
+			if strings.HasPrefix(abspath, `\\?\UNC\`) {
+				return abspath
+			}
+			// Check if \\?\ already exist
+			if strings.HasPrefix(abspath, `\\?\`) {
+				return abspath
+			}
+			// Check if path starts with \\
+			if strings.HasPrefix(abspath, `\\`) {
+				return strings.Replace(abspath, `\\`, `\\?\UNC\`, 1)
+			}
+			// Normal path
+			return `\\?\` + abspath
 		}
 	}
 	return name
@@ -111,6 +125,11 @@ func Create(name string) (*os.File, error) {
 	return os.Create(fixpath(name))
 }
 
+// Open opens a file for reading.
+func Open(name string) (File, error) {
+	return os.Open(fixpath(name))
+}
+
 // OpenFile is the generalized open call; most users will use Open
 // or Create instead.  It opens the named file with specified flag
 // (O_RDONLY etc.) and perm, (0666 etc.) if applicable.  If successful,

src/restic/fs/file_linux.go

@@ -1,59 +0,0 @@
-// +build linux,go1.4
-
-package fs
-
-import (
-	"os"
-	"syscall"
-
-	"restic/errors"
-
-	"golang.org/x/sys/unix"
-)
-
-// Open opens a file for reading, without updating the atime and without caching data on read.
-func Open(name string) (File, error) {
-	file, err := os.OpenFile(name, os.O_RDONLY|syscall.O_NOATIME, 0)
-	if os.IsPermission(errors.Cause(err)) {
-		file, err = os.OpenFile(name, os.O_RDONLY, 0)
-	}
-	return &nonCachingFile{File: file}, err
-}
-
-// nonCachingFile wraps an *os.File and calls fadvise() to instantly forget
-// data that has been read or written.
-type nonCachingFile struct {
-	*os.File
-	readOffset int64
-}
-
-func (f *nonCachingFile) Read(p []byte) (int, error) {
-	n, err := f.File.Read(p)
-
-	if n > 0 {
-		ferr := unix.Fadvise(int(f.File.Fd()), f.readOffset, int64(n), unix.FADV_DONTNEED)
-
-		f.readOffset += int64(n)
-
-		if err == nil {
-			err = ferr
-		}
-	}
-
-	return n, err
-}
-
-// ClearCache syncs and then removes the file's content from the OS cache.
-func ClearCache(file File) error {
-	f, ok := file.(*os.File)
-	if !ok {
-		panic("ClearCache called for file not *os.File")
-	}
-
-	err := f.Sync()
-	if err != nil {
-		return err
-	}
-
-	return unix.Fadvise(int(f.Fd()), 0, 0, unix.FADV_DONTNEED)
-}

src/restic/fs/file_nonlinux.go

@@ -1,15 +0,0 @@
-// +build !linux !go1.4
-
-package fs
-
-import "os"
-
-// Open opens a file for reading.
-func Open(name string) (File, error) {
-	return os.OpenFile(fixpath(name), os.O_RDONLY, 0)
-}
-
-// ClearCache syncs and then removes the file's content from the OS cache.
-func ClearCache(f File) error {
-	return nil
-}

src/restic/fuse/file.go

@@ -4,8 +4,6 @@
 package fuse
 
 import (
-	"sync"
-
 	"restic/errors"
 
 	"restic"
@@ -35,29 +33,23 @@ type file struct {
 	node        *restic.Node
 	ownerIsRoot bool
 
-	sizes []uint
+	sizes []int
 	blobs [][]byte
 }
 
 const defaultBlobSize = 128 * 1024
 
-var blobPool = sync.Pool{
-	New: func() interface{} {
-		return make([]byte, defaultBlobSize)
-	},
-}
-
 func newFile(repo BlobLoader, node *restic.Node, ownerIsRoot bool) (*file, error) {
 	debug.Log("create new file for %v with %d blobs", node.Name, len(node.Content))
 	var bytes uint64
-	sizes := make([]uint, len(node.Content))
+	sizes := make([]int, len(node.Content))
 	for i, id := range node.Content {
 		size, err := repo.LookupBlobSize(id, restic.DataBlob)
 		if err != nil {
 			return nil, err
 		}
 
-		sizes[i] = size
+		sizes[i] = int(size)
 		bytes += uint64(size)
 	}
 
@@ -99,16 +91,12 @@ func (f *file) getBlobAt(i int) (blob []byte, err error) {
 		return f.blobs[i], nil
 	}
 
-	buf := blobPool.Get().([]byte)
-	buf = buf[:cap(buf)]
-
-	if uint(len(buf)) < f.sizes[i] {
-		if len(buf) > defaultBlobSize {
-			blobPool.Put(buf)
-		}
-		buf = make([]byte, f.sizes[i])
+	// release earlier blobs
+	for j := 0; j < i; j++ {
+		f.blobs[j] = nil
 	}
 
+	buf := restic.NewBlobBuffer(f.sizes[i])
 	n, err := f.repo.LoadBlob(restic.DataBlob, f.node.Content[i], buf)
 	if err != nil {
 		debug.Log("LoadBlob(%v, %v) failed: %v", f.node.Name, f.node.Content[i], err)
@@ -169,10 +157,7 @@ func (f *file) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadR
 
 func (f *file) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
 	for i := range f.blobs {
-		if f.blobs[i] != nil {
-			blobPool.Put(f.blobs[i])
-			f.blobs[i] = nil
-		}
+		f.blobs[i] = nil
 	}
 	return nil
 }

src/restic/fuse/file_test.go

@@ -9,7 +9,9 @@ import (
 	"testing"
 	"time"
 
-	"restic/errors"
+	"golang.org/x/net/context"
+
+	"restic/repository"
 
 	"bazil.org/fuse"
 
@@ -17,108 +19,96 @@ import (
 	. "restic/test"
 )
 
-type MockRepo struct {
-	blobs map[restic.ID][]byte
-}
-
-func NewMockRepo(content map[restic.ID][]byte) *MockRepo {
-	return &MockRepo{blobs: content}
-}
-
-func (m *MockRepo) LookupBlobSize(id restic.ID, t restic.BlobType) (uint, error) {
-	buf, ok := m.blobs[id]
-	if !ok {
-		return 0, errors.New("blob not found")
-	}
-
-	return uint(len(buf)), nil
-}
-
-func (m *MockRepo) LoadBlob(t restic.BlobType, id restic.ID, buf []byte) (int, error) {
-	size, err := m.LookupBlobSize(id, t)
-	if err != nil {
-		return 0, err
-	}
-
-	if uint(len(buf)) < size {
-		return 0, errors.New("buffer too small")
-	}
-
-	buf = buf[:size]
-	copy(buf, m.blobs[id])
-	return int(size), nil
-}
-
-type MockContext struct{}
-
-func (m MockContext) Deadline() (time.Time, bool)       { return time.Now(), false }
-func (m MockContext) Done() <-chan struct{}             { return nil }
-func (m MockContext) Err() error                        { return nil }
-func (m MockContext) Value(key interface{}) interface{} { return nil }
-
-var testContent = genTestContent()
-var testContentLengths = []uint{
-	4646 * 1024,
-	655 * 1024,
-	378 * 1024,
-	8108 * 1024,
-	558 * 1024,
-}
-var testMaxFileSize uint
-
-func genTestContent() map[restic.ID][]byte {
-	m := make(map[restic.ID][]byte)
-
-	for _, length := range testContentLengths {
-		buf := Random(int(length), int(length))
-		id := restic.Hash(buf)
-		m[id] = buf
-		testMaxFileSize += length
-	}
-
-	return m
-}
-
-const maxBufSize = 20 * 1024 * 1024
-
-func testRead(t *testing.T, f *file, offset, length int, data []byte) {
-	ctx := MockContext{}
+func testRead(t testing.TB, f *file, offset, length int, data []byte) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
 
 	req := &fuse.ReadRequest{
 		Offset: int64(offset),
 		Size:   length,
 	}
 	resp := &fuse.ReadResponse{
-		Data: make([]byte, length),
+		Data: data,
 	}
 	OK(t, f.Read(ctx, req, resp))
 }
 
-var offsetReadsTests = []struct {
-	offset, length int
-}{
-	{0, 5 * 1024 * 1024},
-	{4000 * 1024, 1000 * 1024},
+func firstSnapshotID(t testing.TB, repo restic.Repository) (first restic.ID) {
+	done := make(chan struct{})
+	defer close(done)
+	for id := range repo.List(restic.SnapshotFile, done) {
+		if first.IsNull() {
+			first = id
+		}
+	}
+	return first
+}
+
+func loadFirstSnapshot(t testing.TB, repo restic.Repository) *restic.Snapshot {
+	id := firstSnapshotID(t, repo)
+	sn, err := restic.LoadSnapshot(repo, id)
+	OK(t, err)
+	return sn
+}
+
+func loadTree(t testing.TB, repo restic.Repository, id restic.ID) *restic.Tree {
+	tree, err := repo.LoadTree(id)
+	OK(t, err)
+	return tree
 }
 
 func TestFuseFile(t *testing.T) {
-	repo := NewMockRepo(testContent)
-	ctx := MockContext{}
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
 
-	memfile := make([]byte, 0, maxBufSize)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	timestamp, err := time.Parse(time.RFC3339, "2017-01-24T10:42:56+01:00")
+	OK(t, err)
+	restic.TestCreateSnapshot(t, repo, timestamp, 2, 0.1)
+
+	sn := loadFirstSnapshot(t, repo)
+	tree := loadTree(t, repo, *sn.Tree)
+
+	var content restic.IDs
+	for _, node := range tree.Nodes {
+		content = append(content, node.Content...)
+	}
+	t.Logf("tree loaded, content: %v", content)
+
+	var (
+		filesize uint64
+		memfile  []byte
+	)
+	for _, id := range content {
+		size, err := repo.LookupBlobSize(id, restic.DataBlob)
+		OK(t, err)
+		filesize += uint64(size)
+
+		buf := restic.NewBlobBuffer(int(size))
+		n, err := repo.LoadBlob(restic.DataBlob, id, buf)
+		OK(t, err)
+
+		if uint(n) != size {
+			t.Fatalf("not enough bytes read for id %v: want %v, got %v", id.Str(), size, n)
+		}
+
+		if uint(len(buf)) != size {
+			t.Fatalf("buffer has wrong length for id %v: want %v, got %v", id.Str(), size, len(buf))
+		}
 
-	var ids restic.IDs
-	for id, buf := range repo.blobs {
-		ids = append(ids, id)
 		memfile = append(memfile, buf...)
 	}
 
+	t.Logf("filesize is %v, memfile has size %v", filesize, len(memfile))
+
 	node := &restic.Node{
 		Name:    "foo",
 		Inode:   23,
 		Mode:    0742,
-		Size:    42,
-		Content: ids,
+		Size:    filesize,
+		Content: content,
 	}
 	f, err := newFile(repo, node, false)
 	OK(t, err)
@@ -131,28 +121,19 @@ func TestFuseFile(t *testing.T) {
 	Equals(t, node.Size, attr.Size)
 	Equals(t, (node.Size/uint64(attr.BlockSize))+1, attr.Blocks)
 
-	for i, test := range offsetReadsTests {
-		b := memfile[test.offset : test.offset+test.length]
-		buf := make([]byte, test.length)
-		testRead(t, f, test.offset, test.length, buf)
-		if !bytes.Equal(b, buf) {
-			t.Errorf("test %d failed, wrong data returned", i)
-		}
-	}
-
 	for i := 0; i < 200; i++ {
-		length := rand.Intn(int(testMaxFileSize) / 2)
-		offset := rand.Intn(int(testMaxFileSize))
-		if length+offset > int(testMaxFileSize) {
-			diff := length + offset - int(testMaxFileSize)
-			length -= diff
-		}
+		offset := rand.Intn(int(filesize))
+		length := rand.Intn(int(filesize)-offset) + 100
 
 		b := memfile[offset : offset+length]
+
 		buf := make([]byte, length)
+
 		testRead(t, f, offset, length, buf)
 		if !bytes.Equal(b, buf) {
-			t.Errorf("test %d failed (offset %d, length %d), wrong data returned", i, offset, length)
+			t.Errorf("test %d failed, wrong data returned (offset %v, length %v)", i, offset, length)
 		}
 	}
+
+	OK(t, f.Release(ctx, nil))
 }

src/restic/hashing/reader.go

@@ -0,0 +1,29 @@
+package hashing
+
+import (
+	"hash"
+	"io"
+)
+
+// Reader hashes all data read from the underlying reader.
+type Reader struct {
+	r io.Reader
+	h hash.Hash
+}
+
+// NewReader returns a new Reader that uses the hash h.
+func NewReader(r io.Reader, h hash.Hash) *Reader {
+	return &Reader{
+		h: h,
+		r: io.TeeReader(r, h),
+	}
+}
+
+func (h *Reader) Read(p []byte) (int, error) {
+	return h.r.Read(p)
+}
+
+// Sum returns the hash of the data read so far.
+func (h *Reader) Sum(d []byte) []byte {
+	return h.h.Sum(d)
+}

src/restic/hashing/reader_test.go

@@ -0,0 +1,73 @@
+package hashing
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/sha256"
+	"io"
+	"io/ioutil"
+	"testing"
+)
+
+func TestReader(t *testing.T) {
+	tests := []int{5, 23, 2<<18 + 23, 1 << 20}
+
+	for _, size := range tests {
+		data := make([]byte, size)
+		_, err := io.ReadFull(rand.Reader, data)
+		if err != nil {
+			t.Fatalf("ReadFull: %v", err)
+		}
+
+		expectedHash := sha256.Sum256(data)
+
+		rd := NewReader(bytes.NewReader(data), sha256.New())
+		n, err := io.Copy(ioutil.Discard, rd)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if n != int64(size) {
+			t.Errorf("Reader: invalid number of bytes written: got %d, expected %d",
+				n, size)
+		}
+
+		resultingHash := rd.Sum(nil)
+
+		if !bytes.Equal(expectedHash[:], resultingHash) {
+			t.Errorf("Reader: hashes do not match: expected %02x, got %02x",
+				expectedHash, resultingHash)
+		}
+	}
+}
+
+func BenchmarkReader(b *testing.B) {
+	buf := make([]byte, 1<<22)
+	_, err := io.ReadFull(rand.Reader, buf)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	expectedHash := sha256.Sum256(buf)
+
+	b.SetBytes(int64(len(buf)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		rd := NewReader(bytes.NewReader(buf), sha256.New())
+		n, err := io.Copy(ioutil.Discard, rd)
+		if err != nil {
+			b.Fatal(err)
+		}
+
+		if n != int64(len(buf)) {
+			b.Errorf("Reader: invalid number of bytes written: got %d, expected %d",
+				n, len(buf))
+		}
+
+		resultingHash := rd.Sum(nil)
+		if !bytes.Equal(expectedHash[:], resultingHash) {
+			b.Errorf("Reader: hashes do not match: expected %02x, got %02x",
+				expectedHash, resultingHash)
+		}
+	}
+}

src/restic/hashing/writer.go

@@ -0,0 +1,31 @@
+package hashing
+
+import (
+	"hash"
+	"io"
+)
+
+// Writer transparently hashes all data while writing it to the underlying writer.
+type Writer struct {
+	w io.Writer
+	h hash.Hash
+}
+
+// NewWriter wraps the writer w and feeds all data written to the hash h.
+func NewWriter(w io.Writer, h hash.Hash) *Writer {
+	return &Writer{
+		h: h,
+		w: io.MultiWriter(w, h),
+	}
+}
+
+// Write wraps the write method of the underlying writer and also hashes all data.
+func (h *Writer) Write(p []byte) (int, error) {
+	n, err := h.w.Write(p)
+	return n, err
+}
+
+// Sum returns the hash of all data written so far.
+func (h *Writer) Sum(d []byte) []byte {
+	return h.h.Sum(d)
+}

src/restic/hashing/writer_test.go

@@ -0,0 +1,74 @@
+package hashing
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/sha256"
+	"io"
+	"io/ioutil"
+	"testing"
+)
+
+func TestWriter(t *testing.T) {
+	tests := []int{5, 23, 2<<18 + 23, 1 << 20}
+
+	for _, size := range tests {
+		data := make([]byte, size)
+		_, err := io.ReadFull(rand.Reader, data)
+		if err != nil {
+			t.Fatalf("ReadFull: %v", err)
+		}
+
+		expectedHash := sha256.Sum256(data)
+
+		wr := NewWriter(ioutil.Discard, sha256.New())
+
+		n, err := io.Copy(wr, bytes.NewReader(data))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if n != int64(size) {
+			t.Errorf("Writer: invalid number of bytes written: got %d, expected %d",
+				n, size)
+		}
+
+		resultingHash := wr.Sum(nil)
+
+		if !bytes.Equal(expectedHash[:], resultingHash) {
+			t.Errorf("Writer: hashes do not match: expected %02x, got %02x",
+				expectedHash, resultingHash)
+		}
+	}
+}
+
+func BenchmarkWriter(b *testing.B) {
+	buf := make([]byte, 1<<22)
+	_, err := io.ReadFull(rand.Reader, buf)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	expectedHash := sha256.Sum256(buf)
+
+	b.SetBytes(int64(len(buf)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		wr := NewWriter(ioutil.Discard, sha256.New())
+		n, err := io.Copy(wr, bytes.NewReader(buf))
+		if err != nil {
+			b.Fatal(err)
+		}
+
+		if n != int64(len(buf)) {
+			b.Errorf("Writer: invalid number of bytes written: got %d, expected %d",
+				n, len(buf))
+		}
+
+		resultingHash := wr.Sum(nil)
+		if !bytes.Equal(expectedHash[:], resultingHash) {
+			b.Errorf("Writer: hashes do not match: expected %02x, got %02x",
+				expectedHash, resultingHash)
+		}
+	}
+}

src/restic/id.go

@@ -114,3 +114,13 @@ func (id *ID) UnmarshalJSON(b []byte) error {
 
 	return nil
 }
+
+// IDFromHash returns the ID for the hash.
+func IDFromHash(hash []byte) (id ID) {
+	if len(hash) != idSize {
+		panic("invalid hash type, not enough/too many bytes")
+	}
+
+	copy(id[:], hash)
+	return id
+}

src/restic/index/index.go

@@ -14,27 +14,20 @@ import (
 
 // Pack contains information about the contents of a pack.
 type Pack struct {
+	ID      restic.ID
 	Size    int64
 	Entries []restic.Blob
 }
 
-// Blob contains information about a blob.
-type Blob struct {
-	Size  int64
-	Packs restic.IDSet
-}
-
 // Index contains information about blobs and packs stored in a repo.
 type Index struct {
 	Packs    map[restic.ID]Pack
-	Blobs    map[restic.BlobHandle]Blob
 	IndexIDs restic.IDSet
 }
 
 func newIndex() *Index {
 	return &Index{
 		Packs:    make(map[restic.ID]Pack),
-		Blobs:    make(map[restic.BlobHandle]Blob),
 		IndexIDs: restic.NewIDSet(),
 	}
 }
@@ -69,9 +62,6 @@ func New(repo restic.Repository, p *restic.Progress) (*Index, error) {
 		if err != nil {
 			return nil, err
 		}
-
-		p := Pack{Entries: j.Entries(), Size: j.Size()}
-		idx.Packs[packID] = p
 	}
 
 	return idx, nil
@@ -179,19 +169,7 @@ func (idx *Index) AddPack(id restic.ID, size int64, entries []restic.Blob) error
 		return errors.Errorf("pack %v already present in the index", id.Str())
 	}
 
-	idx.Packs[id] = Pack{Size: size, Entries: entries}
-
-	for _, entry := range entries {
-		h := restic.BlobHandle{ID: entry.ID, Type: entry.Type}
-		if _, ok := idx.Blobs[h]; !ok {
-			idx.Blobs[h] = Blob{
-				Size:  int64(entry.Length),
-				Packs: restic.NewIDSet(),
-			}
-		}
-
-		idx.Blobs[h].Packs.Insert(id)
-	}
+	idx.Packs[id] = Pack{ID: id, Size: size, Entries: entries}
 
 	return nil
 }
@@ -202,15 +180,6 @@ func (idx *Index) RemovePack(id restic.ID) error {
 		return errors.Errorf("pack %v not found in the index", id.Str())
 	}
 
-	for _, blob := range idx.Packs[id].Entries {
-		h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
-		idx.Blobs[h].Packs.Delete(id)
-
-		if len(idx.Blobs[h].Packs) == 0 {
-			delete(idx.Blobs, h)
-		}
-	}
-
 	delete(idx.Packs, id)
 
 	return nil
@@ -239,14 +208,11 @@ func (idx *Index) DuplicateBlobs() (dups restic.BlobSet) {
 func (idx *Index) PacksForBlobs(blobs restic.BlobSet) (packs restic.IDSet) {
 	packs = restic.NewIDSet()
 
-	for h := range blobs {
-		blob, ok := idx.Blobs[h]
-		if !ok {
-			continue
-		}
-
-		for id := range blob.Packs {
-			packs.Insert(id)
+	for id, p := range idx.Packs {
+		for _, entry := range p.Entries {
+			if blobs.Has(restic.BlobHandle{ID: entry.ID, Type: entry.Type}) {
+				packs.Insert(id)
+			}
 		}
 	}
 
@@ -264,31 +230,20 @@ type Location struct {
 var ErrBlobNotFound = errors.New("blob not found in index")
 
 // FindBlob returns a list of packs and positions the blob can be found in.
-func (idx *Index) FindBlob(h restic.BlobHandle) ([]Location, error) {
-	blob, ok := idx.Blobs[h]
-	if !ok {
-		return nil, ErrBlobNotFound
+func (idx *Index) FindBlob(h restic.BlobHandle) (result []Location, err error) {
+	for id, p := range idx.Packs {
+		for _, entry := range p.Entries {
+			if entry.ID.Equal(h.ID) && entry.Type == h.Type {
+				result = append(result, Location{
+					PackID: id,
+					Blob:   entry,
+				})
+			}
+		}
 	}
 
-	result := make([]Location, 0, len(blob.Packs))
-	for packID := range blob.Packs {
-		pack, ok := idx.Packs[packID]
-		if !ok {
-			return nil, errors.Errorf("pack %v not found in index", packID.Str())
-		}
-
-		for _, entry := range pack.Entries {
-			if entry.Type != h.Type {
-				continue
-			}
-
-			if !entry.ID.Equal(h.ID) {
-				continue
-			}
-
-			loc := Location{PackID: packID, Blob: entry}
-			result = append(result, loc)
-		}
+	if len(result) == 0 {
+		return nil, ErrBlobNotFound
 	}
 
 	return result, nil

src/restic/index/index_test.go

@@ -3,7 +3,9 @@ package index
 import (
 	"math/rand"
 	"restic"
+	"restic/checker"
 	"restic/repository"
+	"restic/test"
 	"testing"
 	"time"
 )
@@ -25,9 +27,14 @@ func createFilledRepo(t testing.TB, snapshots int, dup float32) (restic.Reposito
 
 func validateIndex(t testing.TB, repo restic.Repository, idx *Index) {
 	for id := range repo.List(restic.DataFile, nil) {
-		if _, ok := idx.Packs[id]; !ok {
+		p, ok := idx.Packs[id]
+		if !ok {
 			t.Errorf("pack %v missing from index", id.Str())
 		}
+
+		if !p.ID.Equal(id) {
+			t.Errorf("pack %v has invalid ID: want %v, got %v", id.Str(), id, p.ID)
+		}
 	}
 }
 
@@ -135,6 +142,40 @@ func BenchmarkIndexNew(b *testing.B) {
 		if idx == nil {
 			b.Fatalf("New() returned nil index")
 		}
+		b.Logf("idx %v packs", len(idx.Packs))
+	}
+}
+
+func BenchmarkIndexSave(b *testing.B) {
+	repo, cleanup := repository.TestRepository(b)
+	defer cleanup()
+
+	idx, err := New(repo, nil)
+	test.OK(b, err)
+
+	for i := 0; i < 8000; i++ {
+		entries := make([]restic.Blob, 0, 200)
+		for j := 0; j < cap(entries); j++ {
+			entries = append(entries, restic.Blob{
+				ID:     restic.NewRandomID(),
+				Length: 1000,
+				Offset: 5,
+				Type:   restic.DataBlob,
+			})
+		}
+
+		idx.AddPack(restic.NewRandomID(), 10000, entries)
+	}
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		id, err := idx.Save(repo, nil)
+		if err != nil {
+			b.Fatalf("New() returned error %v", err)
+		}
+
+		b.Logf("saved as %v", id.Str())
 	}
 }
 
@@ -151,7 +192,7 @@ func TestIndexDuplicateBlobs(t *testing.T) {
 	if len(dups) == 0 {
 		t.Errorf("no duplicate blobs found")
 	}
-	t.Logf("%d packs, %d unique blobs", len(idx.Packs), len(idx.Blobs))
+	t.Logf("%d packs, %d duplicate blobs", len(idx.Packs), len(dups))
 
 	packs := idx.PacksForBlobs(dups)
 	if len(packs) == 0 {
@@ -169,7 +210,7 @@ func loadIndex(t testing.TB, repo restic.Repository) *Index {
 	return idx
 }
 
-func TestIndexSave(t *testing.T) {
+func TestSave(t *testing.T) {
 	repo, cleanup := createFilledRepo(t, 3, 0)
 	defer cleanup()
 
@@ -193,7 +234,8 @@ func TestIndexSave(t *testing.T) {
 
 	for id := range idx.IndexIDs {
 		t.Logf("remove index %v", id.Str())
-		err = repo.Backend().Remove(restic.IndexFile, id.String())
+		h := restic.Handle{Type: restic.IndexFile, Name: id.String()}
+		err = repo.Backend().Remove(h)
 		if err != nil {
 			t.Errorf("error removing index %v: %v", id, err)
 		}
@@ -219,6 +261,42 @@ func TestIndexSave(t *testing.T) {
 	}
 }
 
+func TestIndexSave(t *testing.T) {
+	repo, cleanup := createFilledRepo(t, 3, 0)
+	defer cleanup()
+
+	idx := loadIndex(t, repo)
+
+	id, err := idx.Save(repo, idx.IndexIDs.List())
+	if err != nil {
+		t.Fatalf("unable to save new index: %v", err)
+	}
+
+	t.Logf("new index saved as %v", id.Str())
+
+	for id := range idx.IndexIDs {
+		t.Logf("remove index %v", id.Str())
+		h := restic.Handle{Type: restic.IndexFile, Name: id.String()}
+		err = repo.Backend().Remove(h)
+		if err != nil {
+			t.Errorf("error removing index %v: %v", id, err)
+		}
+	}
+
+	idx2 := loadIndex(t, repo)
+	t.Logf("load new index with %d packs", len(idx2.Packs))
+
+	checker := checker.New(repo)
+	hints, errs := checker.LoadIndex()
+	for _, h := range hints {
+		t.Logf("hint: %v\n", h)
+	}
+
+	for _, err := range errs {
+		t.Errorf("checker found error: %v", err)
+	}
+}
+
 func TestIndexAddRemovePack(t *testing.T) {
 	repo, cleanup := createFilledRepo(t, 3, 0)
 	defer cleanup()
@@ -249,12 +327,7 @@ func TestIndexAddRemovePack(t *testing.T) {
 		if err == nil {
 			t.Errorf("removed blob %v found in index", h)
 		}
-
-		if _, ok := idx.Blobs[h]; ok {
-			t.Errorf("removed blob %v found in index.Blobs", h)
-		}
 	}
-
 }
 
 // example index serialization from doc/Design.md

src/restic/lock.go

@@ -186,7 +186,7 @@ func (l *Lock) Unlock() error {
 		return nil
 	}
 
-	return l.repo.Backend().Remove(LockFile, l.lockID.String())
+	return l.repo.Backend().Remove(Handle{Type: LockFile, Name: l.lockID.String()})
 }
 
 var staleTimeout = 30 * time.Minute
@@ -234,7 +234,7 @@ func (l *Lock) Refresh() error {
 		return err
 	}
 
-	err = l.repo.Backend().Remove(LockFile, l.lockID.String())
+	err = l.repo.Backend().Remove(Handle{Type: LockFile, Name: l.lockID.String()})
 	if err != nil {
 		return err
 	}
@@ -289,7 +289,7 @@ func RemoveStaleLocks(repo Repository) error {
 		}
 
 		if lock.Stale() {
-			return repo.Backend().Remove(LockFile, id.String())
+			return repo.Backend().Remove(Handle{Type: LockFile, Name: id.String()})
 		}
 
 		return nil
@@ -299,6 +299,6 @@ func RemoveStaleLocks(repo Repository) error {
 // RemoveAllLocks removes all locks forcefully.
 func RemoveAllLocks(repo Repository) error {
 	return eachLock(repo, func(id ID, lock *Lock, err error) error {
-		return repo.Backend().Remove(LockFile, id.String())
+		return repo.Backend().Remove(Handle{Type: LockFile, Name: id.String()})
 	})
 }

src/restic/lock_test.go

@@ -102,7 +102,8 @@ func createFakeLock(repo restic.Repository, t time.Time, pid int) (restic.ID, er
 }
 
 func removeLock(repo restic.Repository, id restic.ID) error {
-	return repo.Backend().Remove(restic.LockFile, id.String())
+	h := restic.Handle{Type: restic.LockFile, Name: id.String()}
+	return repo.Backend().Remove(h)
 }
 
 var staleLockTests = []struct {
@@ -162,7 +163,8 @@ func TestLockStale(t *testing.T) {
 }
 
 func lockExists(repo restic.Repository, t testing.TB, id restic.ID) bool {
-	exists, err := repo.Backend().Test(restic.LockFile, id.String())
+	h := restic.Handle{Type: restic.LockFile, Name: id.String()}
+	exists, err := repo.Backend().Test(h)
 	OK(t, err)
 
 	return exists

src/restic/mock/backend.go

@@ -1,6 +1,7 @@
 package mock
 
 import (
+	"io"
 	"restic"
 
 	"restic/errors"
@@ -9,12 +10,12 @@ import (
 // Backend implements a mock backend.
 type Backend struct {
 	CloseFn    func() error
-	LoadFn     func(h restic.Handle, p []byte, off int64) (int, error)
-	SaveFn     func(h restic.Handle, p []byte) error
+	SaveFn     func(h restic.Handle, rd io.Reader) error
+	LoadFn     func(h restic.Handle, length int, offset int64) (io.ReadCloser, error)
 	StatFn     func(h restic.Handle) (restic.FileInfo, error)
 	ListFn     func(restic.FileType, <-chan struct{}) <-chan string
-	RemoveFn   func(restic.FileType, string) error
-	TestFn     func(restic.FileType, string) (bool, error)
+	RemoveFn   func(h restic.Handle) error
+	TestFn     func(h restic.Handle) (bool, error)
 	DeleteFn   func() error
 	LocationFn func() string
 }
@@ -37,22 +38,22 @@ func (m *Backend) Location() string {
 	return m.LocationFn()
 }
 
-// Load loads data from the backend.
-func (m *Backend) Load(h restic.Handle, p []byte, off int64) (int, error) {
-	if m.LoadFn == nil {
-		return 0, errors.New("not implemented")
-	}
-
-	return m.LoadFn(h, p, off)
-}
-
 // Save data in the backend.
-func (m *Backend) Save(h restic.Handle, p []byte) error {
+func (m *Backend) Save(h restic.Handle, rd io.Reader) error {
 	if m.SaveFn == nil {
 		return errors.New("not implemented")
 	}
 
-	return m.SaveFn(h, p)
+	return m.SaveFn(h, rd)
+}
+
+// Load loads data from the backend.
+func (m *Backend) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	if m.LoadFn == nil {
+		return nil, errors.New("not implemented")
+	}
+
+	return m.LoadFn(h, length, offset)
 }
 
 // Stat an object in the backend.
@@ -76,21 +77,21 @@ func (m *Backend) List(t restic.FileType, done <-chan struct{}) <-chan string {
 }
 
 // Remove data from the backend.
-func (m *Backend) Remove(t restic.FileType, name string) error {
+func (m *Backend) Remove(h restic.Handle) error {
 	if m.RemoveFn == nil {
 		return errors.New("not implemented")
 	}
 
-	return m.RemoveFn(t, name)
+	return m.RemoveFn(h)
 }
 
 // Test for the existence of a specific item.
-func (m *Backend) Test(t restic.FileType, name string) (bool, error) {
+func (m *Backend) Test(h restic.Handle) (bool, error) {
 	if m.TestFn == nil {
 		return false, errors.New("not implemented")
 	}
 
-	return m.TestFn(t, name)
+	return m.TestFn(h)
 }
 
 // Delete all data.

src/restic/node.go

@@ -184,7 +184,7 @@ func (node Node) RestoreTimestamps(path string) error {
 
 func (node Node) createDirAt(path string) error {
 	err := fs.Mkdir(path, node.Mode)
-	if err != nil {
+	if err != nil && !os.IsExist(err) {
 		return errors.Wrap(err, "Mkdir")
 	}
 
@@ -208,7 +208,7 @@ func (node Node) createFileAt(path string, repo Repository) error {
 
 		buf = buf[:cap(buf)]
 		if uint(len(buf)) < size {
-			buf = make([]byte, size)
+			buf = NewBlobBuffer(int(size))
 		}
 
 		n, err := repo.LoadBlob(DataBlob, id, buf)

src/restic/node_test.go

@@ -137,6 +137,31 @@ var nodeTests = []restic.Node{
 		AccessTime: parseTime("2015-05-14 21:07:24.222"),
 		ChangeTime: parseTime("2015-05-14 21:07:25.333"),
 	},
+
+	// include "testFile" and "testDir" again with slightly different
+	// metadata, so we can test if CreateAt works with pre-existing files.
+	restic.Node{
+		Name:       "testFile",
+		Type:       "file",
+		Content:    restic.IDs{},
+		UID:        uint32(os.Getuid()),
+		GID:        uint32(os.Getgid()),
+		Mode:       0604,
+		ModTime:    parseTime("2005-05-14 21:07:03.111"),
+		AccessTime: parseTime("2005-05-14 21:07:04.222"),
+		ChangeTime: parseTime("2005-05-14 21:07:05.333"),
+	},
+	restic.Node{
+		Name:       "testDir",
+		Type:       "dir",
+		Subtree:    nil,
+		UID:        uint32(os.Getuid()),
+		GID:        uint32(os.Getgid()),
+		Mode:       0750 | os.ModeDir,
+		ModTime:    parseTime("2005-05-14 21:07:03.111"),
+		AccessTime: parseTime("2005-05-14 21:07:04.222"),
+		ChangeTime: parseTime("2005-05-14 21:07:05.333"),
+	},
 }
 
 func TestNodeRestoreAt(t *testing.T) {

src/restic/pack/pack.go

@@ -85,15 +85,15 @@ func (p *Packer) Finalize() (uint, error) {
 		return 0, errors.Wrap(err, "Write")
 	}
 
-	hdrBytes := bytesHeader + crypto.Extension
-	if uint(n) != hdrBytes {
+	hdrBytes := restic.CiphertextLength(int(bytesHeader))
+	if n != hdrBytes {
 		return 0, errors.New("wrong number of bytes written")
 	}
 
-	bytesWritten += hdrBytes
+	bytesWritten += uint(hdrBytes)
 
 	// write length
-	err = binary.Write(p.wr, binary.LittleEndian, uint32(uint(len(p.blobs))*entrySize+crypto.Extension))
+	err = binary.Write(p.wr, binary.LittleEndian, uint32(restic.CiphertextLength(len(p.blobs)*int(entrySize))))
 	if err != nil {
 		return 0, errors.Wrap(err, "binary.Write")
 	}
@@ -233,6 +233,8 @@ func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries []restic.Blob, err
 
 	hdrRd := bytes.NewReader(buf)
 
+	entries = make([]restic.Blob, 0, uint(n)/entrySize)
+
 	pos := uint(0)
 	for {
 		e := headerEntry{}

src/restic/pack/pack_test.go

@@ -54,10 +54,9 @@ func verifyBlobs(t testing.TB, bufs []Buf, k *crypto.Key, rd io.ReaderAt, packSi
 	}
 	// header length
 	written += binary.Size(uint32(0))
-	// header
-	written += len(bufs) * (binary.Size(restic.BlobType(0)) + binary.Size(uint32(0)) + len(restic.ID{}))
-	// header crypto
-	written += crypto.Extension
+	// header + header crypto
+	headerSize := len(bufs) * (binary.Size(restic.BlobType(0)) + binary.Size(uint32(0)) + len(restic.ID{}))
+	written += restic.CiphertextLength(headerSize)
 
 	// check length
 	Equals(t, uint(written), packSize)
@@ -127,7 +126,7 @@ func TestUnpackReadSeeker(t *testing.T) {
 	id := restic.Hash(packData)
 
 	handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
-	OK(t, b.Save(handle, packData))
+	OK(t, b.Save(handle, bytes.NewReader(packData)))
 	verifyBlobs(t, bufs, k, restic.ReaderAt(b, handle), packSize)
 }
 
@@ -140,6 +139,6 @@ func TestShortPack(t *testing.T) {
 	id := restic.Hash(packData)
 
 	handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
-	OK(t, b.Save(handle, packData))
+	OK(t, b.Save(handle, bytes.NewReader(packData)))
 	verifyBlobs(t, bufs, k, restic.ReaderAt(b, handle), packSize)
 }

src/restic/readerat.go

@@ -2,6 +2,7 @@ package restic
 
 import (
 	"io"
+	"restic/debug"
 )
 
 type backendReaderAt struct {
@@ -9,11 +10,30 @@ type backendReaderAt struct {
 	h  Handle
 }
 
-func (brd backendReaderAt) ReadAt(p []byte, off int64) (n int, err error) {
-	return brd.be.Load(brd.h, p, off)
+func (brd backendReaderAt) ReadAt(p []byte, offset int64) (n int, err error) {
+	return ReadAt(brd.be, brd.h, offset, p)
 }
 
 // ReaderAt returns an io.ReaderAt for a file in the backend.
 func ReaderAt(be Backend, h Handle) io.ReaderAt {
 	return backendReaderAt{be: be, h: h}
 }
+
+// ReadAt reads from the backend handle h at the given position.
+func ReadAt(be Backend, h Handle, offset int64, p []byte) (n int, err error) {
+	debug.Log("ReadAt(%v) at %v, len %v", h, offset, len(p))
+	rd, err := be.Load(h, len(p), offset)
+	if err != nil {
+		return 0, err
+	}
+
+	n, err = io.ReadFull(rd, p)
+	e := rd.Close()
+	if err == nil {
+		err = e
+	}
+
+	debug.Log("ReadAt(%v) ReadFull returned %v bytes", h, n)
+
+	return n, err
+}

src/restic/repository/index.go

@@ -1,7 +1,6 @@
 package repository
 
 import (
-	"bytes"
 	"encoding/json"
 	"io"
 	"restic"
@@ -10,7 +9,6 @@ import (
 
 	"restic/errors"
 
-	"restic/crypto"
 	"restic/debug"
 )
 
@@ -177,15 +175,15 @@ func (idx *Index) Has(id restic.ID, tpe restic.BlobType) bool {
 	return false
 }
 
-// LookupSize returns the length of the cleartext content behind the
-// given id
-func (idx *Index) LookupSize(id restic.ID, tpe restic.BlobType) (cleartextLength uint, err error) {
+// LookupSize returns the length of the plaintext content of the blob with the
+// given id.
+func (idx *Index) LookupSize(id restic.ID, tpe restic.BlobType) (plaintextLength uint, err error) {
 	blobs, err := idx.Lookup(id, tpe)
 	if err != nil {
 		return 0, err
 	}
 
-	return blobs[0].Length - crypto.Extension, nil
+	return uint(restic.PlaintextLength(int(blobs[0].Length))), nil
 }
 
 // Supersedes returns the list of indexes this index supersedes, if any.
@@ -452,12 +450,11 @@ func isErrOldIndex(err error) bool {
 var ErrOldIndexFormat = errors.New("index has old format")
 
 // DecodeIndex loads and unserializes an index from rd.
-func DecodeIndex(rd io.Reader) (idx *Index, err error) {
+func DecodeIndex(buf []byte) (idx *Index, err error) {
 	debug.Log("Start decoding index")
-	idxJSON := jsonIndex{}
+	idxJSON := &jsonIndex{}
 
-	dec := json.NewDecoder(rd)
-	err = dec.Decode(&idxJSON)
+	err = json.Unmarshal(buf, idxJSON)
 	if err != nil {
 		debug.Log("Error %v", err)
 
@@ -491,12 +488,11 @@ func DecodeIndex(rd io.Reader) (idx *Index, err error) {
 }
 
 // DecodeOldIndex loads and unserializes an index in the old format from rd.
-func DecodeOldIndex(rd io.Reader) (idx *Index, err error) {
+func DecodeOldIndex(buf []byte) (idx *Index, err error) {
 	debug.Log("Start decoding old index")
 	list := []*packJSON{}
 
-	dec := json.NewDecoder(rd)
-	err = dec.Decode(&list)
+	err = json.Unmarshal(buf, &list)
 	if err != nil {
 		debug.Log("Error %#v", err)
 		return nil, errors.Wrap(err, "Decode")
@@ -523,7 +519,7 @@ func DecodeOldIndex(rd io.Reader) (idx *Index, err error) {
 }
 
 // LoadIndexWithDecoder loads the index and decodes it with fn.
-func LoadIndexWithDecoder(repo restic.Repository, id restic.ID, fn func(io.Reader) (*Index, error)) (idx *Index, err error) {
+func LoadIndexWithDecoder(repo restic.Repository, id restic.ID, fn func([]byte) (*Index, error)) (idx *Index, err error) {
 	debug.Log("Loading index %v", id.Str())
 
 	buf, err := repo.LoadAndDecrypt(restic.IndexFile, id)
@@ -531,7 +527,7 @@ func LoadIndexWithDecoder(repo restic.Repository, id restic.ID, fn func(io.Reade
 		return nil, err
 	}
 
-	idx, err = fn(bytes.NewReader(buf))
+	idx, err = fn(buf)
 	if err != nil {
 		debug.Log("error while decoding index %v: %v", id, err)
 		return nil, err

src/restic/repository/index_rebuild.go

@@ -55,7 +55,8 @@ func RebuildIndex(repo restic.Repository) error {
 	debug.Log("new index saved as %v", id.Str())
 
 	for indexID := range oldIndexes {
-		err := repo.Backend().Remove(restic.IndexFile, indexID.String())
+		h := restic.Handle{Type: restic.IndexFile, Name: indexID.String()}
+		err := repo.Backend().Remove(h)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "unable to remove index %v: %v\n", indexID.Str(), err)
 		}

src/restic/repository/index_test.go

@@ -54,7 +54,7 @@ func TestIndexSerialize(t *testing.T) {
 	err := idx.Encode(wr)
 	OK(t, err)
 
-	idx2, err := repository.DecodeIndex(wr)
+	idx2, err := repository.DecodeIndex(wr.Bytes())
 	OK(t, err)
 	Assert(t, idx2 != nil,
 		"nil returned for decoded index")
@@ -136,7 +136,7 @@ func TestIndexSerialize(t *testing.T) {
 	Assert(t, id2.Equal(id),
 		"wrong ID returned: want %v, got %v", id, id2)
 
-	idx3, err := repository.DecodeIndex(wr3)
+	idx3, err := repository.DecodeIndex(wr3.Bytes())
 	OK(t, err)
 	Assert(t, idx3 != nil,
 		"nil returned for decoded index")
@@ -288,7 +288,7 @@ var exampleLookupTest = struct {
 func TestIndexUnserialize(t *testing.T) {
 	oldIdx := restic.IDs{restic.TestParseID("ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452")}
 
-	idx, err := repository.DecodeIndex(bytes.NewReader(docExample))
+	idx, err := repository.DecodeIndex(docExample)
 	OK(t, err)
 
 	for _, test := range exampleTests {
@@ -326,8 +326,17 @@ func TestIndexUnserialize(t *testing.T) {
 	}
 }
 
+func BenchmarkDecodeIndex(b *testing.B) {
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		_, err := repository.DecodeIndex(docExample)
+		OK(b, err)
+	}
+}
+
 func TestIndexUnserializeOld(t *testing.T) {
-	idx, err := repository.DecodeOldIndex(bytes.NewReader(docOldExample))
+	idx, err := repository.DecodeOldIndex(docOldExample)
 	OK(t, err)
 
 	for _, test := range exampleTests {

src/restic/repository/key.go

@@ -1,6 +1,7 @@
 package repository
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -146,7 +147,7 @@ func SearchKey(s *Repository, password string, maxKeys int) (*Key, error) {
 // LoadKey loads a key from the backend.
 func LoadKey(s *Repository, name string) (k *Key, err error) {
 	h := restic.Handle{Type: restic.KeyFile, Name: name}
-	data, err := backend.LoadAll(s.be, h, nil)
+	data, err := backend.LoadAll(s.be, h)
 	if err != nil {
 		return nil, err
 	}
@@ -232,7 +233,7 @@ func AddKey(s *Repository, password string, template *crypto.Key) (*Key, error)
 		Name: restic.Hash(buf).String(),
 	}
 
-	err = s.be.Save(h, buf)
+	err = s.be.Save(h, bytes.NewReader(buf))
 	if err != nil {
 		return nil, err
 	}

src/restic/repository/master_index.go

@@ -30,8 +30,7 @@ func (mi *MasterIndex) Lookup(id restic.ID, tpe restic.BlobType) (blobs []restic
 	for _, idx := range mi.idx {
 		blobs, err = idx.Lookup(id, tpe)
 		if err == nil {
-			debug.Log("MasterIndex.Lookup",
-				"found id %v: %v", id.Str(), blobs)
+			debug.Log("found id %v: %v", id.Str(), blobs)
 			return
 		}
 	}
@@ -46,9 +45,8 @@ func (mi *MasterIndex) LookupSize(id restic.ID, tpe restic.BlobType) (uint, erro
 	defer mi.idxMutex.RUnlock()
 
 	for _, idx := range mi.idx {
-		length, err := idx.LookupSize(id, tpe)
-		if err == nil {
-			return length, nil
+		if idx.Has(id, tpe) {
+			return idx.LookupSize(id, tpe)
 		}
 	}
 
@@ -119,16 +117,14 @@ func (mi *MasterIndex) Remove(index *Index) {
 	}
 }
 
-// Current returns an index that is not yet finalized, so that new entries can
-// still be added. If all indexes are finalized, a new index is created and
-// returned.
-func (mi *MasterIndex) Current() *Index {
+// Store remembers the id and pack in the index.
+func (mi *MasterIndex) Store(pb restic.PackedBlob) {
 	mi.idxMutex.RLock()
-
 	for _, idx := range mi.idx {
 		if !idx.Final() {
 			mi.idxMutex.RUnlock()
-			return idx
+			idx.Store(pb)
+			return
 		}
 	}
 
@@ -137,9 +133,8 @@ func (mi *MasterIndex) Current() *Index {
 	defer mi.idxMutex.Unlock()
 
 	newIdx := NewIndex()
+	newIdx.Store(pb)
 	mi.idx = append(mi.idx, newIdx)
-
-	return newIdx
 }
 
 // NotFinalIndexes returns all indexes that have not yet been saved.

src/restic/repository/packer_manager.go

@@ -1,6 +1,7 @@
 package repository
 
 import (
+	"crypto/sha256"
 	"io"
 	"io/ioutil"
 	"os"
@@ -8,6 +9,7 @@ import (
 	"sync"
 
 	"restic/errors"
+	"restic/hashing"
 
 	"restic/crypto"
 	"restic/debug"
@@ -17,15 +19,22 @@ import (
 
 // Saver implements saving data in a backend.
 type Saver interface {
-	Save(h restic.Handle, jp []byte) error
+	Save(restic.Handle, io.Reader) error
+}
+
+// Packer holds a pack.Packer together with a hash writer.
+type Packer struct {
+	*pack.Packer
+	hw      *hashing.Writer
+	tmpfile *os.File
 }
 
 // packerManager keeps a list of open packs and creates new on demand.
 type packerManager struct {
-	be    Saver
-	key   *crypto.Key
-	pm    sync.Mutex
-	packs []*pack.Packer
+	be      Saver
+	key     *crypto.Key
+	pm      sync.Mutex
+	packers []*Packer
 
 	pool sync.Pool
 }
@@ -50,18 +59,18 @@ func newPackerManager(be Saver, key *crypto.Key) *packerManager {
 
 // findPacker returns a packer for a new blob of size bytes. Either a new one is
 // created or one is returned that already has some blobs.
-func (r *packerManager) findPacker(size uint) (packer *pack.Packer, err error) {
+func (r *packerManager) findPacker(size uint) (packer *Packer, err error) {
 	r.pm.Lock()
 	defer r.pm.Unlock()
 
 	// search for a suitable packer
-	if len(r.packs) > 0 {
+	if len(r.packers) > 0 {
 		debug.Log("searching packer for %d bytes\n", size)
-		for i, p := range r.packs {
-			if p.Size()+size < maxPackSize {
+		for i, p := range r.packers {
+			if p.Packer.Size()+size < maxPackSize {
 				debug.Log("found packer %v", p)
 				// remove from list
-				r.packs = append(r.packs[:i], r.packs[i+1:]...)
+				r.packers = append(r.packers[:i], r.packers[i+1:]...)
 				return p, nil
 			}
 		}
@@ -74,50 +83,43 @@ func (r *packerManager) findPacker(size uint) (packer *pack.Packer, err error) {
 		return nil, errors.Wrap(err, "ioutil.TempFile")
 	}
 
-	return pack.NewPacker(r.key, tmpfile), nil
+	hw := hashing.NewWriter(tmpfile, sha256.New())
+	p := pack.NewPacker(r.key, hw)
+	packer = &Packer{
+		Packer:  p,
+		hw:      hw,
+		tmpfile: tmpfile,
+	}
+
+	return packer, nil
 }
 
 // insertPacker appends p to s.packs.
-func (r *packerManager) insertPacker(p *pack.Packer) {
+func (r *packerManager) insertPacker(p *Packer) {
 	r.pm.Lock()
 	defer r.pm.Unlock()
 
-	r.packs = append(r.packs, p)
-	debug.Log("%d packers\n", len(r.packs))
+	r.packers = append(r.packers, p)
+	debug.Log("%d packers\n", len(r.packers))
 }
 
 // savePacker stores p in the backend.
-func (r *Repository) savePacker(p *pack.Packer) error {
-	debug.Log("save packer with %d blobs\n", p.Count())
-	n, err := p.Finalize()
+func (r *Repository) savePacker(p *Packer) error {
+	debug.Log("save packer with %d blobs\n", p.Packer.Count())
+	_, err := p.Packer.Finalize()
 	if err != nil {
 		return err
 	}
 
-	tmpfile := p.Writer().(*os.File)
-	f, err := fs.Open(tmpfile.Name())
+	_, err = p.tmpfile.Seek(0, 0)
 	if err != nil {
-		return errors.Wrap(err, "Open")
+		return errors.Wrap(err, "Seek")
 	}
 
-	data := make([]byte, n)
-	m, err := io.ReadFull(f, data)
-	if err != nil {
-		return errors.Wrap(err, "ReadFul")
-	}
-
-	if uint(m) != n {
-		return errors.Errorf("read wrong number of bytes from %v: want %v, got %v", tmpfile.Name(), n, m)
-	}
-
-	if err = f.Close(); err != nil {
-		return errors.Wrap(err, "Close")
-	}
-
-	id := restic.Hash(data)
+	id := restic.IDFromHash(p.hw.Sum(nil))
 	h := restic.Handle{Type: restic.DataFile, Name: id.String()}
 
-	err = r.be.Save(h, data)
+	err = r.be.Save(h, p.tmpfile)
 	if err != nil {
 		debug.Log("Save(%v) error: %v", h, err)
 		return err
@@ -125,15 +127,20 @@ func (r *Repository) savePacker(p *pack.Packer) error {
 
 	debug.Log("saved as %v", h)
 
-	err = fs.Remove(tmpfile.Name())
+	err = p.tmpfile.Close()
+	if err != nil {
+		return errors.Wrap(err, "close tempfile")
+	}
+
+	err = fs.Remove(p.tmpfile.Name())
 	if err != nil {
 		return errors.Wrap(err, "Remove")
 	}
 
 	// update blobs in the index
-	for _, b := range p.Blobs() {
+	for _, b := range p.Packer.Blobs() {
 		debug.Log("  updating blob %v to pack %v", b.ID.Str(), id.Str())
-		r.idx.Current().Store(restic.PackedBlob{
+		r.idx.Store(restic.PackedBlob{
 			Blob: restic.Blob{
 				Type:   b.Type,
 				ID:     b.ID,
@@ -152,5 +159,5 @@ func (r *packerManager) countPacker() int {
 	r.pm.Lock()
 	defer r.pm.Unlock()
 
-	return len(r.packs)
+	return len(r.packers)
 }

src/restic/repository/packer_manager_test.go

@@ -7,6 +7,7 @@ import (
 	"restic"
 	"restic/backend/mem"
 	"restic/crypto"
+	"restic/mock"
 	"testing"
 )
 
@@ -46,32 +47,19 @@ func randomID(rd io.Reader) restic.ID {
 
 const maxBlobSize = 1 << 20
 
-func saveFile(t testing.TB, be Saver, filename string, n int) {
-	f, err := os.Open(filename)
-	if err != nil {
+func saveFile(t testing.TB, be Saver, f *os.File, id restic.ID) {
+	h := restic.Handle{Type: restic.DataFile, Name: id.String()}
+	t.Logf("save file %v", h)
+
+	if err := be.Save(h, f); err != nil {
 		t.Fatal(err)
 	}
 
-	data := make([]byte, n)
-	m, err := io.ReadFull(f, data)
-
-	if m != n {
-		t.Fatalf("read wrong number of bytes from %v: want %v, got %v", filename, m, n)
-	}
-
-	if err = f.Close(); err != nil {
+	if err := f.Close(); err != nil {
 		t.Fatal(err)
 	}
 
-	h := restic.Handle{Type: restic.DataFile, Name: restic.Hash(data).String()}
-
-	err = be.Save(h, data)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = os.Remove(filename)
-	if err != nil {
+	if err := os.Remove(f.Name()); err != nil {
 		t.Fatal(err)
 	}
 }
@@ -105,13 +93,17 @@ func fillPacks(t testing.TB, rnd *randReader, be Saver, pm *packerManager, buf [
 			continue
 		}
 
-		bytesWritten, err := packer.Finalize()
+		_, err = packer.Finalize()
 		if err != nil {
 			t.Fatal(err)
 		}
 
-		tmpfile := packer.Writer().(*os.File)
-		saveFile(t, be, tmpfile.Name(), int(bytesWritten))
+		if _, err = packer.tmpfile.Seek(0, 0); err != nil {
+			t.Fatal(err)
+		}
+
+		packID := restic.IDFromHash(packer.hw.Sum(nil))
+		saveFile(t, be, packer.tmpfile, packID)
 	}
 
 	return bytes
@@ -119,27 +111,21 @@ func fillPacks(t testing.TB, rnd *randReader, be Saver, pm *packerManager, buf [
 
 func flushRemainingPacks(t testing.TB, rnd *randReader, be Saver, pm *packerManager) (bytes int) {
 	if pm.countPacker() > 0 {
-		for _, packer := range pm.packs {
+		for _, packer := range pm.packers {
 			n, err := packer.Finalize()
 			if err != nil {
 				t.Fatal(err)
 			}
 			bytes += int(n)
 
-			tmpfile := packer.Writer().(*os.File)
-			saveFile(t, be, tmpfile.Name(), bytes)
+			packID := restic.IDFromHash(packer.hw.Sum(nil))
+			saveFile(t, be, packer.tmpfile, packID)
 		}
 	}
 
 	return bytes
 }
 
-type fakeBackend struct{}
-
-func (f *fakeBackend) Save(h restic.Handle, data []byte) error {
-	return nil
-}
-
 func TestPackerManager(t *testing.T) {
 	rnd := newRandReader(rand.NewSource(23))
 
@@ -157,17 +143,18 @@ func TestPackerManager(t *testing.T) {
 func BenchmarkPackerManager(t *testing.B) {
 	rnd := newRandReader(rand.NewSource(23))
 
-	be := &fakeBackend{}
-	pm := newPackerManager(be, crypto.NewRandomKey())
+	be := &mock.Backend{
+		SaveFn: func(restic.Handle, io.Reader) error { return nil },
+	}
 	blobBuf := make([]byte, maxBlobSize)
 
 	t.ResetTimer()
 
-	bytes := 0
 	for i := 0; i < t.N; i++ {
+		bytes := 0
+		pm := newPackerManager(be, crypto.NewRandomKey())
 		bytes += fillPacks(t, rnd, be, pm, blobBuf)
+		bytes += flushRemainingPacks(t, rnd, be, pm)
+		t.Logf("saved %d bytes", bytes)
 	}
-
-	bytes += flushRemainingPacks(t, rnd, be, pm)
-	t.Logf("saved %d bytes", bytes)
 }

src/restic/repository/repack.go

@@ -1,11 +1,14 @@
 package repository
 
 import (
-	"bytes"
+	"crypto/sha256"
 	"io"
+	"io/ioutil"
+	"os"
 	"restic"
 	"restic/crypto"
 	"restic/debug"
+	"restic/hashing"
 	"restic/pack"
 
 	"restic/errors"
@@ -18,30 +21,47 @@ import (
 func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet) (err error) {
 	debug.Log("repacking %d packs while keeping %d blobs", len(packs), len(keepBlobs))
 
-	buf := make([]byte, 0, maxPackSize)
 	for packID := range packs {
-		// load the complete pack
+		// load the complete pack into a temp file
 		h := restic.Handle{Type: restic.DataFile, Name: packID.String()}
 
-		l, err := repo.Backend().Load(h, buf[:cap(buf)], 0)
-		if errors.Cause(err) == io.ErrUnexpectedEOF {
-			err = nil
-			buf = buf[:l]
+		tempfile, err := ioutil.TempFile("", "restic-temp-repack-")
+		if err != nil {
+			return errors.Wrap(err, "TempFile")
 		}
 
+		beRd, err := repo.Backend().Load(h, 0, 0)
 		if err != nil {
 			return err
 		}
 
-		debug.Log("pack %v loaded (%d bytes)", packID.Str(), len(buf))
+		defer beRd.Close()
 
-		blobs, err := pack.List(repo.Key(), bytes.NewReader(buf), int64(len(buf)))
+		hrd := hashing.NewReader(beRd, sha256.New())
+		packLength, err := io.Copy(tempfile, hrd)
+		if err != nil {
+			return errors.Wrap(err, "Copy")
+		}
+
+		hash := restic.IDFromHash(hrd.Sum(nil))
+		debug.Log("pack %v loaded (%d bytes), hash %v", packID.Str(), packLength, hash.Str())
+
+		if !packID.Equal(hash) {
+			return errors.Errorf("hash does not match id: want %v, got %v", packID, hash)
+		}
+
+		_, err = tempfile.Seek(0, 0)
+		if err != nil {
+			return errors.Wrap(err, "Seek")
+		}
+
+		blobs, err := pack.List(repo.Key(), tempfile, packLength)
 		if err != nil {
 			return err
 		}
 
 		debug.Log("processing pack %v, blobs: %v", packID.Str(), len(blobs))
-		var plaintext []byte
+		var buf []byte
 		for _, entry := range blobs {
 			h := restic.BlobHandle{ID: entry.ID, Type: entry.Type}
 			if !keepBlobs.Has(h) {
@@ -50,21 +70,36 @@ func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet
 
 			debug.Log("  process blob %v", h)
 
-			ciphertext := buf[entry.Offset : entry.Offset+entry.Length]
-			plaintext = plaintext[:len(plaintext)]
-			if len(plaintext) < len(ciphertext) {
-				plaintext = make([]byte, len(ciphertext))
+			buf = buf[:len(buf)]
+			if uint(len(buf)) < entry.Length {
+				buf = make([]byte, entry.Length)
+			}
+			buf = buf[:entry.Length]
+
+			n, err := tempfile.ReadAt(buf, int64(entry.Offset))
+			if err != nil {
+				return errors.Wrap(err, "ReadAt")
 			}
 
-			debug.Log("  ciphertext %d, plaintext %d", len(plaintext), len(ciphertext))
+			if n != len(buf) {
+				return errors.Errorf("read blob %v from %v: not enough bytes read, want %v, got %v",
+					h, tempfile.Name(), len(buf), n)
+			}
 
-			n, err := crypto.Decrypt(repo.Key(), plaintext, ciphertext)
+			n, err = crypto.Decrypt(repo.Key(), buf, buf)
 			if err != nil {
 				return err
 			}
-			plaintext = plaintext[:n]
 
-			_, err = repo.SaveBlob(entry.Type, plaintext, entry.ID)
+			buf = buf[:n]
+
+			id := restic.Hash(buf)
+			if !id.Equal(entry.ID) {
+				return errors.Errorf("read blob %v from %v: wrong data returned, hash is %v",
+					h, tempfile.Name(), id)
+			}
+
+			_, err = repo.SaveBlob(entry.Type, buf, entry.ID)
 			if err != nil {
 				return err
 			}
@@ -73,6 +108,14 @@ func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet
 
 			keepBlobs.Delete(h)
 		}
+
+		if err = tempfile.Close(); err != nil {
+			return errors.Wrap(err, "Close")
+		}
+
+		if err = os.Remove(tempfile.Name()); err != nil {
+			return errors.Wrap(err, "Remove")
+		}
 	}
 
 	if err := repo.Flush(); err != nil {
@@ -80,7 +123,8 @@ func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet
 	}
 
 	for packID := range packs {
-		err := repo.Backend().Remove(restic.DataFile, packID.String())
+		h := restic.Handle{Type: restic.DataFile, Name: packID.String()}
+		err := repo.Backend().Remove(h)
 		if err != nil {
 			debug.Log("error removing pack %v: %v", packID.Str(), err)
 			return err

src/restic/repository/repository.go

@@ -54,7 +54,7 @@ func (r *Repository) LoadAndDecrypt(t restic.FileType, id restic.ID) ([]byte, er
 	debug.Log("load %v with id %v", t, id.Str())
 
 	h := restic.Handle{Type: t, Name: id.String()}
-	buf, err := backend.LoadAll(r.be, h, nil)
+	buf, err := backend.LoadAll(r.be, h)
 	if err != nil {
 		debug.Log("error loading %v: %v", id.Str(), err)
 		return nil, err
@@ -64,33 +64,20 @@ func (r *Repository) LoadAndDecrypt(t restic.FileType, id restic.ID) ([]byte, er
 		return nil, errors.New("invalid data returned")
 	}
 
-	plain := make([]byte, len(buf))
-
 	// decrypt
-	n, err := r.decryptTo(plain, buf)
+	n, err := r.decryptTo(buf, buf)
 	if err != nil {
 		return nil, err
 	}
 
-	return plain[:n], nil
+	return buf[:n], nil
 }
 
 // loadBlob tries to load and decrypt content identified by t and id from a
 // pack from the backend, the result is stored in plaintextBuf, which must be
 // large enough to hold the complete blob.
 func (r *Repository) loadBlob(id restic.ID, t restic.BlobType, plaintextBuf []byte) (int, error) {
-	debug.Log("load %v with id %v (buf %p, len %d)", t, id.Str(), plaintextBuf, len(plaintextBuf))
-
-	// lookup plaintext size of blob
-	size, err := r.idx.LookupSize(id, t)
-	if err != nil {
-		return 0, err
-	}
-
-	// make sure the plaintext buffer is large enough, extend otherwise
-	if len(plaintextBuf) < int(size) {
-		return 0, errors.Errorf("buffer is too small: %d < %d", len(plaintextBuf), size)
-	}
+	debug.Log("load %v with id %v (buf len %v, cap %d)", t, id.Str(), len(plaintextBuf), cap(plaintextBuf))
 
 	// lookup packs
 	blobs, err := r.idx.Lookup(id, t)
@@ -109,8 +96,14 @@ func (r *Repository) loadBlob(id restic.ID, t restic.BlobType, plaintextBuf []by
 
 		// load blob from pack
 		h := restic.Handle{Type: restic.DataFile, Name: blob.PackID.String()}
-		ciphertextBuf := make([]byte, blob.Length)
-		n, err := r.be.Load(h, ciphertextBuf, int64(blob.Offset))
+
+		if uint(cap(plaintextBuf)) < blob.Length {
+			return 0, errors.Errorf("buffer is too small: %v < %v", cap(plaintextBuf), blob.Length)
+		}
+
+		plaintextBuf = plaintextBuf[:blob.Length]
+
+		n, err := restic.ReadAt(r.be, h, int64(blob.Offset), plaintextBuf)
 		if err != nil {
 			debug.Log("error loading blob %v: %v", blob, err)
 			lastError = err
@@ -125,7 +118,7 @@ func (r *Repository) loadBlob(id restic.ID, t restic.BlobType, plaintextBuf []by
 		}
 
 		// decrypt
-		n, err = r.decryptTo(plaintextBuf, ciphertextBuf)
+		n, err = r.decryptTo(plaintextBuf, plaintextBuf)
 		if err != nil {
 			lastError = errors.Errorf("decrypting blob %v failed: %v", id, err)
 			continue
@@ -224,7 +217,7 @@ func (r *Repository) SaveJSONUnpacked(t restic.FileType, item interface{}) (rest
 // SaveUnpacked encrypts data and stores it in the backend. Returned is the
 // storage hash.
 func (r *Repository) SaveUnpacked(t restic.FileType, p []byte) (id restic.ID, err error) {
-	ciphertext := make([]byte, len(p)+crypto.Extension)
+	ciphertext := restic.NewBlobBuffer(len(p))
 	ciphertext, err = r.Encrypt(ciphertext, p)
 	if err != nil {
 		return restic.ID{}, err
@@ -233,7 +226,7 @@ func (r *Repository) SaveUnpacked(t restic.FileType, p []byte) (id restic.ID, er
 	id = restic.Hash(ciphertext)
 	h := restic.Handle{Type: t, Name: id.String()}
 
-	err = r.be.Save(h, ciphertext)
+	err = r.be.Save(h, bytes.NewReader(ciphertext))
 	if err != nil {
 		debug.Log("error saving blob %v: %v", h, err)
 		return restic.ID{}, err
@@ -248,15 +241,15 @@ func (r *Repository) Flush() error {
 	r.pm.Lock()
 	defer r.pm.Unlock()
 
-	debug.Log("manually flushing %d packs", len(r.packs))
+	debug.Log("manually flushing %d packs", len(r.packerManager.packers))
 
-	for _, p := range r.packs {
+	for _, p := range r.packerManager.packers {
 		err := r.savePacker(p)
 		if err != nil {
 			return err
 		}
 	}
-	r.packs = r.packs[:0]
+	r.packerManager.packers = r.packerManager.packers[:0]
 	return nil
 }
 
@@ -387,7 +380,7 @@ func (r *Repository) SearchKey(password string, maxKeys int) error {
 // Init creates a new master key with the supplied password, initializes and
 // saves the repository config.
 func (r *Repository) Init(password string) error {
-	has, err := r.be.Test(restic.ConfigFile, "")
+	has, err := r.be.Test(restic.Handle{Type: restic.ConfigFile})
 	if err != nil {
 		return err
 	}
@@ -528,16 +521,18 @@ func (r *Repository) Close() error {
 	return r.be.Close()
 }
 
-// LoadBlob loads a blob of type t from the repository to the buffer.
+// LoadBlob loads a blob of type t from the repository to the buffer. buf must
+// be large enough to hold the encrypted blob, since it is used as scratch
+// space.
 func (r *Repository) LoadBlob(t restic.BlobType, id restic.ID, buf []byte) (int, error) {
-	debug.Log("load blob %v into buf %p", id.Str(), buf)
+	debug.Log("load blob %v into buf (len %v, cap %v)", id.Str(), len(buf), cap(buf))
 	size, err := r.idx.LookupSize(id, t)
 	if err != nil {
 		return 0, err
 	}
 
-	if len(buf) < int(size) {
-		return 0, errors.Errorf("buffer is too small for data blob (%d < %d)", len(buf), size)
+	if cap(buf) < restic.CiphertextLength(int(size)) {
+		return 0, errors.Errorf("buffer is too small for data blob (%d < %d)", cap(buf), restic.CiphertextLength(int(size)))
 	}
 
 	n, err := r.loadBlob(id, t, buf)
@@ -571,7 +566,7 @@ func (r *Repository) LoadTree(id restic.ID) (*restic.Tree, error) {
 	}
 
 	debug.Log("size is %d, create buffer", size)
-	buf := make([]byte, size)
+	buf := restic.NewBlobBuffer(int(size))
 
 	n, err := r.loadBlob(id, restic.TreeBlob, buf)
 	if err != nil {

src/restic/repository/repository_test.go

@@ -2,12 +2,12 @@ package repository_test
 
 import (
 	"bytes"
-	"crypto/rand"
 	"crypto/sha256"
 	"io"
-	mrand "math/rand"
+	"math/rand"
 	"path/filepath"
 	"testing"
+	"time"
 
 	"restic"
 	"restic/archiver"
@@ -17,13 +17,15 @@ import (
 
 var testSizes = []int{5, 23, 2<<18 + 23, 1 << 20}
 
+var rnd = rand.New(rand.NewSource(time.Now().UnixNano()))
+
 func TestSave(t *testing.T) {
 	repo, cleanup := repository.TestRepository(t)
 	defer cleanup()
 
 	for _, size := range testSizes {
 		data := make([]byte, size)
-		_, err := io.ReadFull(rand.Reader, data)
+		_, err := io.ReadFull(rnd, data)
 		OK(t, err)
 
 		id := restic.Hash(data)
@@ -38,7 +40,7 @@ func TestSave(t *testing.T) {
 		// OK(t, repo.SaveIndex())
 
 		// read back
-		buf := make([]byte, size)
+		buf := restic.NewBlobBuffer(size)
 		n, err := repo.LoadBlob(restic.DataBlob, id, buf)
 		OK(t, err)
 		Equals(t, len(buf), n)
@@ -59,7 +61,7 @@ func TestSaveFrom(t *testing.T) {
 
 	for _, size := range testSizes {
 		data := make([]byte, size)
-		_, err := io.ReadFull(rand.Reader, data)
+		_, err := io.ReadFull(rnd, data)
 		OK(t, err)
 
 		id := restic.Hash(data)
@@ -72,7 +74,7 @@ func TestSaveFrom(t *testing.T) {
 		OK(t, repo.Flush())
 
 		// read back
-		buf := make([]byte, size)
+		buf := restic.NewBlobBuffer(size)
 		n, err := repo.LoadBlob(restic.DataBlob, id, buf)
 		OK(t, err)
 		Equals(t, len(buf), n)
@@ -94,7 +96,7 @@ func BenchmarkSaveAndEncrypt(t *testing.B) {
 	size := 4 << 20 // 4MiB
 
 	data := make([]byte, size)
-	_, err := io.ReadFull(rand.Reader, data)
+	_, err := io.ReadFull(rnd, data)
 	OK(t, err)
 
 	id := restic.ID(sha256.Sum256(data))
@@ -145,6 +147,113 @@ func BenchmarkLoadTree(t *testing.B) {
 	}
 }
 
+func TestLoadBlob(t *testing.T) {
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
+
+	length := 1000000
+	buf := restic.NewBlobBuffer(length)
+	_, err := io.ReadFull(rnd, buf)
+	OK(t, err)
+
+	id, err := repo.SaveBlob(restic.DataBlob, buf, restic.ID{})
+	OK(t, err)
+	OK(t, repo.Flush())
+
+	// first, test with buffers that are too small
+	for _, testlength := range []int{length - 20, length, restic.CiphertextLength(length) - 1} {
+		buf = make([]byte, 0, testlength)
+		n, err := repo.LoadBlob(restic.DataBlob, id, buf)
+		if err == nil {
+			t.Errorf("LoadBlob() did not return an error for a buffer that is too small to hold the blob")
+			continue
+		}
+
+		if n != 0 {
+			t.Errorf("LoadBlob() returned an error and n > 0")
+			continue
+		}
+	}
+
+	// then use buffers that are large enough
+	base := restic.CiphertextLength(length)
+	for _, testlength := range []int{base, base + 7, base + 15, base + 1000} {
+		buf = make([]byte, 0, testlength)
+		n, err := repo.LoadBlob(restic.DataBlob, id, buf)
+		if err != nil {
+			t.Errorf("LoadBlob() returned an error for buffer size %v: %v", testlength, err)
+			continue
+		}
+
+		if n != length {
+			t.Errorf("LoadBlob() returned the wrong number of bytes: want %v, got %v", length, n)
+			continue
+		}
+	}
+}
+
+func BenchmarkLoadBlob(b *testing.B) {
+	repo, cleanup := repository.TestRepository(b)
+	defer cleanup()
+
+	length := 1000000
+	buf := restic.NewBlobBuffer(length)
+	_, err := io.ReadFull(rnd, buf)
+	OK(b, err)
+
+	id, err := repo.SaveBlob(restic.DataBlob, buf, restic.ID{})
+	OK(b, err)
+	OK(b, repo.Flush())
+
+	b.ResetTimer()
+	b.SetBytes(int64(length))
+
+	for i := 0; i < b.N; i++ {
+		n, err := repo.LoadBlob(restic.DataBlob, id, buf)
+		OK(b, err)
+		if n != length {
+			b.Errorf("wanted %d bytes, got %d", length, n)
+		}
+
+		id2 := restic.Hash(buf[:n])
+		if !id.Equal(id2) {
+			b.Errorf("wrong data returned, wanted %v, got %v", id.Str(), id2.Str())
+		}
+	}
+}
+
+func BenchmarkLoadAndDecrypt(b *testing.B) {
+	repo, cleanup := repository.TestRepository(b)
+	defer cleanup()
+
+	length := 1000000
+	buf := restic.NewBlobBuffer(length)
+	_, err := io.ReadFull(rnd, buf)
+	OK(b, err)
+
+	dataID := restic.Hash(buf)
+
+	storageID, err := repo.SaveUnpacked(restic.DataFile, buf)
+	OK(b, err)
+	// OK(b, repo.Flush())
+
+	b.ResetTimer()
+	b.SetBytes(int64(length))
+
+	for i := 0; i < b.N; i++ {
+		data, err := repo.LoadAndDecrypt(restic.DataFile, storageID)
+		OK(b, err)
+		if len(data) != length {
+			b.Errorf("wanted %d bytes, got %d", length, len(data))
+		}
+
+		id2 := restic.Hash(data)
+		if !dataID.Equal(id2) {
+			b.Errorf("wrong data returned, wanted %v, got %v", storageID.Str(), id2.Str())
+		}
+	}
+}
+
 func TestLoadJSONUnpacked(t *testing.T) {
 	repo, cleanup := repository.TestRepository(t)
 	defer cleanup()
@@ -182,25 +291,48 @@ func TestRepositoryLoadIndex(t *testing.T) {
 }
 
 func BenchmarkLoadIndex(b *testing.B) {
-	repodir, cleanup := Env(b, repoFixture)
+	repository.TestUseLowSecurityKDFParameters(b)
+
+	repo, cleanup := repository.TestRepository(b)
 	defer cleanup()
 
-	repo := repository.TestOpenLocal(b, repodir)
+	idx := repository.NewIndex()
+
+	for i := 0; i < 5000; i++ {
+		idx.Store(restic.PackedBlob{
+			Blob: restic.Blob{
+				Type:   restic.DataBlob,
+				Length: 1234,
+				ID:     restic.NewRandomID(),
+				Offset: 1235,
+			},
+			PackID: restic.NewRandomID(),
+		})
+	}
+
+	id, err := repository.SaveIndex(repo, idx)
+	OK(b, err)
+
+	b.Logf("index saved as %v (%v entries)", id.Str(), idx.Count(restic.DataBlob))
+	fi, err := repo.Backend().Stat(restic.Handle{Type: restic.IndexFile, Name: id.String()})
+	OK(b, err)
+	b.Logf("filesize is %v", fi.Size)
+
 	b.ResetTimer()
 
 	for i := 0; i < b.N; i++ {
-		repo.SetIndex(repository.NewMasterIndex())
-		OK(b, repo.LoadIndex())
+		_, err := repository.LoadIndex(repo, id)
+		OK(b, err)
 	}
 }
 
 // saveRandomDataBlobs generates random data blobs and saves them to the repository.
 func saveRandomDataBlobs(t testing.TB, repo restic.Repository, num int, sizeMax int) {
 	for i := 0; i < num; i++ {
-		size := mrand.Int() % sizeMax
+		size := rand.Int() % sizeMax
 
 		buf := make([]byte, size)
-		_, err := io.ReadFull(rand.Reader, buf)
+		_, err := io.ReadFull(rnd, buf)
 		OK(t, err)
 
 		_, err = repo.SaveBlob(restic.DataBlob, buf, restic.ID{})

src/restic/snapshot.go

@@ -145,8 +145,8 @@ func SamePaths(expected, actual []string) bool {
 // ErrNoSnapshotFound is returned when no snapshot for the given criteria could be found.
 var ErrNoSnapshotFound = errors.New("no snapshot found")
 
-// FindLatestSnapshot finds latest snapshot with optional target/directory and source filters
-func FindLatestSnapshot(repo Repository, targets []string, source string) (ID, error) {
+// FindLatestSnapshot finds latest snapshot with optional target/directory and hostname filters.
+func FindLatestSnapshot(repo Repository, targets []string, hostname string) (ID, error) {
 	var (
 		latest   time.Time
 		latestID ID
@@ -158,7 +158,7 @@ func FindLatestSnapshot(repo Repository, targets []string, source string) (ID, e
 		if err != nil {
 			return ID{}, errors.Errorf("Error listing snapshot: %v", err)
 		}
-		if snapshot.Time.After(latest) && SamePaths(snapshot.Paths, targets) && (source == "" || source == snapshot.Hostname) {
+		if snapshot.Time.After(latest) && SamePaths(snapshot.Paths, targets) && (hostname == "" || hostname == snapshot.Hostname) {
 			latest = snapshot.Time
 			latestID = snapshotID
 			found = true

src/restic/test/helpers.go

@@ -79,7 +79,7 @@ func Random(seed, count int) []byte {
 
 		for j := range data {
 			cur := i + j
-			if len(p) >= cur {
+			if cur >= len(p) {
 				break
 			}
 			p[cur] = data[j]

src/restic/testing.go

@@ -23,16 +23,26 @@ type fakeFileSystem struct {
 	repo        Repository
 	knownBlobs  IDSet
 	duplication float32
+	buf         []byte
+	chunker     *chunker.Chunker
 }
 
 // saveFile reads from rd and saves the blobs in the repository. The list of
 // IDs is returned.
-func (fs fakeFileSystem) saveFile(rd io.Reader) (blobs IDs) {
-	blobs = IDs{}
-	ch := chunker.New(rd, fs.repo.Config().ChunkerPolynomial)
+func (fs *fakeFileSystem) saveFile(rd io.Reader) (blobs IDs) {
+	if fs.buf == nil {
+		fs.buf = make([]byte, chunker.MaxSize)
+	}
 
+	if fs.chunker == nil {
+		fs.chunker = chunker.New(rd, fs.repo.Config().ChunkerPolynomial)
+	} else {
+		fs.chunker.Reset(rd, fs.repo.Config().ChunkerPolynomial)
+	}
+
+	blobs = IDs{}
 	for {
-		chunk, err := ch.Next(getBuf())
+		chunk, err := fs.chunker.Next(fs.buf)
 		if errors.Cause(err) == io.EOF {
 			break
 		}
@@ -50,7 +60,6 @@ func (fs fakeFileSystem) saveFile(rd io.Reader) (blobs IDs) {
 
 			fs.knownBlobs.Insert(id)
 		}
-		freeBuf(chunk.Data)
 
 		blobs = append(blobs, id)
 	}
@@ -64,7 +73,7 @@ const (
 	maxNodes    = 32
 )
 
-func (fs fakeFileSystem) treeIsKnown(tree *Tree) (bool, []byte, ID) {
+func (fs *fakeFileSystem) treeIsKnown(tree *Tree) (bool, []byte, ID) {
 	data, err := json.Marshal(tree)
 	if err != nil {
 		fs.t.Fatalf("json.Marshal(tree) returned error: %v", err)
@@ -76,7 +85,7 @@ func (fs fakeFileSystem) treeIsKnown(tree *Tree) (bool, []byte, ID) {
 	return fs.blobIsKnown(id, TreeBlob), data, id
 }
 
-func (fs fakeFileSystem) blobIsKnown(id ID, t BlobType) bool {
+func (fs *fakeFileSystem) blobIsKnown(id ID, t BlobType) bool {
 	if rand.Float32() < fs.duplication {
 		return false
 	}
@@ -94,7 +103,7 @@ func (fs fakeFileSystem) blobIsKnown(id ID, t BlobType) bool {
 }
 
 // saveTree saves a tree of fake files in the repo and returns the ID.
-func (fs fakeFileSystem) saveTree(seed int64, depth int) ID {
+func (fs *fakeFileSystem) saveTree(seed int64, depth int) ID {
 	rnd := rand.NewSource(seed)
 	numNodes := int(rnd.Int63() % maxNodes)
 

src/restic/testing_test.go

@@ -47,3 +47,14 @@ func TestCreateSnapshot(t *testing.T) {
 
 	checker.TestCheckRepo(t, repo)
 }
+
+func BenchmarkTestCreateSnapshot(t *testing.B) {
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
+
+	t.ResetTimer()
+
+	for i := 0; i < t.N; i++ {
+		restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth, 0)
+	}
+}