Add VERSION file for 0.4.0

This was a temporary fix for Go earlier than 1.5 to run code on all
avaialble cores. We don't need that any more since we require at least
Go 1.6.

CONTRIBUTING.md

@@ -3,7 +3,10 @@ This document describes the way you can contribute to the restic project.
 Ways to Help Out
 ================
 
-Thank you for your contribution!
+Thank you for your contribution! Please **open an issue first** (or add a
+comment to an existing issue) if you plan to work on any code or add a new
+feature. This way, duplicate work is prevented and we can discuss your ideas
+and design first.
 
 There are several ways you can help us out. First of all code contributions and
 bug fixes are most welcome. However even "minor" details as fixing spelling
@@ -83,7 +86,7 @@ The following commands can be used to run all the tests:
     [...]
 
 If you want to run your tests on Linux, OpenBSD or FreeBSD, you can use
-[vagrant](https://www.vagrantup.com/) with the proveded `Vagrantfile` to
+[vagrant](https://www.vagrantup.com/) with the provided `Vagrantfile` to
 quickly set up VMs and run the tests, e.g.:
 
     $ vagrant up freebsd

README.md

@@ -11,7 +11,8 @@ restic is a backup program that is fast, efficient and secure. Detailed
 information can be found in [the documentation](doc/index.md) and [the user
 manual](doc/Manual.md). The [design document](doc/Design.md) lists the
 technical background and gives detailed information about the structure of the
-repository and the data saved therein.
+repository and the data saved therein. The file [FAQ.md](doc/FAQ.md) lists the
+most frequently asked questions.
 
 The latest documentation can be viewed online at
 <https://restic.readthedocs.io/en/latest>. On the bottom left corner there is
@@ -39,10 +40,15 @@ Building restic with gccgo may work, but is not supported.
 Contribute and Documentation
 ============================
 
-Contributions are welcome! More information and a description of the
-development environment can be found in [`CONTRIBUTING.md`](CONTRIBUTING.md). A
-document describing the design of restic and the data structures stored on the
-back end is contained in [`doc/Design.md`](doc/Design.md).
+Contributions are welcome! Please **open an issue first** (or add a comment to
+an existing issue) if you plan to work on any code or add a new feature. This
+way, duplicate work is prevented and we can discuss your ideas and design
+first.
+
+More information and a description of the development environment can be found
+in [`CONTRIBUTING.md`](CONTRIBUTING.md). A document describing the design of
+restic and the data structures stored on the back end is contained in
+[`doc/Design.md`](doc/Design.md).
 
 If you'd like to start contributing to restic, but don't know exactly what do
 to, have a look at this great article by Dave Cheney:

VERSION

@@ -1 +1 @@
-0.3.3
+0.4.0

build.go

@@ -174,6 +174,7 @@ func showUsage(output io.Writer) {
 	fmt.Fprintf(output, "  -t     --tags          specify additional build tags\n")
 	fmt.Fprintf(output, "  -k     --keep-gopath   do not remove the GOPATH after build\n")
 	fmt.Fprintf(output, "  -T     --test          run tests\n")
+	fmt.Fprintf(output, "  -o     --output        set output file name\n")
 	fmt.Fprintf(output, "         --goos value    set GOOS for cross-compilation\n")
 	fmt.Fprintf(output, "         --goarch value  set GOARCH for cross-compilation\n")
 }
@@ -204,7 +205,7 @@ func cleanEnv() (env []string) {
 func build(cwd, goos, goarch, gopath string, args ...string) error {
 	args = append([]string{"build"}, args...)
 	cmd := exec.Command("go", args...)
-	cmd.Env = append(cleanEnv(), "GOPATH="+gopath, "GOARCH="+goarch, "GOOS="+goos)
+	cmd.Env = append(cleanEnv(), "GOPATH="+gopath, "GOARCH="+goarch, "GOOS="+goos, "CGO_ENABLED=0")
 	cmd.Dir = cwd
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
@@ -298,6 +299,8 @@ func main() {
 	targetGOOS := runtime.GOOS
 	targetGOARCH := runtime.GOARCH
 
+	var outputFilename string
+
 	for i, arg := range params {
 		if skipNext {
 			skipNext = false
@@ -315,6 +318,9 @@ func main() {
 			}
 			skipNext = true
 			buildTags = strings.Split(params[i+1], " ")
+		case "-o", "--output":
+			skipNext = true
+			outputFilename = params[i+1]
 		case "-T", "--test":
 			runTests = true
 		case "--goos":
@@ -377,9 +383,11 @@ func main() {
 		}
 	}()
 
-	outputFilename := config.Name
-	if targetGOOS == "windows" {
-		outputFilename += ".exe"
+	if outputFilename == "" {
+		outputFilename = config.Name
+		if targetGOOS == "windows" {
+			outputFilename += ".exe"
+		}
 	}
 
 	cwd, err := os.Getwd()

build_release_binaries.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+set -e
+
+if [[ -z "$VERSION" ]]; then
+    echo '$VERSION unset'
+    exit 1
+fi
+
+dir=$(mktemp -d --tmpdir restic-release-XXXXXX)
+echo "path is ${dir}"
+
+for R in       \
+    darwin/386     \
+    darwin/amd64   \
+    freebsd/386    \
+    freebsd/amd64  \
+    freebsd/arm    \
+    linux/386      \
+    linux/amd64    \
+    linux/arm      \
+    linux/arm64    \
+    openbsd/386   \
+    openbsd/amd64 \
+    windows/386    \
+    windows/amd64  \
+    ; do \
+
+    OS=$(dirname $R)
+    ARCH=$(basename $R)
+    filename=restic_${VERSION}_${OS}_${ARCH}
+
+    if [[ "$OS" == "windows" ]]; then
+        filename="${filename}.exe"
+    fi
+
+    echo $filename
+
+    go run ../build.go --goos $OS --goarch $ARCH --output ${filename}
+    if [[ "$OS" == "windows" ]]; then
+        zip ${filename%.exe}.zip ${filename}
+        rm ${filename}
+        mv ${filename%.exe}.zip ${dir}
+    else
+        bzip2 ${filename}
+        mv ${filename}.bz2 ${dir}
+    fi
+done
+
+echo "packing sources"
+git archive --format=tar --prefix=restic-$VERSION/ v$VERSION | gzip -n > restic-$VERSION.tar.gz
+mv restic-$VERSION.tar.gz ${dir}
+
+echo "creating checksums"
+pushd ${dir}
+sha256sum restic_*.{zip,bz2} > SHA256SUMS
+gpg --armor --detach-sign SHA256SUMS
+popd
+
+echo "creating source signature file"
+gpg --armor --detach-sign ${dir}/restic-$VERSION.tar.gz
+
+echo
+echo "done, path is ${dir}"

doc/Design.md

@@ -327,10 +327,11 @@ A snapshot references a tree by the SHA-256 hash of the JSON string
 representation of its contents. Trees and data are saved in pack files in a
 subdirectory of the directory `data`.
 
-The command `restic cat tree` can be used to inspect the tree referenced above:
+The command `restic cat blob` can be used to inspect the tree referenced above
+(piping the output of the command to `jq .` so that the JSON is indented):
 
 ```console
-$ restic -r /tmp/restic-repo cat tree b8138ab08a4722596ac89c917827358da4672eac68e3c03a8115b88dbf4bfb59
+$ restic -r /tmp/restic-repo cat blob b8138ab08a4722596ac89c917827358da4672eac68e3c03a8115b88dbf4bfb59 | jq .
 enter password for repository:
 {
   "nodes": [
@@ -356,11 +357,11 @@ A tree contains a list of entries (in the field `nodes`) which contain meta
 data like a name and timestamps. When the entry references a directory, the
 field `subtree` contains the plain text ID of another tree object.
 
-When the command `restic cat tree` is used, the storage hash is needed to print
+When the command `restic cat blob` is used, the plaintext ID is needed to print
 a tree. The tree referenced above can be dumped as follows:
 
 ```console
-$ restic -r /tmp/restic-repo cat tree 8b238c8811cc362693e91a857460c78d3acf7d9edb2f111048691976803cf16e
+$ restic -r /tmp/restic-repo cat blob 8b238c8811cc362693e91a857460c78d3acf7d9edb2f111048691976803cf16e
 enter password for repository:
 {
   "nodes": [
@@ -389,8 +390,8 @@ enter password for repository:
 This tree contains a file entry. This time, the `subtree` field is not present
 and the `content` field contains a list with one plain text SHA-256 hash.
 
-The command `restic cat data` can be used to extract and decrypt data given a
-plaintext ID, e.g. for the data mentioned above:
+The command `restic cat blob` can also be used to extract and decrypt data
+given a plaintext ID, e.g. for the data mentioned above:
 
 ```console
 $ restic -r /tmp/restic-repo cat blob 50f77b3b4291e8411a027b9f9b9e64658181cc676ce6ba9958b95f268cb1109d | sha256sum

doc/FAQ.md

@@ -0,0 +1,20 @@
+FAQ
+===
+
+This is the list of Frequently Asked Questions for restic.
+
+`restic check` reports packs that aren't referenced in any index, is my repository broken?
+------------------------------------------------------------------------------------------
+
+When `restic check` reports that there are pack files in the repository that are not referenced in any index, that's (in contrast to what restic reports at the moment) not a source for concern. The output looks like this:
+
+    $ restic check
+    Create exclusive lock for repository
+    Load indexes
+    Check all packs
+    pack 819a9a52e4f51230afa89aefbf90df37fb70996337ae57e6f7a822959206a85e: not referenced in any index
+    pack de299e69fb075354a3775b6b045d152387201f1cdc229c31d1caa34c3b340141: not referenced in any index
+    Check snapshots, trees and blobs
+    Fatal: repository contains errors
+
+The message means that there is more data stored in the repo than strictly necessary. With high probability this is duplicate data. In order to clean it up, the command `restic prune` can be used. The cause of this bug is not yet known.

doc/Manual.md

@@ -77,7 +77,7 @@ Available Commands:
 Flags:
       --no-lock                do not lock the repo, this allows some operations on read-only repos
   -p, --password-file string   read the repository password from a file
-  -q, --quiet                  do not outputcomprehensive progress report
+  -q, --quiet                  do not output comprehensive progress report
   -r, --repo string            repository to backup to or restore from (default: $RESTIC_REPOSITORY)
 
 Use "restic [command] --help" for more information about a command.
@@ -110,7 +110,7 @@ Flags:
 Global Flags:
       --no-lock                do not lock the repo, this allows some operations on read-only repos
   -p, --password-file string   read the repository password from a file
-  -q, --quiet                  do not outputcomprehensive progress report
+  -q, --quiet                  do not output comprehensive progress report
   -r, --repo string            repository to backup to or restore from (default: $RESTIC_REPOSITORY)
 ```
 

doc/REST_backend.md

@@ -6,6 +6,19 @@ following values are valid for `{type}`: `data`, `keys`, `locks`, `snapshots`,
 `index`, `config`. `{path}` is a path to the repository, so that multiple
 different repositories can be accessed. The default path is `/`.
 
+## POST {path}?create=true
+
+This request is used to initially create a new repository. The server responds
+with "200 OK" if the repository structure was created successfully or already
+exists, otherwise an error is returned.
+
+## DELETE {path}
+
+Deletes the repository on the server side. The server responds with "200 OK" if
+the repository was successfully removed. If this function is not implemented
+the server returns "501 Not Implemented", if this it is denied by the server it
+returns "403 Forbidden".
+
 ## HEAD {path}/config
 
 Returns "200 OK" if the repository has a configuration,

run_integration_tests.go

@@ -176,7 +176,7 @@ func (env *TravisEnvironment) Prepare() error {
 				"windows/386", "windows/amd64",
 				"darwin/386", "darwin/amd64",
 				"freebsd/386", "freebsd/amd64",
-				"opendbsd/386", "opendbsd/amd64",
+				"openbsd/386", "openbsd/amd64",
 			}
 			if !strings.HasPrefix(runtime.Version(), "go1.3") {
 				env.goxOSArch = append(env.goxOSArch,

src/cmds/restic/cmd_cat.go

@@ -9,13 +9,12 @@ import (
 
 	"restic"
 	"restic/backend"
-	"restic/debug"
 	"restic/errors"
 	"restic/repository"
 )
 
 var cmdCat = &cobra.Command{
-	Use:   "cat [flags] [pack|blob|tree|snapshot|key|masterkey|config|lock] ID",
+	Use:   "cat [flags] [pack|blob|snapshot|index|key|masterkey|config|lock] ID",
 	Short: "print internal objects to stdout",
 	Long: `
 The "cat" command is used to print internal objects to stdout.
@@ -99,7 +98,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 		return nil
 	case "key":
 		h := restic.Handle{Type: restic.KeyFile, Name: id.String()}
-		buf, err := backend.LoadAll(repo.Backend(), h, nil)
+		buf, err := backend.LoadAll(repo.Backend(), h)
 		if err != nil {
 			return err
 		}
@@ -150,7 +149,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 	switch tpe {
 	case "pack":
 		h := restic.Handle{Type: restic.DataFile, Name: id.String()}
-		buf, err := backend.LoadAll(repo.Backend(), h, nil)
+		buf, err := backend.LoadAll(repo.Backend(), h)
 		if err != nil {
 			return err
 		}
@@ -172,7 +171,7 @@ func runCat(gopts GlobalOptions, args []string) error {
 			blob := list[0]
 
 			buf := make([]byte, blob.Length)
-			n, err := repo.LoadBlob(restic.DataBlob, id, buf)
+			n, err := repo.LoadBlob(t, id, buf)
 			if err != nil {
 				return err
 			}
@@ -184,23 +183,6 @@ func runCat(gopts GlobalOptions, args []string) error {
 
 		return errors.Fatal("blob not found")
 
-	case "tree":
-		debug.Log("cat tree %v", id.Str())
-		tree, err := repo.LoadTree(id)
-		if err != nil {
-			debug.Log("unable to load tree %v: %v", id.Str(), err)
-			return err
-		}
-
-		buf, err := json.MarshalIndent(&tree, "", "  ")
-		if err != nil {
-			debug.Log("error json.MarshalIndent(): %v", err)
-			return err
-		}
-
-		_, err = os.Stdout.Write(append(buf, '\n'))
-		return nil
-
 	default:
 		return errors.Fatal("invalid type")
 	}

src/cmds/restic/cmd_forget.go

@@ -133,7 +133,8 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 		}
 
 		if !opts.DryRun {
-			err = repo.Backend().Remove(restic.SnapshotFile, id.String())
+			h := restic.Handle{Type: restic.SnapshotFile, Name: id.String()}
+			err = repo.Backend().Remove(h)
 			if err != nil {
 				return err
 			}
@@ -201,7 +202,8 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 
 		if !opts.DryRun {
 			for _, sn := range remove {
-				err = repo.Backend().Remove(restic.SnapshotFile, sn.ID().String())
+				h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
+				err = repo.Backend().Remove(h)
 				if err != nil {
 					return err
 				}

src/cmds/restic/cmd_key.go

@@ -87,7 +87,8 @@ func deleteKey(repo *repository.Repository, name string) error {
 		return errors.Fatal("refusing to remove key currently used to access repository")
 	}
 
-	err := repo.Backend().Remove(restic.KeyFile, name)
+	h := restic.Handle{Type: restic.KeyFile, Name: name}
+	err := repo.Backend().Remove(h)
 	if err != nil {
 		return err
 	}
@@ -107,7 +108,8 @@ func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
 		return errors.Fatalf("creating new key failed: %v\n", err)
 	}
 
-	err = repo.Backend().Remove(restic.KeyFile, repo.KeyName())
+	h := restic.Handle{Type: restic.KeyFile, Name: repo.KeyName()}
+	err = repo.Backend().Remove(h)
 	if err != nil {
 		return err
 	}

src/cmds/restic/cmd_list.go

@@ -1,8 +1,10 @@
 package main
 
 import (
+	"fmt"
 	"restic"
 	"restic/errors"
+	"restic/index"
 
 	"github.com/spf13/cobra"
 )
@@ -52,6 +54,19 @@ func runList(opts GlobalOptions, args []string) error {
 		t = restic.KeyFile
 	case "locks":
 		t = restic.LockFile
+	case "blobs":
+		idx, err := index.Load(repo, nil)
+		if err != nil {
+			return err
+		}
+
+		for _, pack := range idx.Packs {
+			for _, entry := range pack.Entries {
+				fmt.Printf("%v %v\n", entry.Type, entry.ID)
+			}
+		}
+
+		return nil
 	default:
 		return errors.Fatal("invalid type")
 	}

src/cmds/restic/cmd_ls.go

@@ -17,22 +17,35 @@ var cmdLs = &cobra.Command{
 	Short: "list files in a snapshot",
 	Long: `
 The "ls" command allows listing files and directories in a snapshot.
+
+The special snapshot-ID "latest" can be used to list files and directories of the latest snapshot in the repository.
 `,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runLs(globalOptions, args)
 	},
 }
 
-var listLong bool
+// LsOptions collects all options for the ls command.
+type LsOptions struct {
+	ListLong bool
+	Host     string
+	Paths    []string
+}
+
+var lsOptions LsOptions
 
 func init() {
 	cmdRoot.AddCommand(cmdLs)
 
-	cmdLs.Flags().BoolVarP(&listLong, "long", "l", false, "use a long listing format showing size and mode")
+	flags := cmdLs.Flags()
+	flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
+
+	flags.StringVarP(&lsOptions.Host, "host", "H", "", `only consider snapshots for this host when the snapshot ID is "latest"`)
+	flags.StringSliceVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
 }
 
 func printNode(prefix string, n *restic.Node) string {
-	if !listLong {
+	if !lsOptions.ListLong {
 		return filepath.Join(prefix, n.Name)
 	}
 
@@ -86,9 +99,19 @@ func runLs(gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	id, err := restic.FindSnapshot(repo, args[0])
-	if err != nil {
-		return err
+	snapshotIDString := args[0]
+	var id restic.ID
+
+	if snapshotIDString == "latest" {
+		id, err = restic.FindLatestSnapshot(repo, lsOptions.Paths, lsOptions.Host)
+		if err != nil {
+			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Host:%v", err, lsOptions.Paths, lsOptions.Host)
+		}
+	} else {
+		id, err = restic.FindSnapshot(repo, snapshotIDString)
+		if err != nil {
+			Exitf(1, "invalid id %q: %v", snapshotIDString, err)
+		}
 	}
 
 	sn, err := restic.LoadSnapshot(repo, id)

src/cmds/restic/cmd_prune.go

@@ -103,11 +103,13 @@ func runPrune(gopts GlobalOptions) error {
 		return err
 	}
 
+	blobs := 0
 	for _, pack := range idx.Packs {
 		stats.bytes += pack.Size
+		blobs += len(pack.Entries)
 	}
 	Verbosef("repository contains %v packs (%v blobs) with %v bytes\n",
-		len(idx.Packs), len(idx.Blobs), formatBytes(uint64(stats.bytes)))
+		len(idx.Packs), blobs, formatBytes(uint64(stats.bytes)))
 
 	blobCount := make(map[restic.BlobHandle]int)
 	duplicateBlobs := 0
@@ -164,14 +166,17 @@ func runPrune(gopts GlobalOptions) error {
 
 	// find packs that need a rewrite
 	rewritePacks := restic.NewIDSet()
-	for h, blob := range idx.Blobs {
-		if !usedBlobs.Has(h) {
-			rewritePacks.Merge(blob.Packs)
-			continue
-		}
+	for _, pack := range idx.Packs {
+		for _, blob := range pack.Entries {
+			h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
+			if !usedBlobs.Has(h) {
+				rewritePacks.Insert(pack.ID)
+				continue
+			}
 
-		if blobCount[h] > 1 {
-			rewritePacks.Merge(blob.Packs)
+			if blobCount[h] > 1 {
+				rewritePacks.Insert(pack.ID)
+			}
 		}
 	}
 
@@ -214,7 +219,8 @@ func runPrune(gopts GlobalOptions) error {
 	}
 
 	for packID := range removePacks {
-		err = repo.Backend().Remove(restic.DataFile, packID.String())
+		h := restic.Handle{Type: restic.DataFile, Name: packID.String()}
+		err = repo.Backend().Remove(h)
 		if err != nil {
 			Warnf("unable to remove file %v from the repository\n", packID.Str())
 		}
@@ -234,7 +240,8 @@ func runPrune(gopts GlobalOptions) error {
 
 	var supersedes restic.IDs
 	for idxID := range repo.List(restic.IndexFile, done) {
-		err := repo.Backend().Remove(restic.IndexFile, idxID.String())
+		h := restic.Handle{Type: restic.IndexFile, Name: idxID.String()}
+		err := repo.Backend().Remove(h)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "unable to remove index %v: %v\n", idxID.Str(), err)
 		}

src/cmds/restic/cmd_snapshots.go

@@ -57,8 +57,8 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 	}
 
 	tab := NewTable()
-	tab.Header = fmt.Sprintf("%-8s  %-19s  %-10s  %-10s  %s", "ID", "Date", "Host", "Tags", "Directory")
-	tab.RowFormat = "%-8s  %-19s  %-10s  %-10s  %s"
+	tab.Header = fmt.Sprintf("%-8s  %-19s  %-10s  %-10s  %-3s %s", "ID", "Date", "Host", "Tags", "", "Directory")
+	tab.RowFormat = "%-8s  %-19s  %-10s  %-10s  %-3s %s"
 
 	done := make(chan struct{})
 	defer close(done)
@@ -97,9 +97,15 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 			firstTag = sn.Tags[0]
 		}
 
-		tab.Rows = append(tab.Rows, []interface{}{sn.ID().Str(), sn.Time.Format(TimeFormat), sn.Hostname, firstTag, sn.Paths[0]})
-
 		rows := len(sn.Paths)
+
+		treeElement := "   "
+		if rows != 1 {
+			treeElement = "┌──"
+		}
+
+		tab.Rows = append(tab.Rows, []interface{}{sn.ID().Str(), sn.Time.Format(TimeFormat), sn.Hostname, firstTag, treeElement, sn.Paths[0]})
+
 		if len(sn.Tags) > rows {
 			rows = len(sn.Tags)
 		}
@@ -115,7 +121,12 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 				tag = sn.Tags[i]
 			}
 
-			tab.Rows = append(tab.Rows, []interface{}{"", "", "", tag, path})
+			treeElement := "│"
+			if i == (rows - 1) {
+				treeElement = "└──"
+			}
+
+			tab.Rows = append(tab.Rows, []interface{}{"", "", "", tag, treeElement, path})
 		}
 	}
 

src/cmds/restic/global.go

@@ -51,7 +51,7 @@ func init() {
 	f := cmdRoot.PersistentFlags()
 	f.StringVarP(&globalOptions.Repo, "repo", "r", os.Getenv("RESTIC_REPOSITORY"), "repository to backup to or restore from (default: $RESTIC_REPOSITORY)")
 	f.StringVarP(&globalOptions.PasswordFile, "password-file", "p", "", "read the repository password from a file")
-	f.BoolVarP(&globalOptions.Quiet, "quiet", "q", false, "do not outputcomprehensive progress report")
+	f.BoolVarP(&globalOptions.Quiet, "quiet", "q", false, "do not output comprehensive progress report")
 	f.BoolVar(&globalOptions.NoLock, "no-lock", false, "do not lock the repo, this allows some operations on read-only repos")
 
 	restoreTerminal()

src/cmds/restic/global_debug.go

@@ -0,0 +1,60 @@
+// +build debug
+
+package main
+
+import (
+	"fmt"
+	"net/http"
+	_ "net/http/pprof"
+	"os"
+	"restic/errors"
+
+	"github.com/pkg/profile"
+)
+
+var (
+	listenMemoryProfile string
+	memProfilePath      string
+	cpuProfilePath      string
+
+	prof interface {
+		Stop()
+	}
+)
+
+func init() {
+	f := cmdRoot.PersistentFlags()
+	f.StringVar(&listenMemoryProfile, "listen-profile", "", "listen on this `address:port` for memory profiling")
+	f.StringVar(&memProfilePath, "mem-profile", "", "write memory profile to `dir`")
+	f.StringVar(&cpuProfilePath, "cpu-profile", "", "write cpu profile to `dir`")
+}
+
+func runDebug() error {
+	if listenMemoryProfile != "" {
+		fmt.Fprintf(os.Stderr, "running memory profile HTTP server on %v\n", listenMemoryProfile)
+		go func() {
+			err := http.ListenAndServe(listenMemoryProfile, nil)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "memory profile listen failed: %v\n", err)
+			}
+		}()
+	}
+
+	if memProfilePath != "" && cpuProfilePath != "" {
+		return errors.Fatal("only one profile (memory or CPU) may be activated at the same time")
+	}
+
+	if memProfilePath != "" {
+		prof = profile.Start(profile.Quiet, profile.MemProfile, profile.ProfilePath(memProfilePath))
+	} else if memProfilePath != "" {
+		prof = profile.Start(profile.Quiet, profile.CPUProfile, profile.ProfilePath(memProfilePath))
+	}
+
+	return nil
+}
+
+func shutdownDebug() {
+	if prof != nil {
+		prof.Stop()
+	}
+}

src/cmds/restic/global_release.go

@@ -0,0 +1,9 @@
+// +build !debug
+
+package main
+
+// runDebug is a noop without the debug tag.
+func runDebug() error { return nil }
+
+// shutdownDebug is a noop without the debug tag.
+func shutdownDebug() {}

src/cmds/restic/integration_test.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	mrand "math/rand"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -159,6 +160,15 @@ func testRunFind(t testing.TB, gopts GlobalOptions, pattern string) []string {
 	return strings.Split(string(buf.Bytes()), "\n")
 }
 
+func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) {
+	opts := ForgetOptions{}
+	OK(t, runForget(opts, gopts, args))
+}
+
+func testRunPrune(t testing.TB, gopts GlobalOptions) {
+	OK(t, runPrune(gopts))
+}
+
 func TestBackup(t *testing.T) {
 	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
 		datafile := filepath.Join("testdata", "backup-data.tar.gz")
@@ -730,6 +740,30 @@ func TestRestoreFilter(t *testing.T) {
 	})
 }
 
+func TestRestore(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+		testRunInit(t, gopts)
+
+		for i := 0; i < 10; i++ {
+			p := filepath.Join(env.testdata, fmt.Sprintf("foo/bar/testfile%v", i))
+			OK(t, os.MkdirAll(filepath.Dir(p), 0755))
+			OK(t, appendRandomData(p, uint(mrand.Intn(5<<21))))
+		}
+
+		opts := BackupOptions{}
+
+		testRunBackup(t, []string{env.testdata}, opts, gopts)
+		testRunCheck(t, gopts)
+
+		// Restore latest without any filters
+		restoredir := filepath.Join(env.base, "restore")
+		testRunRestoreLatest(t, gopts, restoredir, nil, "")
+
+		Assert(t, directoriesEqualContents(env.testdata, filepath.Join(restoredir, filepath.Base(env.testdata))),
+			"directories are not equal")
+	})
+}
+
 func TestRestoreLatest(t *testing.T) {
 
 	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
@@ -947,3 +981,33 @@ func TestCheckRestoreNoLock(t *testing.T) {
 		testRunRestore(t, gopts, filepath.Join(env.base, "restore"), snapshotIDs[0])
 	})
 }
+
+func TestPrune(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		fd, err := os.Open(datafile)
+		if os.IsNotExist(errors.Cause(err)) {
+			t.Skipf("unable to find data file %q, skipping", datafile)
+			return
+		}
+		OK(t, err)
+		OK(t, fd.Close())
+
+		testRunInit(t, gopts)
+
+		SetupTarTestFixture(t, env.testdata, datafile)
+		opts := BackupOptions{}
+
+		testRunBackup(t, []string{filepath.Join(env.testdata, "0", "0", "1")}, opts, gopts)
+		testRunBackup(t, []string{filepath.Join(env.testdata, "0", "0", "2")}, opts, gopts)
+		testRunBackup(t, []string{filepath.Join(env.testdata, "0", "0", "3")}, opts, gopts)
+
+		snapshotIDs := testRunList(t, "snapshots", gopts)
+		Assert(t, len(snapshotIDs) == 3,
+			"expected one snapshot, got %v", snapshotIDs)
+
+		testRunForget(t, gopts, snapshotIDs[0].String())
+		testRunPrune(t, gopts)
+		testRunCheck(t, gopts)
+	})
+}

src/cmds/restic/main.go

@@ -5,7 +5,6 @@ import (
 	"os"
 	"restic"
 	"restic/debug"
-	"runtime"
 
 	"github.com/spf13/cobra"
 
@@ -22,17 +21,15 @@ directories in an encrypted repository stored on different backends.
 `,
 	SilenceErrors: true,
 	SilenceUsage:  true,
-}
 
-func init() {
-	// set GOMAXPROCS to number of CPUs
-	if runtime.Version() < "go1.5" {
-		gomaxprocs := os.Getenv("GOMAXPROCS")
-		debug.Log("read GOMAXPROCS from env variable, value: %s", gomaxprocs)
-		if gomaxprocs == "" {
-			runtime.GOMAXPROCS(runtime.NumCPU())
-		}
-	}
+	// run the debug functions for all subcommands (if build tag "debug" is
+	// enabled)
+	PersistentPreRunE: func(*cobra.Command, []string) error {
+		return runDebug()
+	},
+	PersistentPostRun: func(*cobra.Command, []string) {
+		shutdownDebug()
+	},
 }
 
 func main() {

src/restic/archiver/archive_reader_test.go

@@ -44,7 +44,7 @@ func checkSavedFile(t *testing.T, repo restic.Repository, treeID restic.ID, name
 			t.Fatal(err)
 		}
 
-		buf := make([]byte, int(size))
+		buf := restic.NewBlobBuffer(int(size))
 		n := loadBlob(t, repo, id, buf)
 		if n != len(buf) {
 			t.Errorf("wrong number of bytes read, want %d, got %d", len(buf), n)

src/restic/archiver/archiver_duplication_test.go

@@ -39,15 +39,15 @@ func randomID() restic.ID {
 func forgetfulBackend() restic.Backend {
 	be := &mock.Backend{}
 
-	be.TestFn = func(t restic.FileType, name string) (bool, error) {
+	be.TestFn = func(h restic.Handle) (bool, error) {
 		return false, nil
 	}
 
-	be.LoadFn = func(h restic.Handle, p []byte, off int64) (int, error) {
-		return 0, errors.New("not found")
+	be.LoadFn = func(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+		return nil, errors.New("not found")
 	}
 
-	be.SaveFn = func(h restic.Handle, p []byte) error {
+	be.SaveFn = func(h restic.Handle, rd io.Reader) error {
 		return nil
 	}
 
@@ -55,7 +55,7 @@ func forgetfulBackend() restic.Backend {
 		return restic.FileInfo{}, errors.New("not found")
 	}
 
-	be.RemoveFn = func(t restic.FileType, name string) error {
+	be.RemoveFn = func(h restic.Handle) error {
 		return nil
 	}
 
@@ -142,6 +142,11 @@ func testArchiverDuplication(t *testing.T) {
 	close(done)
 
 	wg.Wait()
+
+	err = repo.Flush()
+	if err != nil {
+		t.Fatal(err)
+	}
 }
 
 func TestArchiverDuplication(t *testing.T) {

src/restic/backend.go

@@ -1,5 +1,7 @@
 package restic
 
+import "io"
+
 // Backend is used to store and access data.
 type Backend interface {
 	// Location returns a string that describes the type and location of the
@@ -7,22 +9,22 @@ type Backend interface {
 	Location() string
 
 	// Test a boolean value whether a File with the name and type exists.
-	Test(t FileType, name string) (bool, error)
+	Test(h Handle) (bool, error)
 
 	// Remove removes a File with type t and name.
-	Remove(t FileType, name string) error
+	Remove(h Handle) error
 
 	// Close the backend
 	Close() error
 
-	// Load returns the data stored in the backend for h at the given offset
-	// and saves it in p. Load has the same semantics as io.ReaderAt, except
-	// that a negative offset is also allowed. In this case it references a
-	// position relative to the end of the file (similar to Seek()).
-	Load(h Handle, p []byte, off int64) (int, error)
-
 	// Save stores the data in the backend under the given handle.
-	Save(h Handle, p []byte) error
+	Save(h Handle, rd io.Reader) error
+
+	// Load returns a reader that yields the contents of the file at h at the
+	// given offset. If length is larger than zero, only a portion of the file
+	// is returned. rd must be closed after use. If an error is returned, the
+	// ReadCloser must be nil.
+	Load(h Handle, length int, offset int64) (io.ReadCloser, error)
 
 	// Stat returns information about the File identified by h.
 	Stat(h Handle) (FileInfo, error)

src/restic/backend/local/backend_test.go

@@ -51,13 +51,6 @@ func TestLocalBackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestLocalBackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestLocalBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/local/local.go

@@ -101,58 +101,18 @@ func dirname(base string, t restic.FileType, name string) string {
 	return filepath.Join(base, n)
 }
 
-// Load returns the data stored in the backend for h at the given offset and
-// saves it in p. Load has the same semantics as io.ReaderAt, with one
-// exception: when off is lower than zero, it is treated as an offset relative
-// to the end of the file.
-func (b *Local) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
-	debug.Log("Load %v, length %v at %v", h, len(p), off)
-	if err := h.Valid(); err != nil {
-		return 0, err
-	}
-
-	f, err := fs.Open(filename(b.p, h.Type, h.Name))
-	if err != nil {
-		return 0, errors.Wrap(err, "Open")
-	}
-
-	defer func() {
-		e := f.Close()
-		if err == nil {
-			err = errors.Wrap(e, "Close")
-		}
-	}()
-
-	switch {
-	case off > 0:
-		_, err = f.Seek(off, 0)
-	case off < 0:
-		_, err = f.Seek(off, 2)
-	}
-
-	if err != nil {
-		return 0, errors.Wrap(err, "Seek")
-	}
-
-	return io.ReadFull(f, p)
-}
-
-// writeToTempfile saves p into a tempfile in tempdir.
-func writeToTempfile(tempdir string, p []byte) (filename string, err error) {
+// copyToTempfile saves p into a tempfile in tempdir.
+func copyToTempfile(tempdir string, rd io.Reader) (filename string, err error) {
 	tmpfile, err := ioutil.TempFile(tempdir, "temp-")
 	if err != nil {
 		return "", errors.Wrap(err, "TempFile")
 	}
 
-	n, err := tmpfile.Write(p)
+	_, err = io.Copy(tmpfile, rd)
 	if err != nil {
 		return "", errors.Wrap(err, "Write")
 	}
 
-	if n != len(p) {
-		return "", errors.New("not all bytes writen")
-	}
-
 	if err = tmpfile.Sync(); err != nil {
 		return "", errors.Wrap(err, "Syncn")
 	}
@@ -166,14 +126,14 @@ func writeToTempfile(tempdir string, p []byte) (filename string, err error) {
 }
 
 // Save stores data in the backend at the handle.
-func (b *Local) Save(h restic.Handle, p []byte) (err error) {
-	debug.Log("Save %v, length %v", h, len(p))
+func (b *Local) Save(h restic.Handle, rd io.Reader) (err error) {
+	debug.Log("Save %v", h)
 	if err := h.Valid(); err != nil {
 		return err
 	}
 
-	tmpfile, err := writeToTempfile(filepath.Join(b.p, backend.Paths.Temp), p)
-	debug.Log("saved %v (%d bytes) to %v", h, len(p), tmpfile)
+	tmpfile, err := copyToTempfile(filepath.Join(b.p, backend.Paths.Temp), rd)
+	debug.Log("saved %v to %v", h, tmpfile)
 	if err != nil {
 		return err
 	}
@@ -210,6 +170,39 @@ func (b *Local) Save(h restic.Handle, p []byte) (err error) {
 	return setNewFileMode(filename, fi)
 }
 
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (b *Local) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	f, err := os.Open(filename(b.p, h.Type, h.Name))
+	if err != nil {
+		return nil, err
+	}
+
+	if offset > 0 {
+		_, err = f.Seek(offset, 0)
+		if err != nil {
+			f.Close()
+			return nil, err
+		}
+	}
+
+	if length > 0 {
+		return backend.LimitReadCloser(f, int64(length)), nil
+	}
+
+	return f, nil
+}
+
 // Stat returns information about a blob.
 func (b *Local) Stat(h restic.Handle) (restic.FileInfo, error) {
 	debug.Log("Stat %v", h)
@@ -226,9 +219,9 @@ func (b *Local) Stat(h restic.Handle) (restic.FileInfo, error) {
 }
 
 // Test returns true if a blob of the given type and name exists in the backend.
-func (b *Local) Test(t restic.FileType, name string) (bool, error) {
-	debug.Log("Test %v %v", t, name)
-	_, err := fs.Stat(filename(b.p, t, name))
+func (b *Local) Test(h restic.Handle) (bool, error) {
+	debug.Log("Test %v", h)
+	_, err := fs.Stat(filename(b.p, h.Type, h.Name))
 	if err != nil {
 		if os.IsNotExist(errors.Cause(err)) {
 			return false, nil
@@ -240,9 +233,9 @@ func (b *Local) Test(t restic.FileType, name string) (bool, error) {
 }
 
 // Remove removes the blob with the given name and type.
-func (b *Local) Remove(t restic.FileType, name string) error {
-	debug.Log("Remove %v %v", t, name)
-	fn := filename(b.p, t, name)
+func (b *Local) Remove(h restic.Handle) error {
+	debug.Log("Remove %v", h)
+	fn := filename(b.p, h.Type, h.Name)
 
 	// reset read-only flag
 	err := fs.Chmod(fn, 0666)

src/restic/backend/mem/backend_test.go

@@ -51,13 +51,6 @@ func TestMemBackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestMemBackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestMemBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/mem/mem_backend.go

@@ -1,21 +1,19 @@
 package mem
 
 import (
+	"bytes"
 	"io"
+	"io/ioutil"
 	"restic"
 	"sync"
 
+	"restic/backend"
 	"restic/errors"
 
 	"restic/debug"
 )
 
-type entry struct {
-	Type restic.FileType
-	Name string
-}
-
-type memMap map[entry][]byte
+type memMap map[restic.Handle][]byte
 
 // make sure that MemoryBackend implements backend.Backend
 var _ restic.Backend = &MemoryBackend{}
@@ -39,61 +37,21 @@ func New() *MemoryBackend {
 }
 
 // Test returns whether a file exists.
-func (be *MemoryBackend) Test(t restic.FileType, name string) (bool, error) {
+func (be *MemoryBackend) Test(h restic.Handle) (bool, error) {
 	be.m.Lock()
 	defer be.m.Unlock()
 
-	debug.Log("test %v %v", t, name)
+	debug.Log("Test %v", h)
 
-	if _, ok := be.data[entry{t, name}]; ok {
+	if _, ok := be.data[h]; ok {
 		return true, nil
 	}
 
 	return false, nil
 }
 
-// Load reads data from the backend.
-func (be *MemoryBackend) Load(h restic.Handle, p []byte, off int64) (int, error) {
-	if err := h.Valid(); err != nil {
-		return 0, err
-	}
-
-	be.m.Lock()
-	defer be.m.Unlock()
-
-	if h.Type == restic.ConfigFile {
-		h.Name = ""
-	}
-
-	debug.Log("get %v offset %v len %v", h, off, len(p))
-
-	if _, ok := be.data[entry{h.Type, h.Name}]; !ok {
-		return 0, errors.New("no such data")
-	}
-
-	buf := be.data[entry{h.Type, h.Name}]
-	switch {
-	case off > int64(len(buf)):
-		return 0, errors.New("offset beyond end of file")
-	case off < -int64(len(buf)):
-		off = 0
-	case off < 0:
-		off = int64(len(buf)) + off
-	}
-
-	buf = buf[off:]
-
-	n := copy(p, buf)
-
-	if len(p) > len(buf) {
-		return n, io.ErrUnexpectedEOF
-	}
-
-	return n, nil
-}
-
 // Save adds new Data to the backend.
-func (be *MemoryBackend) Save(h restic.Handle, p []byte) error {
+func (be *MemoryBackend) Save(h restic.Handle, rd io.Reader) error {
 	if err := h.Valid(); err != nil {
 		return err
 	}
@@ -105,18 +63,59 @@ func (be *MemoryBackend) Save(h restic.Handle, p []byte) error {
 		h.Name = ""
 	}
 
-	if _, ok := be.data[entry{h.Type, h.Name}]; ok {
+	if _, ok := be.data[h]; ok {
 		return errors.New("file already exists")
 	}
 
-	debug.Log("save %v bytes at %v", len(p), h)
-	buf := make([]byte, len(p))
-	copy(buf, p)
-	be.data[entry{h.Type, h.Name}] = buf
+	buf, err := ioutil.ReadAll(rd)
+	if err != nil {
+		return err
+	}
+
+	be.data[h] = buf
+	debug.Log("saved %v bytes at %v", len(buf), h)
 
 	return nil
 }
 
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (be *MemoryBackend) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	be.m.Lock()
+	defer be.m.Unlock()
+
+	if h.Type == restic.ConfigFile {
+		h.Name = ""
+	}
+
+	debug.Log("Load %v offset %v len %v", h, offset, length)
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	if _, ok := be.data[h]; !ok {
+		return nil, errors.New("no such data")
+	}
+
+	buf := be.data[h]
+	if offset > int64(len(buf)) {
+		return nil, errors.New("offset beyond end of file")
+	}
+
+	buf = buf[offset:]
+	if length > 0 && len(buf) > length {
+		buf = buf[:length]
+	}
+
+	return backend.Closer{Reader: bytes.NewReader(buf)}, nil
+}
+
 // Stat returns information about a file in the backend.
 func (be *MemoryBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 	be.m.Lock()
@@ -132,7 +131,7 @@ func (be *MemoryBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 
 	debug.Log("stat %v", h)
 
-	e, ok := be.data[entry{h.Type, h.Name}]
+	e, ok := be.data[h]
 	if !ok {
 		return restic.FileInfo{}, errors.New("no such data")
 	}
@@ -141,17 +140,17 @@ func (be *MemoryBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 }
 
 // Remove deletes a file from the backend.
-func (be *MemoryBackend) Remove(t restic.FileType, name string) error {
+func (be *MemoryBackend) Remove(h restic.Handle) error {
 	be.m.Lock()
 	defer be.m.Unlock()
 
-	debug.Log("get %v %v", t, name)
+	debug.Log("Remove %v", h)
 
-	if _, ok := be.data[entry{t, name}]; !ok {
+	if _, ok := be.data[h]; !ok {
 		return errors.New("no such data")
 	}
 
-	delete(be.data, entry{t, name})
+	delete(be.data, h)
 
 	return nil
 }

src/restic/backend/rest/backend_test.go

@@ -51,13 +51,6 @@ func TestRestBackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestRestBackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestRestBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/rest/rest.go

@@ -1,7 +1,6 @@
 package rest
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -12,6 +11,7 @@ import (
 	"restic"
 	"strings"
 
+	"restic/debug"
 	"restic/errors"
 
 	"restic/backend"
@@ -19,6 +19,9 @@ import (
 
 const connLimit = 10
 
+// make sure the rest backend implements restic.Backend
+var _ restic.Backend = &restBackend{}
+
 // restPath returns the path to the given resource.
 func restPath(url *url.URL, h restic.Handle) string {
 	u := *url
@@ -71,65 +74,18 @@ func (b *restBackend) Location() string {
 	return b.url.String()
 }
 
-// Load returns the data stored in the backend for h at the given offset
-// and saves it in p. Load has the same semantics as io.ReaderAt.
-func (b *restBackend) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
-	if err := h.Valid(); err != nil {
-		return 0, err
-	}
-
-	// invert offset
-	if off < 0 {
-		info, err := b.Stat(h)
-		if err != nil {
-			return 0, errors.Wrap(err, "Stat")
-		}
-
-		if -off > info.Size {
-			off = 0
-		} else {
-			off = info.Size + off
-		}
-	}
-
-	req, err := http.NewRequest("GET", restPath(b.url, h), nil)
-	if err != nil {
-		return 0, errors.Wrap(err, "http.NewRequest")
-	}
-	req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", off, off+int64(len(p))))
-	<-b.connChan
-	resp, err := b.client.Do(req)
-	b.connChan <- struct{}{}
-
-	if resp != nil {
-		defer func() {
-			io.Copy(ioutil.Discard, resp.Body)
-			e := resp.Body.Close()
-
-			if err == nil {
-				err = errors.Wrap(e, "Close")
-			}
-		}()
-	}
-
-	if err != nil {
-		return 0, errors.Wrap(err, "client.Do")
-	}
-	if resp.StatusCode != 200 && resp.StatusCode != 206 {
-		return 0, errors.Errorf("unexpected HTTP response code %v", resp.StatusCode)
-	}
-
-	return io.ReadFull(resp.Body, p)
-}
-
 // Save stores data in the backend at the handle.
-func (b *restBackend) Save(h restic.Handle, p []byte) (err error) {
+func (b *restBackend) Save(h restic.Handle, rd io.Reader) (err error) {
 	if err := h.Valid(); err != nil {
 		return err
 	}
 
+	// make sure that client.Post() cannot close the reader by wrapping it in
+	// backend.Closer, which has a noop method.
+	rd = backend.Closer{Reader: rd}
+
 	<-b.connChan
-	resp, err := b.client.Post(restPath(b.url, h), "binary/octet-stream", bytes.NewReader(p))
+	resp, err := b.client.Post(restPath(b.url, h), "binary/octet-stream", rd)
 	b.connChan <- struct{}{}
 
 	if resp != nil {
@@ -154,6 +110,56 @@ func (b *restBackend) Save(h restic.Handle, p []byte) (err error) {
 	return nil
 }
 
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (b *restBackend) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	if length < 0 {
+		return nil, errors.Errorf("invalid length %d", length)
+	}
+
+	req, err := http.NewRequest("GET", restPath(b.url, h), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "http.NewRequest")
+	}
+
+	byteRange := fmt.Sprintf("bytes=%d-", offset)
+	if length > 0 {
+		byteRange = fmt.Sprintf("bytes=%d-%d", offset, offset+int64(length)-1)
+	}
+	req.Header.Add("Range", byteRange)
+	debug.Log("Load(%v) send range %v", h, byteRange)
+
+	<-b.connChan
+	resp, err := b.client.Do(req)
+	b.connChan <- struct{}{}
+
+	if err != nil {
+		if resp != nil {
+			io.Copy(ioutil.Discard, resp.Body)
+			resp.Body.Close()
+		}
+		return nil, errors.Wrap(err, "client.Do")
+	}
+
+	if resp.StatusCode != 200 && resp.StatusCode != 206 {
+		io.Copy(ioutil.Discard, resp.Body)
+		resp.Body.Close()
+		return nil, errors.Errorf("unexpected HTTP response code %v", resp.StatusCode)
+	}
+
+	return resp.Body, nil
+}
+
 // Stat returns information about a blob.
 func (b *restBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 	if err := h.Valid(); err != nil {
@@ -188,8 +194,8 @@ func (b *restBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 }
 
 // Test returns true if a blob of the given type and name exists in the backend.
-func (b *restBackend) Test(t restic.FileType, name string) (bool, error) {
-	_, err := b.Stat(restic.Handle{Type: t, Name: name})
+func (b *restBackend) Test(h restic.Handle) (bool, error) {
+	_, err := b.Stat(h)
 	if err != nil {
 		return false, nil
 	}
@@ -198,8 +204,7 @@ func (b *restBackend) Test(t restic.FileType, name string) (bool, error) {
 }
 
 // Remove removes the blob with the given name and type.
-func (b *restBackend) Remove(t restic.FileType, name string) error {
-	h := restic.Handle{Type: t, Name: name}
+func (b *restBackend) Remove(h restic.Handle) error {
 	if err := h.Valid(); err != nil {
 		return err
 	}

src/restic/backend/rest/rest_test.go

@@ -37,7 +37,7 @@ func init() {
 			return nil, err
 		}
 
-		exists, err := be.Test(restic.ConfigFile, "")
+		exists, err := be.Test(restic.Handle{Type: restic.ConfigFile, Name: ""})
 		if err != nil {
 			return nil, err
 		}

src/restic/backend/s3/backend_test.go

@@ -51,13 +51,6 @@ func TestS3BackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestS3BackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestS3BackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/s3/s3.go

@@ -7,6 +7,7 @@ import (
 	"restic"
 	"strings"
 
+	"restic/backend"
 	"restic/errors"
 
 	"github.com/minio/minio-go"
@@ -54,11 +55,11 @@ func Open(cfg Config) (restic.Backend, error) {
 	return be, nil
 }
 
-func (be *s3) s3path(t restic.FileType, name string) string {
-	if t == restic.ConfigFile {
-		return path.Join(be.prefix, string(t))
+func (be *s3) s3path(h restic.Handle) string {
+	if h.Type == restic.ConfigFile {
+		return path.Join(be.prefix, string(h.Type))
 	}
-	return path.Join(be.prefix, string(t), name)
+	return path.Join(be.prefix, string(h.Type), h.Name)
 }
 
 func (be *s3) createConnections() {
@@ -73,88 +74,15 @@ func (be *s3) Location() string {
 	return be.bucketname
 }
 
-// Load returns the data stored in the backend for h at the given offset
-// and saves it in p. Load has the same semantics as io.ReaderAt.
-func (be s3) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
-	var obj *minio.Object
-
-	debug.Log("%v, offset %v, len %v", h, off, len(p))
-	objName := be.s3path(h.Type, h.Name)
-
-	<-be.connChan
-	defer func() {
-		be.connChan <- struct{}{}
-	}()
-
-	obj, err = be.client.GetObject(be.bucketname, objName)
-	if err != nil {
-		debug.Log("  err %v", err)
-		return 0, errors.Wrap(err, "client.GetObject")
-	}
-
-	// make sure that the object is closed properly.
-	defer func() {
-		e := obj.Close()
-		if err == nil {
-			err = errors.Wrap(e, "Close")
-		}
-	}()
-
-	info, err := obj.Stat()
-	if err != nil {
-		return 0, errors.Wrap(err, "obj.Stat")
-	}
-
-	// handle negative offsets
-	if off < 0 {
-		// if the negative offset is larger than the object itself, read from
-		// the beginning.
-		if -off > info.Size {
-			off = 0
-		} else {
-			// otherwise compute the offset from the end of the file.
-			off = info.Size + off
-		}
-	}
-
-	// return an error if the offset is beyond the end of the file
-	if off > info.Size {
-		return 0, errors.Wrap(io.EOF, "")
-	}
-
-	var nextError error
-
-	// manually create an io.ErrUnexpectedEOF
-	if off+int64(len(p)) > info.Size {
-		newlen := info.Size - off
-		p = p[:newlen]
-
-		nextError = io.ErrUnexpectedEOF
-
-		debug.Log("    capped buffer to %v byte", len(p))
-	}
-
-	n, err = obj.ReadAt(p, off)
-	if int64(n) == info.Size-off && errors.Cause(err) == io.EOF {
-		err = nil
-	}
-
-	if err == nil {
-		err = nextError
-	}
-
-	return n, err
-}
-
 // Save stores data in the backend at the handle.
-func (be s3) Save(h restic.Handle, p []byte) (err error) {
+func (be *s3) Save(h restic.Handle, rd io.Reader) (err error) {
 	if err := h.Valid(); err != nil {
 		return err
 	}
 
-	debug.Log("%v with %d bytes", h, len(p))
+	debug.Log("Save %v", h)
 
-	objName := be.s3path(h.Type, h.Name)
+	objName := be.s3path(h)
 
 	// Check key does not already exist
 	_, err = be.client.StatObject(be.bucketname, objName)
@@ -168,19 +96,97 @@ func (be s3) Save(h restic.Handle, p []byte) (err error) {
 		be.connChan <- struct{}{}
 	}()
 
-	debug.Log("PutObject(%v, %v, %v, %v)",
-		be.bucketname, objName, int64(len(p)), "binary/octet-stream")
-	n, err := be.client.PutObject(be.bucketname, objName, bytes.NewReader(p), "binary/octet-stream")
+	debug.Log("PutObject(%v, %v)",
+		be.bucketname, objName)
+	n, err := be.client.PutObject(be.bucketname, objName, rd, "binary/octet-stream")
 	debug.Log("%v -> %v bytes, err %#v", objName, n, err)
 
 	return errors.Wrap(err, "client.PutObject")
 }
 
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	if length < 0 {
+		return nil, errors.Errorf("invalid length %d", length)
+	}
+
+	var obj *minio.Object
+
+	objName := be.s3path(h)
+
+	<-be.connChan
+	defer func() {
+		be.connChan <- struct{}{}
+	}()
+
+	obj, err := be.client.GetObject(be.bucketname, objName)
+	if err != nil {
+		debug.Log("  err %v", err)
+		return nil, errors.Wrap(err, "client.GetObject")
+	}
+
+	// if we're going to read the whole object, just pass it on.
+	if length == 0 {
+		debug.Log("Load %v: pass on object", h)
+		_, err = obj.Seek(offset, 0)
+		if err != nil {
+			_ = obj.Close()
+			return nil, errors.Wrap(err, "obj.Seek")
+		}
+
+		return obj, nil
+	}
+
+	// otherwise use a buffer with ReadAt
+	info, err := obj.Stat()
+	if err != nil {
+		_ = obj.Close()
+		return nil, errors.Wrap(err, "obj.Stat")
+	}
+
+	if offset > info.Size {
+		_ = obj.Close()
+		return nil, errors.Errorf("offset larger than file size")
+	}
+
+	l := int64(length)
+	if offset+l > info.Size {
+		l = info.Size - offset
+	}
+
+	buf := make([]byte, l)
+	n, err := obj.ReadAt(buf, offset)
+	debug.Log("Load %v: use buffer with ReadAt: %v, %v", h, n, err)
+	if err == io.EOF {
+		debug.Log("Load %v: shorten buffer %v -> %v", h, len(buf), n)
+		buf = buf[:n]
+		err = nil
+	}
+
+	if err != nil {
+		_ = obj.Close()
+		return nil, errors.Wrap(err, "obj.ReadAt")
+	}
+
+	return backend.Closer{Reader: bytes.NewReader(buf)}, nil
+}
+
 // Stat returns information about a blob.
-func (be s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) {
+func (be *s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) {
 	debug.Log("%v", h)
 
-	objName := be.s3path(h.Type, h.Name)
+	objName := be.s3path(h)
 	var obj *minio.Object
 
 	obj, err = be.client.GetObject(be.bucketname, objName)
@@ -207,9 +213,9 @@ func (be s3) Stat(h restic.Handle) (bi restic.FileInfo, err error) {
 }
 
 // Test returns true if a blob of the given type and name exists in the backend.
-func (be *s3) Test(t restic.FileType, name string) (bool, error) {
+func (be *s3) Test(h restic.Handle) (bool, error) {
 	found := false
-	objName := be.s3path(t, name)
+	objName := be.s3path(h)
 	_, err := be.client.StatObject(be.bucketname, objName)
 	if err == nil {
 		found = true
@@ -220,10 +226,10 @@ func (be *s3) Test(t restic.FileType, name string) (bool, error) {
 }
 
 // Remove removes the blob with the given name and type.
-func (be *s3) Remove(t restic.FileType, name string) error {
-	objName := be.s3path(t, name)
+func (be *s3) Remove(h restic.Handle) error {
+	objName := be.s3path(h)
 	err := be.client.RemoveObject(be.bucketname, objName)
-	debug.Log("%v %v -> err %v", t, name, err)
+	debug.Log("Remove(%v) -> err %v", h, err)
 	return errors.Wrap(err, "client.RemoveObject")
 }
 
@@ -234,7 +240,7 @@ func (be *s3) List(t restic.FileType, done <-chan struct{}) <-chan string {
 	debug.Log("listing %v", t)
 	ch := make(chan string)
 
-	prefix := be.s3path(t, "") + "/"
+	prefix := be.s3path(restic.Handle{Type: t}) + "/"
 
 	listresp := be.client.ListObjects(be.bucketname, prefix, true, done)
 
@@ -262,7 +268,7 @@ func (be *s3) removeKeys(t restic.FileType) error {
 	done := make(chan struct{})
 	defer close(done)
 	for key := range be.List(restic.DataFile, done) {
-		err := be.Remove(restic.DataFile, key)
+		err := be.Remove(restic.Handle{Type: restic.DataFile, Name: key})
 		if err != nil {
 			return err
 		}
@@ -287,7 +293,7 @@ func (be *s3) Delete() error {
 		}
 	}
 
-	return be.Remove(restic.ConfigFile, "")
+	return be.Remove(restic.Handle{Type: restic.ConfigFile})
 }
 
 // Close does nothing

src/restic/backend/s3/s3_test.go

@@ -44,7 +44,7 @@ func init() {
 			return nil, err
 		}
 
-		exists, err := be.Test(restic.ConfigFile, "")
+		exists, err := be.Test(restic.Handle{Type: restic.ConfigFile})
 		if err != nil {
 			return nil, err
 		}

src/restic/backend/sftp/backend_test.go

@@ -51,13 +51,6 @@ func TestSftpBackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestSftpBackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestSftpBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/sftp/sftp.go

@@ -259,11 +259,11 @@ func (r *SFTP) mkdirAll(dir string, mode os.FileMode) error {
 }
 
 // Rename temp file to final name according to type and name.
-func (r *SFTP) renameFile(oldname string, t restic.FileType, name string) error {
-	filename := r.filename(t, name)
+func (r *SFTP) renameFile(oldname string, h restic.Handle) error {
+	filename := r.filename(h)
 
 	// create directories if necessary
-	if t == restic.DataFile {
+	if h.Type == restic.DataFile {
 		err := r.mkdirAll(path.Dir(filename), backend.Modes.Dir)
 		if err != nil {
 			return err
@@ -297,22 +297,22 @@ func Join(parts ...string) string {
 }
 
 // Construct path for given restic.Type and name.
-func (r *SFTP) filename(t restic.FileType, name string) string {
-	if t == restic.ConfigFile {
+func (r *SFTP) filename(h restic.Handle) string {
+	if h.Type == restic.ConfigFile {
 		return Join(r.p, "config")
 	}
 
-	return Join(r.dirname(t, name), name)
+	return Join(r.dirname(h), h.Name)
 }
 
 // Construct directory for given backend.Type.
-func (r *SFTP) dirname(t restic.FileType, name string) string {
+func (r *SFTP) dirname(h restic.Handle) string {
 	var n string
-	switch t {
+	switch h.Type {
 	case restic.DataFile:
 		n = backend.Paths.Data
-		if len(name) > 2 {
-			n = Join(n, name[:2])
+		if len(h.Name) > 2 {
+			n = Join(n, h.Name[:2])
 		}
 	case restic.SnapshotFile:
 		n = backend.Paths.Snapshots
@@ -326,47 +326,9 @@ func (r *SFTP) dirname(t restic.FileType, name string) string {
 	return Join(r.p, n)
 }
 
-// Load returns the data stored in the backend for h at the given offset
-// and saves it in p. Load has the same semantics as io.ReaderAt.
-func (r *SFTP) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
-	debug.Log("load %v, %d bytes, offset %v", h, len(p), off)
-	if err := r.clientError(); err != nil {
-		return 0, err
-	}
-
-	if err := h.Valid(); err != nil {
-		return 0, err
-	}
-
-	f, err := r.c.Open(r.filename(h.Type, h.Name))
-	if err != nil {
-		return 0, errors.Wrap(err, "Open")
-	}
-
-	defer func() {
-		e := f.Close()
-		if err == nil {
-			err = errors.Wrap(e, "Close")
-		}
-	}()
-
-	switch {
-	case off > 0:
-		_, err = f.Seek(off, 0)
-	case off < 0:
-		_, err = f.Seek(off, 2)
-	}
-
-	if err != nil {
-		return 0, errors.Wrap(err, "Seek")
-	}
-
-	return io.ReadFull(f, p)
-}
-
 // Save stores data in the backend at the handle.
-func (r *SFTP) Save(h restic.Handle, p []byte) (err error) {
-	debug.Log("save %v bytes to %v", h, len(p))
+func (r *SFTP) Save(h restic.Handle, rd io.Reader) (err error) {
+	debug.Log("save to %v", h)
 	if err := r.clientError(); err != nil {
 		return err
 	}
@@ -380,31 +342,60 @@ func (r *SFTP) Save(h restic.Handle, p []byte) (err error) {
 		return err
 	}
 
-	debug.Log("save %v (%d bytes) to %v", h, len(p), filename)
-
-	n, err := tmpfile.Write(p)
+	n, err := io.Copy(tmpfile, rd)
 	if err != nil {
 		return errors.Wrap(err, "Write")
 	}
 
-	if n != len(p) {
-		return errors.New("not all bytes writen")
-	}
+	debug.Log("saved %v (%d bytes) to %v", h, n, filename)
 
 	err = tmpfile.Close()
 	if err != nil {
 		return errors.Wrap(err, "Close")
 	}
 
-	err = r.renameFile(filename, h.Type, h.Name)
+	err = r.renameFile(filename, h)
 	debug.Log("save %v: rename %v: %v",
 		h, path.Base(filename), err)
 	return err
 }
 
+// Load returns a reader that yields the contents of the file at h at the
+// given offset. If length is nonzero, only a portion of the file is
+// returned. rd must be closed after use.
+func (r *SFTP) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	debug.Log("Load %v, length %v, offset %v", h, length, offset)
+	if err := h.Valid(); err != nil {
+		return nil, err
+	}
+
+	if offset < 0 {
+		return nil, errors.New("offset is negative")
+	}
+
+	f, err := r.c.Open(r.filename(h))
+	if err != nil {
+		return nil, err
+	}
+
+	if offset > 0 {
+		_, err = f.Seek(offset, 0)
+		if err != nil {
+			_ = f.Close()
+			return nil, err
+		}
+	}
+
+	if length > 0 {
+		return backend.LimitReadCloser(f, int64(length)), nil
+	}
+
+	return f, nil
+}
+
 // Stat returns information about a blob.
 func (r *SFTP) Stat(h restic.Handle) (restic.FileInfo, error) {
-	debug.Log("stat %v", h)
+	debug.Log("Stat(%v)", h)
 	if err := r.clientError(); err != nil {
 		return restic.FileInfo{}, err
 	}
@@ -413,7 +404,7 @@ func (r *SFTP) Stat(h restic.Handle) (restic.FileInfo, error) {
 		return restic.FileInfo{}, err
 	}
 
-	fi, err := r.c.Lstat(r.filename(h.Type, h.Name))
+	fi, err := r.c.Lstat(r.filename(h))
 	if err != nil {
 		return restic.FileInfo{}, errors.Wrap(err, "Lstat")
 	}
@@ -422,13 +413,13 @@ func (r *SFTP) Stat(h restic.Handle) (restic.FileInfo, error) {
 }
 
 // Test returns true if a blob of the given type and name exists in the backend.
-func (r *SFTP) Test(t restic.FileType, name string) (bool, error) {
-	debug.Log("type %v, name %v", t, name)
+func (r *SFTP) Test(h restic.Handle) (bool, error) {
+	debug.Log("Test(%v)", h)
 	if err := r.clientError(); err != nil {
 		return false, err
 	}
 
-	_, err := r.c.Lstat(r.filename(t, name))
+	_, err := r.c.Lstat(r.filename(h))
 	if os.IsNotExist(errors.Cause(err)) {
 		return false, nil
 	}
@@ -441,13 +432,13 @@ func (r *SFTP) Test(t restic.FileType, name string) (bool, error) {
 }
 
 // Remove removes the content stored at name.
-func (r *SFTP) Remove(t restic.FileType, name string) error {
-	debug.Log("type %v, name %v", t, name)
+func (r *SFTP) Remove(h restic.Handle) error {
+	debug.Log("Remove(%v)", h)
 	if err := r.clientError(); err != nil {
 		return err
 	}
 
-	return r.c.Remove(r.filename(t, name))
+	return r.c.Remove(r.filename(h))
 }
 
 // List returns a channel that yields all names of blobs of type t. A
@@ -462,7 +453,7 @@ func (r *SFTP) List(t restic.FileType, done <-chan struct{}) <-chan string {
 
 		if t == restic.DataFile {
 			// read first level
-			basedir := r.dirname(t, "")
+			basedir := r.dirname(restic.Handle{Type: t})
 
 			list1, err := r.c.ReadDir(basedir)
 			if err != nil {
@@ -495,7 +486,7 @@ func (r *SFTP) List(t restic.FileType, done <-chan struct{}) <-chan string {
 				}
 			}
 		} else {
-			entries, err := r.c.ReadDir(r.dirname(t, ""))
+			entries, err := r.c.ReadDir(r.dirname(restic.Handle{Type: t}))
 			if err != nil {
 				return
 			}

src/restic/backend/test/backend_test.go

@@ -51,13 +51,6 @@ func TestTestBackendLoad(t *testing.T) {
 	test.TestLoad(t)
 }
 
-func TestTestBackendLoadNegativeOffset(t *testing.T) {
-	if SkipMessage != "" {
-		t.Skip(SkipMessage)
-	}
-	test.TestLoadNegativeOffset(t)
-}
-
 func TestTestBackendSave(t *testing.T) {
 	if SkipMessage != "" {
 		t.Skip(SkipMessage)

src/restic/backend/test/tests.go

@@ -4,13 +4,16 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"math/rand"
+	"os"
 	"reflect"
 	"restic"
+	"restic/errors"
 	"sort"
+	"strings"
 	"testing"
 
-	"restic/errors"
 	"restic/test"
 
 	"restic/backend"
@@ -127,7 +130,7 @@ func TestCreateWithConfig(t testing.TB) {
 	}
 
 	// remove config
-	err = b.Remove(restic.ConfigFile, "")
+	err = b.Remove(restic.Handle{Type: restic.ConfigFile, Name: ""})
 	if err != nil {
 		t.Fatalf("unexpected error removing config: %v", err)
 	}
@@ -152,12 +155,12 @@ func TestConfig(t testing.TB) {
 	var testString = "Config"
 
 	// create config and read it back
-	_, err := backend.LoadAll(b, restic.Handle{Type: restic.ConfigFile}, nil)
+	_, err := backend.LoadAll(b, restic.Handle{Type: restic.ConfigFile})
 	if err == nil {
 		t.Fatalf("did not get expected error for non-existing config")
 	}
 
-	err = b.Save(restic.Handle{Type: restic.ConfigFile}, []byte(testString))
+	err = b.Save(restic.Handle{Type: restic.ConfigFile}, strings.NewReader(testString))
 	if err != nil {
 		t.Fatalf("Save() error: %v", err)
 	}
@@ -166,7 +169,7 @@ func TestConfig(t testing.TB) {
 	// same config
 	for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} {
 		h := restic.Handle{Type: restic.ConfigFile, Name: name}
-		buf, err := backend.LoadAll(b, h, nil)
+		buf, err := backend.LoadAll(b, h)
 		if err != nil {
 			t.Fatalf("unable to read config with name %q: %v", name, err)
 		}
@@ -182,12 +185,12 @@ func TestLoad(t testing.TB) {
 	b := open(t)
 	defer close(t)
 
-	_, err := b.Load(restic.Handle{}, nil, 0)
+	_, err := b.Load(restic.Handle{}, 0, 0)
 	if err == nil {
 		t.Fatalf("Load() did not return an error for invalid handle")
 	}
 
-	_, err = b.Load(restic.Handle{Type: restic.DataFile, Name: "foobar"}, nil, 0)
+	_, err = b.Load(restic.Handle{Type: restic.DataFile, Name: "foobar"}, 0, 0)
 	if err == nil {
 		t.Fatalf("Load() did not return an error for non-existing blob")
 	}
@@ -198,11 +201,20 @@ func TestLoad(t testing.TB) {
 	id := restic.Hash(data)
 
 	handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
-	err = b.Save(handle, data)
+	err = b.Save(handle, bytes.NewReader(data))
 	if err != nil {
 		t.Fatalf("Save() error: %v", err)
 	}
 
+	rd, err := b.Load(handle, 100, -1)
+	if err == nil {
+		t.Fatalf("Load() returned no error for negative offset!")
+	}
+
+	if rd != nil {
+		t.Fatalf("Load() returned a non-nil reader for negative offset!")
+	}
+
 	for i := 0; i < 50; i++ {
 		l := rand.Intn(length + 2000)
 		o := rand.Intn(length + 2000)
@@ -215,157 +227,60 @@ func TestLoad(t testing.TB) {
 			d = d[:0]
 		}
 
-		if l > 0 && l < len(d) {
-			d = d[:l]
-		}
-
-		buf := make([]byte, l)
-		n, err := b.Load(handle, buf, int64(o))
-
-		// if we requested data beyond the end of the file, require
-		// ErrUnexpectedEOF error
-		if l > len(d) {
-			if errors.Cause(err) != io.ErrUnexpectedEOF {
-				t.Errorf("Load(%d, %d) did not return io.ErrUnexpectedEOF", len(buf), int64(o))
-			}
-			err = nil
-			buf = buf[:len(d)]
-		}
-
-		if err != nil {
-			t.Errorf("Load(%d, %d): unexpected error: %v", len(buf), int64(o), err)
-			continue
-		}
-
-		if n != len(buf) {
-			t.Errorf("Load(%d, %d): wrong length returned, want %d, got %d",
-				len(buf), int64(o), len(buf), n)
-			continue
-		}
-
-		buf = buf[:n]
-		if !bytes.Equal(buf, d) {
-			t.Errorf("Load(%d, %d) returned wrong bytes", len(buf), int64(o))
-			continue
-		}
-	}
-
-	// test with negative offset
-	for i := 0; i < 50; i++ {
-		l := rand.Intn(length + 2000)
-		o := rand.Intn(length + 2000)
-
-		d := data
-		if o < len(d) {
-			d = d[len(d)-o:]
-		} else {
-			o = 0
+		getlen := l
+		if l >= len(d) && rand.Float32() >= 0.5 {
+			getlen = 0
 		}
 
 		if l > 0 && l < len(d) {
 			d = d[:l]
 		}
 
-		buf := make([]byte, l)
-		n, err := b.Load(handle, buf, -int64(o))
-
-		// if we requested data beyond the end of the file, require
-		// ErrUnexpectedEOF error
-		if l > len(d) {
-			if errors.Cause(err) != io.ErrUnexpectedEOF {
-				t.Errorf("Load(%d, %d) did not return io.ErrUnexpectedEOF", len(buf), int64(o))
-				continue
-			}
-			err = nil
-			buf = buf[:len(d)]
-		}
-
+		rd, err := b.Load(handle, getlen, int64(o))
 		if err != nil {
-			t.Errorf("Load(%d, %d): unexpected error: %v", len(buf), int64(o), err)
+			t.Errorf("Load(%d, %d) returned unexpected error: %v", l, o, err)
 			continue
 		}
 
-		if n != len(buf) {
-			t.Errorf("Load(%d, %d): wrong length returned, want %d, got %d",
-				len(buf), int64(o), len(buf), n)
+		buf, err := ioutil.ReadAll(rd)
+		if err != nil {
+			t.Errorf("Load(%d, %d) ReadAll() returned unexpected error: %v", l, o, err)
+			continue
+		}
+
+		if l <= len(d) && len(buf) != l {
+			t.Errorf("Load(%d, %d) wrong number of bytes read: want %d, got %d", l, o, l, len(buf))
+			continue
+		}
+
+		if l > len(d) && len(buf) != len(d) {
+			t.Errorf("Load(%d, %d) wrong number of bytes read for overlong read: want %d, got %d", l, o, l, len(buf))
 			continue
 		}
 
-		buf = buf[:n]
 		if !bytes.Equal(buf, d) {
-			t.Errorf("Load(%d, %d) returned wrong bytes", len(buf), int64(o))
+			t.Errorf("Load(%d, %d) returned wrong bytes", l, o)
+			continue
+		}
+
+		err = rd.Close()
+		if err != nil {
+			t.Errorf("Load(%d, %d) rd.Close() returned unexpected error: %v", l, o, err)
 			continue
 		}
 	}
 
-	// load with a too-large buffer, this should return io.ErrUnexpectedEOF
-	buf := make([]byte, length+100)
-	n, err := b.Load(handle, buf, 0)
-	if n != length {
-		t.Errorf("wrong length for larger buffer returned, want %d, got %d", length, n)
-	}
-
-	if errors.Cause(err) != io.ErrUnexpectedEOF {
-		t.Errorf("wrong error returned for larger buffer: want io.ErrUnexpectedEOF, got %#v", err)
-	}
-
-	test.OK(t, b.Remove(restic.DataFile, id.String()))
+	test.OK(t, b.Remove(handle))
 }
 
-// TestLoadNegativeOffset tests the backend's Load function with negative offsets.
-func TestLoadNegativeOffset(t testing.TB) {
-	b := open(t)
-	defer close(t)
+type errorCloser struct {
+	io.Reader
+	t testing.TB
+}
 
-	length := rand.Intn(1<<24) + 2000
-
-	data := test.Random(23, length)
-	id := restic.Hash(data)
-
-	handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
-	err := b.Save(handle, data)
-	if err != nil {
-		t.Fatalf("Save() error: %v", err)
-	}
-
-	// test normal reads
-	for i := 0; i < 50; i++ {
-		l := rand.Intn(length + 2000)
-		o := -rand.Intn(length + 2000)
-
-		buf := make([]byte, l)
-		n, err := b.Load(handle, buf, int64(o))
-
-		// if we requested data beyond the end of the file, require
-		// ErrUnexpectedEOF error
-		if len(buf) > -o {
-			if errors.Cause(err) != io.ErrUnexpectedEOF {
-				t.Errorf("Load(%d, %d) did not return io.ErrUnexpectedEOF", len(buf), o)
-				continue
-			}
-			err = nil
-			buf = buf[:-o]
-		}
-
-		if err != nil {
-			t.Errorf("Load(%d, %d) returned error: %v", len(buf), o, err)
-			continue
-		}
-
-		if n != len(buf) {
-			t.Errorf("Load(%d, %d) returned short read, only got %d bytes", len(buf), o, n)
-			continue
-		}
-
-		p := len(data) + o
-		if !bytes.Equal(buf, data[p:p+len(buf)]) {
-			t.Errorf("Load(%d, %d) returned wrong bytes", len(buf), o)
-			continue
-		}
-
-	}
-
-	test.OK(t, b.Remove(restic.DataFile, id.String()))
+func (ec errorCloser) Close() error {
+	ec.t.Error("forbidden method close was called")
+	return errors.New("forbidden method close was called")
 }
 
 // TestSave tests saving data in the backend.
@@ -384,10 +299,10 @@ func TestSave(t testing.TB) {
 			Type: restic.DataFile,
 			Name: fmt.Sprintf("%s-%d", id, i),
 		}
-		err := b.Save(h, data)
+		err := b.Save(h, bytes.NewReader(data))
 		test.OK(t, err)
 
-		buf, err := backend.LoadAll(b, h, nil)
+		buf, err := backend.LoadAll(b, h)
 		test.OK(t, err)
 		if len(buf) != len(data) {
 			t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf))
@@ -404,11 +319,51 @@ func TestSave(t testing.TB) {
 			t.Fatalf("Stat() returned different size, want %q, got %d", len(data), fi.Size)
 		}
 
-		err = b.Remove(h.Type, h.Name)
+		err = b.Remove(h)
 		if err != nil {
 			t.Fatalf("error removing item: %v", err)
 		}
 	}
+
+	// test saving from a tempfile
+	tmpfile, err := ioutil.TempFile("", "restic-backend-save-test-")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	length := rand.Intn(1<<23) + 200000
+	data := test.Random(23, length)
+	copy(id[:], data)
+
+	if _, err = tmpfile.Write(data); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err = tmpfile.Seek(0, 0); err != nil {
+		t.Fatal(err)
+	}
+
+	h := restic.Handle{Type: restic.DataFile, Name: id.String()}
+
+	// wrap the tempfile in an errorCloser, so we can detect if the backend
+	// closes the reader
+	err = b.Save(h, errorCloser{t: t, Reader: tmpfile})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err = tmpfile.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	if err = os.Remove(tmpfile.Name()); err != nil {
+		t.Fatal(err)
+	}
+
+	err = b.Remove(h)
+	if err != nil {
+		t.Fatalf("error removing item: %v", err)
+	}
 }
 
 var filenameTests = []struct {
@@ -430,13 +385,13 @@ func TestSaveFilenames(t testing.TB) {
 
 	for i, test := range filenameTests {
 		h := restic.Handle{Name: test.name, Type: restic.DataFile}
-		err := b.Save(h, []byte(test.data))
+		err := b.Save(h, strings.NewReader(test.data))
 		if err != nil {
 			t.Errorf("test %d failed: Save() returned %v", i, err)
 			continue
 		}
 
-		buf, err := backend.LoadAll(b, h, nil)
+		buf, err := backend.LoadAll(b, h)
 		if err != nil {
 			t.Errorf("test %d failed: Load() returned %v", i, err)
 			continue
@@ -446,7 +401,7 @@ func TestSaveFilenames(t testing.TB) {
 			t.Errorf("test %d: returned wrong bytes", i)
 		}
 
-		err = b.Remove(h.Type, h.Name)
+		err = b.Remove(h)
 		if err != nil {
 			t.Errorf("test %d failed: Remove() returned %v", i, err)
 			continue
@@ -464,10 +419,12 @@ var testStrings = []struct {
 	{"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"},
 }
 
-func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) {
+func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) restic.Handle {
 	id := restic.Hash(data)
-	err := b.Save(restic.Handle{Name: id.String(), Type: tpe}, data)
+	h := restic.Handle{Name: id.String(), Type: tpe}
+	err := b.Save(h, bytes.NewReader(data))
 	test.OK(t, err)
+	return h
 }
 
 // TestBackend tests all functions of the backend.
@@ -485,21 +442,21 @@ func TestBackend(t testing.TB) {
 			test.OK(t, err)
 
 			// test if blob is already in repository
-			ret, err := b.Test(tpe, id.String())
+			h := restic.Handle{Type: tpe, Name: id.String()}
+			ret, err := b.Test(h)
 			test.OK(t, err)
 			test.Assert(t, !ret, "blob was found to exist before creating")
 
 			// try to stat a not existing blob
-			h := restic.Handle{Type: tpe, Name: id.String()}
 			_, err = b.Stat(h)
 			test.Assert(t, err != nil, "blob data could be extracted before creation")
 
 			// try to read not existing blob
-			_, err = b.Load(h, nil, 0)
+			_, err = b.Load(h, 0, 0)
 			test.Assert(t, err != nil, "blob reader could be obtained before creation")
 
 			// try to get string out, should fail
-			ret, err = b.Test(tpe, id.String())
+			ret, err = b.Test(h)
 			test.OK(t, err)
 			test.Assert(t, !ret, "id %q was found (but should not have)", ts.id)
 		}
@@ -510,7 +467,7 @@ func TestBackend(t testing.TB) {
 
 			// test Load()
 			h := restic.Handle{Type: tpe, Name: ts.id}
-			buf, err := backend.LoadAll(b, h, nil)
+			buf, err := backend.LoadAll(b, h)
 			test.OK(t, err)
 			test.Equals(t, ts.data, string(buf))
 
@@ -520,9 +477,18 @@ func TestBackend(t testing.TB) {
 			length := end - start
 
 			buf2 := make([]byte, length)
-			n, err := b.Load(h, buf2, int64(start))
+			rd, err := b.Load(h, len(buf2), int64(start))
 			test.OK(t, err)
-			test.Equals(t, length, n)
+			n, err := io.ReadFull(rd, buf2)
+			test.OK(t, err)
+			test.Equals(t, len(buf2), n)
+
+			remaining, err := io.Copy(ioutil.Discard, rd)
+			test.OK(t, err)
+			test.Equals(t, int64(0), remaining)
+
+			test.OK(t, rd.Close())
+
 			test.Equals(t, ts.data[start:end], string(buf2))
 		}
 
@@ -530,20 +496,21 @@ func TestBackend(t testing.TB) {
 		ts := testStrings[0]
 
 		// create blob
-		err := b.Save(restic.Handle{Type: tpe, Name: ts.id}, []byte(ts.data))
+		err := b.Save(restic.Handle{Type: tpe, Name: ts.id}, strings.NewReader(ts.data))
 		test.Assert(t, err != nil, "expected error, got %v", err)
 
 		// remove and recreate
-		err = b.Remove(tpe, ts.id)
+		h := restic.Handle{Type: tpe, Name: ts.id}
+		err = b.Remove(h)
 		test.OK(t, err)
 
 		// test that the blob is gone
-		ok, err := b.Test(tpe, ts.id)
+		ok, err := b.Test(h)
 		test.OK(t, err)
 		test.Assert(t, ok == false, "removed blob still present")
 
 		// create blob
-		err = b.Save(restic.Handle{Type: tpe, Name: ts.id}, []byte(ts.data))
+		err = b.Save(h, strings.NewReader(ts.data))
 		test.OK(t, err)
 
 		// list items
@@ -578,12 +545,14 @@ func TestBackend(t testing.TB) {
 				id, err := restic.ParseID(ts.id)
 				test.OK(t, err)
 
-				found, err := b.Test(tpe, id.String())
+				h := restic.Handle{Type: tpe, Name: id.String()}
+
+				found, err := b.Test(h)
 				test.OK(t, err)
 
-				test.OK(t, b.Remove(tpe, id.String()))
+				test.OK(t, b.Remove(h))
 
-				found, err = b.Test(tpe, id.String())
+				found, err = b.Test(h)
 				test.OK(t, err)
 				test.Assert(t, !found, fmt.Sprintf("id %q not found after removal", id))
 			}

src/restic/backend/utils.go

@@ -2,29 +2,51 @@ package backend
 
 import (
 	"io"
+	"io/ioutil"
 	"restic"
-
-	"restic/errors"
 )
 
-// LoadAll reads all data stored in the backend for the handle. The buffer buf
-// is resized to accomodate all data in the blob. Errors returned by be.Load()
-// are passed on, except io.ErrUnexpectedEOF is silenced and nil returned
-// instead, since it means this function is working properly.
-func LoadAll(be restic.Backend, h restic.Handle, buf []byte) ([]byte, error) {
-	fi, err := be.Stat(h)
+// LoadAll reads all data stored in the backend for the handle.
+func LoadAll(be restic.Backend, h restic.Handle) (buf []byte, err error) {
+	rd, err := be.Load(h, 0, 0)
 	if err != nil {
-		return nil, errors.Wrap(err, "Stat")
+		return nil, err
 	}
 
-	if fi.Size > int64(len(buf)) {
-		buf = make([]byte, int(fi.Size))
-	}
+	defer func() {
+		io.Copy(ioutil.Discard, rd)
+		e := rd.Close()
+		if err == nil {
+			err = e
+		}
+	}()
 
-	n, err := be.Load(h, buf, 0)
-	if errors.Cause(err) == io.ErrUnexpectedEOF {
-		err = nil
-	}
-	buf = buf[:n]
-	return buf, err
+	return ioutil.ReadAll(rd)
+}
+
+// Closer wraps an io.Reader and adds a Close() method that does nothing.
+type Closer struct {
+	io.Reader
+}
+
+// Close is a no-op.
+func (c Closer) Close() error {
+	return nil
+}
+
+// LimitedReadCloser wraps io.LimitedReader and exposes the Close() method.
+type LimitedReadCloser struct {
+	io.ReadCloser
+	io.Reader
+}
+
+// Read reads data from the limited reader.
+func (l *LimitedReadCloser) Read(p []byte) (int, error) {
+	return l.Reader.Read(p)
+}
+
+// LimitReadCloser returns a new reader wraps r in an io.LimitReader, but also
+// exposes the Close() method.
+func LimitReadCloser(r io.ReadCloser, n int64) *LimitedReadCloser {
+	return &LimitedReadCloser{ReadCloser: r, Reader: io.LimitReader(r, n)}
 }

src/restic/backend/utils_test.go

@@ -21,10 +21,10 @@ func TestLoadAll(t *testing.T) {
 		data := Random(23+i, rand.Intn(MiB)+500*KiB)
 
 		id := restic.Hash(data)
-		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, data)
+		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data))
 		OK(t, err)
 
-		buf, err := backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()}, nil)
+		buf, err := backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()})
 		OK(t, err)
 
 		if len(buf) != len(data) {
@@ -46,11 +46,11 @@ func TestLoadSmallBuffer(t *testing.T) {
 		data := Random(23+i, rand.Intn(MiB)+500*KiB)
 
 		id := restic.Hash(data)
-		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, data)
+		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data))
 		OK(t, err)
 
 		buf := make([]byte, len(data)-23)
-		buf, err = backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()}, buf)
+		buf, err = backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()})
 		OK(t, err)
 
 		if len(buf) != len(data) {
@@ -72,11 +72,11 @@ func TestLoadLargeBuffer(t *testing.T) {
 		data := Random(23+i, rand.Intn(MiB)+500*KiB)
 
 		id := restic.Hash(data)
-		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, data)
+		err := b.Save(restic.Handle{Name: id.String(), Type: restic.DataFile}, bytes.NewReader(data))
 		OK(t, err)
 
 		buf := make([]byte, len(data)+100)
-		buf, err = backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()}, buf)
+		buf, err = backend.LoadAll(b, restic.Handle{Type: restic.DataFile, Name: id.String()})
 		OK(t, err)
 
 		if len(buf) != len(data) {

src/restic/buffer.go

@@ -0,0 +1,21 @@
+package restic
+
+import "restic/crypto"
+
+// NewBlobBuffer returns a buffer that is large enough to hold a blob of size
+// plaintext bytes, including the crypto overhead.
+func NewBlobBuffer(size int) []byte {
+	return make([]byte, size, size+crypto.Extension)
+}
+
+// PlaintextLength returns the plaintext length of a blob with ciphertextSize
+// bytes.
+func PlaintextLength(ciphertextSize int) int {
+	return ciphertextSize - crypto.Extension
+}
+
+// CiphertextLength returns the encrypted length of a blob with plaintextSize
+// bytes.
+func CiphertextLength(plaintextSize int) int {
+	return plaintextSize + crypto.Extension
+}

src/restic/checker/checker.go

@@ -187,7 +187,8 @@ func packIDTester(repo restic.Repository, inChan <-chan restic.ID, errChan chan<
 	defer wg.Done()
 
 	for id := range inChan {
-		ok, err := repo.Backend().Test(restic.DataFile, id.String())
+		h := restic.Handle{Type: restic.DataFile, Name: id.String()}
+		ok, err := repo.Backend().Test(h)
 		if err != nil {
 			err = PackError{ID: id, Err: err}
 		} else {
@@ -658,7 +659,7 @@ func (c *Checker) CountPacks() uint64 {
 func checkPack(r restic.Repository, id restic.ID) error {
 	debug.Log("checking pack %v", id.Str())
 	h := restic.Handle{Type: restic.DataFile, Name: id.String()}
-	buf, err := backend.LoadAll(r.Backend(), h, nil)
+	buf, err := backend.LoadAll(r.Backend(), h)
 	if err != nil {
 		return err
 	}

src/restic/checker/checker_test.go

@@ -1,6 +1,7 @@
 package checker_test
 
 import (
+	"io"
 	"math/rand"
 	"path/filepath"
 	"sort"
@@ -8,7 +9,6 @@ import (
 
 	"restic"
 	"restic/archiver"
-	"restic/backend/mem"
 	"restic/checker"
 	"restic/repository"
 	"restic/test"
@@ -73,8 +73,11 @@ func TestMissingPack(t *testing.T) {
 
 	repo := repository.TestOpenLocal(t, repodir)
 
-	packID := "657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6"
-	test.OK(t, repo.Backend().Remove(restic.DataFile, packID))
+	packHandle := restic.Handle{
+		Type: restic.DataFile,
+		Name: "657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6",
+	}
+	test.OK(t, repo.Backend().Remove(packHandle))
 
 	chkr := checker.New(repo)
 	hints, errs := chkr.LoadIndex()
@@ -92,7 +95,7 @@ func TestMissingPack(t *testing.T) {
 		"expected exactly one error, got %v", len(errs))
 
 	if err, ok := errs[0].(checker.PackError); ok {
-		test.Equals(t, packID, err.ID.String())
+		test.Equals(t, packHandle.Name, err.ID.String())
 	} else {
 		t.Errorf("expected error returned by checker.Packs() to be PackError, got %v", err)
 	}
@@ -105,9 +108,12 @@ func TestUnreferencedPack(t *testing.T) {
 	repo := repository.TestOpenLocal(t, repodir)
 
 	// index 3f1a only references pack 60e0
-	indexID := "3f1abfcb79c6f7d0a3be517d2c83c8562fba64ef2c8e9a3544b4edaf8b5e3b44"
 	packID := "60e0438dcb978ec6860cc1f8c43da648170ee9129af8f650f876bad19f8f788e"
-	test.OK(t, repo.Backend().Remove(restic.IndexFile, indexID))
+	indexHandle := restic.Handle{
+		Type: restic.IndexFile,
+		Name: "3f1abfcb79c6f7d0a3be517d2c83c8562fba64ef2c8e9a3544b4edaf8b5e3b44",
+	}
+	test.OK(t, repo.Backend().Remove(indexHandle))
 
 	chkr := checker.New(repo)
 	hints, errs := chkr.LoadIndex()
@@ -137,8 +143,11 @@ func TestUnreferencedBlobs(t *testing.T) {
 
 	repo := repository.TestOpenLocal(t, repodir)
 
-	snID := "51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02"
-	test.OK(t, repo.Backend().Remove(restic.SnapshotFile, snID))
+	snapshotHandle := restic.Handle{
+		Type: restic.SnapshotFile,
+		Name: "51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02",
+	}
+	test.OK(t, repo.Backend().Remove(snapshotHandle))
 
 	unusedBlobsBySnapshot := restic.IDs{
 		restic.TestParseID("58c748bbe2929fdf30c73262bd8313fe828f8925b05d1d4a87fe109082acb849"),
@@ -208,15 +217,35 @@ type errorBackend struct {
 	ProduceErrors bool
 }
 
-func (b errorBackend) Load(h restic.Handle, p []byte, off int64) (int, error) {
-	n, err := b.Backend.Load(h, p, off)
+func (b errorBackend) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	rd, err := b.Backend.Load(h, length, offset)
+	if err != nil {
+		return rd, err
+	}
 
 	if b.ProduceErrors {
-		induceError(p)
+		return errorReadCloser{rd}, err
+	}
+
+	return rd, nil
+}
+
+type errorReadCloser struct {
+	io.ReadCloser
+}
+
+func (erd errorReadCloser) Read(p []byte) (int, error) {
+	n, err := erd.ReadCloser.Read(p)
+	if n > 0 {
+		induceError(p[:n])
 	}
 	return n, err
 }
 
+func (erd errorReadCloser) Close() error {
+	return erd.ReadCloser.Close()
+}
+
 // induceError flips a bit in the slice.
 func induceError(data []byte) {
 	if rand.Float32() < 0.2 {
@@ -228,19 +257,15 @@ func induceError(data []byte) {
 }
 
 func TestCheckerModifiedData(t *testing.T) {
-	be := mem.New()
-
-	repository.TestUseLowSecurityKDFParameters(t)
-
-	repo := repository.New(be)
-	test.OK(t, repo.Init(test.TestPassword))
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
 
 	arch := archiver.New(repo)
 	_, id, err := arch.Snapshot(nil, []string{"."}, nil, nil)
 	test.OK(t, err)
 	t.Logf("archived as %v", id.Str())
 
-	beError := &errorBackend{Backend: be}
+	beError := &errorBackend{Backend: repo.Backend()}
 	checkRepo := repository.New(beError)
 	test.OK(t, checkRepo.SearchKey(test.TestPassword, 5))
 
@@ -266,7 +291,7 @@ func TestCheckerModifiedData(t *testing.T) {
 	}
 
 	for _, err := range checkData(chkr) {
-		t.Logf("struct error: %v", err)
+		t.Logf("data error: %v", err)
 		errFound = true
 	}
 

src/restic/fuse/file.go

@@ -4,8 +4,6 @@
 package fuse
 
 import (
-	"sync"
-
 	"restic/errors"
 
 	"restic"
@@ -35,29 +33,23 @@ type file struct {
 	node        *restic.Node
 	ownerIsRoot bool
 
-	sizes []uint
+	sizes []int
 	blobs [][]byte
 }
 
 const defaultBlobSize = 128 * 1024
 
-var blobPool = sync.Pool{
-	New: func() interface{} {
-		return make([]byte, defaultBlobSize)
-	},
-}
-
 func newFile(repo BlobLoader, node *restic.Node, ownerIsRoot bool) (*file, error) {
 	debug.Log("create new file for %v with %d blobs", node.Name, len(node.Content))
 	var bytes uint64
-	sizes := make([]uint, len(node.Content))
+	sizes := make([]int, len(node.Content))
 	for i, id := range node.Content {
 		size, err := repo.LookupBlobSize(id, restic.DataBlob)
 		if err != nil {
 			return nil, err
 		}
 
-		sizes[i] = size
+		sizes[i] = int(size)
 		bytes += uint64(size)
 	}
 
@@ -99,16 +91,12 @@ func (f *file) getBlobAt(i int) (blob []byte, err error) {
 		return f.blobs[i], nil
 	}
 
-	buf := blobPool.Get().([]byte)
-	buf = buf[:cap(buf)]
-
-	if uint(len(buf)) < f.sizes[i] {
-		if len(buf) > defaultBlobSize {
-			blobPool.Put(buf)
-		}
-		buf = make([]byte, f.sizes[i])
+	// release earlier blobs
+	for j := 0; j < i; j++ {
+		f.blobs[j] = nil
 	}
 
+	buf := restic.NewBlobBuffer(f.sizes[i])
 	n, err := f.repo.LoadBlob(restic.DataBlob, f.node.Content[i], buf)
 	if err != nil {
 		debug.Log("LoadBlob(%v, %v) failed: %v", f.node.Name, f.node.Content[i], err)
@@ -169,10 +157,7 @@ func (f *file) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadR
 
 func (f *file) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
 	for i := range f.blobs {
-		if f.blobs[i] != nil {
-			blobPool.Put(f.blobs[i])
-			f.blobs[i] = nil
-		}
+		f.blobs[i] = nil
 	}
 	return nil
 }

src/restic/fuse/file_test.go

@@ -9,7 +9,9 @@ import (
 	"testing"
 	"time"
 
-	"restic/errors"
+	"golang.org/x/net/context"
+
+	"restic/repository"
 
 	"bazil.org/fuse"
 
@@ -17,108 +19,96 @@ import (
 	. "restic/test"
 )
 
-type MockRepo struct {
-	blobs map[restic.ID][]byte
-}
-
-func NewMockRepo(content map[restic.ID][]byte) *MockRepo {
-	return &MockRepo{blobs: content}
-}
-
-func (m *MockRepo) LookupBlobSize(id restic.ID, t restic.BlobType) (uint, error) {
-	buf, ok := m.blobs[id]
-	if !ok {
-		return 0, errors.New("blob not found")
-	}
-
-	return uint(len(buf)), nil
-}
-
-func (m *MockRepo) LoadBlob(t restic.BlobType, id restic.ID, buf []byte) (int, error) {
-	size, err := m.LookupBlobSize(id, t)
-	if err != nil {
-		return 0, err
-	}
-
-	if uint(len(buf)) < size {
-		return 0, errors.New("buffer too small")
-	}
-
-	buf = buf[:size]
-	copy(buf, m.blobs[id])
-	return int(size), nil
-}
-
-type MockContext struct{}
-
-func (m MockContext) Deadline() (time.Time, bool)       { return time.Now(), false }
-func (m MockContext) Done() <-chan struct{}             { return nil }
-func (m MockContext) Err() error                        { return nil }
-func (m MockContext) Value(key interface{}) interface{} { return nil }
-
-var testContent = genTestContent()
-var testContentLengths = []uint{
-	4646 * 1024,
-	655 * 1024,
-	378 * 1024,
-	8108 * 1024,
-	558 * 1024,
-}
-var testMaxFileSize uint
-
-func genTestContent() map[restic.ID][]byte {
-	m := make(map[restic.ID][]byte)
-
-	for _, length := range testContentLengths {
-		buf := Random(int(length), int(length))
-		id := restic.Hash(buf)
-		m[id] = buf
-		testMaxFileSize += length
-	}
-
-	return m
-}
-
-const maxBufSize = 20 * 1024 * 1024
-
-func testRead(t *testing.T, f *file, offset, length int, data []byte) {
-	ctx := MockContext{}
+func testRead(t testing.TB, f *file, offset, length int, data []byte) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
 
 	req := &fuse.ReadRequest{
 		Offset: int64(offset),
 		Size:   length,
 	}
 	resp := &fuse.ReadResponse{
-		Data: make([]byte, length),
+		Data: data,
 	}
 	OK(t, f.Read(ctx, req, resp))
 }
 
-var offsetReadsTests = []struct {
-	offset, length int
-}{
-	{0, 5 * 1024 * 1024},
-	{4000 * 1024, 1000 * 1024},
+func firstSnapshotID(t testing.TB, repo restic.Repository) (first restic.ID) {
+	done := make(chan struct{})
+	defer close(done)
+	for id := range repo.List(restic.SnapshotFile, done) {
+		if first.IsNull() {
+			first = id
+		}
+	}
+	return first
+}
+
+func loadFirstSnapshot(t testing.TB, repo restic.Repository) *restic.Snapshot {
+	id := firstSnapshotID(t, repo)
+	sn, err := restic.LoadSnapshot(repo, id)
+	OK(t, err)
+	return sn
+}
+
+func loadTree(t testing.TB, repo restic.Repository, id restic.ID) *restic.Tree {
+	tree, err := repo.LoadTree(id)
+	OK(t, err)
+	return tree
 }
 
 func TestFuseFile(t *testing.T) {
-	repo := NewMockRepo(testContent)
-	ctx := MockContext{}
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
 
-	memfile := make([]byte, 0, maxBufSize)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	timestamp, err := time.Parse(time.RFC3339, "2017-01-24T10:42:56+01:00")
+	OK(t, err)
+	restic.TestCreateSnapshot(t, repo, timestamp, 2, 0.1)
+
+	sn := loadFirstSnapshot(t, repo)
+	tree := loadTree(t, repo, *sn.Tree)
+
+	var content restic.IDs
+	for _, node := range tree.Nodes {
+		content = append(content, node.Content...)
+	}
+	t.Logf("tree loaded, content: %v", content)
+
+	var (
+		filesize uint64
+		memfile  []byte
+	)
+	for _, id := range content {
+		size, err := repo.LookupBlobSize(id, restic.DataBlob)
+		OK(t, err)
+		filesize += uint64(size)
+
+		buf := restic.NewBlobBuffer(int(size))
+		n, err := repo.LoadBlob(restic.DataBlob, id, buf)
+		OK(t, err)
+
+		if uint(n) != size {
+			t.Fatalf("not enough bytes read for id %v: want %v, got %v", id.Str(), size, n)
+		}
+
+		if uint(len(buf)) != size {
+			t.Fatalf("buffer has wrong length for id %v: want %v, got %v", id.Str(), size, len(buf))
+		}
 
-	var ids restic.IDs
-	for id, buf := range repo.blobs {
-		ids = append(ids, id)
 		memfile = append(memfile, buf...)
 	}
 
+	t.Logf("filesize is %v, memfile has size %v", filesize, len(memfile))
+
 	node := &restic.Node{
 		Name:    "foo",
 		Inode:   23,
 		Mode:    0742,
-		Size:    42,
-		Content: ids,
+		Size:    filesize,
+		Content: content,
 	}
 	f, err := newFile(repo, node, false)
 	OK(t, err)
@@ -131,28 +121,19 @@ func TestFuseFile(t *testing.T) {
 	Equals(t, node.Size, attr.Size)
 	Equals(t, (node.Size/uint64(attr.BlockSize))+1, attr.Blocks)
 
-	for i, test := range offsetReadsTests {
-		b := memfile[test.offset : test.offset+test.length]
-		buf := make([]byte, test.length)
-		testRead(t, f, test.offset, test.length, buf)
-		if !bytes.Equal(b, buf) {
-			t.Errorf("test %d failed, wrong data returned", i)
-		}
-	}
-
 	for i := 0; i < 200; i++ {
-		length := rand.Intn(int(testMaxFileSize) / 2)
-		offset := rand.Intn(int(testMaxFileSize))
-		if length+offset > int(testMaxFileSize) {
-			diff := length + offset - int(testMaxFileSize)
-			length -= diff
-		}
+		offset := rand.Intn(int(filesize))
+		length := rand.Intn(int(filesize)-offset) + 100
 
 		b := memfile[offset : offset+length]
+
 		buf := make([]byte, length)
+
 		testRead(t, f, offset, length, buf)
 		if !bytes.Equal(b, buf) {
-			t.Errorf("test %d failed (offset %d, length %d), wrong data returned", i, offset, length)
+			t.Errorf("test %d failed, wrong data returned (offset %v, length %v)", i, offset, length)
 		}
 	}
+
+	OK(t, f.Release(ctx, nil))
 }

src/restic/hashing/reader.go

@@ -0,0 +1,29 @@
+package hashing
+
+import (
+	"hash"
+	"io"
+)
+
+// Reader hashes all data read from the underlying reader.
+type Reader struct {
+	r io.Reader
+	h hash.Hash
+}
+
+// NewReader returns a new Reader that uses the hash h.
+func NewReader(r io.Reader, h hash.Hash) *Reader {
+	return &Reader{
+		h: h,
+		r: io.TeeReader(r, h),
+	}
+}
+
+func (h *Reader) Read(p []byte) (int, error) {
+	return h.r.Read(p)
+}
+
+// Sum returns the hash of the data read so far.
+func (h *Reader) Sum(d []byte) []byte {
+	return h.h.Sum(d)
+}

src/restic/hashing/reader_test.go

@@ -0,0 +1,73 @@
+package hashing
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/sha256"
+	"io"
+	"io/ioutil"
+	"testing"
+)
+
+func TestReader(t *testing.T) {
+	tests := []int{5, 23, 2<<18 + 23, 1 << 20}
+
+	for _, size := range tests {
+		data := make([]byte, size)
+		_, err := io.ReadFull(rand.Reader, data)
+		if err != nil {
+			t.Fatalf("ReadFull: %v", err)
+		}
+
+		expectedHash := sha256.Sum256(data)
+
+		rd := NewReader(bytes.NewReader(data), sha256.New())
+		n, err := io.Copy(ioutil.Discard, rd)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if n != int64(size) {
+			t.Errorf("Reader: invalid number of bytes written: got %d, expected %d",
+				n, size)
+		}
+
+		resultingHash := rd.Sum(nil)
+
+		if !bytes.Equal(expectedHash[:], resultingHash) {
+			t.Errorf("Reader: hashes do not match: expected %02x, got %02x",
+				expectedHash, resultingHash)
+		}
+	}
+}
+
+func BenchmarkReader(b *testing.B) {
+	buf := make([]byte, 1<<22)
+	_, err := io.ReadFull(rand.Reader, buf)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	expectedHash := sha256.Sum256(buf)
+
+	b.SetBytes(int64(len(buf)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		rd := NewReader(bytes.NewReader(buf), sha256.New())
+		n, err := io.Copy(ioutil.Discard, rd)
+		if err != nil {
+			b.Fatal(err)
+		}
+
+		if n != int64(len(buf)) {
+			b.Errorf("Reader: invalid number of bytes written: got %d, expected %d",
+				n, len(buf))
+		}
+
+		resultingHash := rd.Sum(nil)
+		if !bytes.Equal(expectedHash[:], resultingHash) {
+			b.Errorf("Reader: hashes do not match: expected %02x, got %02x",
+				expectedHash, resultingHash)
+		}
+	}
+}

src/restic/hashing/writer.go

@@ -0,0 +1,31 @@
+package hashing
+
+import (
+	"hash"
+	"io"
+)
+
+// Writer transparently hashes all data while writing it to the underlying writer.
+type Writer struct {
+	w io.Writer
+	h hash.Hash
+}
+
+// NewWriter wraps the writer w and feeds all data written to the hash h.
+func NewWriter(w io.Writer, h hash.Hash) *Writer {
+	return &Writer{
+		h: h,
+		w: io.MultiWriter(w, h),
+	}
+}
+
+// Write wraps the write method of the underlying writer and also hashes all data.
+func (h *Writer) Write(p []byte) (int, error) {
+	n, err := h.w.Write(p)
+	return n, err
+}
+
+// Sum returns the hash of all data written so far.
+func (h *Writer) Sum(d []byte) []byte {
+	return h.h.Sum(d)
+}

src/restic/hashing/writer_test.go

@@ -0,0 +1,74 @@
+package hashing
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/sha256"
+	"io"
+	"io/ioutil"
+	"testing"
+)
+
+func TestWriter(t *testing.T) {
+	tests := []int{5, 23, 2<<18 + 23, 1 << 20}
+
+	for _, size := range tests {
+		data := make([]byte, size)
+		_, err := io.ReadFull(rand.Reader, data)
+		if err != nil {
+			t.Fatalf("ReadFull: %v", err)
+		}
+
+		expectedHash := sha256.Sum256(data)
+
+		wr := NewWriter(ioutil.Discard, sha256.New())
+
+		n, err := io.Copy(wr, bytes.NewReader(data))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if n != int64(size) {
+			t.Errorf("Writer: invalid number of bytes written: got %d, expected %d",
+				n, size)
+		}
+
+		resultingHash := wr.Sum(nil)
+
+		if !bytes.Equal(expectedHash[:], resultingHash) {
+			t.Errorf("Writer: hashes do not match: expected %02x, got %02x",
+				expectedHash, resultingHash)
+		}
+	}
+}
+
+func BenchmarkWriter(b *testing.B) {
+	buf := make([]byte, 1<<22)
+	_, err := io.ReadFull(rand.Reader, buf)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	expectedHash := sha256.Sum256(buf)
+
+	b.SetBytes(int64(len(buf)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		wr := NewWriter(ioutil.Discard, sha256.New())
+		n, err := io.Copy(wr, bytes.NewReader(buf))
+		if err != nil {
+			b.Fatal(err)
+		}
+
+		if n != int64(len(buf)) {
+			b.Errorf("Writer: invalid number of bytes written: got %d, expected %d",
+				n, len(buf))
+		}
+
+		resultingHash := wr.Sum(nil)
+		if !bytes.Equal(expectedHash[:], resultingHash) {
+			b.Errorf("Writer: hashes do not match: expected %02x, got %02x",
+				expectedHash, resultingHash)
+		}
+	}
+}

src/restic/id.go

@@ -114,3 +114,13 @@ func (id *ID) UnmarshalJSON(b []byte) error {
 
 	return nil
 }
+
+// IDFromHash returns the ID for the hash.
+func IDFromHash(hash []byte) (id ID) {
+	if len(hash) != idSize {
+		panic("invalid hash type, not enough/too many bytes")
+	}
+
+	copy(id[:], hash)
+	return id
+}

src/restic/index/index.go

@@ -14,27 +14,20 @@ import (
 
 // Pack contains information about the contents of a pack.
 type Pack struct {
+	ID      restic.ID
 	Size    int64
 	Entries []restic.Blob
 }
 
-// Blob contains information about a blob.
-type Blob struct {
-	Size  int64
-	Packs restic.IDSet
-}
-
 // Index contains information about blobs and packs stored in a repo.
 type Index struct {
 	Packs    map[restic.ID]Pack
-	Blobs    map[restic.BlobHandle]Blob
 	IndexIDs restic.IDSet
 }
 
 func newIndex() *Index {
 	return &Index{
 		Packs:    make(map[restic.ID]Pack),
-		Blobs:    make(map[restic.BlobHandle]Blob),
 		IndexIDs: restic.NewIDSet(),
 	}
 }
@@ -69,9 +62,6 @@ func New(repo restic.Repository, p *restic.Progress) (*Index, error) {
 		if err != nil {
 			return nil, err
 		}
-
-		p := Pack{Entries: j.Entries(), Size: j.Size()}
-		idx.Packs[packID] = p
 	}
 
 	return idx, nil
@@ -179,19 +169,7 @@ func (idx *Index) AddPack(id restic.ID, size int64, entries []restic.Blob) error
 		return errors.Errorf("pack %v already present in the index", id.Str())
 	}
 
-	idx.Packs[id] = Pack{Size: size, Entries: entries}
-
-	for _, entry := range entries {
-		h := restic.BlobHandle{ID: entry.ID, Type: entry.Type}
-		if _, ok := idx.Blobs[h]; !ok {
-			idx.Blobs[h] = Blob{
-				Size:  int64(entry.Length),
-				Packs: restic.NewIDSet(),
-			}
-		}
-
-		idx.Blobs[h].Packs.Insert(id)
-	}
+	idx.Packs[id] = Pack{ID: id, Size: size, Entries: entries}
 
 	return nil
 }
@@ -202,15 +180,6 @@ func (idx *Index) RemovePack(id restic.ID) error {
 		return errors.Errorf("pack %v not found in the index", id.Str())
 	}
 
-	for _, blob := range idx.Packs[id].Entries {
-		h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
-		idx.Blobs[h].Packs.Delete(id)
-
-		if len(idx.Blobs[h].Packs) == 0 {
-			delete(idx.Blobs, h)
-		}
-	}
-
 	delete(idx.Packs, id)
 
 	return nil
@@ -239,14 +208,11 @@ func (idx *Index) DuplicateBlobs() (dups restic.BlobSet) {
 func (idx *Index) PacksForBlobs(blobs restic.BlobSet) (packs restic.IDSet) {
 	packs = restic.NewIDSet()
 
-	for h := range blobs {
-		blob, ok := idx.Blobs[h]
-		if !ok {
-			continue
-		}
-
-		for id := range blob.Packs {
-			packs.Insert(id)
+	for id, p := range idx.Packs {
+		for _, entry := range p.Entries {
+			if blobs.Has(restic.BlobHandle{ID: entry.ID, Type: entry.Type}) {
+				packs.Insert(id)
+			}
 		}
 	}
 
@@ -264,31 +230,20 @@ type Location struct {
 var ErrBlobNotFound = errors.New("blob not found in index")
 
 // FindBlob returns a list of packs and positions the blob can be found in.
-func (idx *Index) FindBlob(h restic.BlobHandle) ([]Location, error) {
-	blob, ok := idx.Blobs[h]
-	if !ok {
-		return nil, ErrBlobNotFound
+func (idx *Index) FindBlob(h restic.BlobHandle) (result []Location, err error) {
+	for id, p := range idx.Packs {
+		for _, entry := range p.Entries {
+			if entry.ID.Equal(h.ID) && entry.Type == h.Type {
+				result = append(result, Location{
+					PackID: id,
+					Blob:   entry,
+				})
+			}
+		}
 	}
 
-	result := make([]Location, 0, len(blob.Packs))
-	for packID := range blob.Packs {
-		pack, ok := idx.Packs[packID]
-		if !ok {
-			return nil, errors.Errorf("pack %v not found in index", packID.Str())
-		}
-
-		for _, entry := range pack.Entries {
-			if entry.Type != h.Type {
-				continue
-			}
-
-			if !entry.ID.Equal(h.ID) {
-				continue
-			}
-
-			loc := Location{PackID: packID, Blob: entry}
-			result = append(result, loc)
-		}
+	if len(result) == 0 {
+		return nil, ErrBlobNotFound
 	}
 
 	return result, nil

src/restic/index/index_test.go

@@ -3,7 +3,9 @@ package index
 import (
 	"math/rand"
 	"restic"
+	"restic/checker"
 	"restic/repository"
+	"restic/test"
 	"testing"
 	"time"
 )
@@ -25,9 +27,14 @@ func createFilledRepo(t testing.TB, snapshots int, dup float32) (restic.Reposito
 
 func validateIndex(t testing.TB, repo restic.Repository, idx *Index) {
 	for id := range repo.List(restic.DataFile, nil) {
-		if _, ok := idx.Packs[id]; !ok {
+		p, ok := idx.Packs[id]
+		if !ok {
 			t.Errorf("pack %v missing from index", id.Str())
 		}
+
+		if !p.ID.Equal(id) {
+			t.Errorf("pack %v has invalid ID: want %v, got %v", id.Str(), id, p.ID)
+		}
 	}
 }
 
@@ -135,6 +142,40 @@ func BenchmarkIndexNew(b *testing.B) {
 		if idx == nil {
 			b.Fatalf("New() returned nil index")
 		}
+		b.Logf("idx %v packs", len(idx.Packs))
+	}
+}
+
+func BenchmarkIndexSave(b *testing.B) {
+	repo, cleanup := repository.TestRepository(b)
+	defer cleanup()
+
+	idx, err := New(repo, nil)
+	test.OK(b, err)
+
+	for i := 0; i < 8000; i++ {
+		entries := make([]restic.Blob, 0, 200)
+		for j := 0; j < cap(entries); j++ {
+			entries = append(entries, restic.Blob{
+				ID:     restic.NewRandomID(),
+				Length: 1000,
+				Offset: 5,
+				Type:   restic.DataBlob,
+			})
+		}
+
+		idx.AddPack(restic.NewRandomID(), 10000, entries)
+	}
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		id, err := idx.Save(repo, nil)
+		if err != nil {
+			b.Fatalf("New() returned error %v", err)
+		}
+
+		b.Logf("saved as %v", id.Str())
 	}
 }
 
@@ -151,7 +192,7 @@ func TestIndexDuplicateBlobs(t *testing.T) {
 	if len(dups) == 0 {
 		t.Errorf("no duplicate blobs found")
 	}
-	t.Logf("%d packs, %d unique blobs", len(idx.Packs), len(idx.Blobs))
+	t.Logf("%d packs, %d duplicate blobs", len(idx.Packs), len(dups))
 
 	packs := idx.PacksForBlobs(dups)
 	if len(packs) == 0 {
@@ -169,7 +210,7 @@ func loadIndex(t testing.TB, repo restic.Repository) *Index {
 	return idx
 }
 
-func TestIndexSave(t *testing.T) {
+func TestSave(t *testing.T) {
 	repo, cleanup := createFilledRepo(t, 3, 0)
 	defer cleanup()
 
@@ -193,7 +234,8 @@ func TestIndexSave(t *testing.T) {
 
 	for id := range idx.IndexIDs {
 		t.Logf("remove index %v", id.Str())
-		err = repo.Backend().Remove(restic.IndexFile, id.String())
+		h := restic.Handle{Type: restic.IndexFile, Name: id.String()}
+		err = repo.Backend().Remove(h)
 		if err != nil {
 			t.Errorf("error removing index %v: %v", id, err)
 		}
@@ -219,6 +261,42 @@ func TestIndexSave(t *testing.T) {
 	}
 }
 
+func TestIndexSave(t *testing.T) {
+	repo, cleanup := createFilledRepo(t, 3, 0)
+	defer cleanup()
+
+	idx := loadIndex(t, repo)
+
+	id, err := idx.Save(repo, idx.IndexIDs.List())
+	if err != nil {
+		t.Fatalf("unable to save new index: %v", err)
+	}
+
+	t.Logf("new index saved as %v", id.Str())
+
+	for id := range idx.IndexIDs {
+		t.Logf("remove index %v", id.Str())
+		h := restic.Handle{Type: restic.IndexFile, Name: id.String()}
+		err = repo.Backend().Remove(h)
+		if err != nil {
+			t.Errorf("error removing index %v: %v", id, err)
+		}
+	}
+
+	idx2 := loadIndex(t, repo)
+	t.Logf("load new index with %d packs", len(idx2.Packs))
+
+	checker := checker.New(repo)
+	hints, errs := checker.LoadIndex()
+	for _, h := range hints {
+		t.Logf("hint: %v\n", h)
+	}
+
+	for _, err := range errs {
+		t.Errorf("checker found error: %v", err)
+	}
+}
+
 func TestIndexAddRemovePack(t *testing.T) {
 	repo, cleanup := createFilledRepo(t, 3, 0)
 	defer cleanup()
@@ -249,12 +327,7 @@ func TestIndexAddRemovePack(t *testing.T) {
 		if err == nil {
 			t.Errorf("removed blob %v found in index", h)
 		}
-
-		if _, ok := idx.Blobs[h]; ok {
-			t.Errorf("removed blob %v found in index.Blobs", h)
-		}
 	}
-
 }
 
 // example index serialization from doc/Design.md

src/restic/lock.go

@@ -186,7 +186,7 @@ func (l *Lock) Unlock() error {
 		return nil
 	}
 
-	return l.repo.Backend().Remove(LockFile, l.lockID.String())
+	return l.repo.Backend().Remove(Handle{Type: LockFile, Name: l.lockID.String()})
 }
 
 var staleTimeout = 30 * time.Minute
@@ -234,7 +234,7 @@ func (l *Lock) Refresh() error {
 		return err
 	}
 
-	err = l.repo.Backend().Remove(LockFile, l.lockID.String())
+	err = l.repo.Backend().Remove(Handle{Type: LockFile, Name: l.lockID.String()})
 	if err != nil {
 		return err
 	}
@@ -289,7 +289,7 @@ func RemoveStaleLocks(repo Repository) error {
 		}
 
 		if lock.Stale() {
-			return repo.Backend().Remove(LockFile, id.String())
+			return repo.Backend().Remove(Handle{Type: LockFile, Name: id.String()})
 		}
 
 		return nil
@@ -299,6 +299,6 @@ func RemoveStaleLocks(repo Repository) error {
 // RemoveAllLocks removes all locks forcefully.
 func RemoveAllLocks(repo Repository) error {
 	return eachLock(repo, func(id ID, lock *Lock, err error) error {
-		return repo.Backend().Remove(LockFile, id.String())
+		return repo.Backend().Remove(Handle{Type: LockFile, Name: id.String()})
 	})
 }

src/restic/lock_test.go

@@ -102,7 +102,8 @@ func createFakeLock(repo restic.Repository, t time.Time, pid int) (restic.ID, er
 }
 
 func removeLock(repo restic.Repository, id restic.ID) error {
-	return repo.Backend().Remove(restic.LockFile, id.String())
+	h := restic.Handle{Type: restic.LockFile, Name: id.String()}
+	return repo.Backend().Remove(h)
 }
 
 var staleLockTests = []struct {
@@ -162,7 +163,8 @@ func TestLockStale(t *testing.T) {
 }
 
 func lockExists(repo restic.Repository, t testing.TB, id restic.ID) bool {
-	exists, err := repo.Backend().Test(restic.LockFile, id.String())
+	h := restic.Handle{Type: restic.LockFile, Name: id.String()}
+	exists, err := repo.Backend().Test(h)
 	OK(t, err)
 
 	return exists

src/restic/mock/backend.go

@@ -1,6 +1,7 @@
 package mock
 
 import (
+	"io"
 	"restic"
 
 	"restic/errors"
@@ -9,12 +10,12 @@ import (
 // Backend implements a mock backend.
 type Backend struct {
 	CloseFn    func() error
-	LoadFn     func(h restic.Handle, p []byte, off int64) (int, error)
-	SaveFn     func(h restic.Handle, p []byte) error
+	SaveFn     func(h restic.Handle, rd io.Reader) error
+	LoadFn     func(h restic.Handle, length int, offset int64) (io.ReadCloser, error)
 	StatFn     func(h restic.Handle) (restic.FileInfo, error)
 	ListFn     func(restic.FileType, <-chan struct{}) <-chan string
-	RemoveFn   func(restic.FileType, string) error
-	TestFn     func(restic.FileType, string) (bool, error)
+	RemoveFn   func(h restic.Handle) error
+	TestFn     func(h restic.Handle) (bool, error)
 	DeleteFn   func() error
 	LocationFn func() string
 }
@@ -37,22 +38,22 @@ func (m *Backend) Location() string {
 	return m.LocationFn()
 }
 
-// Load loads data from the backend.
-func (m *Backend) Load(h restic.Handle, p []byte, off int64) (int, error) {
-	if m.LoadFn == nil {
-		return 0, errors.New("not implemented")
-	}
-
-	return m.LoadFn(h, p, off)
-}
-
 // Save data in the backend.
-func (m *Backend) Save(h restic.Handle, p []byte) error {
+func (m *Backend) Save(h restic.Handle, rd io.Reader) error {
 	if m.SaveFn == nil {
 		return errors.New("not implemented")
 	}
 
-	return m.SaveFn(h, p)
+	return m.SaveFn(h, rd)
+}
+
+// Load loads data from the backend.
+func (m *Backend) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+	if m.LoadFn == nil {
+		return nil, errors.New("not implemented")
+	}
+
+	return m.LoadFn(h, length, offset)
 }
 
 // Stat an object in the backend.
@@ -76,21 +77,21 @@ func (m *Backend) List(t restic.FileType, done <-chan struct{}) <-chan string {
 }
 
 // Remove data from the backend.
-func (m *Backend) Remove(t restic.FileType, name string) error {
+func (m *Backend) Remove(h restic.Handle) error {
 	if m.RemoveFn == nil {
 		return errors.New("not implemented")
 	}
 
-	return m.RemoveFn(t, name)
+	return m.RemoveFn(h)
 }
 
 // Test for the existence of a specific item.
-func (m *Backend) Test(t restic.FileType, name string) (bool, error) {
+func (m *Backend) Test(h restic.Handle) (bool, error) {
 	if m.TestFn == nil {
 		return false, errors.New("not implemented")
 	}
 
-	return m.TestFn(t, name)
+	return m.TestFn(h)
 }
 
 // Delete all data.

src/restic/node.go

@@ -208,7 +208,7 @@ func (node Node) createFileAt(path string, repo Repository) error {
 
 		buf = buf[:cap(buf)]
 		if uint(len(buf)) < size {
-			buf = make([]byte, size)
+			buf = NewBlobBuffer(int(size))
 		}
 
 		n, err := repo.LoadBlob(DataBlob, id, buf)

src/restic/pack/pack.go

@@ -85,15 +85,15 @@ func (p *Packer) Finalize() (uint, error) {
 		return 0, errors.Wrap(err, "Write")
 	}
 
-	hdrBytes := bytesHeader + crypto.Extension
-	if uint(n) != hdrBytes {
+	hdrBytes := restic.CiphertextLength(int(bytesHeader))
+	if n != hdrBytes {
 		return 0, errors.New("wrong number of bytes written")
 	}
 
-	bytesWritten += hdrBytes
+	bytesWritten += uint(hdrBytes)
 
 	// write length
-	err = binary.Write(p.wr, binary.LittleEndian, uint32(uint(len(p.blobs))*entrySize+crypto.Extension))
+	err = binary.Write(p.wr, binary.LittleEndian, uint32(restic.CiphertextLength(len(p.blobs)*int(entrySize))))
 	if err != nil {
 		return 0, errors.Wrap(err, "binary.Write")
 	}
@@ -233,6 +233,8 @@ func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries []restic.Blob, err
 
 	hdrRd := bytes.NewReader(buf)
 
+	entries = make([]restic.Blob, 0, uint(n)/entrySize)
+
 	pos := uint(0)
 	for {
 		e := headerEntry{}

src/restic/pack/pack_test.go

@@ -54,10 +54,9 @@ func verifyBlobs(t testing.TB, bufs []Buf, k *crypto.Key, rd io.ReaderAt, packSi
 	}
 	// header length
 	written += binary.Size(uint32(0))
-	// header
-	written += len(bufs) * (binary.Size(restic.BlobType(0)) + binary.Size(uint32(0)) + len(restic.ID{}))
-	// header crypto
-	written += crypto.Extension
+	// header + header crypto
+	headerSize := len(bufs) * (binary.Size(restic.BlobType(0)) + binary.Size(uint32(0)) + len(restic.ID{}))
+	written += restic.CiphertextLength(headerSize)
 
 	// check length
 	Equals(t, uint(written), packSize)
@@ -127,7 +126,7 @@ func TestUnpackReadSeeker(t *testing.T) {
 	id := restic.Hash(packData)
 
 	handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
-	OK(t, b.Save(handle, packData))
+	OK(t, b.Save(handle, bytes.NewReader(packData)))
 	verifyBlobs(t, bufs, k, restic.ReaderAt(b, handle), packSize)
 }
 
@@ -140,6 +139,6 @@ func TestShortPack(t *testing.T) {
 	id := restic.Hash(packData)
 
 	handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
-	OK(t, b.Save(handle, packData))
+	OK(t, b.Save(handle, bytes.NewReader(packData)))
 	verifyBlobs(t, bufs, k, restic.ReaderAt(b, handle), packSize)
 }

src/restic/readerat.go

@@ -2,6 +2,7 @@ package restic
 
 import (
 	"io"
+	"restic/debug"
 )
 
 type backendReaderAt struct {
@@ -9,11 +10,30 @@ type backendReaderAt struct {
 	h  Handle
 }
 
-func (brd backendReaderAt) ReadAt(p []byte, off int64) (n int, err error) {
-	return brd.be.Load(brd.h, p, off)
+func (brd backendReaderAt) ReadAt(p []byte, offset int64) (n int, err error) {
+	return ReadAt(brd.be, brd.h, offset, p)
 }
 
 // ReaderAt returns an io.ReaderAt for a file in the backend.
 func ReaderAt(be Backend, h Handle) io.ReaderAt {
 	return backendReaderAt{be: be, h: h}
 }
+
+// ReadAt reads from the backend handle h at the given position.
+func ReadAt(be Backend, h Handle, offset int64, p []byte) (n int, err error) {
+	debug.Log("ReadAt(%v) at %v, len %v", h, offset, len(p))
+	rd, err := be.Load(h, len(p), offset)
+	if err != nil {
+		return 0, err
+	}
+
+	n, err = io.ReadFull(rd, p)
+	e := rd.Close()
+	if err == nil {
+		err = e
+	}
+
+	debug.Log("ReadAt(%v) ReadFull returned %v bytes", h, n)
+
+	return n, err
+}

src/restic/repository/index.go

@@ -1,7 +1,6 @@
 package repository
 
 import (
-	"bytes"
 	"encoding/json"
 	"io"
 	"restic"
@@ -10,7 +9,6 @@ import (
 
 	"restic/errors"
 
-	"restic/crypto"
 	"restic/debug"
 )
 
@@ -177,15 +175,15 @@ func (idx *Index) Has(id restic.ID, tpe restic.BlobType) bool {
 	return false
 }
 
-// LookupSize returns the length of the cleartext content behind the
-// given id
-func (idx *Index) LookupSize(id restic.ID, tpe restic.BlobType) (cleartextLength uint, err error) {
+// LookupSize returns the length of the plaintext content of the blob with the
+// given id.
+func (idx *Index) LookupSize(id restic.ID, tpe restic.BlobType) (plaintextLength uint, err error) {
 	blobs, err := idx.Lookup(id, tpe)
 	if err != nil {
 		return 0, err
 	}
 
-	return blobs[0].Length - crypto.Extension, nil
+	return uint(restic.PlaintextLength(int(blobs[0].Length))), nil
 }
 
 // Supersedes returns the list of indexes this index supersedes, if any.
@@ -452,12 +450,11 @@ func isErrOldIndex(err error) bool {
 var ErrOldIndexFormat = errors.New("index has old format")
 
 // DecodeIndex loads and unserializes an index from rd.
-func DecodeIndex(rd io.Reader) (idx *Index, err error) {
+func DecodeIndex(buf []byte) (idx *Index, err error) {
 	debug.Log("Start decoding index")
-	idxJSON := jsonIndex{}
+	idxJSON := &jsonIndex{}
 
-	dec := json.NewDecoder(rd)
-	err = dec.Decode(&idxJSON)
+	err = json.Unmarshal(buf, idxJSON)
 	if err != nil {
 		debug.Log("Error %v", err)
 
@@ -491,12 +488,11 @@ func DecodeIndex(rd io.Reader) (idx *Index, err error) {
 }
 
 // DecodeOldIndex loads and unserializes an index in the old format from rd.
-func DecodeOldIndex(rd io.Reader) (idx *Index, err error) {
+func DecodeOldIndex(buf []byte) (idx *Index, err error) {
 	debug.Log("Start decoding old index")
 	list := []*packJSON{}
 
-	dec := json.NewDecoder(rd)
-	err = dec.Decode(&list)
+	err = json.Unmarshal(buf, &list)
 	if err != nil {
 		debug.Log("Error %#v", err)
 		return nil, errors.Wrap(err, "Decode")
@@ -523,7 +519,7 @@ func DecodeOldIndex(rd io.Reader) (idx *Index, err error) {
 }
 
 // LoadIndexWithDecoder loads the index and decodes it with fn.
-func LoadIndexWithDecoder(repo restic.Repository, id restic.ID, fn func(io.Reader) (*Index, error)) (idx *Index, err error) {
+func LoadIndexWithDecoder(repo restic.Repository, id restic.ID, fn func([]byte) (*Index, error)) (idx *Index, err error) {
 	debug.Log("Loading index %v", id.Str())
 
 	buf, err := repo.LoadAndDecrypt(restic.IndexFile, id)
@@ -531,7 +527,7 @@ func LoadIndexWithDecoder(repo restic.Repository, id restic.ID, fn func(io.Reade
 		return nil, err
 	}
 
-	idx, err = fn(bytes.NewReader(buf))
+	idx, err = fn(buf)
 	if err != nil {
 		debug.Log("error while decoding index %v: %v", id, err)
 		return nil, err

src/restic/repository/index_rebuild.go

@@ -55,7 +55,8 @@ func RebuildIndex(repo restic.Repository) error {
 	debug.Log("new index saved as %v", id.Str())
 
 	for indexID := range oldIndexes {
-		err := repo.Backend().Remove(restic.IndexFile, indexID.String())
+		h := restic.Handle{Type: restic.IndexFile, Name: indexID.String()}
+		err := repo.Backend().Remove(h)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "unable to remove index %v: %v\n", indexID.Str(), err)
 		}

src/restic/repository/index_test.go

@@ -54,7 +54,7 @@ func TestIndexSerialize(t *testing.T) {
 	err := idx.Encode(wr)
 	OK(t, err)
 
-	idx2, err := repository.DecodeIndex(wr)
+	idx2, err := repository.DecodeIndex(wr.Bytes())
 	OK(t, err)
 	Assert(t, idx2 != nil,
 		"nil returned for decoded index")
@@ -136,7 +136,7 @@ func TestIndexSerialize(t *testing.T) {
 	Assert(t, id2.Equal(id),
 		"wrong ID returned: want %v, got %v", id, id2)
 
-	idx3, err := repository.DecodeIndex(wr3)
+	idx3, err := repository.DecodeIndex(wr3.Bytes())
 	OK(t, err)
 	Assert(t, idx3 != nil,
 		"nil returned for decoded index")
@@ -288,7 +288,7 @@ var exampleLookupTest = struct {
 func TestIndexUnserialize(t *testing.T) {
 	oldIdx := restic.IDs{restic.TestParseID("ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452")}
 
-	idx, err := repository.DecodeIndex(bytes.NewReader(docExample))
+	idx, err := repository.DecodeIndex(docExample)
 	OK(t, err)
 
 	for _, test := range exampleTests {
@@ -326,8 +326,17 @@ func TestIndexUnserialize(t *testing.T) {
 	}
 }
 
+func BenchmarkDecodeIndex(b *testing.B) {
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		_, err := repository.DecodeIndex(docExample)
+		OK(b, err)
+	}
+}
+
 func TestIndexUnserializeOld(t *testing.T) {
-	idx, err := repository.DecodeOldIndex(bytes.NewReader(docOldExample))
+	idx, err := repository.DecodeOldIndex(docOldExample)
 	OK(t, err)
 
 	for _, test := range exampleTests {

src/restic/repository/key.go

@@ -1,6 +1,7 @@
 package repository
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -146,7 +147,7 @@ func SearchKey(s *Repository, password string, maxKeys int) (*Key, error) {
 // LoadKey loads a key from the backend.
 func LoadKey(s *Repository, name string) (k *Key, err error) {
 	h := restic.Handle{Type: restic.KeyFile, Name: name}
-	data, err := backend.LoadAll(s.be, h, nil)
+	data, err := backend.LoadAll(s.be, h)
 	if err != nil {
 		return nil, err
 	}
@@ -232,7 +233,7 @@ func AddKey(s *Repository, password string, template *crypto.Key) (*Key, error)
 		Name: restic.Hash(buf).String(),
 	}
 
-	err = s.be.Save(h, buf)
+	err = s.be.Save(h, bytes.NewReader(buf))
 	if err != nil {
 		return nil, err
 	}

src/restic/repository/master_index.go

@@ -30,8 +30,7 @@ func (mi *MasterIndex) Lookup(id restic.ID, tpe restic.BlobType) (blobs []restic
 	for _, idx := range mi.idx {
 		blobs, err = idx.Lookup(id, tpe)
 		if err == nil {
-			debug.Log("MasterIndex.Lookup",
-				"found id %v: %v", id.Str(), blobs)
+			debug.Log("found id %v: %v", id.Str(), blobs)
 			return
 		}
 	}
@@ -46,9 +45,8 @@ func (mi *MasterIndex) LookupSize(id restic.ID, tpe restic.BlobType) (uint, erro
 	defer mi.idxMutex.RUnlock()
 
 	for _, idx := range mi.idx {
-		length, err := idx.LookupSize(id, tpe)
-		if err == nil {
-			return length, nil
+		if idx.Has(id, tpe) {
+			return idx.LookupSize(id, tpe)
 		}
 	}
 

src/restic/repository/packer_manager.go

@@ -1,6 +1,7 @@
 package repository
 
 import (
+	"crypto/sha256"
 	"io"
 	"io/ioutil"
 	"os"
@@ -8,6 +9,7 @@ import (
 	"sync"
 
 	"restic/errors"
+	"restic/hashing"
 
 	"restic/crypto"
 	"restic/debug"
@@ -17,15 +19,22 @@ import (
 
 // Saver implements saving data in a backend.
 type Saver interface {
-	Save(h restic.Handle, jp []byte) error
+	Save(restic.Handle, io.Reader) error
+}
+
+// Packer holds a pack.Packer together with a hash writer.
+type Packer struct {
+	*pack.Packer
+	hw      *hashing.Writer
+	tmpfile *os.File
 }
 
 // packerManager keeps a list of open packs and creates new on demand.
 type packerManager struct {
-	be    Saver
-	key   *crypto.Key
-	pm    sync.Mutex
-	packs []*pack.Packer
+	be      Saver
+	key     *crypto.Key
+	pm      sync.Mutex
+	packers []*Packer
 
 	pool sync.Pool
 }
@@ -50,18 +59,18 @@ func newPackerManager(be Saver, key *crypto.Key) *packerManager {
 
 // findPacker returns a packer for a new blob of size bytes. Either a new one is
 // created or one is returned that already has some blobs.
-func (r *packerManager) findPacker(size uint) (packer *pack.Packer, err error) {
+func (r *packerManager) findPacker(size uint) (packer *Packer, err error) {
 	r.pm.Lock()
 	defer r.pm.Unlock()
 
 	// search for a suitable packer
-	if len(r.packs) > 0 {
+	if len(r.packers) > 0 {
 		debug.Log("searching packer for %d bytes\n", size)
-		for i, p := range r.packs {
-			if p.Size()+size < maxPackSize {
+		for i, p := range r.packers {
+			if p.Packer.Size()+size < maxPackSize {
 				debug.Log("found packer %v", p)
 				// remove from list
-				r.packs = append(r.packs[:i], r.packs[i+1:]...)
+				r.packers = append(r.packers[:i], r.packers[i+1:]...)
 				return p, nil
 			}
 		}
@@ -74,50 +83,43 @@ func (r *packerManager) findPacker(size uint) (packer *pack.Packer, err error) {
 		return nil, errors.Wrap(err, "ioutil.TempFile")
 	}
 
-	return pack.NewPacker(r.key, tmpfile), nil
+	hw := hashing.NewWriter(tmpfile, sha256.New())
+	p := pack.NewPacker(r.key, hw)
+	packer = &Packer{
+		Packer:  p,
+		hw:      hw,
+		tmpfile: tmpfile,
+	}
+
+	return packer, nil
 }
 
 // insertPacker appends p to s.packs.
-func (r *packerManager) insertPacker(p *pack.Packer) {
+func (r *packerManager) insertPacker(p *Packer) {
 	r.pm.Lock()
 	defer r.pm.Unlock()
 
-	r.packs = append(r.packs, p)
-	debug.Log("%d packers\n", len(r.packs))
+	r.packers = append(r.packers, p)
+	debug.Log("%d packers\n", len(r.packers))
 }
 
 // savePacker stores p in the backend.
-func (r *Repository) savePacker(p *pack.Packer) error {
-	debug.Log("save packer with %d blobs\n", p.Count())
-	n, err := p.Finalize()
+func (r *Repository) savePacker(p *Packer) error {
+	debug.Log("save packer with %d blobs\n", p.Packer.Count())
+	_, err := p.Packer.Finalize()
 	if err != nil {
 		return err
 	}
 
-	tmpfile := p.Writer().(*os.File)
-	f, err := fs.Open(tmpfile.Name())
+	_, err = p.tmpfile.Seek(0, 0)
 	if err != nil {
-		return errors.Wrap(err, "Open")
+		return errors.Wrap(err, "Seek")
 	}
 
-	data := make([]byte, n)
-	m, err := io.ReadFull(f, data)
-	if err != nil {
-		return errors.Wrap(err, "ReadFul")
-	}
-
-	if uint(m) != n {
-		return errors.Errorf("read wrong number of bytes from %v: want %v, got %v", tmpfile.Name(), n, m)
-	}
-
-	if err = f.Close(); err != nil {
-		return errors.Wrap(err, "Close")
-	}
-
-	id := restic.Hash(data)
+	id := restic.IDFromHash(p.hw.Sum(nil))
 	h := restic.Handle{Type: restic.DataFile, Name: id.String()}
 
-	err = r.be.Save(h, data)
+	err = r.be.Save(h, p.tmpfile)
 	if err != nil {
 		debug.Log("Save(%v) error: %v", h, err)
 		return err
@@ -125,13 +127,18 @@ func (r *Repository) savePacker(p *pack.Packer) error {
 
 	debug.Log("saved as %v", h)
 
-	err = fs.Remove(tmpfile.Name())
+	err = p.tmpfile.Close()
+	if err != nil {
+		return errors.Wrap(err, "close tempfile")
+	}
+
+	err = fs.Remove(p.tmpfile.Name())
 	if err != nil {
 		return errors.Wrap(err, "Remove")
 	}
 
 	// update blobs in the index
-	for _, b := range p.Blobs() {
+	for _, b := range p.Packer.Blobs() {
 		debug.Log("  updating blob %v to pack %v", b.ID.Str(), id.Str())
 		r.idx.Store(restic.PackedBlob{
 			Blob: restic.Blob{
@@ -152,5 +159,5 @@ func (r *packerManager) countPacker() int {
 	r.pm.Lock()
 	defer r.pm.Unlock()
 
-	return len(r.packs)
+	return len(r.packers)
 }

src/restic/repository/packer_manager_test.go

@@ -7,6 +7,7 @@ import (
 	"restic"
 	"restic/backend/mem"
 	"restic/crypto"
+	"restic/mock"
 	"testing"
 )
 
@@ -46,32 +47,19 @@ func randomID(rd io.Reader) restic.ID {
 
 const maxBlobSize = 1 << 20
 
-func saveFile(t testing.TB, be Saver, filename string, n int) {
-	f, err := os.Open(filename)
-	if err != nil {
+func saveFile(t testing.TB, be Saver, f *os.File, id restic.ID) {
+	h := restic.Handle{Type: restic.DataFile, Name: id.String()}
+	t.Logf("save file %v", h)
+
+	if err := be.Save(h, f); err != nil {
 		t.Fatal(err)
 	}
 
-	data := make([]byte, n)
-	m, err := io.ReadFull(f, data)
-
-	if m != n {
-		t.Fatalf("read wrong number of bytes from %v: want %v, got %v", filename, m, n)
-	}
-
-	if err = f.Close(); err != nil {
+	if err := f.Close(); err != nil {
 		t.Fatal(err)
 	}
 
-	h := restic.Handle{Type: restic.DataFile, Name: restic.Hash(data).String()}
-
-	err = be.Save(h, data)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	err = os.Remove(filename)
-	if err != nil {
+	if err := os.Remove(f.Name()); err != nil {
 		t.Fatal(err)
 	}
 }
@@ -105,13 +93,17 @@ func fillPacks(t testing.TB, rnd *randReader, be Saver, pm *packerManager, buf [
 			continue
 		}
 
-		bytesWritten, err := packer.Finalize()
+		_, err = packer.Finalize()
 		if err != nil {
 			t.Fatal(err)
 		}
 
-		tmpfile := packer.Writer().(*os.File)
-		saveFile(t, be, tmpfile.Name(), int(bytesWritten))
+		if _, err = packer.tmpfile.Seek(0, 0); err != nil {
+			t.Fatal(err)
+		}
+
+		packID := restic.IDFromHash(packer.hw.Sum(nil))
+		saveFile(t, be, packer.tmpfile, packID)
 	}
 
 	return bytes
@@ -119,27 +111,21 @@ func fillPacks(t testing.TB, rnd *randReader, be Saver, pm *packerManager, buf [
 
 func flushRemainingPacks(t testing.TB, rnd *randReader, be Saver, pm *packerManager) (bytes int) {
 	if pm.countPacker() > 0 {
-		for _, packer := range pm.packs {
+		for _, packer := range pm.packers {
 			n, err := packer.Finalize()
 			if err != nil {
 				t.Fatal(err)
 			}
 			bytes += int(n)
 
-			tmpfile := packer.Writer().(*os.File)
-			saveFile(t, be, tmpfile.Name(), bytes)
+			packID := restic.IDFromHash(packer.hw.Sum(nil))
+			saveFile(t, be, packer.tmpfile, packID)
 		}
 	}
 
 	return bytes
 }
 
-type fakeBackend struct{}
-
-func (f *fakeBackend) Save(h restic.Handle, data []byte) error {
-	return nil
-}
-
 func TestPackerManager(t *testing.T) {
 	rnd := newRandReader(rand.NewSource(23))
 
@@ -157,17 +143,18 @@ func TestPackerManager(t *testing.T) {
 func BenchmarkPackerManager(t *testing.B) {
 	rnd := newRandReader(rand.NewSource(23))
 
-	be := &fakeBackend{}
-	pm := newPackerManager(be, crypto.NewRandomKey())
+	be := &mock.Backend{
+		SaveFn: func(restic.Handle, io.Reader) error { return nil },
+	}
 	blobBuf := make([]byte, maxBlobSize)
 
 	t.ResetTimer()
 
-	bytes := 0
 	for i := 0; i < t.N; i++ {
+		bytes := 0
+		pm := newPackerManager(be, crypto.NewRandomKey())
 		bytes += fillPacks(t, rnd, be, pm, blobBuf)
+		bytes += flushRemainingPacks(t, rnd, be, pm)
+		t.Logf("saved %d bytes", bytes)
 	}
-
-	bytes += flushRemainingPacks(t, rnd, be, pm)
-	t.Logf("saved %d bytes", bytes)
 }

src/restic/repository/repack.go

@@ -1,11 +1,14 @@
 package repository
 
 import (
-	"bytes"
+	"crypto/sha256"
 	"io"
+	"io/ioutil"
+	"os"
 	"restic"
 	"restic/crypto"
 	"restic/debug"
+	"restic/hashing"
 	"restic/pack"
 
 	"restic/errors"
@@ -18,30 +21,47 @@ import (
 func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet) (err error) {
 	debug.Log("repacking %d packs while keeping %d blobs", len(packs), len(keepBlobs))
 
-	buf := make([]byte, 0, maxPackSize)
 	for packID := range packs {
-		// load the complete pack
+		// load the complete pack into a temp file
 		h := restic.Handle{Type: restic.DataFile, Name: packID.String()}
 
-		l, err := repo.Backend().Load(h, buf[:cap(buf)], 0)
-		if errors.Cause(err) == io.ErrUnexpectedEOF {
-			err = nil
-			buf = buf[:l]
+		tempfile, err := ioutil.TempFile("", "restic-temp-repack-")
+		if err != nil {
+			return errors.Wrap(err, "TempFile")
 		}
 
+		beRd, err := repo.Backend().Load(h, 0, 0)
 		if err != nil {
 			return err
 		}
 
-		debug.Log("pack %v loaded (%d bytes)", packID.Str(), len(buf))
+		defer beRd.Close()
 
-		blobs, err := pack.List(repo.Key(), bytes.NewReader(buf), int64(len(buf)))
+		hrd := hashing.NewReader(beRd, sha256.New())
+		packLength, err := io.Copy(tempfile, hrd)
+		if err != nil {
+			return errors.Wrap(err, "Copy")
+		}
+
+		hash := restic.IDFromHash(hrd.Sum(nil))
+		debug.Log("pack %v loaded (%d bytes), hash %v", packID.Str(), packLength, hash.Str())
+
+		if !packID.Equal(hash) {
+			return errors.Errorf("hash does not match id: want %v, got %v", packID, hash)
+		}
+
+		_, err = tempfile.Seek(0, 0)
+		if err != nil {
+			return errors.Wrap(err, "Seek")
+		}
+
+		blobs, err := pack.List(repo.Key(), tempfile, packLength)
 		if err != nil {
 			return err
 		}
 
 		debug.Log("processing pack %v, blobs: %v", packID.Str(), len(blobs))
-		var plaintext []byte
+		var buf []byte
 		for _, entry := range blobs {
 			h := restic.BlobHandle{ID: entry.ID, Type: entry.Type}
 			if !keepBlobs.Has(h) {
@@ -50,21 +70,36 @@ func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet
 
 			debug.Log("  process blob %v", h)
 
-			ciphertext := buf[entry.Offset : entry.Offset+entry.Length]
-			plaintext = plaintext[:len(plaintext)]
-			if len(plaintext) < len(ciphertext) {
-				plaintext = make([]byte, len(ciphertext))
+			buf = buf[:len(buf)]
+			if uint(len(buf)) < entry.Length {
+				buf = make([]byte, entry.Length)
+			}
+			buf = buf[:entry.Length]
+
+			n, err := tempfile.ReadAt(buf, int64(entry.Offset))
+			if err != nil {
+				return errors.Wrap(err, "ReadAt")
 			}
 
-			debug.Log("  ciphertext %d, plaintext %d", len(plaintext), len(ciphertext))
+			if n != len(buf) {
+				return errors.Errorf("read blob %v from %v: not enough bytes read, want %v, got %v",
+					h, tempfile.Name(), len(buf), n)
+			}
 
-			n, err := crypto.Decrypt(repo.Key(), plaintext, ciphertext)
+			n, err = crypto.Decrypt(repo.Key(), buf, buf)
 			if err != nil {
 				return err
 			}
-			plaintext = plaintext[:n]
 
-			_, err = repo.SaveBlob(entry.Type, plaintext, entry.ID)
+			buf = buf[:n]
+
+			id := restic.Hash(buf)
+			if !id.Equal(entry.ID) {
+				return errors.Errorf("read blob %v from %v: wrong data returned, hash is %v",
+					h, tempfile.Name(), id)
+			}
+
+			_, err = repo.SaveBlob(entry.Type, buf, entry.ID)
 			if err != nil {
 				return err
 			}
@@ -73,6 +108,14 @@ func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet
 
 			keepBlobs.Delete(h)
 		}
+
+		if err = tempfile.Close(); err != nil {
+			return errors.Wrap(err, "Close")
+		}
+
+		if err = os.Remove(tempfile.Name()); err != nil {
+			return errors.Wrap(err, "Remove")
+		}
 	}
 
 	if err := repo.Flush(); err != nil {
@@ -80,7 +123,8 @@ func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet
 	}
 
 	for packID := range packs {
-		err := repo.Backend().Remove(restic.DataFile, packID.String())
+		h := restic.Handle{Type: restic.DataFile, Name: packID.String()}
+		err := repo.Backend().Remove(h)
 		if err != nil {
 			debug.Log("error removing pack %v: %v", packID.Str(), err)
 			return err

src/restic/repository/repository.go

@@ -54,7 +54,7 @@ func (r *Repository) LoadAndDecrypt(t restic.FileType, id restic.ID) ([]byte, er
 	debug.Log("load %v with id %v", t, id.Str())
 
 	h := restic.Handle{Type: t, Name: id.String()}
-	buf, err := backend.LoadAll(r.be, h, nil)
+	buf, err := backend.LoadAll(r.be, h)
 	if err != nil {
 		debug.Log("error loading %v: %v", id.Str(), err)
 		return nil, err
@@ -64,33 +64,20 @@ func (r *Repository) LoadAndDecrypt(t restic.FileType, id restic.ID) ([]byte, er
 		return nil, errors.New("invalid data returned")
 	}
 
-	plain := make([]byte, len(buf))
-
 	// decrypt
-	n, err := r.decryptTo(plain, buf)
+	n, err := r.decryptTo(buf, buf)
 	if err != nil {
 		return nil, err
 	}
 
-	return plain[:n], nil
+	return buf[:n], nil
 }
 
 // loadBlob tries to load and decrypt content identified by t and id from a
 // pack from the backend, the result is stored in plaintextBuf, which must be
 // large enough to hold the complete blob.
 func (r *Repository) loadBlob(id restic.ID, t restic.BlobType, plaintextBuf []byte) (int, error) {
-	debug.Log("load %v with id %v (buf %p, len %d)", t, id.Str(), plaintextBuf, len(plaintextBuf))
-
-	// lookup plaintext size of blob
-	size, err := r.idx.LookupSize(id, t)
-	if err != nil {
-		return 0, err
-	}
-
-	// make sure the plaintext buffer is large enough, extend otherwise
-	if len(plaintextBuf) < int(size) {
-		return 0, errors.Errorf("buffer is too small: %d < %d", len(plaintextBuf), size)
-	}
+	debug.Log("load %v with id %v (buf len %v, cap %d)", t, id.Str(), len(plaintextBuf), cap(plaintextBuf))
 
 	// lookup packs
 	blobs, err := r.idx.Lookup(id, t)
@@ -109,8 +96,14 @@ func (r *Repository) loadBlob(id restic.ID, t restic.BlobType, plaintextBuf []by
 
 		// load blob from pack
 		h := restic.Handle{Type: restic.DataFile, Name: blob.PackID.String()}
-		ciphertextBuf := make([]byte, blob.Length)
-		n, err := r.be.Load(h, ciphertextBuf, int64(blob.Offset))
+
+		if uint(cap(plaintextBuf)) < blob.Length {
+			return 0, errors.Errorf("buffer is too small: %v < %v", cap(plaintextBuf), blob.Length)
+		}
+
+		plaintextBuf = plaintextBuf[:blob.Length]
+
+		n, err := restic.ReadAt(r.be, h, int64(blob.Offset), plaintextBuf)
 		if err != nil {
 			debug.Log("error loading blob %v: %v", blob, err)
 			lastError = err
@@ -125,7 +118,7 @@ func (r *Repository) loadBlob(id restic.ID, t restic.BlobType, plaintextBuf []by
 		}
 
 		// decrypt
-		n, err = r.decryptTo(plaintextBuf, ciphertextBuf)
+		n, err = r.decryptTo(plaintextBuf, plaintextBuf)
 		if err != nil {
 			lastError = errors.Errorf("decrypting blob %v failed: %v", id, err)
 			continue
@@ -224,7 +217,7 @@ func (r *Repository) SaveJSONUnpacked(t restic.FileType, item interface{}) (rest
 // SaveUnpacked encrypts data and stores it in the backend. Returned is the
 // storage hash.
 func (r *Repository) SaveUnpacked(t restic.FileType, p []byte) (id restic.ID, err error) {
-	ciphertext := make([]byte, len(p)+crypto.Extension)
+	ciphertext := restic.NewBlobBuffer(len(p))
 	ciphertext, err = r.Encrypt(ciphertext, p)
 	if err != nil {
 		return restic.ID{}, err
@@ -233,7 +226,7 @@ func (r *Repository) SaveUnpacked(t restic.FileType, p []byte) (id restic.ID, er
 	id = restic.Hash(ciphertext)
 	h := restic.Handle{Type: t, Name: id.String()}
 
-	err = r.be.Save(h, ciphertext)
+	err = r.be.Save(h, bytes.NewReader(ciphertext))
 	if err != nil {
 		debug.Log("error saving blob %v: %v", h, err)
 		return restic.ID{}, err
@@ -248,15 +241,15 @@ func (r *Repository) Flush() error {
 	r.pm.Lock()
 	defer r.pm.Unlock()
 
-	debug.Log("manually flushing %d packs", len(r.packs))
+	debug.Log("manually flushing %d packs", len(r.packerManager.packers))
 
-	for _, p := range r.packs {
+	for _, p := range r.packerManager.packers {
 		err := r.savePacker(p)
 		if err != nil {
 			return err
 		}
 	}
-	r.packs = r.packs[:0]
+	r.packerManager.packers = r.packerManager.packers[:0]
 	return nil
 }
 
@@ -387,7 +380,7 @@ func (r *Repository) SearchKey(password string, maxKeys int) error {
 // Init creates a new master key with the supplied password, initializes and
 // saves the repository config.
 func (r *Repository) Init(password string) error {
-	has, err := r.be.Test(restic.ConfigFile, "")
+	has, err := r.be.Test(restic.Handle{Type: restic.ConfigFile})
 	if err != nil {
 		return err
 	}
@@ -528,16 +521,18 @@ func (r *Repository) Close() error {
 	return r.be.Close()
 }
 
-// LoadBlob loads a blob of type t from the repository to the buffer.
+// LoadBlob loads a blob of type t from the repository to the buffer. buf must
+// be large enough to hold the encrypted blob, since it is used as scratch
+// space.
 func (r *Repository) LoadBlob(t restic.BlobType, id restic.ID, buf []byte) (int, error) {
-	debug.Log("load blob %v into buf %p", id.Str(), buf)
+	debug.Log("load blob %v into buf (len %v, cap %v)", id.Str(), len(buf), cap(buf))
 	size, err := r.idx.LookupSize(id, t)
 	if err != nil {
 		return 0, err
 	}
 
-	if len(buf) < int(size) {
-		return 0, errors.Errorf("buffer is too small for data blob (%d < %d)", len(buf), size)
+	if cap(buf) < restic.CiphertextLength(int(size)) {
+		return 0, errors.Errorf("buffer is too small for data blob (%d < %d)", cap(buf), restic.CiphertextLength(int(size)))
 	}
 
 	n, err := r.loadBlob(id, t, buf)
@@ -571,7 +566,7 @@ func (r *Repository) LoadTree(id restic.ID) (*restic.Tree, error) {
 	}
 
 	debug.Log("size is %d, create buffer", size)
-	buf := make([]byte, size)
+	buf := restic.NewBlobBuffer(int(size))
 
 	n, err := r.loadBlob(id, restic.TreeBlob, buf)
 	if err != nil {

src/restic/repository/repository_test.go

@@ -2,12 +2,12 @@ package repository_test
 
 import (
 	"bytes"
-	"crypto/rand"
 	"crypto/sha256"
 	"io"
-	mrand "math/rand"
+	"math/rand"
 	"path/filepath"
 	"testing"
+	"time"
 
 	"restic"
 	"restic/archiver"
@@ -17,13 +17,15 @@ import (
 
 var testSizes = []int{5, 23, 2<<18 + 23, 1 << 20}
 
+var rnd = rand.New(rand.NewSource(time.Now().UnixNano()))
+
 func TestSave(t *testing.T) {
 	repo, cleanup := repository.TestRepository(t)
 	defer cleanup()
 
 	for _, size := range testSizes {
 		data := make([]byte, size)
-		_, err := io.ReadFull(rand.Reader, data)
+		_, err := io.ReadFull(rnd, data)
 		OK(t, err)
 
 		id := restic.Hash(data)
@@ -38,7 +40,7 @@ func TestSave(t *testing.T) {
 		// OK(t, repo.SaveIndex())
 
 		// read back
-		buf := make([]byte, size)
+		buf := restic.NewBlobBuffer(size)
 		n, err := repo.LoadBlob(restic.DataBlob, id, buf)
 		OK(t, err)
 		Equals(t, len(buf), n)
@@ -59,7 +61,7 @@ func TestSaveFrom(t *testing.T) {
 
 	for _, size := range testSizes {
 		data := make([]byte, size)
-		_, err := io.ReadFull(rand.Reader, data)
+		_, err := io.ReadFull(rnd, data)
 		OK(t, err)
 
 		id := restic.Hash(data)
@@ -72,7 +74,7 @@ func TestSaveFrom(t *testing.T) {
 		OK(t, repo.Flush())
 
 		// read back
-		buf := make([]byte, size)
+		buf := restic.NewBlobBuffer(size)
 		n, err := repo.LoadBlob(restic.DataBlob, id, buf)
 		OK(t, err)
 		Equals(t, len(buf), n)
@@ -94,7 +96,7 @@ func BenchmarkSaveAndEncrypt(t *testing.B) {
 	size := 4 << 20 // 4MiB
 
 	data := make([]byte, size)
-	_, err := io.ReadFull(rand.Reader, data)
+	_, err := io.ReadFull(rnd, data)
 	OK(t, err)
 
 	id := restic.ID(sha256.Sum256(data))
@@ -145,6 +147,113 @@ func BenchmarkLoadTree(t *testing.B) {
 	}
 }
 
+func TestLoadBlob(t *testing.T) {
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
+
+	length := 1000000
+	buf := restic.NewBlobBuffer(length)
+	_, err := io.ReadFull(rnd, buf)
+	OK(t, err)
+
+	id, err := repo.SaveBlob(restic.DataBlob, buf, restic.ID{})
+	OK(t, err)
+	OK(t, repo.Flush())
+
+	// first, test with buffers that are too small
+	for _, testlength := range []int{length - 20, length, restic.CiphertextLength(length) - 1} {
+		buf = make([]byte, 0, testlength)
+		n, err := repo.LoadBlob(restic.DataBlob, id, buf)
+		if err == nil {
+			t.Errorf("LoadBlob() did not return an error for a buffer that is too small to hold the blob")
+			continue
+		}
+
+		if n != 0 {
+			t.Errorf("LoadBlob() returned an error and n > 0")
+			continue
+		}
+	}
+
+	// then use buffers that are large enough
+	base := restic.CiphertextLength(length)
+	for _, testlength := range []int{base, base + 7, base + 15, base + 1000} {
+		buf = make([]byte, 0, testlength)
+		n, err := repo.LoadBlob(restic.DataBlob, id, buf)
+		if err != nil {
+			t.Errorf("LoadBlob() returned an error for buffer size %v: %v", testlength, err)
+			continue
+		}
+
+		if n != length {
+			t.Errorf("LoadBlob() returned the wrong number of bytes: want %v, got %v", length, n)
+			continue
+		}
+	}
+}
+
+func BenchmarkLoadBlob(b *testing.B) {
+	repo, cleanup := repository.TestRepository(b)
+	defer cleanup()
+
+	length := 1000000
+	buf := restic.NewBlobBuffer(length)
+	_, err := io.ReadFull(rnd, buf)
+	OK(b, err)
+
+	id, err := repo.SaveBlob(restic.DataBlob, buf, restic.ID{})
+	OK(b, err)
+	OK(b, repo.Flush())
+
+	b.ResetTimer()
+	b.SetBytes(int64(length))
+
+	for i := 0; i < b.N; i++ {
+		n, err := repo.LoadBlob(restic.DataBlob, id, buf)
+		OK(b, err)
+		if n != length {
+			b.Errorf("wanted %d bytes, got %d", length, n)
+		}
+
+		id2 := restic.Hash(buf[:n])
+		if !id.Equal(id2) {
+			b.Errorf("wrong data returned, wanted %v, got %v", id.Str(), id2.Str())
+		}
+	}
+}
+
+func BenchmarkLoadAndDecrypt(b *testing.B) {
+	repo, cleanup := repository.TestRepository(b)
+	defer cleanup()
+
+	length := 1000000
+	buf := restic.NewBlobBuffer(length)
+	_, err := io.ReadFull(rnd, buf)
+	OK(b, err)
+
+	dataID := restic.Hash(buf)
+
+	storageID, err := repo.SaveUnpacked(restic.DataFile, buf)
+	OK(b, err)
+	// OK(b, repo.Flush())
+
+	b.ResetTimer()
+	b.SetBytes(int64(length))
+
+	for i := 0; i < b.N; i++ {
+		data, err := repo.LoadAndDecrypt(restic.DataFile, storageID)
+		OK(b, err)
+		if len(data) != length {
+			b.Errorf("wanted %d bytes, got %d", length, len(data))
+		}
+
+		id2 := restic.Hash(data)
+		if !dataID.Equal(id2) {
+			b.Errorf("wrong data returned, wanted %v, got %v", storageID.Str(), id2.Str())
+		}
+	}
+}
+
 func TestLoadJSONUnpacked(t *testing.T) {
 	repo, cleanup := repository.TestRepository(t)
 	defer cleanup()
@@ -182,25 +291,48 @@ func TestRepositoryLoadIndex(t *testing.T) {
 }
 
 func BenchmarkLoadIndex(b *testing.B) {
-	repodir, cleanup := Env(b, repoFixture)
+	repository.TestUseLowSecurityKDFParameters(b)
+
+	repo, cleanup := repository.TestRepository(b)
 	defer cleanup()
 
-	repo := repository.TestOpenLocal(b, repodir)
+	idx := repository.NewIndex()
+
+	for i := 0; i < 5000; i++ {
+		idx.Store(restic.PackedBlob{
+			Blob: restic.Blob{
+				Type:   restic.DataBlob,
+				Length: 1234,
+				ID:     restic.NewRandomID(),
+				Offset: 1235,
+			},
+			PackID: restic.NewRandomID(),
+		})
+	}
+
+	id, err := repository.SaveIndex(repo, idx)
+	OK(b, err)
+
+	b.Logf("index saved as %v (%v entries)", id.Str(), idx.Count(restic.DataBlob))
+	fi, err := repo.Backend().Stat(restic.Handle{Type: restic.IndexFile, Name: id.String()})
+	OK(b, err)
+	b.Logf("filesize is %v", fi.Size)
+
 	b.ResetTimer()
 
 	for i := 0; i < b.N; i++ {
-		repo.SetIndex(repository.NewMasterIndex())
-		OK(b, repo.LoadIndex())
+		_, err := repository.LoadIndex(repo, id)
+		OK(b, err)
 	}
 }
 
 // saveRandomDataBlobs generates random data blobs and saves them to the repository.
 func saveRandomDataBlobs(t testing.TB, repo restic.Repository, num int, sizeMax int) {
 	for i := 0; i < num; i++ {
-		size := mrand.Int() % sizeMax
+		size := rand.Int() % sizeMax
 
 		buf := make([]byte, size)
-		_, err := io.ReadFull(rand.Reader, buf)
+		_, err := io.ReadFull(rnd, buf)
 		OK(t, err)
 
 		_, err = repo.SaveBlob(restic.DataBlob, buf, restic.ID{})

src/restic/test/helpers.go

@@ -79,7 +79,7 @@ func Random(seed, count int) []byte {
 
 		for j := range data {
 			cur := i + j
-			if len(p) >= cur {
+			if cur >= len(p) {
 				break
 			}
 			p[cur] = data[j]

src/restic/testing.go

@@ -23,16 +23,26 @@ type fakeFileSystem struct {
 	repo        Repository
 	knownBlobs  IDSet
 	duplication float32
+	buf         []byte
+	chunker     *chunker.Chunker
 }
 
 // saveFile reads from rd and saves the blobs in the repository. The list of
 // IDs is returned.
-func (fs fakeFileSystem) saveFile(rd io.Reader) (blobs IDs) {
-	blobs = IDs{}
-	ch := chunker.New(rd, fs.repo.Config().ChunkerPolynomial)
+func (fs *fakeFileSystem) saveFile(rd io.Reader) (blobs IDs) {
+	if fs.buf == nil {
+		fs.buf = make([]byte, chunker.MaxSize)
+	}
 
+	if fs.chunker == nil {
+		fs.chunker = chunker.New(rd, fs.repo.Config().ChunkerPolynomial)
+	} else {
+		fs.chunker.Reset(rd, fs.repo.Config().ChunkerPolynomial)
+	}
+
+	blobs = IDs{}
 	for {
-		chunk, err := ch.Next(getBuf())
+		chunk, err := fs.chunker.Next(fs.buf)
 		if errors.Cause(err) == io.EOF {
 			break
 		}
@@ -50,7 +60,6 @@ func (fs fakeFileSystem) saveFile(rd io.Reader) (blobs IDs) {
 
 			fs.knownBlobs.Insert(id)
 		}
-		freeBuf(chunk.Data)
 
 		blobs = append(blobs, id)
 	}
@@ -64,7 +73,7 @@ const (
 	maxNodes    = 32
 )
 
-func (fs fakeFileSystem) treeIsKnown(tree *Tree) (bool, []byte, ID) {
+func (fs *fakeFileSystem) treeIsKnown(tree *Tree) (bool, []byte, ID) {
 	data, err := json.Marshal(tree)
 	if err != nil {
 		fs.t.Fatalf("json.Marshal(tree) returned error: %v", err)
@@ -76,7 +85,7 @@ func (fs fakeFileSystem) treeIsKnown(tree *Tree) (bool, []byte, ID) {
 	return fs.blobIsKnown(id, TreeBlob), data, id
 }
 
-func (fs fakeFileSystem) blobIsKnown(id ID, t BlobType) bool {
+func (fs *fakeFileSystem) blobIsKnown(id ID, t BlobType) bool {
 	if rand.Float32() < fs.duplication {
 		return false
 	}
@@ -94,7 +103,7 @@ func (fs fakeFileSystem) blobIsKnown(id ID, t BlobType) bool {
 }
 
 // saveTree saves a tree of fake files in the repo and returns the ID.
-func (fs fakeFileSystem) saveTree(seed int64, depth int) ID {
+func (fs *fakeFileSystem) saveTree(seed int64, depth int) ID {
 	rnd := rand.NewSource(seed)
 	numNodes := int(rnd.Int63() % maxNodes)
 

src/restic/testing_test.go

@@ -47,3 +47,14 @@ func TestCreateSnapshot(t *testing.T) {
 
 	checker.TestCheckRepo(t, repo)
 }
+
+func BenchmarkTestCreateSnapshot(t *testing.B) {
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
+
+	t.ResetTimer()
+
+	for i := 0; i < t.N; i++ {
+		restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth, 0)
+	}
+}

vendor/manifest

@@ -37,6 +37,12 @@
 			"revision": "17b591df37844cde689f4d5813e5cea0927d8dd2",
 			"branch": "master"
 		},
+		{
+			"importpath": "github.com/pkg/profile",
+			"repository": "https://github.com/pkg/profile",
+			"revision": "1c16f117a3ab788fdf0e334e623b8bccf5679866",
+			"branch": "HEAD"
+		},
 		{
 			"importpath": "github.com/pkg/sftp",
 			"repository": "https://github.com/pkg/sftp",

vendor/src/github.com/pkg/profile/AUTHORS

@@ -0,0 +1 @@
+Dave Cheney <dave@cheney.net>

vendor/src/github.com/pkg/profile/LICENSE

@@ -0,0 +1,24 @@
+Copyright (c) 2013 Dave Cheney. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/src/github.com/pkg/profile/README.md

@@ -0,0 +1,47 @@
+profile
+=======
+
+Simple profiling support package for Go
+
+[![Build Status](https://travis-ci.org/pkg/profile.svg?branch=master)](https://travis-ci.org/pkg/profile) [![GoDoc](http://godoc.org/github.com/pkg/profile?status.svg)](http://godoc.org/github.com/pkg/profile)
+
+
+installation
+------------
+
+    go get github.com/pkg/profile
+
+usage
+-----
+
+Enabling profiling in your application is as simple as one line at the top of your main function
+
+```go
+import "github.com/pkg/profile"
+
+func main() {
+    defer profile.Start().Stop()
+    ...
+}
+```
+
+options
+-------
+
+What to profile is controlled by config value passed to profile.Start. 
+By default CPU profiling is enabled.
+
+```go
+import "github.com/pkg/profile"
+
+func main() {
+    // p.Stop() must be called before the program exits to
+    // ensure profiling information is written to disk.
+    p := profile.Start(profile.MemProfile, profile.ProfilePath("."), profile.NoShutdownHook)
+    ...
+}
+```
+
+Several convenience package level values are provided for cpu, memory, and block (contention) profiling.
+
+For more complex options, consult the [documentation](http://godoc.org/github.com/pkg/profile).

vendor/src/github.com/pkg/profile/example_test.go

@@ -0,0 +1,56 @@
+package profile_test
+
+import (
+	"flag"
+	"os"
+
+	"github.com/pkg/profile"
+)
+
+func ExampleStart() {
+	// start a simple CPU profile and register
+	// a defer to Stop (flush) the profiling data.
+	defer profile.Start().Stop()
+}
+
+func ExampleCPUProfile() {
+	// CPU profiling is the default profiling mode, but you can specify it
+	// explicitly for completeness.
+	defer profile.Start(profile.CPUProfile).Stop()
+}
+
+func ExampleMemProfile() {
+	// use memory profiling, rather than the default cpu profiling.
+	defer profile.Start(profile.MemProfile).Stop()
+}
+
+func ExampleMemProfileRate() {
+	// use memory profiling with custom rate.
+	defer profile.Start(profile.MemProfileRate(2048)).Stop()
+}
+
+func ExampleProfilePath() {
+	// set the location that the profile will be written to
+	defer profile.Start(profile.ProfilePath(os.Getenv("HOME")))
+}
+
+func ExampleNoShutdownHook() {
+	// disable the automatic shutdown hook.
+	defer profile.Start(profile.NoShutdownHook).Stop()
+}
+
+func ExampleStart_withFlags() {
+	// use the flags package to selectively enable profiling.
+	mode := flag.String("profile.mode", "", "enable profiling mode, one of [cpu, mem, block]")
+	flag.Parse()
+	switch *mode {
+	case "cpu":
+		defer profile.Start(profile.CPUProfile).Stop()
+	case "mem":
+		defer profile.Start(profile.MemProfile).Stop()
+	case "block":
+		defer profile.Start(profile.BlockProfile).Stop()
+	default:
+		// do nothing
+	}
+}

vendor/src/github.com/pkg/profile/profile.go

@@ -0,0 +1,216 @@
+// Package profile provides a simple way to manage runtime/pprof
+// profiling of your Go application.
+package profile
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"runtime"
+	"runtime/pprof"
+	"sync/atomic"
+)
+
+const (
+	cpuMode = iota
+	memMode
+	blockMode
+	traceMode
+)
+
+type profile struct {
+	// quiet suppresses informational messages during profiling.
+	quiet bool
+
+	// noShutdownHook controls whether the profiling package should
+	// hook SIGINT to write profiles cleanly.
+	noShutdownHook bool
+
+	// mode holds the type of profiling that will be made
+	mode int
+
+	// path holds the base path where various profiling files are  written.
+	// If blank, the base path will be generated by ioutil.TempDir.
+	path string
+
+	// memProfileRate holds the rate for the memory profile.
+	memProfileRate int
+
+	// closer holds a cleanup function that run after each profile
+	closer func()
+
+	// stopped records if a call to profile.Stop has been made
+	stopped uint32
+}
+
+// NoShutdownHook controls whether the profiling package should
+// hook SIGINT to write profiles cleanly.
+// Programs with more sophisticated signal handling should set
+// this to true and ensure the Stop() function returned from Start()
+// is called during shutdown.
+func NoShutdownHook(p *profile) { p.noShutdownHook = true }
+
+// Quiet suppresses informational messages during profiling.
+func Quiet(p *profile) { p.quiet = true }
+
+// CPUProfile enables cpu profiling.
+// It disables any previous profiling settings.
+func CPUProfile(p *profile) { p.mode = cpuMode }
+
+// DefaultMemProfileRate is the default memory profiling rate.
+// See also http://golang.org/pkg/runtime/#pkg-variables
+const DefaultMemProfileRate = 4096
+
+// MemProfile enables memory profiling.
+// It disables any previous profiling settings.
+func MemProfile(p *profile) {
+	p.memProfileRate = DefaultMemProfileRate
+	p.mode = memMode
+}
+
+// MemProfileRate enables memory profiling at the preferred rate.
+// It disables any previous profiling settings.
+func MemProfileRate(rate int) func(*profile) {
+	return func(p *profile) {
+		p.memProfileRate = rate
+		p.mode = memMode
+	}
+}
+
+// BlockProfile enables block (contention) profiling.
+// It disables any previous profiling settings.
+func BlockProfile(p *profile) { p.mode = blockMode }
+
+// ProfilePath controls the base path where various profiling
+// files are written. If blank, the base path will be generated
+// by ioutil.TempDir.
+func ProfilePath(path string) func(*profile) {
+	return func(p *profile) {
+		p.path = path
+	}
+}
+
+// Stop stops the profile and flushes any unwritten data.
+func (p *profile) Stop() {
+	if !atomic.CompareAndSwapUint32(&p.stopped, 0, 1) {
+		// someone has already called close
+		return
+	}
+	p.closer()
+	atomic.StoreUint32(&started, 0)
+}
+
+// started is non zero if a profile is running.
+var started uint32
+
+// Start starts a new profiling session.
+// The caller should call the Stop method on the value returned
+// to cleanly stop profiling.
+func Start(options ...func(*profile)) interface {
+	Stop()
+} {
+	if !atomic.CompareAndSwapUint32(&started, 0, 1) {
+		log.Fatal("profile: Start() already called")
+	}
+
+	var prof profile
+	for _, option := range options {
+		option(&prof)
+	}
+
+	path, err := func() (string, error) {
+		if p := prof.path; p != "" {
+			return p, os.MkdirAll(p, 0777)
+		}
+		return ioutil.TempDir("", "profile")
+	}()
+
+	if err != nil {
+		log.Fatalf("profile: could not create initial output directory: %v", err)
+	}
+
+	logf := func(format string, args ...interface{}) {
+		if !prof.quiet {
+			log.Printf(format, args...)
+		}
+	}
+
+	switch prof.mode {
+	case cpuMode:
+		fn := filepath.Join(path, "cpu.pprof")
+		f, err := os.Create(fn)
+		if err != nil {
+			log.Fatalf("profile: could not create cpu profile %q: %v", fn, err)
+		}
+		logf("profile: cpu profiling enabled, %s", fn)
+		pprof.StartCPUProfile(f)
+		prof.closer = func() {
+			pprof.StopCPUProfile()
+			f.Close()
+			logf("profile: cpu profiling disabled, %s", fn)
+		}
+
+	case memMode:
+		fn := filepath.Join(path, "mem.pprof")
+		f, err := os.Create(fn)
+		if err != nil {
+			log.Fatalf("profile: could not create memory profile %q: %v", fn, err)
+		}
+		old := runtime.MemProfileRate
+		runtime.MemProfileRate = prof.memProfileRate
+		logf("profile: memory profiling enabled (rate %d), %s", runtime.MemProfileRate, fn)
+		prof.closer = func() {
+			pprof.Lookup("heap").WriteTo(f, 0)
+			f.Close()
+			runtime.MemProfileRate = old
+			logf("profile: memory profiling disabled, %s", fn)
+		}
+
+	case blockMode:
+		fn := filepath.Join(path, "block.pprof")
+		f, err := os.Create(fn)
+		if err != nil {
+			log.Fatalf("profile: could not create block profile %q: %v", fn, err)
+		}
+		runtime.SetBlockProfileRate(1)
+		logf("profile: block profiling enabled, %s", fn)
+		prof.closer = func() {
+			pprof.Lookup("block").WriteTo(f, 0)
+			f.Close()
+			runtime.SetBlockProfileRate(0)
+			logf("profile: block profiling disabled, %s", fn)
+		}
+
+	case traceMode:
+		fn := filepath.Join(path, "trace.out")
+		f, err := os.Create(fn)
+		if err != nil {
+			log.Fatalf("profile: could not create trace output file %q: %v", fn, err)
+		}
+		if err := startTrace(f); err != nil {
+			log.Fatalf("profile: could not start trace: %v", err)
+		}
+		logf("profile: trace enabled, %s", fn)
+		prof.closer = func() {
+			stopTrace()
+			logf("profile: trace disabled, %s", fn)
+		}
+	}
+
+	if !prof.noShutdownHook {
+		go func() {
+			c := make(chan os.Signal, 1)
+			signal.Notify(c, os.Interrupt)
+			<-c
+
+			log.Println("profile: caught interrupt, stopping profiles")
+			prof.Stop()
+
+			os.Exit(0)
+		}()
+	}
+
+	return &prof
+}

vendor/src/github.com/pkg/profile/profile_test.go

@@ -0,0 +1,304 @@
+package profile
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+type checkFn func(t *testing.T, stdout, stderr []byte, err error)
+
+var profileTests = []struct {
+	name   string
+	code   string
+	checks []checkFn
+}{{
+	name: "default profile (cpu)",
+	code: `
+package main
+
+import "github.com/pkg/profile"
+
+func main() {
+	defer profile.Start().Stop()
+}	
+`,
+	checks: []checkFn{
+		NoStdout,
+		Stderr("profile: cpu profiling enabled"),
+		NoErr,
+	},
+}, {
+	name: "memory profile",
+	code: `
+package main
+
+import "github.com/pkg/profile"
+
+func main() {
+	defer profile.Start(profile.MemProfile).Stop()
+}	
+`,
+	checks: []checkFn{
+		NoStdout,
+		Stderr("profile: memory profiling enabled"),
+		NoErr,
+	},
+}, {
+	name: "memory profile (rate 2048)",
+	code: `
+package main
+
+import "github.com/pkg/profile"
+
+func main() {
+	defer profile.Start(profile.MemProfileRate(2048)).Stop()
+}	
+`,
+	checks: []checkFn{
+		NoStdout,
+		Stderr("profile: memory profiling enabled (rate 2048)"),
+		NoErr,
+	},
+}, {
+	name: "double start",
+	code: `
+package main
+
+import "github.com/pkg/profile"
+
+func main() {
+	profile.Start()
+	profile.Start()
+}	
+`,
+	checks: []checkFn{
+		NoStdout,
+		Stderr("cpu profiling enabled", "profile: Start() already called"),
+		Err,
+	},
+}, {
+	name: "block profile",
+	code: `
+package main
+
+import "github.com/pkg/profile"
+
+func main() {
+	defer profile.Start(profile.BlockProfile).Stop()
+}	
+`,
+	checks: []checkFn{
+		NoStdout,
+		Stderr("profile: block profiling enabled"),
+		NoErr,
+	},
+}, {
+	name: "profile path",
+	code: `
+package main
+
+import "github.com/pkg/profile"
+
+func main() {
+	defer profile.Start(profile.ProfilePath(".")).Stop()
+}	
+`,
+	checks: []checkFn{
+		NoStdout,
+		Stderr("profile: cpu profiling enabled, cpu.pprof"),
+		NoErr,
+	},
+}, {
+	name: "profile path error",
+	code: `
+package main
+
+import "github.com/pkg/profile"
+
+func main() {
+		defer profile.Start(profile.ProfilePath("README.md")).Stop()
+}	
+`,
+	checks: []checkFn{
+		NoStdout,
+		Stderr("could not create initial output"),
+		Err,
+	},
+}, {
+	name: "multiple profile sessions",
+	code: `
+package main
+
+import "github.com/pkg/profile"
+
+func main() {
+	profile.Start(profile.CPUProfile).Stop()
+	profile.Start(profile.MemProfile).Stop()
+	profile.Start(profile.BlockProfile).Stop()
+	profile.Start(profile.CPUProfile).Stop()
+}
+`,
+	checks: []checkFn{
+		NoStdout,
+		Stderr("profile: cpu profiling enabled",
+			"profile: cpu profiling disabled",
+			"profile: memory profiling enabled",
+			"profile: memory profiling disabled",
+			"profile: block profiling enabled",
+			"profile: block profiling disabled"),
+		NoErr,
+	},
+}, {
+	name: "profile quiet",
+	code: `
+package main
+
+import "github.com/pkg/profile"
+
+func main() {
+        defer profile.Start(profile.Quiet).Stop()
+}       
+`,
+	checks: []checkFn{NoStdout, NoStderr, NoErr},
+}}
+
+func TestProfile(t *testing.T) {
+	for _, tt := range profileTests {
+		t.Log(tt.name)
+		stdout, stderr, err := runTest(t, tt.code)
+		for _, f := range tt.checks {
+			f(t, stdout, stderr, err)
+		}
+	}
+}
+
+// NoStdout checks that stdout was blank.
+func NoStdout(t *testing.T, stdout, _ []byte, _ error) {
+	if len := len(stdout); len > 0 {
+		t.Errorf("stdout: wanted 0 bytes, got %d", len)
+	}
+}
+
+// Stderr verifies that the given lines match the output from stderr
+func Stderr(lines ...string) checkFn {
+	return func(t *testing.T, _, stderr []byte, _ error) {
+		r := bytes.NewReader(stderr)
+		if !validateOutput(r, lines) {
+			t.Errorf("stderr: wanted '%s', got '%s'", lines, stderr)
+		}
+	}
+}
+
+// NoStderr checks that stderr was blank.
+func NoStderr(t *testing.T, _, stderr []byte, _ error) {
+	if len := len(stderr); len > 0 {
+		t.Errorf("stderr: wanted 0 bytes, got %d", len)
+	}
+}
+
+// Err checks that there was an error returned
+func Err(t *testing.T, _, _ []byte, err error) {
+	if err == nil {
+		t.Errorf("expected error")
+	}
+}
+
+// NoErr checks that err was nil
+func NoErr(t *testing.T, _, _ []byte, err error) {
+	if err != nil {
+		t.Errorf("error: expected nil, got %v", err)
+	}
+}
+
+// validatedOutput validates the given slice of lines against data from the given reader.
+func validateOutput(r io.Reader, want []string) bool {
+	s := bufio.NewScanner(r)
+	for _, line := range want {
+		if !s.Scan() || !strings.Contains(s.Text(), line) {
+			return false
+		}
+	}
+	return true
+}
+
+var validateOutputTests = []struct {
+	input string
+	lines []string
+	want  bool
+}{{
+	input: "",
+	want:  true,
+}, {
+	input: `profile: yes
+`,
+	want: true,
+}, {
+	input: `profile: yes
+`,
+	lines: []string{"profile: yes"},
+	want:  true,
+}, {
+	input: `profile: yes
+profile: no
+`,
+	lines: []string{"profile: yes"},
+	want:  true,
+}, {
+	input: `profile: yes
+profile: no
+`,
+	lines: []string{"profile: yes", "profile: no"},
+	want:  true,
+}, {
+	input: `profile: yes
+profile: no
+`,
+	lines: []string{"profile: no"},
+	want:  false,
+}}
+
+func TestValidateOutput(t *testing.T) {
+	for _, tt := range validateOutputTests {
+		r := strings.NewReader(tt.input)
+		got := validateOutput(r, tt.lines)
+		if tt.want != got {
+			t.Errorf("validateOutput(%q, %q), want %v, got %v", tt.input, tt.lines, tt.want, got)
+		}
+	}
+}
+
+// runTest executes the go program supplied and returns the contents of stdout,
+// stderr, and an error which may contain status information about the result
+// of the program.
+func runTest(t *testing.T, code string) ([]byte, []byte, error) {
+	chk := func(err error) {
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	gopath, err := ioutil.TempDir("", "profile-gopath")
+	chk(err)
+	defer os.RemoveAll(gopath)
+
+	srcdir := filepath.Join(gopath, "src")
+	err = os.Mkdir(srcdir, 0755)
+	chk(err)
+	src := filepath.Join(srcdir, "main.go")
+	err = ioutil.WriteFile(src, []byte(code), 0644)
+	chk(err)
+
+	cmd := exec.Command("go", "run", src)
+
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	err = cmd.Run()
+	return stdout.Bytes(), stderr.Bytes(), err
+}

vendor/src/github.com/pkg/profile/trace.go

@@ -0,0 +1,11 @@
+// +build go1.7
+
+package profile
+
+import "runtime/trace"
+
+// Trace profile controls if execution tracing will be enabled. It disables any previous profiling settings.
+func TraceProfile(p *profile) { p.mode = traceMode }
+
+var startTrace = trace.Start
+var stopTrace = trace.Stop

vendor/src/github.com/pkg/profile/trace16.go

@@ -0,0 +1,10 @@
+// +build !go1.7
+
+package profile
+
+import "io"
+
+// mock trace support for Go 1.6 and earlier.
+
+func startTrace(w io.Writer) error { return nil }
+func stopTrace()                   {}

vendor/src/github.com/pkg/profile/trace_test.go

@@ -0,0 +1,10 @@
+// +build go1.7
+
+package profile_test
+
+import "github.com/pkg/profile"
+
+func ExampleTraceProfile() {
+	// use execution tracing, rather than the default cpu profiling.
+	defer profile.Start(profile.TraceProfile).Stop()
+}