Move commands to src/cmds

src/cmds/restic-server/.gitignore

@@ -0,0 +1,24 @@
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+*.test
+*.prof

src/cmds/restic-server/LICENSE

@@ -0,0 +1,24 @@
+Copyright (c) 2015, Bertil Chapuis
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+

src/cmds/restic-server/README.md

@@ -0,0 +1,29 @@
+# Restic Server
+
+Restic Server is a sample server that implement restic's rest backend api.
+It has been developed for demonstration purpose and is not intented to be used in production.
+
+## Getting started
+
+By default the server persists backup data in `/tmp/restic`. 
+Build and start the server with a custom persistence directory:
+
+```
+go build
+./restic-server -path /user/home/backup
+```
+
+The server use an `.htpasswd` file to specify users. You can create such a file at the root of the persistence directory by executing the following command. In order to append new user to the file, just omit the `-c` argument.
+
+```
+htpasswd -s -c .htpasswd username
+```
+
+By default the server uses http. This is not very secure since with Basic Authentication, username and passwords will be present in every request. In order to enable TLS support just add the `-tls` argument and add a private and public key at the root of your persistence directory. 
+
+Signed certificate are required by the restic backend but if you just want to test the feature you can generate unsigned keys with the following commands:
+
+```
+openssl genrsa -out private_key 2048
+openssl req -new -x509 -key private_key -out public_key -days 365
+```

src/cmds/restic-server/handlers.go

@@ -0,0 +1,192 @@
+// +build go1.4
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+// Context contains repository meta-data.
+type Context struct {
+	path string
+}
+
+// AuthHandler wraps h with a http.HandlerFunc that performs basic
+// authentication against the user/passwords pairs stored in f and returns the
+// http.HandlerFunc.
+func AuthHandler(f *HtpasswdFile, h http.Handler) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		username, password, ok := r.BasicAuth()
+		if !ok {
+			http.Error(w, "401 unauthorized", 401)
+			return
+		}
+		if !f.Validate(username, password) {
+			http.Error(w, "401 unauthorized", 401)
+			return
+		}
+		h.ServeHTTP(w, r)
+	}
+}
+
+// CheckConfig returns a http.HandlerFunc that checks whether
+// a configuration exists.
+func CheckConfig(c *Context) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		config := filepath.Join(c.path, "config")
+		st, err := os.Stat(config)
+		if err != nil {
+			http.Error(w, "404 not found", 404)
+			return
+		}
+		w.Header().Add("Content-Length", fmt.Sprint(st.Size()))
+	}
+}
+
+// GetConfig returns a http.HandlerFunc that allows for a
+// config to be retrieved.
+func GetConfig(c *Context) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		config := filepath.Join(c.path, "config")
+		bytes, err := ioutil.ReadFile(config)
+		if err != nil {
+			http.Error(w, "404 not found", 404)
+			return
+		}
+		w.Write(bytes)
+	}
+}
+
+// SaveConfig returns a http.HandlerFunc that allows for a
+// config to be saved.
+func SaveConfig(c *Context) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		config := filepath.Join(c.path, "config")
+		bytes, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			http.Error(w, "400 bad request", 400)
+			return
+		}
+		errw := ioutil.WriteFile(config, bytes, 0600)
+		if errw != nil {
+			http.Error(w, "500 internal server error", 500)
+			return
+		}
+		w.Write([]byte("200 ok"))
+	}
+}
+
+// ListBlobs returns a http.HandlerFunc that lists
+// all blobs of a given type in an arbitrary order.
+func ListBlobs(c *Context) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		vars := strings.Split(r.RequestURI, "/")
+		dir := vars[1]
+		path := filepath.Join(c.path, dir)
+		files, err := ioutil.ReadDir(path)
+		if err != nil {
+			http.Error(w, "404 not found", 404)
+			return
+		}
+		names := make([]string, len(files))
+		for i, f := range files {
+			names[i] = f.Name()
+		}
+		data, err := json.Marshal(names)
+		if err != nil {
+			http.Error(w, "500 internal server error", 500)
+			return
+		}
+		w.Write(data)
+	}
+}
+
+// CheckBlob reutrns a http.HandlerFunc that tests whether a blob exists
+// and returns 200, if it does, or 404 otherwise.
+func CheckBlob(c *Context) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		vars := strings.Split(r.RequestURI, "/")
+		dir := vars[1]
+		name := vars[2]
+		path := filepath.Join(c.path, dir, name)
+		st, err := os.Stat(path)
+		if err != nil {
+			http.Error(w, "404 not found", 404)
+			return
+		}
+		w.Header().Add("Content-Length", fmt.Sprint(st.Size()))
+	}
+}
+
+// GetBlob returns a http.HandlerFunc that retrieves a blob
+// from the repository.
+func GetBlob(c *Context) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		vars := strings.Split(r.RequestURI, "/")
+		dir := vars[1]
+		name := vars[2]
+		path := filepath.Join(c.path, dir, name)
+		file, err := os.Open(path)
+		if err != nil {
+			http.Error(w, "404 not found", 404)
+			return
+		}
+		defer file.Close()
+		http.ServeContent(w, r, "", time.Unix(0, 0), file)
+	}
+}
+
+// SaveBlob returns a http.HandlerFunc that saves a blob to the repository.
+func SaveBlob(c *Context) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		vars := strings.Split(r.RequestURI, "/")
+		dir := vars[1]
+		name := vars[2]
+		path := filepath.Join(c.path, dir, name)
+		tmp := path + "_tmp"
+		tf, err := os.OpenFile(tmp, os.O_CREATE|os.O_WRONLY, 0600)
+		if err != nil {
+			http.Error(w, "500 internal server error", 500)
+			return
+		}
+		if _, err := io.Copy(tf, r.Body); err != nil {
+			http.Error(w, "400 bad request", 400)
+			tf.Close()
+			os.Remove(tmp)
+			return
+		}
+		if err := tf.Close(); err != nil {
+			http.Error(w, "500 internal server error", 500)
+		}
+		if err := os.Rename(tmp, path); err != nil {
+			http.Error(w, "500 internal server error", 500)
+			return
+		}
+		w.Write([]byte("200 ok"))
+	}
+}
+
+// DeleteBlob returns a http.HandlerFunc that deletes a blob from the
+// repository.
+func DeleteBlob(c *Context) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		vars := strings.Split(r.RequestURI, "/")
+		dir := vars[1]
+		name := vars[2]
+		path := filepath.Join(c.path, dir, name)
+		err := os.Remove(path)
+		if err != nil {
+			http.Error(w, "500 internal server error", 500)
+			return
+		}
+		w.Write([]byte("200 ok"))
+	}
+}

src/cmds/restic-server/htpasswd.go

@@ -0,0 +1,96 @@
+// +build go1.4
+
+package main
+
+/*
+Copied from: github.com/bitly/oauth2_proxy
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+import (
+	"crypto/sha1"
+	"encoding/base64"
+	"encoding/csv"
+	"io"
+	"log"
+	"os"
+)
+
+// lookup passwords in a htpasswd file
+// The entries must have been created with -s for SHA encryption
+
+// HtpasswdFile is a map for usernames to passwords.
+type HtpasswdFile struct {
+	Users map[string]string
+}
+
+// NewHtpasswdFromFile reads the users and passwords from a htpasswd
+// file and returns them. If an error is encountered, it is returned, together
+// with a nil-Pointer for the HtpasswdFile.
+func NewHtpasswdFromFile(path string) (*HtpasswdFile, error) {
+	r, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer r.Close()
+	return NewHtpasswd(r)
+}
+
+// NewHtpasswd reads the users and passwords from a htpasswd
+// datastream in file and returns them. If an error is encountered,
+// it is returned, together with a nil-Pointer for the HtpasswdFile.
+func NewHtpasswd(file io.Reader) (*HtpasswdFile, error) {
+	cr := csv.NewReader(file)
+	cr.Comma = ':'
+	cr.Comment = '#'
+	cr.TrimLeadingSpace = true
+
+	records, err := cr.ReadAll()
+	if err != nil {
+		return nil, err
+	}
+	h := &HtpasswdFile{Users: make(map[string]string)}
+	for _, record := range records {
+		h.Users[record[0]] = record[1]
+	}
+	return h, nil
+}
+
+// Validate returns true if password matches the stored password
+// for user. If no password for user is stored, or the password
+// is wrong, false is returned.
+func (h *HtpasswdFile) Validate(user string, password string) bool {
+	realPassword, exists := h.Users[user]
+	if !exists {
+		return false
+	}
+	if realPassword[:5] == "{SHA}" {
+		d := sha1.New()
+		d.Write([]byte(password))
+		if realPassword[5:] == base64.StdEncoding.EncodeToString(d.Sum(nil)) {
+			return true
+		}
+	} else {
+		log.Printf("Invalid htpasswd entry for %s. Must be a SHA entry.", user)
+	}
+	return false
+}

src/cmds/restic-server/router.go

@@ -0,0 +1,137 @@
+// +build go1.4
+
+package main
+
+import (
+	"log"
+	"net/http"
+	"strings"
+)
+
+// Route is a handler for a path that was already split.
+type Route struct {
+	path    []string
+	handler http.Handler
+}
+
+// Router maps HTTP methods to a slice of Route handlers.
+type Router struct {
+	routes map[string][]Route
+}
+
+// NewRouter creates a new Router and returns a pointer to it.
+func NewRouter() *Router {
+	return &Router{make(map[string][]Route)}
+}
+
+// Options registers handler for path with method "OPTIONS".
+func (router *Router) Options(path string, handler http.Handler) {
+	router.Handle("OPTIONS", path, handler)
+}
+
+// OptionsFunc registers handler for path with method "OPTIONS".
+func (router *Router) OptionsFunc(path string, handler http.HandlerFunc) {
+	router.Handle("OPTIONS", path, handler)
+}
+
+// Get registers handler for path with method "GET".
+func (router *Router) Get(path string, handler http.Handler) {
+	router.Handle("GET", path, handler)
+}
+
+// GetFunc registers handler for path with method "GET".
+func (router *Router) GetFunc(path string, handler http.HandlerFunc) {
+	router.Handle("GET", path, handler)
+}
+
+// Head registers handler for path with method "HEAD".
+func (router *Router) Head(path string, handler http.Handler) {
+	router.Handle("HEAD", path, handler)
+}
+
+// HeadFunc registers handler for path with method "HEAD".
+func (router *Router) HeadFunc(path string, handler http.HandlerFunc) {
+	router.Handle("HEAD", path, handler)
+}
+
+// Post registers handler for path with method "POST".
+func (router *Router) Post(path string, handler http.Handler) {
+	router.Handle("POST", path, handler)
+}
+
+// PostFunc registers handler for path with method "POST".
+func (router *Router) PostFunc(path string, handler http.HandlerFunc) {
+	router.Handle("POST", path, handler)
+}
+
+// Put registers handler for path with method "PUT".
+func (router *Router) Put(path string, handler http.Handler) {
+	router.Handle("PUT", path, handler)
+}
+
+// PutFunc registers handler for path with method "PUT".
+func (router *Router) PutFunc(path string, handler http.HandlerFunc) {
+	router.Handle("PUT", path, handler)
+}
+
+// Delete registers handler for path with method "DELETE".
+func (router *Router) Delete(path string, handler http.Handler) {
+	router.Handle("DELETE", path, handler)
+}
+
+// DeleteFunc registers handler for path with method "DELETE".
+func (router *Router) DeleteFunc(path string, handler http.HandlerFunc) {
+	router.Handle("DELETE", path, handler)
+}
+
+// Trace registers handler for path with method "TRACE".
+func (router *Router) Trace(path string, handler http.Handler) {
+	router.Handle("TRACE", path, handler)
+}
+
+// TraceFunc registers handler for path with method "TRACE".
+func (router *Router) TraceFunc(path string, handler http.HandlerFunc) {
+	router.Handle("TRACE", path, handler)
+}
+
+// Connect registers handler for path with method "Connect".
+func (router *Router) Connect(path string, handler http.Handler) {
+	router.Handle("Connect", path, handler)
+}
+
+// ConnectFunc registers handler for path with method "Connect".
+func (router *Router) ConnectFunc(path string, handler http.HandlerFunc) {
+	router.Handle("Connect", path, handler)
+}
+
+// Handle registers a http.Handler for method and uri
+func (router *Router) Handle(method string, uri string, handler http.Handler) {
+	routes := router.routes[method]
+	path := strings.Split(uri, "/")
+	routes = append(routes, Route{path, handler})
+	router.routes[method] = routes
+}
+
+func (router *Router) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	method := r.Method
+	uri := r.RequestURI
+	path := strings.Split(uri, "/")
+
+	log.Printf("%s %s", method, uri)
+
+ROUTE:
+	for _, route := range router.routes[method] {
+		if len(route.path) != len(path) {
+			continue
+		}
+		for i := 0; i < len(route.path); i++ {
+			if !strings.HasPrefix(route.path[i], ":") && route.path[i] != path[i] {
+				continue ROUTE
+			}
+		}
+		route.handler.ServeHTTP(w, r)
+		return
+	}
+
+	http.Error(w, "404 not found", 404)
+}

src/cmds/restic-server/router_test.go

@@ -0,0 +1,74 @@
+// +build go1.4
+
+package main
+
+import (
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+func TestRouter(t *testing.T) {
+	router := NewRouter()
+
+	getConfig := []byte("GET /config")
+	router.GetFunc("/config", func(w http.ResponseWriter, r *http.Request) {
+		w.Write(getConfig)
+	})
+
+	postConfig := []byte("POST /config")
+	router.PostFunc("/config", func(w http.ResponseWriter, r *http.Request) {
+		w.Write(postConfig)
+	})
+
+	getBlobs := []byte("GET /blobs/")
+	router.GetFunc("/blobs/", func(w http.ResponseWriter, r *http.Request) {
+		w.Write(getBlobs)
+	})
+
+	getBlob := []byte("GET /blobs/:sha")
+	router.GetFunc("/blobs/:sha", func(w http.ResponseWriter, r *http.Request) {
+		w.Write(getBlob)
+	})
+
+	server := httptest.NewServer(router)
+	defer server.Close()
+
+	getConfigResp, _ := http.Get(server.URL + "/config")
+	getConfigBody, _ := ioutil.ReadAll(getConfigResp.Body)
+	if getConfigResp.StatusCode != 200 {
+		t.Fatalf("Wanted HTTP Status 200, got %d", getConfigResp.StatusCode)
+	}
+	if string(getConfig) != string(getConfigBody) {
+		t.Fatalf("Config wrong:\nWanted '%s'\nGot: '%s'", string(getConfig), string(getConfigBody))
+	}
+
+	postConfigResp, _ := http.Post(server.URL+"/config", "binary/octet-stream", strings.NewReader("post test"))
+	postConfigBody, _ := ioutil.ReadAll(postConfigResp.Body)
+	if postConfigResp.StatusCode != 200 {
+		t.Fatalf("Wanted HTTP Status 200, got %d", postConfigResp.StatusCode)
+	}
+	if string(postConfig) != string(postConfigBody) {
+		t.Fatalf("Config wrong:\nWanted '%s'\nGot: '%s'", string(postConfig), string(postConfigBody))
+	}
+
+	getBlobsResp, _ := http.Get(server.URL + "/blobs/")
+	getBlobsBody, _ := ioutil.ReadAll(getBlobsResp.Body)
+	if getBlobsResp.StatusCode != 200 {
+		t.Fatalf("Wanted HTTP Status 200, got %d", getBlobsResp.StatusCode)
+	}
+	if string(getBlobs) != string(getBlobsBody) {
+		t.Fatalf("Config wrong:\nWanted '%s'\nGot: '%s'", string(getBlobs), string(getBlobsBody))
+	}
+
+	getBlobResp, _ := http.Get(server.URL + "/blobs/test")
+	getBlobBody, _ := ioutil.ReadAll(getBlobResp.Body)
+	if getBlobResp.StatusCode != 200 {
+		t.Fatalf("Wanted HTTP Status 200, got %d", getBlobResp.StatusCode)
+	}
+	if string(getBlob) != string(getBlobBody) {
+		t.Fatalf("Config wrong:\nWanted '%s'\nGot: '%s'", string(getBlob), string(getBlobBody))
+	}
+}

src/cmds/restic-server/server.go

@@ -0,0 +1,73 @@
+// +build go1.4
+
+package main
+
+import (
+	"flag"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+)
+
+const (
+	defaultHTTPPort  = ":8000"
+	defaultHTTPSPort = ":8443"
+)
+
+func main() {
+	// Parse command-line args
+	var path = flag.String("path", "/tmp/restic", "specifies the path of the data directory")
+	var tls = flag.Bool("tls", false, "turns on tls support")
+	flag.Parse()
+
+	// Create the missing directories
+	dirs := []string{
+		"data",
+		"snapshots",
+		"index",
+		"locks",
+		"keys",
+		"tmp",
+	}
+	for _, d := range dirs {
+		os.MkdirAll(filepath.Join(*path, d), 0700)
+	}
+
+	// Define the routes
+	context := &Context{*path}
+	router := NewRouter()
+	router.HeadFunc("/config", CheckConfig(context))
+	router.GetFunc("/config", GetConfig(context))
+	router.PostFunc("/config", SaveConfig(context))
+	router.GetFunc("/:dir/", ListBlobs(context))
+	router.HeadFunc("/:dir/:name", CheckBlob(context))
+	router.GetFunc("/:type/:name", GetBlob(context))
+	router.PostFunc("/:type/:name", SaveBlob(context))
+	router.DeleteFunc("/:type/:name", DeleteBlob(context))
+
+	// Check for a password file
+	var handler http.Handler
+	htpasswdFile, err := NewHtpasswdFromFile(filepath.Join(*path, ".htpasswd"))
+	if err != nil {
+		log.Println("Authentication disabled")
+		handler = router
+	} else {
+		log.Println("Authentication enabled")
+		handler = AuthHandler(htpasswdFile, router)
+	}
+
+	// start the server
+	if !*tls {
+		log.Printf("start server on port %s\n", defaultHTTPPort)
+		http.ListenAndServe(defaultHTTPPort, handler)
+	} else {
+		privateKey := filepath.Join(*path, "private_key")
+		publicKey := filepath.Join(*path, "public_key")
+		log.Println("TLS enabled")
+		log.Printf("private key: %s", privateKey)
+		log.Printf("public key: %s", publicKey)
+		log.Printf("start server on port %s\n", defaultHTTPSPort)
+		http.ListenAndServeTLS(defaultHTTPSPort, publicKey, privateKey, handler)
+	}
+}

src/cmds/restic/.gitignore

@@ -0,0 +1 @@
+config.mk

src/cmds/restic/cleanup.go

@@ -0,0 +1,65 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+
+	"restic/debug"
+)
+
+var cleanupHandlers struct {
+	sync.Mutex
+	list []func() error
+	done bool
+}
+
+var stderr = os.Stderr
+
+func init() {
+	c := make(chan os.Signal)
+	signal.Notify(c, syscall.SIGINT)
+
+	go CleanupHandler(c)
+}
+
+// AddCleanupHandler adds the function f to the list of cleanup handlers so
+// that it is executed when all the cleanup handlers are run, e.g. when SIGINT
+// is received.
+func AddCleanupHandler(f func() error) {
+	cleanupHandlers.Lock()
+	defer cleanupHandlers.Unlock()
+
+	cleanupHandlers.list = append(cleanupHandlers.list, f)
+}
+
+// RunCleanupHandlers runs all registered cleanup handlers
+func RunCleanupHandlers() {
+	cleanupHandlers.Lock()
+	defer cleanupHandlers.Unlock()
+
+	if cleanupHandlers.done {
+		return
+	}
+	cleanupHandlers.done = true
+
+	for _, f := range cleanupHandlers.list {
+		err := f()
+		if err != nil {
+			fmt.Fprintf(stderr, "error in cleanup handler: %v\n", err)
+		}
+	}
+}
+
+// CleanupHandler handles the SIGINT signal.
+func CleanupHandler(c <-chan os.Signal) {
+	for s := range c {
+		debug.Log("CleanupHandler", "signal %v received, cleaning up", s)
+		fmt.Println("\x1b[2KInterrupt received, cleaning up")
+		RunCleanupHandlers()
+		fmt.Println("exiting")
+		os.Exit(0)
+	}
+}

src/cmds/restic/cmd_backup.go

@@ -0,0 +1,338 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"golang.org/x/crypto/ssh/terminal"
+	"restic"
+	"restic/backend"
+	"restic/debug"
+	"restic/filter"
+	"restic/repository"
+)
+
+type CmdBackup struct {
+	Parent   string   `short:"p" long:"parent"  description:"use this parent snapshot (default: last snapshot in repo that has the same target)"`
+	Force    bool     `short:"f" long:"force"   description:"Force re-reading the target. Overrides the \"parent\" flag"`
+	Excludes []string `short:"e" long:"exclude" description:"Exclude a pattern (can be specified multiple times)"`
+
+	global *GlobalOptions
+}
+
+func init() {
+	_, err := parser.AddCommand("backup",
+		"save file/directory",
+		"The backup command creates a snapshot of a file or directory",
+		&CmdBackup{global: &globalOpts})
+	if err != nil {
+		panic(err)
+	}
+}
+
+func formatBytes(c uint64) string {
+	b := float64(c)
+
+	switch {
+	case c > 1<<40:
+		return fmt.Sprintf("%.3f TiB", b/(1<<40))
+	case c > 1<<30:
+		return fmt.Sprintf("%.3f GiB", b/(1<<30))
+	case c > 1<<20:
+		return fmt.Sprintf("%.3f MiB", b/(1<<20))
+	case c > 1<<10:
+		return fmt.Sprintf("%.3f KiB", b/(1<<10))
+	default:
+		return fmt.Sprintf("%dB", c)
+	}
+}
+
+func formatSeconds(sec uint64) string {
+	hours := sec / 3600
+	sec -= hours * 3600
+	min := sec / 60
+	sec -= min * 60
+	if hours > 0 {
+		return fmt.Sprintf("%d:%02d:%02d", hours, min, sec)
+	}
+
+	return fmt.Sprintf("%d:%02d", min, sec)
+}
+
+func formatPercent(numerator uint64, denominator uint64) string {
+	if denominator == 0 {
+		return ""
+	}
+
+	percent := 100.0 * float64(numerator) / float64(denominator)
+
+	if percent > 100 {
+		percent = 100
+	}
+
+	return fmt.Sprintf("%3.2f%%", percent)
+}
+
+func formatRate(bytes uint64, duration time.Duration) string {
+	sec := float64(duration) / float64(time.Second)
+	rate := float64(bytes) / sec / (1 << 20)
+	return fmt.Sprintf("%.2fMiB/s", rate)
+}
+
+func formatDuration(d time.Duration) string {
+	sec := uint64(d / time.Second)
+	return formatSeconds(sec)
+}
+
+func printTree2(indent int, t *restic.Tree) {
+	for _, node := range t.Nodes {
+		if node.Tree() != nil {
+			fmt.Printf("%s%s/\n", strings.Repeat("  ", indent), node.Name)
+			printTree2(indent+1, node.Tree())
+		} else {
+			fmt.Printf("%s%s\n", strings.Repeat("  ", indent), node.Name)
+		}
+	}
+}
+
+func (cmd CmdBackup) Usage() string {
+	return "DIR/FILE [DIR/FILE] [...]"
+}
+
+func (cmd CmdBackup) newScanProgress() *restic.Progress {
+	if !cmd.global.ShowProgress() {
+		return nil
+	}
+
+	p := restic.NewProgress(time.Second)
+	p.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
+		fmt.Printf("\x1b[2K[%s] %d directories, %d files, %s\r", formatDuration(d), s.Dirs, s.Files, formatBytes(s.Bytes))
+	}
+	p.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
+		fmt.Printf("\x1b[2Kscanned %d directories, %d files in %s\n", s.Dirs, s.Files, formatDuration(d))
+	}
+
+	return p
+}
+
+func (cmd CmdBackup) newArchiveProgress(todo restic.Stat) *restic.Progress {
+	if !cmd.global.ShowProgress() {
+		return nil
+	}
+
+	archiveProgress := restic.NewProgress(time.Second)
+
+	var bps, eta uint64
+	itemsTodo := todo.Files + todo.Dirs
+
+	archiveProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
+		sec := uint64(d / time.Second)
+		if todo.Bytes > 0 && sec > 0 && ticker {
+			bps = s.Bytes / sec
+			if s.Bytes >= todo.Bytes {
+				eta = 0
+			} else if bps > 0 {
+				eta = (todo.Bytes - s.Bytes) / bps
+			}
+		}
+
+		itemsDone := s.Files + s.Dirs
+
+		status1 := fmt.Sprintf("[%s] %s  %s/s  %s / %s  %d / %d items  %d errors  ",
+			formatDuration(d),
+			formatPercent(s.Bytes, todo.Bytes),
+			formatBytes(bps),
+			formatBytes(s.Bytes), formatBytes(todo.Bytes),
+			itemsDone, itemsTodo,
+			s.Errors)
+		status2 := fmt.Sprintf("ETA %s ", formatSeconds(eta))
+
+		w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
+		if err == nil {
+			maxlen := w - len(status2)
+
+			if maxlen < 4 {
+				status1 = ""
+			} else if len(status1) > maxlen {
+				status1 = status1[:maxlen-4]
+				status1 += "... "
+			}
+		}
+
+		fmt.Printf("\x1b[2K%s%s\r", status1, status2)
+	}
+
+	archiveProgress.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
+		fmt.Printf("\nduration: %s, %s\n", formatDuration(d), formatRate(todo.Bytes, d))
+	}
+
+	return archiveProgress
+}
+
+func samePaths(expected, actual []string) bool {
+	if expected == nil || actual == nil {
+		return true
+	}
+
+	if len(expected) != len(actual) {
+		return false
+	}
+	for i := range expected {
+		if expected[i] != actual[i] {
+			return false
+		}
+	}
+
+	return true
+}
+
+var errNoSnapshotFound = errors.New("no snapshot found")
+
+func findLatestSnapshot(repo *repository.Repository, targets []string) (backend.ID, error) {
+	var (
+		latest   time.Time
+		latestID backend.ID
+		found    bool
+	)
+
+	for snapshotID := range repo.List(backend.Snapshot, make(chan struct{})) {
+		snapshot, err := restic.LoadSnapshot(repo, snapshotID)
+		if err != nil {
+			return backend.ID{}, fmt.Errorf("Error listing snapshot: %v", err)
+		}
+		if snapshot.Time.After(latest) && samePaths(snapshot.Paths, targets) {
+			latest = snapshot.Time
+			latestID = snapshotID
+			found = true
+		}
+	}
+
+	if !found {
+		return backend.ID{}, errNoSnapshotFound
+	}
+
+	return latestID, nil
+}
+
+// filterExisting returns a slice of all existing items, or an error if no
+// items exist at all.
+func filterExisting(items []string) (result []string, err error) {
+	for _, item := range items {
+		_, err := os.Lstat(item)
+		if err != nil && os.IsNotExist(err) {
+			continue
+		}
+
+		result = append(result, item)
+	}
+
+	if len(result) == 0 {
+		return nil, errors.New("all target directories/files do not exist")
+	}
+
+	return
+}
+
+func (cmd CmdBackup) Execute(args []string) error {
+	if len(args) == 0 {
+		return fmt.Errorf("wrong number of parameters, Usage: %s", cmd.Usage())
+	}
+
+	target := make([]string, 0, len(args))
+	for _, d := range args {
+		if a, err := filepath.Abs(d); err == nil {
+			d = a
+		}
+		target = append(target, d)
+	}
+
+	target, err := filterExisting(target)
+	if err != nil {
+		return err
+	}
+
+	repo, err := cmd.global.OpenRepository()
+	if err != nil {
+		return err
+	}
+
+	lock, err := lockRepo(repo)
+	defer unlockRepo(lock)
+	if err != nil {
+		return err
+	}
+
+	err = repo.LoadIndex()
+	if err != nil {
+		return err
+	}
+
+	var parentSnapshotID *backend.ID
+
+	// Force using a parent
+	if !cmd.Force && cmd.Parent != "" {
+		id, err := restic.FindSnapshot(repo, cmd.Parent)
+		if err != nil {
+			return fmt.Errorf("invalid id %q: %v", cmd.Parent, err)
+		}
+
+		parentSnapshotID = &id
+	}
+
+	// Find last snapshot to set it as parent, if not already set
+	if !cmd.Force && parentSnapshotID == nil {
+		id, err := findLatestSnapshot(repo, target)
+		if err == nil {
+			parentSnapshotID = &id
+		} else if err != errNoSnapshotFound {
+			return err
+		}
+	}
+
+	if parentSnapshotID != nil {
+		cmd.global.Verbosef("using parent snapshot %v\n", parentSnapshotID.Str())
+	}
+
+	cmd.global.Verbosef("scan %v\n", target)
+
+	selectFilter := func(item string, fi os.FileInfo) bool {
+		matched, err := filter.List(cmd.Excludes, item)
+		if err != nil {
+			cmd.global.Warnf("error for exclude pattern: %v", err)
+		}
+
+		if matched {
+			debug.Log("backup.Execute", "path %q excluded by a filter", item)
+		}
+
+		return !matched
+	}
+
+	stat, err := restic.Scan(target, selectFilter, cmd.newScanProgress())
+	if err != nil {
+		return err
+	}
+
+	arch := restic.NewArchiver(repo)
+	arch.Excludes = cmd.Excludes
+	arch.SelectFilter = selectFilter
+
+	arch.Error = func(dir string, fi os.FileInfo, err error) error {
+		// TODO: make ignoring errors configurable
+		cmd.global.Warnf("\x1b[2K\rerror for %s: %v\n", dir, err)
+		return nil
+	}
+
+	_, id, err := arch.Snapshot(cmd.newArchiveProgress(stat), target, parentSnapshotID)
+	if err != nil {
+		return err
+	}
+
+	cmd.global.Verbosef("snapshot %s saved\n", id.Str())
+
+	return nil
+}

src/cmds/restic/cmd_cache.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"fmt"
+
+	"restic"
+)
+
+type CmdCache struct {
+	global *GlobalOptions
+}
+
+func init() {
+	_, err := parser.AddCommand("cache",
+		"manage cache",
+		"The cache command creates and manages the local cache",
+		&CmdCache{global: &globalOpts})
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (cmd CmdCache) Usage() string {
+	return "[update|clear]"
+}
+
+func (cmd CmdCache) Execute(args []string) error {
+	// if len(args) == 0 || len(args) > 2 {
+	// 	return fmt.Errorf("wrong number of parameters, Usage: %s", cmd.Usage())
+	// }
+
+	repo, err := cmd.global.OpenRepository()
+	if err != nil {
+		return err
+	}
+
+	lock, err := lockRepo(repo)
+	defer unlockRepo(lock)
+	if err != nil {
+		return err
+	}
+
+	cache, err := restic.NewCache(repo, cmd.global.CacheDir)
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("clear cache for old snapshots\n")
+	err = cache.Clear(repo)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("done\n")
+
+	return nil
+}

src/cmds/restic/cmd_cat.go

@@ -0,0 +1,199 @@
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+
+	"restic"
+	"restic/backend"
+	"restic/debug"
+	"restic/pack"
+	"restic/repository"
+)
+
+type CmdCat struct {
+	global *GlobalOptions
+}
+
+func init() {
+	_, err := parser.AddCommand("cat",
+		"dump something",
+		"The cat command dumps data structures or data from a repository",
+		&CmdCat{global: &globalOpts})
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (cmd CmdCat) Usage() string {
+	return "[pack|blob|tree|snapshot|key|masterkey|config|lock] ID"
+}
+
+func (cmd CmdCat) Execute(args []string) error {
+	if len(args) < 1 || (args[0] != "masterkey" && args[0] != "config" && len(args) != 2) {
+		return fmt.Errorf("type or ID not specified, Usage: %s", cmd.Usage())
+	}
+
+	repo, err := cmd.global.OpenRepository()
+	if err != nil {
+		return err
+	}
+
+	lock, err := lockRepo(repo)
+	defer unlockRepo(lock)
+	if err != nil {
+		return err
+	}
+
+	tpe := args[0]
+
+	var id backend.ID
+	if tpe != "masterkey" && tpe != "config" {
+		id, err = backend.ParseID(args[1])
+		if err != nil {
+			if tpe != "snapshot" {
+				return err
+			}
+
+			// find snapshot id with prefix
+			id, err = restic.FindSnapshot(repo, args[1])
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	// handle all types that don't need an index
+	switch tpe {
+	case "config":
+		buf, err := json.MarshalIndent(repo.Config, "", "  ")
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(string(buf))
+		return nil
+	case "index":
+		buf, err := repo.LoadAndDecrypt(backend.Index, id)
+		if err != nil {
+			return err
+		}
+
+		_, err = os.Stdout.Write(append(buf, '\n'))
+		return err
+
+	case "snapshot":
+		sn := &restic.Snapshot{}
+		err = repo.LoadJSONUnpacked(backend.Snapshot, id, sn)
+		if err != nil {
+			return err
+		}
+
+		buf, err := json.MarshalIndent(&sn, "", "  ")
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(string(buf))
+
+		return nil
+	case "key":
+		h := backend.Handle{Type: backend.Key, Name: id.String()}
+		buf, err := backend.LoadAll(repo.Backend(), h, nil)
+		if err != nil {
+			return err
+		}
+
+		key := &repository.Key{}
+		err = json.Unmarshal(buf, key)
+		if err != nil {
+			return err
+		}
+
+		buf, err = json.MarshalIndent(&key, "", "  ")
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(string(buf))
+		return nil
+	case "masterkey":
+		buf, err := json.MarshalIndent(repo.Key(), "", "  ")
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(string(buf))
+		return nil
+	case "lock":
+		lock, err := restic.LoadLock(repo, id)
+		if err != nil {
+			return err
+		}
+
+		buf, err := json.MarshalIndent(&lock, "", "  ")
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(string(buf))
+
+		return nil
+	}
+
+	// load index, handle all the other types
+	err = repo.LoadIndex()
+	if err != nil {
+		return err
+	}
+
+	switch tpe {
+	case "pack":
+		h := backend.Handle{Type: backend.Data, Name: id.String()}
+		buf, err := backend.LoadAll(repo.Backend(), h, nil)
+		if err != nil {
+			return err
+		}
+
+		_, err = os.Stdout.Write(buf)
+		return err
+
+	case "blob":
+		blob, err := repo.Index().Lookup(id)
+		if err != nil {
+			return err
+		}
+
+		buf := make([]byte, blob.Length)
+		data, err := repo.LoadBlob(blob.Type, id, buf)
+		if err != nil {
+			return err
+		}
+
+		_, err = os.Stdout.Write(data)
+		return err
+
+	case "tree":
+		debug.Log("cat", "cat tree %v", id.Str())
+		tree := restic.NewTree()
+		err = repo.LoadJSONPack(pack.Tree, id, tree)
+		if err != nil {
+			debug.Log("cat", "unable to load tree %v: %v", id.Str(), err)
+			return err
+		}
+
+		buf, err := json.MarshalIndent(&tree, "", "  ")
+		if err != nil {
+			debug.Log("cat", "error json.MarshalIndent(): %v", err)
+			return err
+		}
+
+		_, err = os.Stdout.Write(append(buf, '\n'))
+		return nil
+
+	default:
+		return errors.New("invalid type")
+	}
+}

src/cmds/restic/cmd_check.go

@@ -0,0 +1,165 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"time"
+
+	"golang.org/x/crypto/ssh/terminal"
+
+	"restic"
+	"restic/checker"
+)
+
+type CmdCheck struct {
+	ReadData    bool `long:"read-data"    default:"false" description:"Read data blobs"`
+	CheckUnused bool `long:"check-unused" default:"false" description:"Check for unused blobs"`
+
+	global *GlobalOptions
+}
+
+func init() {
+	_, err := parser.AddCommand("check",
+		"check the repository",
+		"The check command check the integrity and consistency of the repository",
+		&CmdCheck{global: &globalOpts})
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (cmd CmdCheck) Usage() string {
+	return "[check-options]"
+}
+
+func (cmd CmdCheck) newReadProgress(todo restic.Stat) *restic.Progress {
+	if !cmd.global.ShowProgress() {
+		return nil
+	}
+
+	readProgress := restic.NewProgress(time.Second)
+
+	readProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
+		status := fmt.Sprintf("[%s] %s  %d / %d items",
+			formatDuration(d),
+			formatPercent(s.Blobs, todo.Blobs),
+			s.Blobs, todo.Blobs)
+
+		w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
+		if err == nil {
+			if len(status) > w {
+				max := w - len(status) - 4
+				status = status[:max] + "... "
+			}
+		}
+
+		fmt.Printf("\x1b[2K%s\r", status)
+	}
+
+	readProgress.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
+		fmt.Printf("\nduration: %s\n", formatDuration(d))
+	}
+
+	return readProgress
+}
+
+func (cmd CmdCheck) Execute(args []string) error {
+	if len(args) != 0 {
+		return errors.New("check has no arguments")
+	}
+
+	repo, err := cmd.global.OpenRepository()
+	if err != nil {
+		return err
+	}
+
+	if !cmd.global.NoLock {
+		cmd.global.Verbosef("Create exclusive lock for repository\n")
+		lock, err := lockRepoExclusive(repo)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+	}
+
+	chkr := checker.New(repo)
+
+	cmd.global.Verbosef("Load indexes\n")
+	hints, errs := chkr.LoadIndex()
+
+	dupFound := false
+	for _, hint := range hints {
+		cmd.global.Printf("%v\n", hint)
+		if _, ok := hint.(checker.ErrDuplicatePacks); ok {
+			dupFound = true
+		}
+	}
+
+	if dupFound {
+		cmd.global.Printf("\nrun `restic rebuild-index' to correct this\n")
+	}
+
+	if len(errs) > 0 {
+		for _, err := range errs {
+			cmd.global.Warnf("error: %v\n", err)
+		}
+		return fmt.Errorf("LoadIndex returned errors")
+	}
+
+	done := make(chan struct{})
+	defer close(done)
+
+	errorsFound := false
+	errChan := make(chan error)
+
+	cmd.global.Verbosef("Check all packs\n")
+	go chkr.Packs(errChan, done)
+
+	for err := range errChan {
+		errorsFound = true
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+	}
+
+	cmd.global.Verbosef("Check snapshots, trees and blobs\n")
+	errChan = make(chan error)
+	go chkr.Structure(errChan, done)
+
+	for err := range errChan {
+		errorsFound = true
+		if e, ok := err.(checker.TreeError); ok {
+			fmt.Fprintf(os.Stderr, "error for tree %v:\n", e.ID.Str())
+			for _, treeErr := range e.Errors {
+				fmt.Fprintf(os.Stderr, "  %v\n", treeErr)
+			}
+		} else {
+			fmt.Fprintf(os.Stderr, "error: %v\n", err)
+		}
+	}
+
+	if cmd.CheckUnused {
+		for _, id := range chkr.UnusedBlobs() {
+			cmd.global.Verbosef("unused blob %v\n", id.Str())
+			errorsFound = true
+		}
+	}
+
+	if cmd.ReadData {
+		cmd.global.Verbosef("Read all data\n")
+
+		p := cmd.newReadProgress(restic.Stat{Blobs: chkr.CountPacks()})
+		errChan := make(chan error)
+
+		go chkr.ReadData(p, errChan, done)
+
+		for err := range errChan {
+			errorsFound = true
+			fmt.Fprintf(os.Stderr, "%v\n", err)
+		}
+	}
+
+	if errorsFound {
+		return errors.New("repository contains errors")
+	}
+	return nil
+}

src/cmds/restic/cmd_dump.go

@@ -0,0 +1,177 @@
+// +build debug
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/juju/errors"
+	"restic"
+	"restic/backend"
+	"restic/pack"
+	"restic/repository"
+)
+
+type CmdDump struct {
+	global *GlobalOptions
+
+	repo *repository.Repository
+}
+
+func init() {
+	_, err := parser.AddCommand("dump",
+		"dump data structures",
+		"The dump command dumps data structures from a repository as JSON documents",
+		&CmdDump{global: &globalOpts})
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (cmd CmdDump) Usage() string {
+	return "[indexes|snapshots|trees|all]"
+}
+
+func prettyPrintJSON(wr io.Writer, item interface{}) error {
+	buf, err := json.MarshalIndent(item, "", "  ")
+	if err != nil {
+		return err
+	}
+
+	_, err = wr.Write(append(buf, '\n'))
+	return err
+}
+
+func printSnapshots(repo *repository.Repository, wr io.Writer) error {
+	done := make(chan struct{})
+	defer close(done)
+
+	for id := range repo.List(backend.Snapshot, done) {
+		snapshot, err := restic.LoadSnapshot(repo, id)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "LoadSnapshot(%v): %v", id.Str(), err)
+			continue
+		}
+
+		fmt.Fprintf(wr, "snapshot_id: %v\n", id)
+
+		err = prettyPrintJSON(wr, snapshot)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func printTrees(repo *repository.Repository, wr io.Writer) error {
+	done := make(chan struct{})
+	defer close(done)
+
+	trees := []backend.ID{}
+
+	for _, idx := range repo.Index().All() {
+		for blob := range idx.Each(nil) {
+			if blob.Type != pack.Tree {
+				continue
+			}
+
+			trees = append(trees, blob.ID)
+		}
+	}
+
+	for _, id := range trees {
+		tree, err := restic.LoadTree(repo, id)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "LoadTree(%v): %v", id.Str(), err)
+			continue
+		}
+
+		fmt.Fprintf(wr, "tree_id: %v\n", id)
+
+		prettyPrintJSON(wr, tree)
+	}
+
+	return nil
+}
+
+func (cmd CmdDump) DumpIndexes() error {
+	done := make(chan struct{})
+	defer close(done)
+
+	for id := range cmd.repo.List(backend.Index, done) {
+		fmt.Printf("index_id: %v\n", id)
+
+		idx, err := repository.LoadIndex(cmd.repo, id.String())
+		if err != nil {
+			return err
+		}
+
+		err = idx.Dump(os.Stdout)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (cmd CmdDump) Execute(args []string) error {
+	if len(args) != 1 {
+		return fmt.Errorf("type not specified, Usage: %s", cmd.Usage())
+	}
+
+	repo, err := cmd.global.OpenRepository()
+	if err != nil {
+		return err
+	}
+	cmd.repo = repo
+
+	lock, err := lockRepo(repo)
+	defer unlockRepo(lock)
+	if err != nil {
+		return err
+	}
+
+	err = repo.LoadIndex()
+	if err != nil {
+		return err
+	}
+
+	tpe := args[0]
+
+	switch tpe {
+	case "indexes":
+		return cmd.DumpIndexes()
+	case "snapshots":
+		return printSnapshots(repo, os.Stdout)
+	case "trees":
+		return printTrees(repo, os.Stdout)
+	case "all":
+		fmt.Printf("snapshots:\n")
+		err := printSnapshots(repo, os.Stdout)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("\ntrees:\n")
+
+		err = printTrees(repo, os.Stdout)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("\nindexes:\n")
+		err = cmd.DumpIndexes()
+		if err != nil {
+			return err
+		}
+
+		return nil
+	default:
+		return errors.Errorf("no such type %q", tpe)
+	}
+}

src/cmds/restic/cmd_find.go

@@ -0,0 +1,197 @@
+package main
+
+import (
+	"fmt"
+	"path/filepath"
+	"time"
+
+	"restic"
+	"restic/backend"
+	"restic/debug"
+	"restic/repository"
+)
+
+type findResult struct {
+	node *restic.Node
+	path string
+}
+
+type CmdFind struct {
+	Oldest   string `short:"o" long:"oldest" description:"Oldest modification date/time"`
+	Newest   string `short:"n" long:"newest" description:"Newest modification date/time"`
+	Snapshot string `short:"s" long:"snapshot" description:"Snapshot ID to search in"`
+
+	oldest, newest time.Time
+	pattern        string
+	global         *GlobalOptions
+}
+
+var timeFormats = []string{
+	"2006-01-02",
+	"2006-01-02 15:04",
+	"2006-01-02 15:04:05",
+	"2006-01-02 15:04:05 -0700",
+	"2006-01-02 15:04:05 MST",
+	"02.01.2006",
+	"02.01.2006 15:04",
+	"02.01.2006 15:04:05",
+	"02.01.2006 15:04:05 -0700",
+	"02.01.2006 15:04:05 MST",
+	"Mon Jan 2 15:04:05 -0700 MST 2006",
+}
+
+func init() {
+	_, err := parser.AddCommand("find",
+		"find a file/directory",
+		"The find command searches for files or directories in snapshots",
+		&CmdFind{global: &globalOpts})
+	if err != nil {
+		panic(err)
+	}
+}
+
+func parseTime(str string) (time.Time, error) {
+	for _, fmt := range timeFormats {
+		if t, err := time.ParseInLocation(fmt, str, time.Local); err == nil {
+			return t, nil
+		}
+	}
+
+	return time.Time{}, fmt.Errorf("unable to parse time: %q", str)
+}
+
+func (c CmdFind) findInTree(repo *repository.Repository, id backend.ID, path string) ([]findResult, error) {
+	debug.Log("restic.find", "checking tree %v\n", id)
+	tree, err := restic.LoadTree(repo, id)
+	if err != nil {
+		return nil, err
+	}
+
+	results := []findResult{}
+	for _, node := range tree.Nodes {
+		debug.Log("restic.find", "  testing entry %q\n", node.Name)
+
+		m, err := filepath.Match(c.pattern, node.Name)
+		if err != nil {
+			return nil, err
+		}
+
+		if m {
+			debug.Log("restic.find", "    pattern matches\n")
+			if !c.oldest.IsZero() && node.ModTime.Before(c.oldest) {
+				debug.Log("restic.find", "    ModTime is older than %s\n", c.oldest)
+				continue
+			}
+
+			if !c.newest.IsZero() && node.ModTime.After(c.newest) {
+				debug.Log("restic.find", "    ModTime is newer than %s\n", c.newest)
+				continue
+			}
+
+			results = append(results, findResult{node: node, path: path})
+		} else {
+			debug.Log("restic.find", "    pattern does not match\n")
+		}
+
+		if node.Type == "dir" {
+			subdirResults, err := c.findInTree(repo, *node.Subtree, filepath.Join(path, node.Name))
+			if err != nil {
+				return nil, err
+			}
+
+			results = append(results, subdirResults...)
+		}
+	}
+
+	return results, nil
+}
+
+func (c CmdFind) findInSnapshot(repo *repository.Repository, id backend.ID) error {
+	debug.Log("restic.find", "searching in snapshot %s\n  for entries within [%s %s]", id.Str(), c.oldest, c.newest)
+
+	sn, err := restic.LoadSnapshot(repo, id)
+	if err != nil {
+		return err
+	}
+
+	results, err := c.findInTree(repo, *sn.Tree, "")
+	if err != nil {
+		return err
+	}
+
+	if len(results) == 0 {
+		return nil
+	}
+	c.global.Verbosef("found %d matching entries in snapshot %s\n", len(results), id)
+	for _, res := range results {
+		res.node.Name = filepath.Join(res.path, res.node.Name)
+		c.global.Printf("  %s\n", res.node)
+	}
+
+	return nil
+}
+
+func (CmdFind) Usage() string {
+	return "[find-OPTIONS] PATTERN"
+}
+
+func (c CmdFind) Execute(args []string) error {
+	if len(args) != 1 {
+		return fmt.Errorf("wrong number of arguments, Usage: %s", c.Usage())
+	}
+
+	var err error
+
+	if c.Oldest != "" {
+		c.oldest, err = parseTime(c.Oldest)
+		if err != nil {
+			return err
+		}
+	}
+
+	if c.Newest != "" {
+		c.newest, err = parseTime(c.Newest)
+		if err != nil {
+			return err
+		}
+	}
+
+	repo, err := c.global.OpenRepository()
+	if err != nil {
+		return err
+	}
+
+	lock, err := lockRepo(repo)
+	defer unlockRepo(lock)
+	if err != nil {
+		return err
+	}
+
+	err = repo.LoadIndex()
+	if err != nil {
+		return err
+	}
+
+	c.pattern = args[0]
+
+	if c.Snapshot != "" {
+		snapshotID, err := restic.FindSnapshot(repo, c.Snapshot)
+		if err != nil {
+			return fmt.Errorf("invalid id %q: %v", args[1], err)
+		}
+
+		return c.findInSnapshot(repo, snapshotID)
+	}
+
+	done := make(chan struct{})
+	defer close(done)
+	for snapshotID := range repo.List(backend.Snapshot, done) {
+		err := c.findInSnapshot(repo, snapshotID)
+
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

src/cmds/restic/cmd_init.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"errors"
+
+	"restic/repository"
+)
+
+type CmdInit struct {
+	global *GlobalOptions
+}
+
+func (cmd CmdInit) Execute(args []string) error {
+	if cmd.global.Repo == "" {
+		return errors.New("Please specify repository location (-r)")
+	}
+
+	be, err := create(cmd.global.Repo)
+	if err != nil {
+		cmd.global.Exitf(1, "creating backend at %s failed: %v\n", cmd.global.Repo, err)
+	}
+
+	if cmd.global.password == "" {
+		cmd.global.password = cmd.global.ReadPasswordTwice(
+			"enter password for new backend: ",
+			"enter password again: ")
+	}
+
+	s := repository.New(be)
+	err = s.Init(cmd.global.password)
+	if err != nil {
+		cmd.global.Exitf(1, "creating key in backend at %s failed: %v\n", cmd.global.Repo, err)
+	}
+
+	cmd.global.Verbosef("created restic backend %v at %s\n", s.Config.ID[:10], cmd.global.Repo)
+	cmd.global.Verbosef("\n")
+	cmd.global.Verbosef("Please note that knowledge of your password is required to access\n")
+	cmd.global.Verbosef("the repository. Losing your password means that your data is\n")
+	cmd.global.Verbosef("irrecoverably lost.\n")
+
+	return nil
+}
+
+func init() {
+	_, err := parser.AddCommand("init",
+		"create repository",
+		"The init command creates a new repository",
+		&CmdInit{global: &globalOpts})
+	if err != nil {
+		panic(err)
+	}
+}

src/cmds/restic/cmd_key.go

@@ -0,0 +1,165 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+
+	"restic/backend"
+	"restic/repository"
+)
+
+type CmdKey struct {
+	global      *GlobalOptions
+	newPassword string
+}
+
+func init() {
+	_, err := parser.AddCommand("key",
+		"manage keys",
+		"The key command manages keys (passwords) of a repository",
+		&CmdKey{global: &globalOpts})
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (cmd CmdKey) listKeys(s *repository.Repository) error {
+	tab := NewTable()
+	tab.Header = fmt.Sprintf(" %-10s  %-10s  %-10s  %s", "ID", "User", "Host", "Created")
+	tab.RowFormat = "%s%-10s  %-10s  %-10s  %s"
+
+	plen, err := s.PrefixLength(backend.Key)
+	if err != nil {
+		return err
+	}
+
+	done := make(chan struct{})
+	defer close(done)
+
+	for id := range s.List(backend.Key, done) {
+		k, err := repository.LoadKey(s, id.String())
+		if err != nil {
+			cmd.global.Warnf("LoadKey() failed: %v\n", err)
+			continue
+		}
+
+		var current string
+		if id.String() == s.KeyName() {
+			current = "*"
+		} else {
+			current = " "
+		}
+		tab.Rows = append(tab.Rows, []interface{}{current, id.String()[:plen],
+			k.Username, k.Hostname, k.Created.Format(TimeFormat)})
+	}
+
+	return tab.Write(cmd.global.stdout)
+}
+
+func (cmd CmdKey) getNewPassword() string {
+	if cmd.newPassword != "" {
+		return cmd.newPassword
+	}
+
+	return cmd.global.ReadPasswordTwice(
+		"enter password for new key: ",
+		"enter password again: ")
+}
+
+func (cmd CmdKey) addKey(repo *repository.Repository) error {
+	id, err := repository.AddKey(repo, cmd.getNewPassword(), repo.Key())
+	if err != nil {
+		return fmt.Errorf("creating new key failed: %v\n", err)
+	}
+
+	cmd.global.Verbosef("saved new key as %s\n", id)
+
+	return nil
+}
+
+func (cmd CmdKey) deleteKey(repo *repository.Repository, name string) error {
+	if name == repo.KeyName() {
+		return errors.New("refusing to remove key currently used to access repository")
+	}
+
+	err := repo.Backend().Remove(backend.Key, name)
+	if err != nil {
+		return err
+	}
+
+	cmd.global.Verbosef("removed key %v\n", name)
+	return nil
+}
+
+func (cmd CmdKey) changePassword(repo *repository.Repository) error {
+	id, err := repository.AddKey(repo, cmd.getNewPassword(), repo.Key())
+	if err != nil {
+		return fmt.Errorf("creating new key failed: %v\n", err)
+	}
+
+	err = repo.Backend().Remove(backend.Key, repo.KeyName())
+	if err != nil {
+		return err
+	}
+
+	cmd.global.Verbosef("saved new key as %s\n", id)
+
+	return nil
+}
+
+func (cmd CmdKey) Usage() string {
+	return "[list|add|rm|passwd] [ID]"
+}
+
+func (cmd CmdKey) Execute(args []string) error {
+	if len(args) < 1 || (args[0] == "rm" && len(args) != 2) {
+		return fmt.Errorf("wrong number of arguments, Usage: %s", cmd.Usage())
+	}
+
+	repo, err := cmd.global.OpenRepository()
+	if err != nil {
+		return err
+	}
+
+	switch args[0] {
+	case "list":
+		lock, err := lockRepo(repo)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+
+		return cmd.listKeys(repo)
+	case "add":
+		lock, err := lockRepo(repo)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+
+		return cmd.addKey(repo)
+	case "rm":
+		lock, err := lockRepoExclusive(repo)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+
+		id, err := backend.Find(repo.Backend(), backend.Key, args[1])
+		if err != nil {
+			return err
+		}
+
+		return cmd.deleteKey(repo, id)
+	case "passwd":
+		lock, err := lockRepoExclusive(repo)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+
+		return cmd.changePassword(repo)
+	}
+
+	return nil
+}

src/cmds/restic/cmd_list.go

@@ -0,0 +1,80 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+
+	"restic/backend"
+)
+
+type CmdList struct {
+	global *GlobalOptions
+}
+
+func init() {
+	_, err := parser.AddCommand("list",
+		"lists data",
+		"The list command lists structures or data of a repository",
+		&CmdList{global: &globalOpts})
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (cmd CmdList) Usage() string {
+	return "[blobs|packs|index|snapshots|keys|locks]"
+}
+
+func (cmd CmdList) Execute(args []string) error {
+	if len(args) != 1 {
+		return fmt.Errorf("type not specified, Usage: %s", cmd.Usage())
+	}
+
+	repo, err := cmd.global.OpenRepository()
+	if err != nil {
+		return err
+	}
+
+	if !cmd.global.NoLock {
+		lock, err := lockRepo(repo)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+	}
+
+	var t backend.Type
+	switch args[0] {
+	case "blobs":
+		err = repo.LoadIndex()
+		if err != nil {
+			return err
+		}
+
+		for _, idx := range repo.Index().All() {
+			for blob := range idx.Each(nil) {
+				cmd.global.Printf("%s\n", blob.ID)
+			}
+		}
+
+		return nil
+	case "packs":
+		t = backend.Data
+	case "index":
+		t = backend.Index
+	case "snapshots":
+		t = backend.Snapshot
+	case "keys":
+		t = backend.Key
+	case "locks":
+		t = backend.Lock
+	default:
+		return errors.New("invalid type")
+	}
+
+	for id := range repo.List(t, nil) {
+		cmd.global.Printf("%s\n", id)
+	}
+
+	return nil
+}

src/cmds/restic/cmd_ls.go

@@ -0,0 +1,101 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"restic"
+	"restic/backend"
+	"restic/repository"
+)
+
+type CmdLs struct {
+	Long bool `short:"l" long:"long" description:"Use a long listing format showing size and mode"`
+
+	global *GlobalOptions
+}
+
+func init() {
+	_, err := parser.AddCommand("ls",
+		"list files",
+		"The ls command lists all files and directories in a snapshot",
+		&CmdLs{global: &globalOpts})
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (cmd CmdLs) printNode(prefix string, n *restic.Node) string {
+	if !cmd.Long {
+		return filepath.Join(prefix, n.Name)
+	}
+
+	switch n.Type {
+	case "file":
+		return fmt.Sprintf("%s %5d %5d %6d %s %s",
+			n.Mode, n.UID, n.GID, n.Size, n.ModTime, filepath.Join(prefix, n.Name))
+	case "dir":
+		return fmt.Sprintf("%s %5d %5d %6d %s %s",
+			n.Mode|os.ModeDir, n.UID, n.GID, n.Size, n.ModTime, filepath.Join(prefix, n.Name))
+	case "symlink":
+		return fmt.Sprintf("%s %5d %5d %6d %s %s -> %s",
+			n.Mode|os.ModeSymlink, n.UID, n.GID, n.Size, n.ModTime, filepath.Join(prefix, n.Name), n.LinkTarget)
+	default:
+		return fmt.Sprintf("<Node(%s) %s>", n.Type, n.Name)
+	}
+}
+
+func (cmd CmdLs) printTree(prefix string, repo *repository.Repository, id backend.ID) error {
+	tree, err := restic.LoadTree(repo, id)
+	if err != nil {
+		return err
+	}
+
+	for _, entry := range tree.Nodes {
+		cmd.global.Printf(cmd.printNode(prefix, entry) + "\n")
+
+		if entry.Type == "dir" && entry.Subtree != nil {
+			err = cmd.printTree(filepath.Join(prefix, entry.Name), repo, *entry.Subtree)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (cmd CmdLs) Usage() string {
+	return "snapshot-ID [DIR]"
+}
+
+func (cmd CmdLs) Execute(args []string) error {
+	if len(args) < 1 || len(args) > 2 {
+		return fmt.Errorf("wrong number of arguments, Usage: %s", cmd.Usage())
+	}
+
+	repo, err := cmd.global.OpenRepository()
+	if err != nil {
+		return err
+	}
+
+	err = repo.LoadIndex()
+	if err != nil {
+		return err
+	}
+
+	id, err := restic.FindSnapshot(repo, args[0])
+	if err != nil {
+		return err
+	}
+
+	sn, err := restic.LoadSnapshot(repo, id)
+	if err != nil {
+		return err
+	}
+
+	cmd.global.Verbosef("snapshot of %v at %s:\n", sn.Paths, sn.Time)
+
+	return cmd.printTree("", repo, *sn.Tree)
+}

src/cmds/restic/cmd_mount.go

@@ -0,0 +1,107 @@
+// +build !openbsd
+// +build !windows
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"restic/fuse"
+
+	systemFuse "bazil.org/fuse"
+	"bazil.org/fuse/fs"
+)
+
+type CmdMount struct {
+	Root bool `long:"owner-root" description:"use 'root' as the owner of files and dirs" default:"false"`
+
+	global *GlobalOptions
+	ready  chan struct{}
+	done   chan struct{}
+}
+
+func init() {
+	_, err := parser.AddCommand("mount",
+		"mount a repository",
+		"The mount command mounts a repository read-only to a given directory",
+		&CmdMount{
+			global: &globalOpts,
+			ready:  make(chan struct{}, 1),
+			done:   make(chan struct{}),
+		})
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (cmd CmdMount) Usage() string {
+	return "MOUNTPOINT"
+}
+
+func (cmd CmdMount) Execute(args []string) error {
+	if len(args) == 0 {
+		return fmt.Errorf("wrong number of parameters, Usage: %s", cmd.Usage())
+	}
+
+	repo, err := cmd.global.OpenRepository()
+	if err != nil {
+		return err
+	}
+
+	err = repo.LoadIndex()
+	if err != nil {
+		return err
+	}
+
+	mountpoint := args[0]
+	if _, err := os.Stat(mountpoint); os.IsNotExist(err) {
+		cmd.global.Verbosef("Mountpoint %s doesn't exist, creating it\n", mountpoint)
+		err = os.Mkdir(mountpoint, os.ModeDir|0700)
+		if err != nil {
+			return err
+		}
+	}
+	c, err := systemFuse.Mount(
+		mountpoint,
+		systemFuse.ReadOnly(),
+		systemFuse.FSName("restic"),
+	)
+	if err != nil {
+		return err
+	}
+
+	root := fs.Tree{}
+	root.Add("snapshots", fuse.NewSnapshotsDir(repo, cmd.Root))
+
+	cmd.global.Printf("Now serving %s at %s\n", repo.Backend().Location(), mountpoint)
+	cmd.global.Printf("Don't forget to umount after quitting!\n")
+
+	AddCleanupHandler(func() error {
+		return systemFuse.Unmount(mountpoint)
+	})
+
+	cmd.ready <- struct{}{}
+
+	errServe := make(chan error)
+	go func() {
+		err = fs.Serve(c, &root)
+		if err != nil {
+			errServe <- err
+		}
+
+		<-c.Ready
+		errServe <- c.MountError
+	}()
+
+	select {
+	case err := <-errServe:
+		return err
+	case <-cmd.done:
+		err := c.Close()
+		if err != nil {
+			cmd.global.Printf("Error closing fuse connection: %s\n", err)
+		}
+		return systemFuse.Unmount(mountpoint)
+	}
+}

src/cmds/restic/cmd_optimize.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+
+	"restic/backend"
+	"restic/checker"
+)
+
+type CmdOptimize struct {
+	global *GlobalOptions
+}
+
+func init() {
+	_, err := parser.AddCommand("optimize",
+		"optimize the repository",
+		"The optimize command reorganizes the repository and removes uneeded data",
+		&CmdOptimize{global: &globalOpts})
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (cmd CmdOptimize) Usage() string {
+	return "[optimize-options]"
+}
+
+func (cmd CmdOptimize) Execute(args []string) error {
+	if len(args) != 0 {
+		return errors.New("optimize has no arguments")
+	}
+
+	repo, err := cmd.global.OpenRepository()
+	if err != nil {
+		return err
+	}
+
+	cmd.global.Verbosef("Create exclusive lock for repository\n")
+	lock, err := lockRepoExclusive(repo)
+	defer unlockRepo(lock)
+	if err != nil {
+		return err
+	}
+
+	chkr := checker.New(repo)
+
+	cmd.global.Verbosef("Load indexes\n")
+	_, errs := chkr.LoadIndex()
+
+	if len(errs) > 0 {
+		for _, err := range errs {
+			cmd.global.Warnf("error: %v\n", err)
+		}
+		return fmt.Errorf("LoadIndex returned errors")
+	}
+
+	done := make(chan struct{})
+	errChan := make(chan error)
+	go chkr.Structure(errChan, done)
+
+	for err := range errChan {
+		if e, ok := err.(checker.TreeError); ok {
+			cmd.global.Warnf("error for tree %v:\n", e.ID.Str())
+			for _, treeErr := range e.Errors {
+				cmd.global.Warnf("  %v\n", treeErr)
+			}
+		} else {
+			cmd.global.Warnf("error: %v\n", err)
+		}
+	}
+
+	unusedBlobs := backend.NewIDSet(chkr.UnusedBlobs()...)
+	cmd.global.Verbosef("%d unused blobs found, repacking...\n", len(unusedBlobs))
+
+	repacker := checker.NewRepacker(repo, unusedBlobs)
+	err = repacker.Repack()
+	if err != nil {
+		return err
+	}
+
+	cmd.global.Verbosef("repacking done\n")
+	return nil
+}

src/cmds/restic/cmd_rebuild_index.go

@@ -0,0 +1,204 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+
+	"restic/backend"
+	"restic/debug"
+	"restic/pack"
+	"restic/repository"
+)
+
+type CmdRebuildIndex struct {
+	global *GlobalOptions
+
+	repo *repository.Repository
+}
+
+func init() {
+	_, err := parser.AddCommand("rebuild-index",
+		"rebuild the index",
+		"The rebuild-index command builds a new index",
+		&CmdRebuildIndex{global: &globalOpts})
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (cmd CmdRebuildIndex) storeIndex(index *repository.Index) (*repository.Index, error) {
+	debug.Log("RebuildIndex.RebuildIndex", "saving index")
+
+	cmd.global.Printf("  saving new index\n")
+	id, err := repository.SaveIndex(cmd.repo, index)
+	if err != nil {
+		debug.Log("RebuildIndex.RebuildIndex", "error saving index: %v", err)
+		return nil, err
+	}
+
+	debug.Log("RebuildIndex.RebuildIndex", "index saved as %v", id.Str())
+	index = repository.NewIndex()
+
+	return index, nil
+}
+
+func (cmd CmdRebuildIndex) RebuildIndex() error {
+	debug.Log("RebuildIndex.RebuildIndex", "start")
+
+	done := make(chan struct{})
+	defer close(done)
+
+	indexIDs := backend.NewIDSet()
+	for id := range cmd.repo.List(backend.Index, done) {
+		indexIDs.Insert(id)
+	}
+
+	cmd.global.Printf("rebuilding index from %d indexes\n", len(indexIDs))
+
+	debug.Log("RebuildIndex.RebuildIndex", "found %v indexes", len(indexIDs))
+
+	combinedIndex := repository.NewIndex()
+	packsDone := backend.NewIDSet()
+
+	type Blob struct {
+		id  backend.ID
+		tpe pack.BlobType
+	}
+	blobsDone := make(map[Blob]struct{})
+
+	i := 0
+	for indexID := range indexIDs {
+		cmd.global.Printf("  loading index %v\n", i)
+
+		debug.Log("RebuildIndex.RebuildIndex", "load index %v", indexID.Str())
+		idx, err := repository.LoadIndex(cmd.repo, indexID.String())
+		if err != nil {
+			return err
+		}
+
+		debug.Log("RebuildIndex.RebuildIndex", "adding blobs from index %v", indexID.Str())
+
+		for packedBlob := range idx.Each(done) {
+			packsDone.Insert(packedBlob.PackID)
+			b := Blob{
+				id:  packedBlob.ID,
+				tpe: packedBlob.Type,
+			}
+			if _, ok := blobsDone[b]; ok {
+				continue
+			}
+
+			blobsDone[b] = struct{}{}
+			combinedIndex.Store(packedBlob)
+		}
+
+		combinedIndex.AddToSupersedes(indexID)
+
+		if repository.IndexFull(combinedIndex) {
+			combinedIndex, err = cmd.storeIndex(combinedIndex)
+			if err != nil {
+				return err
+			}
+		}
+
+		i++
+	}
+
+	var err error
+	if combinedIndex.Length() > 0 {
+		combinedIndex, err = cmd.storeIndex(combinedIndex)
+		if err != nil {
+			return err
+		}
+	}
+
+	cmd.global.Printf("removing %d old indexes\n", len(indexIDs))
+	for id := range indexIDs {
+		debug.Log("RebuildIndex.RebuildIndex", "remove index %v", id.Str())
+
+		err := cmd.repo.Backend().Remove(backend.Index, id.String())
+		if err != nil {
+			debug.Log("RebuildIndex.RebuildIndex", "error removing index %v: %v", id.Str(), err)
+			return err
+		}
+	}
+
+	cmd.global.Printf("checking for additional packs\n")
+	newPacks := 0
+	var buf []byte
+	for packID := range cmd.repo.List(backend.Data, done) {
+		if packsDone.Has(packID) {
+			continue
+		}
+
+		debug.Log("RebuildIndex.RebuildIndex", "pack %v not indexed", packID.Str())
+		newPacks++
+
+		var err error
+
+		h := backend.Handle{Type: backend.Data, Name: packID.String()}
+		buf, err = backend.LoadAll(cmd.repo.Backend(), h, buf)
+		if err != nil {
+			debug.Log("RebuildIndex.RebuildIndex", "error while loading pack %v", packID.Str())
+			return fmt.Errorf("error while loading pack %v: %v", packID.Str(), err)
+		}
+
+		hash := backend.Hash(buf)
+		if !hash.Equal(packID) {
+			debug.Log("RebuildIndex.RebuildIndex", "Pack ID does not match, want %v, got %v", packID.Str(), hash.Str())
+			return fmt.Errorf("Pack ID does not match, want %v, got %v", packID.Str(), hash.Str())
+		}
+
+		up, err := pack.NewUnpacker(cmd.repo.Key(), bytes.NewReader(buf))
+		if err != nil {
+			debug.Log("RebuildIndex.RebuildIndex", "error while unpacking pack %v", packID.Str())
+			return err
+		}
+
+		for _, blob := range up.Entries {
+			debug.Log("RebuildIndex.RebuildIndex", "pack %v: blob %v", packID.Str(), blob)
+			combinedIndex.Store(repository.PackedBlob{
+				Type:   blob.Type,
+				ID:     blob.ID,
+				PackID: packID,
+				Offset: blob.Offset,
+				Length: blob.Length,
+			})
+		}
+
+		if repository.IndexFull(combinedIndex) {
+			combinedIndex, err = cmd.storeIndex(combinedIndex)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	if combinedIndex.Length() > 0 {
+		combinedIndex, err = cmd.storeIndex(combinedIndex)
+		if err != nil {
+			return err
+		}
+	}
+
+	cmd.global.Printf("added %d packs to the index\n", newPacks)
+
+	debug.Log("RebuildIndex.RebuildIndex", "done")
+	return nil
+}
+
+func (cmd CmdRebuildIndex) Execute(args []string) error {
+	repo, err := cmd.global.OpenRepository()
+	if err != nil {
+		return err
+	}
+	cmd.repo = repo
+
+	lock, err := lockRepoExclusive(repo)
+	defer unlockRepo(lock)
+	if err != nil {
+		return err
+	}
+
+	return cmd.RebuildIndex()
+}

src/cmds/restic/cmd_restore.go

@@ -0,0 +1,116 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+
+	"restic"
+	"restic/debug"
+	"restic/filter"
+)
+
+type CmdRestore struct {
+	Exclude []string `short:"e" long:"exclude" description:"Exclude a pattern (can be specified multiple times)"`
+	Include []string `short:"i" long:"include" description:"Include a pattern, exclude everything else (can be specified multiple times)"`
+	Target  string   `short:"t" long:"target"  description:"Directory to restore to"`
+
+	global *GlobalOptions
+}
+
+func init() {
+	_, err := parser.AddCommand("restore",
+		"restore a snapshot",
+		"The restore command restores a snapshot to a directory",
+		&CmdRestore{global: &globalOpts})
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (cmd CmdRestore) Usage() string {
+	return "snapshot-ID"
+}
+
+func (cmd CmdRestore) Execute(args []string) error {
+	if len(args) != 1 {
+		return fmt.Errorf("wrong number of arguments, Usage: %s", cmd.Usage())
+	}
+
+	if cmd.Target == "" {
+		return errors.New("please specify a directory to restore to (--target)")
+	}
+
+	if len(cmd.Exclude) > 0 && len(cmd.Include) > 0 {
+		return errors.New("exclude and include patterns are mutually exclusive")
+	}
+
+	snapshotIDString := args[0]
+
+	debug.Log("restore", "restore %v to %v", snapshotIDString, cmd.Target)
+
+	repo, err := cmd.global.OpenRepository()
+	if err != nil {
+		return err
+	}
+
+	if !cmd.global.NoLock {
+		lock, err := lockRepo(repo)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+	}
+
+	err = repo.LoadIndex()
+	if err != nil {
+		return err
+	}
+
+	id, err := restic.FindSnapshot(repo, snapshotIDString)
+	if err != nil {
+		cmd.global.Exitf(1, "invalid id %q: %v", snapshotIDString, err)
+	}
+
+	res, err := restic.NewRestorer(repo, id)
+	if err != nil {
+		cmd.global.Exitf(2, "creating restorer failed: %v\n", err)
+	}
+
+	res.Error = func(dir string, node *restic.Node, err error) error {
+		cmd.global.Warnf("error for %s: %+v\n", dir, err)
+		return nil
+	}
+
+	selectExcludeFilter := func(item string, dstpath string, node *restic.Node) bool {
+		matched, err := filter.List(cmd.Exclude, item)
+		if err != nil {
+			cmd.global.Warnf("error for exclude pattern: %v", err)
+		}
+
+		return !matched
+	}
+
+	selectIncludeFilter := func(item string, dstpath string, node *restic.Node) bool {
+		matched, err := filter.List(cmd.Include, item)
+		if err != nil {
+			cmd.global.Warnf("error for include pattern: %v", err)
+		}
+
+		return matched
+	}
+
+	if len(cmd.Exclude) > 0 {
+		res.SelectFilter = selectExcludeFilter
+	} else if len(cmd.Include) > 0 {
+		res.SelectFilter = selectIncludeFilter
+	}
+
+	cmd.global.Verbosef("restoring %s to %s\n", res.Snapshot(), cmd.Target)
+
+	err = res.RestoreTo(cmd.Target)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

src/cmds/restic/cmd_snapshots.go

@@ -0,0 +1,134 @@
+package main
+
+import (
+	"encoding/hex"
+	"fmt"
+	"io"
+	"os"
+	"sort"
+	"strings"
+
+	"restic"
+	"restic/backend"
+)
+
+type Table struct {
+	Header string
+	Rows   [][]interface{}
+
+	RowFormat string
+}
+
+func NewTable() Table {
+	return Table{
+		Rows: [][]interface{}{},
+	}
+}
+
+func (t Table) Write(w io.Writer) error {
+	_, err := fmt.Fprintln(w, t.Header)
+	if err != nil {
+		return err
+	}
+	_, err = fmt.Fprintln(w, strings.Repeat("-", 70))
+	if err != nil {
+		return err
+	}
+
+	for _, row := range t.Rows {
+		_, err = fmt.Fprintf(w, t.RowFormat+"\n", row...)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+const TimeFormat = "2006-01-02 15:04:05"
+
+type CmdSnapshots struct {
+	global *GlobalOptions
+}
+
+func init() {
+	_, err := parser.AddCommand("snapshots",
+		"show snapshots",
+		"The snapshots command lists all snapshots stored in a repository",
+		&CmdSnapshots{global: &globalOpts})
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (cmd CmdSnapshots) Usage() string {
+	return ""
+}
+
+func (cmd CmdSnapshots) Execute(args []string) error {
+	if len(args) != 0 {
+		return fmt.Errorf("wrong number of arguments, usage: %s", cmd.Usage())
+	}
+
+	repo, err := cmd.global.OpenRepository()
+	if err != nil {
+		return err
+	}
+
+	lock, err := lockRepo(repo)
+	defer unlockRepo(lock)
+	if err != nil {
+		return err
+	}
+
+	tab := NewTable()
+	tab.Header = fmt.Sprintf("%-8s  %-19s  %-10s  %s", "ID", "Date", "Source", "Directory")
+	tab.RowFormat = "%-8s  %-19s  %-10s  %s"
+
+	done := make(chan struct{})
+	defer close(done)
+
+	list := []*restic.Snapshot{}
+	for id := range repo.List(backend.Snapshot, done) {
+		sn, err := restic.LoadSnapshot(repo, id)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "error loading snapshot %s: %v\n", id, err)
+			continue
+		}
+
+		pos := sort.Search(len(list), func(i int) bool {
+			return list[i].Time.After(sn.Time)
+		})
+
+		if pos < len(list) {
+			list = append(list, nil)
+			copy(list[pos+1:], list[pos:])
+			list[pos] = sn
+		} else {
+			list = append(list, sn)
+		}
+	}
+
+	plen, err := repo.PrefixLength(backend.Snapshot)
+	if err != nil {
+		return err
+	}
+
+	for _, sn := range list {
+		if len(sn.Paths) == 0 {
+			continue
+		}
+		id := sn.ID()
+		tab.Rows = append(tab.Rows, []interface{}{hex.EncodeToString(id[:plen/2]), sn.Time.Format(TimeFormat), sn.Hostname, sn.Paths[0]})
+
+		if len(sn.Paths) > 1 {
+			for _, path := range sn.Paths[1:] {
+				tab.Rows = append(tab.Rows, []interface{}{"", "", "", path})
+			}
+		}
+	}
+
+	tab.Write(os.Stdout)
+
+	return nil
+}

src/cmds/restic/cmd_unlock.go

@@ -0,0 +1,43 @@
+package main
+
+import "restic"
+
+type CmdUnlock struct {
+	RemoveAll bool `long:"remove-all" description:"Remove all locks, even stale ones"`
+
+	global *GlobalOptions
+}
+
+func init() {
+	_, err := parser.AddCommand("unlock",
+		"remove locks",
+		"The unlock command checks for stale locks and removes them",
+		&CmdUnlock{global: &globalOpts})
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (cmd CmdUnlock) Usage() string {
+	return "[unlock-options]"
+}
+
+func (cmd CmdUnlock) Execute(args []string) error {
+	repo, err := cmd.global.OpenRepository()
+	if err != nil {
+		return err
+	}
+
+	fn := restic.RemoveStaleLocks
+	if cmd.RemoveAll {
+		fn = restic.RemoveAllLocks
+	}
+
+	err = fn(repo)
+	if err != nil {
+		return err
+	}
+
+	cmd.global.Verbosef("successfully removed locks\n")
+	return nil
+}

src/cmds/restic/cmd_version.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"fmt"
+	"runtime"
+)
+
+type CmdVersion struct{}
+
+func init() {
+	_, err := parser.AddCommand("version",
+		"display version",
+		"The version command displays detailed information about the version",
+		&CmdVersion{})
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (cmd CmdVersion) Execute(args []string) error {
+	fmt.Printf("restic %s\ncompiled at %s with %v\n",
+		version, compiledAt, runtime.Version())
+
+	return nil
+}

src/cmds/restic/doc.go

@@ -0,0 +1,2 @@
+// This package contains the code for the restic executable.
+package main

src/cmds/restic/global.go

@@ -0,0 +1,293 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"syscall"
+
+	"restic/backend"
+	"restic/backend/local"
+	"restic/backend/rest"
+	"restic/backend/s3"
+	"restic/backend/sftp"
+	"restic/debug"
+	"restic/location"
+	"restic/repository"
+
+	"github.com/jessevdk/go-flags"
+	"golang.org/x/crypto/ssh/terminal"
+)
+
+var version = "compiled manually"
+var compiledAt = "unknown time"
+
+// GlobalOptions holds all those options that can be set for every command.
+type GlobalOptions struct {
+	Repo     string   `short:"r" long:"repo"                      description:"Repository directory to backup to/restore from"`
+	CacheDir string   `          long:"cache-dir"                 description:"Directory to use as a local cache"`
+	Quiet    bool     `short:"q" long:"quiet"     default:"false" description:"Do not output comprehensive progress report"`
+	NoLock   bool     `          long:"no-lock"   default:"false" description:"Do not lock the repo, this allows some operations on read-only repos."`
+	Options  []string `short:"o" long:"option"                    description:"Specify options in the form 'foo.key=value'"`
+
+	password string
+	stdout   io.Writer
+	stderr   io.Writer
+}
+
+func init() {
+	restoreTerminal()
+}
+
+// checkErrno returns nil when err is set to syscall.Errno(0), since this is no
+// error condition.
+func checkErrno(err error) error {
+	e, ok := err.(syscall.Errno)
+	if !ok {
+		return err
+	}
+
+	if e == 0 {
+		return nil
+	}
+
+	return err
+}
+
+// restoreTerminal installs a cleanup handler that restores the previous
+// terminal state on exit.
+func restoreTerminal() {
+	fd := int(os.Stdout.Fd())
+	if !terminal.IsTerminal(fd) {
+		return
+	}
+
+	state, err := terminal.GetState(fd)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to get terminal state: %v\n", err)
+		return
+	}
+
+	AddCleanupHandler(func() error {
+		err := checkErrno(terminal.Restore(fd, state))
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "unable to get restore terminal state: %#+v\n", err)
+		}
+		return err
+	})
+}
+
+var globalOpts = GlobalOptions{stdout: os.Stdout, stderr: os.Stderr}
+var parser = flags.NewParser(&globalOpts, flags.HelpFlag|flags.PassDoubleDash)
+
+// Printf writes the message to the configured stdout stream.
+func (o GlobalOptions) Printf(format string, args ...interface{}) {
+	_, err := fmt.Fprintf(o.stdout, format, args...)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to write to stdout: %v\n", err)
+		os.Exit(100)
+	}
+}
+
+// Verbosef calls Printf to write the message when the verbose flag is set.
+func (o GlobalOptions) Verbosef(format string, args ...interface{}) {
+	if o.Quiet {
+		return
+	}
+
+	o.Printf(format, args...)
+}
+
+// ShowProgress returns true iff the progress status should be written, i.e.
+// the quiet flag is not set and the output is a terminal.
+func (o GlobalOptions) ShowProgress() bool {
+	if o.Quiet {
+		return false
+	}
+
+	if !terminal.IsTerminal(int(os.Stdout.Fd())) {
+		return false
+	}
+
+	return true
+}
+
+// Warnf writes the message to the configured stderr stream.
+func (o GlobalOptions) Warnf(format string, args ...interface{}) {
+	_, err := fmt.Fprintf(o.stderr, format, args...)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to write to stderr: %v\n", err)
+		os.Exit(100)
+	}
+}
+
+// Exitf uses Warnf to write the message and then calls os.Exit(exitcode).
+func (o GlobalOptions) Exitf(exitcode int, format string, args ...interface{}) {
+	if format[len(format)-1] != '\n' {
+		format += "\n"
+	}
+
+	o.Warnf(format, args...)
+	os.Exit(exitcode)
+}
+
+// readPassword reads the password from the given reader directly.
+func readPassword(in io.Reader) (password string, err error) {
+	buf := make([]byte, 1000)
+	n, err := io.ReadFull(in, buf)
+	buf = buf[:n]
+
+	if err != nil && err != io.ErrUnexpectedEOF {
+		return "", err
+	}
+
+	return strings.TrimRight(string(buf), "\r\n"), nil
+}
+
+// readPasswordTerminal reads the password from the given reader which must be a
+// tty. Prompt is printed on the writer out before attempting to read the
+// password.
+func readPasswordTerminal(in *os.File, out io.Writer, prompt string) (password string, err error) {
+	fmt.Fprint(out, prompt)
+	buf, err := terminal.ReadPassword(int(in.Fd()))
+	fmt.Fprintln(out)
+	if err != nil {
+		return "", err
+	}
+
+	password = string(buf)
+	return password, nil
+}
+
+// ReadPassword reads the password from stdin.
+func (o GlobalOptions) ReadPassword(prompt string) string {
+	var (
+		password string
+		err      error
+	)
+
+	if terminal.IsTerminal(int(os.Stdin.Fd())) {
+		password, err = readPasswordTerminal(os.Stdin, os.Stderr, prompt)
+	} else {
+		password, err = readPassword(os.Stdin)
+	}
+
+	if err != nil {
+		o.Exitf(2, "unable to read password: %v", err)
+	}
+
+	if len(password) == 0 {
+		o.Exitf(1, "an empty password is not a password")
+	}
+
+	return password
+}
+
+// ReadPasswordTwice calls ReadPassword two times and returns an error when the
+// passwords don't match.
+func (o GlobalOptions) ReadPasswordTwice(prompt1, prompt2 string) string {
+	pw1 := o.ReadPassword(prompt1)
+	pw2 := o.ReadPassword(prompt2)
+	if pw1 != pw2 {
+		o.Exitf(1, "passwords do not match")
+	}
+
+	return pw1
+}
+
+// OpenRepository reads the password and opens the repository.
+func (o GlobalOptions) OpenRepository() (*repository.Repository, error) {
+	if o.Repo == "" {
+		return nil, errors.New("Please specify repository location (-r)")
+	}
+
+	be, err := open(o.Repo)
+	if err != nil {
+		return nil, err
+	}
+
+	s := repository.New(be)
+
+	if o.password == "" {
+		o.password = o.ReadPassword("enter password for repository: ")
+	}
+
+	err = s.SearchKey(o.password)
+	if err != nil {
+		return nil, fmt.Errorf("unable to open repo: %v", err)
+	}
+
+	return s, nil
+}
+
+// Open the backend specified by a location config.
+func open(s string) (backend.Backend, error) {
+	debug.Log("open", "parsing location %v", s)
+	loc, err := location.Parse(s)
+	if err != nil {
+		return nil, err
+	}
+
+	switch loc.Scheme {
+	case "local":
+		debug.Log("open", "opening local repository at %#v", loc.Config)
+		return local.Open(loc.Config.(string))
+	case "sftp":
+		debug.Log("open", "opening sftp repository at %#v", loc.Config)
+		return sftp.OpenWithConfig(loc.Config.(sftp.Config))
+	case "s3":
+		cfg := loc.Config.(s3.Config)
+		if cfg.KeyID == "" {
+			cfg.KeyID = os.Getenv("AWS_ACCESS_KEY_ID")
+
+		}
+		if cfg.Secret == "" {
+			cfg.Secret = os.Getenv("AWS_SECRET_ACCESS_KEY")
+		}
+
+		debug.Log("open", "opening s3 repository at %#v", cfg)
+		return s3.Open(cfg)
+	case "rest":
+		return rest.Open(loc.Config.(rest.Config))
+	}
+
+	debug.Log("open", "invalid repository location: %v", s)
+	return nil, fmt.Errorf("invalid scheme %q", loc.Scheme)
+}
+
+// Create the backend specified by URI.
+func create(s string) (backend.Backend, error) {
+	debug.Log("open", "parsing location %v", s)
+	loc, err := location.Parse(s)
+	if err != nil {
+		return nil, err
+	}
+
+	switch loc.Scheme {
+	case "local":
+		debug.Log("open", "create local repository at %#v", loc.Config)
+		return local.Create(loc.Config.(string))
+	case "sftp":
+		debug.Log("open", "create sftp repository at %#v", loc.Config)
+		return sftp.CreateWithConfig(loc.Config.(sftp.Config))
+	case "s3":
+		cfg := loc.Config.(s3.Config)
+		if cfg.KeyID == "" {
+			cfg.KeyID = os.Getenv("AWS_ACCESS_KEY_ID")
+
+		}
+		if cfg.Secret == "" {
+			cfg.Secret = os.Getenv("AWS_SECRET_ACCESS_KEY")
+		}
+
+		debug.Log("open", "create s3 repository at %#v", loc.Config)
+		return s3.Open(cfg)
+	case "rest":
+		return rest.Open(loc.Config.(rest.Config))
+	}
+
+	debug.Log("open", "invalid repository scheme: %v", s)
+	return nil, fmt.Errorf("invalid scheme %q", loc.Scheme)
+}

src/cmds/restic/integration_fuse_test.go

@@ -0,0 +1,162 @@
+// +build !openbsd
+// +build !windows
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"restic"
+	"restic/backend"
+	"restic/repository"
+	. "restic/test"
+)
+
+const (
+	mountWait       = 20
+	mountSleep      = 100 * time.Millisecond
+	mountTestSubdir = "snapshots"
+)
+
+// waitForMount blocks (max mountWait * mountSleep) until the subdir
+// "snapshots" appears in the dir.
+func waitForMount(dir string) error {
+	for i := 0; i < mountWait; i++ {
+		f, err := os.Open(dir)
+		if err != nil {
+			return err
+		}
+
+		names, err := f.Readdirnames(-1)
+		if err != nil {
+			return err
+		}
+
+		if err = f.Close(); err != nil {
+			return err
+		}
+
+		for _, name := range names {
+			if name == mountTestSubdir {
+				return nil
+			}
+		}
+
+		time.Sleep(mountSleep)
+	}
+
+	return fmt.Errorf("subdir %q of dir %s never appeared", mountTestSubdir, dir)
+}
+
+func cmdMount(t testing.TB, global GlobalOptions, dir string, ready, done chan struct{}) {
+	defer func() {
+		ready <- struct{}{}
+	}()
+
+	cmd := &CmdMount{global: &global, ready: ready, done: done}
+	OK(t, cmd.Execute([]string{dir}))
+	if TestCleanupTempDirs {
+		RemoveAll(t, dir)
+	}
+}
+
+func TestMount(t *testing.T) {
+	if !RunFuseTest {
+		t.Skip("Skipping fuse tests")
+	}
+
+	checkSnapshots := func(repo *repository.Repository, mountpoint string, snapshotIDs []backend.ID) {
+		snapshotsDir, err := os.Open(filepath.Join(mountpoint, "snapshots"))
+		OK(t, err)
+		namesInSnapshots, err := snapshotsDir.Readdirnames(-1)
+		OK(t, err)
+		Assert(t,
+			len(namesInSnapshots) == len(snapshotIDs),
+			"Invalid number of snapshots: expected %d, got %d", len(snapshotIDs), len(namesInSnapshots))
+
+		namesMap := make(map[string]bool)
+		for _, name := range namesInSnapshots {
+			namesMap[name] = false
+		}
+
+		for _, id := range snapshotIDs {
+			snapshot, err := restic.LoadSnapshot(repo, id)
+			OK(t, err)
+			_, ok := namesMap[snapshot.Time.Format(time.RFC3339)]
+			Assert(t, ok, "Snapshot %s isn't present in fuse dir", snapshot.Time.Format(time.RFC3339))
+			namesMap[snapshot.Time.Format(time.RFC3339)] = true
+		}
+		for name, present := range namesMap {
+			Assert(t, present, "Directory %s is present in fuse dir but is not a snapshot", name)
+		}
+		OK(t, snapshotsDir.Close())
+	}
+
+	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+		cmdInit(t, global)
+		repo, err := global.OpenRepository()
+		OK(t, err)
+
+		mountpoint, err := ioutil.TempDir(TestTempDir, "restic-test-mount-")
+		OK(t, err)
+
+		// We remove the mountpoint now to check that cmdMount creates it
+		RemoveAll(t, mountpoint)
+
+		ready := make(chan struct{}, 2)
+		done := make(chan struct{})
+		go cmdMount(t, global, mountpoint, ready, done)
+		<-ready
+		defer close(done)
+		OK(t, waitForMount(mountpoint))
+
+		mountpointDir, err := os.Open(mountpoint)
+		OK(t, err)
+		names, err := mountpointDir.Readdirnames(-1)
+		OK(t, err)
+		Assert(t, len(names) == 1 && names[0] == "snapshots", `The fuse virtual directory "snapshots" doesn't exist`)
+		OK(t, mountpointDir.Close())
+
+		checkSnapshots(repo, mountpoint, []backend.ID{})
+
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		fd, err := os.Open(datafile)
+		if os.IsNotExist(err) {
+			t.Skipf("unable to find data file %q, skipping", datafile)
+			return
+		}
+		OK(t, err)
+		OK(t, fd.Close())
+
+		SetupTarTestFixture(t, env.testdata, datafile)
+
+		// first backup
+		cmdBackup(t, global, []string{env.testdata}, nil)
+		snapshotIDs := cmdList(t, global, "snapshots")
+		Assert(t, len(snapshotIDs) == 1,
+			"expected one snapshot, got %v", snapshotIDs)
+
+		checkSnapshots(repo, mountpoint, snapshotIDs)
+
+		// second backup, implicit incremental
+		cmdBackup(t, global, []string{env.testdata}, nil)
+		snapshotIDs = cmdList(t, global, "snapshots")
+		Assert(t, len(snapshotIDs) == 2,
+			"expected two snapshots, got %v", snapshotIDs)
+
+		checkSnapshots(repo, mountpoint, snapshotIDs)
+
+		// third backup, explicit incremental
+		cmdBackup(t, global, []string{env.testdata}, &snapshotIDs[0])
+		snapshotIDs = cmdList(t, global, "snapshots")
+		Assert(t, len(snapshotIDs) == 3,
+			"expected three snapshots, got %v", snapshotIDs)
+
+		checkSnapshots(repo, mountpoint, snapshotIDs)
+	})
+}

src/cmds/restic/integration_helpers_test.go

@@ -0,0 +1,228 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	. "restic/test"
+)
+
+type dirEntry struct {
+	path string
+	fi   os.FileInfo
+}
+
+func walkDir(dir string) <-chan *dirEntry {
+	ch := make(chan *dirEntry, 100)
+
+	go func() {
+		err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "error: %v\n", err)
+				return nil
+			}
+
+			name, err := filepath.Rel(dir, path)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "error: %v\n", err)
+				return nil
+			}
+
+			ch <- &dirEntry{
+				path: name,
+				fi:   info,
+			}
+
+			return nil
+		})
+
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Walk() error: %v\n", err)
+		}
+
+		close(ch)
+	}()
+
+	// first element is root
+	_ = <-ch
+
+	return ch
+}
+
+func isSymlink(fi os.FileInfo) bool {
+	mode := fi.Mode() & (os.ModeType | os.ModeCharDevice)
+	return mode == os.ModeSymlink
+}
+
+func sameModTime(fi1, fi2 os.FileInfo) bool {
+	switch runtime.GOOS {
+	case "darwin", "freebsd", "openbsd":
+		if isSymlink(fi1) && isSymlink(fi2) {
+			return true
+		}
+	}
+
+	return fi1.ModTime() == fi2.ModTime()
+}
+
+// directoriesEqualContents checks if both directories contain exactly the same
+// contents.
+func directoriesEqualContents(dir1, dir2 string) bool {
+	ch1 := walkDir(dir1)
+	ch2 := walkDir(dir2)
+
+	changes := false
+
+	var a, b *dirEntry
+	for {
+		var ok bool
+
+		if ch1 != nil && a == nil {
+			a, ok = <-ch1
+			if !ok {
+				ch1 = nil
+			}
+		}
+
+		if ch2 != nil && b == nil {
+			b, ok = <-ch2
+			if !ok {
+				ch2 = nil
+			}
+		}
+
+		if ch1 == nil && ch2 == nil {
+			break
+		}
+
+		if ch1 == nil {
+			fmt.Printf("+%v\n", b.path)
+			changes = true
+		} else if ch2 == nil {
+			fmt.Printf("-%v\n", a.path)
+			changes = true
+		} else if !a.equals(b) {
+			if a.path < b.path {
+				fmt.Printf("-%v\n", a.path)
+				changes = true
+				a = nil
+				continue
+			} else if a.path > b.path {
+				fmt.Printf("+%v\n", b.path)
+				changes = true
+				b = nil
+				continue
+			} else {
+				fmt.Printf("%%%v\n", a.path)
+				changes = true
+			}
+		}
+
+		a, b = nil, nil
+	}
+
+	if changes {
+		return false
+	}
+
+	return true
+}
+
+type dirStat struct {
+	files, dirs, other uint
+	size               uint64
+}
+
+func isFile(fi os.FileInfo) bool {
+	return fi.Mode()&(os.ModeType|os.ModeCharDevice) == 0
+}
+
+// dirStats walks dir and collects stats.
+func dirStats(dir string) (stat dirStat) {
+	for entry := range walkDir(dir) {
+		if isFile(entry.fi) {
+			stat.files++
+			stat.size += uint64(entry.fi.Size())
+			continue
+		}
+
+		if entry.fi.IsDir() {
+			stat.dirs++
+			continue
+		}
+
+		stat.other++
+	}
+
+	return stat
+}
+
+type testEnvironment struct {
+	base, cache, repo, testdata string
+}
+
+func configureRestic(t testing.TB, cache, repo string) GlobalOptions {
+	return GlobalOptions{
+		CacheDir: cache,
+		Repo:     repo,
+		Quiet:    true,
+
+		password: TestPassword,
+		stdout:   os.Stdout,
+		stderr:   os.Stderr,
+	}
+}
+
+func cleanupTempdir(t testing.TB, tempdir string) {
+	if !TestCleanupTempDirs {
+		t.Logf("leaving temporary directory %v used for test", tempdir)
+		return
+	}
+
+	RemoveAll(t, tempdir)
+}
+
+// withTestEnvironment creates a test environment and calls f with it. After f has
+// returned, the temporary directory is removed.
+func withTestEnvironment(t testing.TB, f func(*testEnvironment, GlobalOptions)) {
+	if !RunIntegrationTest {
+		t.Skip("integration tests disabled")
+	}
+
+	tempdir, err := ioutil.TempDir(TestTempDir, "restic-test-")
+	OK(t, err)
+
+	env := testEnvironment{
+		base:     tempdir,
+		cache:    filepath.Join(tempdir, "cache"),
+		repo:     filepath.Join(tempdir, "repo"),
+		testdata: filepath.Join(tempdir, "testdata"),
+	}
+
+	OK(t, os.MkdirAll(env.testdata, 0700))
+	OK(t, os.MkdirAll(env.cache, 0700))
+	OK(t, os.MkdirAll(env.repo, 0700))
+
+	f(&env, configureRestic(t, env.cache, env.repo))
+
+	if !TestCleanupTempDirs {
+		t.Logf("leaving temporary directory %v used for test", tempdir)
+		return
+	}
+
+	RemoveAll(t, tempdir)
+}
+
+// removeFile resets the read-only flag and then deletes the file.
+func removeFile(fn string) error {
+	err := os.Chmod(fn, 0666)
+	if err != nil {
+		return err
+	}
+
+	return os.Remove(fn)
+}

src/cmds/restic/integration_helpers_unix_test.go

@@ -0,0 +1,41 @@
+//+build !windows
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"syscall"
+)
+
+func (e *dirEntry) equals(other *dirEntry) bool {
+	if e.path != other.path {
+		fmt.Fprintf(os.Stderr, "%v: path does not match (%v != %v)\n", e.path, e.path, other.path)
+		return false
+	}
+
+	if e.fi.Mode() != other.fi.Mode() {
+		fmt.Fprintf(os.Stderr, "%v: mode does not match (%v != %v)\n", e.path, e.fi.Mode(), other.fi.Mode())
+		return false
+	}
+
+	if !sameModTime(e.fi, other.fi) {
+		fmt.Fprintf(os.Stderr, "%v: ModTime does not match (%v != %v)\n", e.path, e.fi.ModTime(), other.fi.ModTime())
+		return false
+	}
+
+	stat, _ := e.fi.Sys().(*syscall.Stat_t)
+	stat2, _ := other.fi.Sys().(*syscall.Stat_t)
+
+	if stat.Uid != stat2.Uid {
+		fmt.Fprintf(os.Stderr, "%v: UID does not match (%v != %v)\n", e.path, stat.Uid, stat2.Uid)
+		return false
+	}
+
+	if stat.Gid != stat2.Gid {
+		fmt.Fprintf(os.Stderr, "%v: GID does not match (%v != %v)\n", e.path, stat.Gid, stat2.Gid)
+		return false
+	}
+
+	return true
+}

src/cmds/restic/integration_helpers_windows_test.go

@@ -0,0 +1,27 @@
+//+build windows
+
+package main
+
+import (
+	"fmt"
+	"os"
+)
+
+func (e *dirEntry) equals(other *dirEntry) bool {
+	if e.path != other.path {
+		fmt.Fprintf(os.Stderr, "%v: path does not match (%v != %v)\n", e.path, e.path, other.path)
+		return false
+	}
+
+	if e.fi.Mode() != other.fi.Mode() {
+		fmt.Fprintf(os.Stderr, "%v: mode does not match (%v != %v)\n", e.path, e.fi.Mode(), other.fi.Mode())
+		return false
+	}
+
+	if !sameModTime(e.fi, other.fi) {
+		fmt.Fprintf(os.Stderr, "%v: ModTime does not match (%v != %v)\n", e.path, e.fi.ModTime(), other.fi.ModTime())
+		return false
+	}
+
+	return true
+}

src/cmds/restic/integration_test.go

@@ -0,0 +1,816 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"crypto/rand"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+
+	"restic/backend"
+	"restic/debug"
+	"restic/filter"
+	"restic/repository"
+	. "restic/test"
+)
+
+func parseIDsFromReader(t testing.TB, rd io.Reader) backend.IDs {
+	IDs := backend.IDs{}
+	sc := bufio.NewScanner(rd)
+
+	for sc.Scan() {
+		id, err := backend.ParseID(sc.Text())
+		if err != nil {
+			t.Logf("parse id %v: %v", sc.Text(), err)
+			continue
+		}
+
+		IDs = append(IDs, id)
+	}
+
+	return IDs
+}
+
+func cmdInit(t testing.TB, global GlobalOptions) {
+	cmd := &CmdInit{global: &global}
+	OK(t, cmd.Execute(nil))
+
+	t.Logf("repository initialized at %v", global.Repo)
+}
+
+func cmdBackup(t testing.TB, global GlobalOptions, target []string, parentID *backend.ID) {
+	cmdBackupExcludes(t, global, target, parentID, nil)
+}
+
+func cmdBackupExcludes(t testing.TB, global GlobalOptions, target []string, parentID *backend.ID, excludes []string) {
+	cmd := &CmdBackup{global: &global, Excludes: excludes}
+	if parentID != nil {
+		cmd.Parent = parentID.String()
+	}
+
+	t.Logf("backing up %v", target)
+
+	OK(t, cmd.Execute(target))
+}
+
+func cmdList(t testing.TB, global GlobalOptions, tpe string) backend.IDs {
+	cmd := &CmdList{global: &global}
+	return executeAndParseIDs(t, cmd, tpe)
+}
+
+func executeAndParseIDs(t testing.TB, cmd *CmdList, args ...string) backend.IDs {
+	buf := bytes.NewBuffer(nil)
+	cmd.global.stdout = buf
+	OK(t, cmd.Execute(args))
+	return parseIDsFromReader(t, buf)
+}
+
+func cmdRestore(t testing.TB, global GlobalOptions, dir string, snapshotID backend.ID) {
+	cmdRestoreExcludes(t, global, dir, snapshotID, nil)
+}
+
+func cmdRestoreExcludes(t testing.TB, global GlobalOptions, dir string, snapshotID backend.ID, excludes []string) {
+	cmd := &CmdRestore{global: &global, Target: dir, Exclude: excludes}
+	OK(t, cmd.Execute([]string{snapshotID.String()}))
+}
+
+func cmdRestoreIncludes(t testing.TB, global GlobalOptions, dir string, snapshotID backend.ID, includes []string) {
+	cmd := &CmdRestore{global: &global, Target: dir, Include: includes}
+	OK(t, cmd.Execute([]string{snapshotID.String()}))
+}
+
+func cmdCheck(t testing.TB, global GlobalOptions) {
+	cmd := &CmdCheck{
+		global:      &global,
+		ReadData:    true,
+		CheckUnused: true,
+	}
+	OK(t, cmd.Execute(nil))
+}
+
+func cmdCheckOutput(t testing.TB, global GlobalOptions) string {
+	buf := bytes.NewBuffer(nil)
+	global.stdout = buf
+	cmd := &CmdCheck{global: &global, ReadData: true}
+	OK(t, cmd.Execute(nil))
+	return string(buf.Bytes())
+}
+
+func cmdRebuildIndex(t testing.TB, global GlobalOptions) {
+	global.stdout = ioutil.Discard
+	cmd := &CmdRebuildIndex{global: &global}
+	OK(t, cmd.Execute(nil))
+}
+
+func cmdOptimize(t testing.TB, global GlobalOptions) {
+	cmd := &CmdOptimize{global: &global}
+	OK(t, cmd.Execute(nil))
+}
+
+func cmdLs(t testing.TB, global GlobalOptions, snapshotID string) []string {
+	var buf bytes.Buffer
+	global.stdout = &buf
+
+	cmd := &CmdLs{global: &global}
+	OK(t, cmd.Execute([]string{snapshotID}))
+
+	return strings.Split(string(buf.Bytes()), "\n")
+}
+
+func cmdFind(t testing.TB, global GlobalOptions, pattern string) []string {
+	var buf bytes.Buffer
+	global.stdout = &buf
+
+	cmd := &CmdFind{global: &global}
+	OK(t, cmd.Execute([]string{pattern}))
+
+	return strings.Split(string(buf.Bytes()), "\n")
+}
+
+func TestBackup(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		fd, err := os.Open(datafile)
+		if os.IsNotExist(err) {
+			t.Skipf("unable to find data file %q, skipping", datafile)
+			return
+		}
+		OK(t, err)
+		OK(t, fd.Close())
+
+		cmdInit(t, global)
+
+		SetupTarTestFixture(t, env.testdata, datafile)
+
+		// first backup
+		cmdBackup(t, global, []string{env.testdata}, nil)
+		snapshotIDs := cmdList(t, global, "snapshots")
+		Assert(t, len(snapshotIDs) == 1,
+			"expected one snapshot, got %v", snapshotIDs)
+
+		cmdCheck(t, global)
+		stat1 := dirStats(env.repo)
+
+		// second backup, implicit incremental
+		cmdBackup(t, global, []string{env.testdata}, nil)
+		snapshotIDs = cmdList(t, global, "snapshots")
+		Assert(t, len(snapshotIDs) == 2,
+			"expected two snapshots, got %v", snapshotIDs)
+
+		stat2 := dirStats(env.repo)
+		if stat2.size > stat1.size+stat1.size/10 {
+			t.Error("repository size has grown by more than 10 percent")
+		}
+		t.Logf("repository grown by %d bytes", stat2.size-stat1.size)
+
+		cmdCheck(t, global)
+		// third backup, explicit incremental
+		cmdBackup(t, global, []string{env.testdata}, &snapshotIDs[0])
+		snapshotIDs = cmdList(t, global, "snapshots")
+		Assert(t, len(snapshotIDs) == 3,
+			"expected three snapshots, got %v", snapshotIDs)
+
+		stat3 := dirStats(env.repo)
+		if stat3.size > stat1.size+stat1.size/10 {
+			t.Error("repository size has grown by more than 10 percent")
+		}
+		t.Logf("repository grown by %d bytes", stat3.size-stat2.size)
+
+		// restore all backups and compare
+		for i, snapshotID := range snapshotIDs {
+			restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i))
+			t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir)
+			cmdRestore(t, global, restoredir, snapshotIDs[0])
+			Assert(t, directoriesEqualContents(env.testdata, filepath.Join(restoredir, "testdata")),
+				"directories are not equal")
+		}
+
+		cmdCheck(t, global)
+	})
+}
+
+func TestBackupNonExistingFile(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		fd, err := os.Open(datafile)
+		if os.IsNotExist(err) {
+			t.Skipf("unable to find data file %q, skipping", datafile)
+			return
+		}
+		OK(t, err)
+		OK(t, fd.Close())
+
+		SetupTarTestFixture(t, env.testdata, datafile)
+
+		cmdInit(t, global)
+
+		global.stderr = ioutil.Discard
+
+		p := filepath.Join(env.testdata, "0", "0")
+		dirs := []string{
+			filepath.Join(p, "0"),
+			filepath.Join(p, "1"),
+			filepath.Join(p, "nonexisting"),
+			filepath.Join(p, "5"),
+		}
+		cmdBackup(t, global, dirs, nil)
+	})
+}
+
+func TestBackupMissingFile1(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		fd, err := os.Open(datafile)
+		if os.IsNotExist(err) {
+			t.Skipf("unable to find data file %q, skipping", datafile)
+			return
+		}
+		OK(t, err)
+		OK(t, fd.Close())
+
+		SetupTarTestFixture(t, env.testdata, datafile)
+
+		cmdInit(t, global)
+
+		global.stderr = ioutil.Discard
+		ranHook := false
+		debug.Hook("pipe.walk1", func(context interface{}) {
+			pathname := context.(string)
+
+			if pathname != filepath.Join("testdata", "0", "0", "9") {
+				return
+			}
+
+			t.Logf("in hook, removing test file testdata/0/0/9/37")
+			ranHook = true
+
+			OK(t, os.Remove(filepath.Join(env.testdata, "0", "0", "9", "37")))
+		})
+
+		cmdBackup(t, global, []string{env.testdata}, nil)
+		cmdCheck(t, global)
+
+		Assert(t, ranHook, "hook did not run")
+		debug.RemoveHook("pipe.walk1")
+	})
+}
+
+func TestBackupMissingFile2(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		fd, err := os.Open(datafile)
+		if os.IsNotExist(err) {
+			t.Skipf("unable to find data file %q, skipping", datafile)
+			return
+		}
+		OK(t, err)
+		OK(t, fd.Close())
+
+		SetupTarTestFixture(t, env.testdata, datafile)
+
+		cmdInit(t, global)
+
+		global.stderr = ioutil.Discard
+		ranHook := false
+		debug.Hook("pipe.walk2", func(context interface{}) {
+			pathname := context.(string)
+
+			if pathname != filepath.Join("testdata", "0", "0", "9", "37") {
+				return
+			}
+
+			t.Logf("in hook, removing test file testdata/0/0/9/37")
+			ranHook = true
+
+			OK(t, os.Remove(filepath.Join(env.testdata, "0", "0", "9", "37")))
+		})
+
+		cmdBackup(t, global, []string{env.testdata}, nil)
+		cmdCheck(t, global)
+
+		Assert(t, ranHook, "hook did not run")
+		debug.RemoveHook("pipe.walk2")
+	})
+}
+
+func TestBackupDirectoryError(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		fd, err := os.Open(datafile)
+		if os.IsNotExist(err) {
+			t.Skipf("unable to find data file %q, skipping", datafile)
+			return
+		}
+		OK(t, err)
+		OK(t, fd.Close())
+
+		SetupTarTestFixture(t, env.testdata, datafile)
+
+		cmdInit(t, global)
+
+		global.stderr = ioutil.Discard
+		ranHook := false
+
+		testdir := filepath.Join(env.testdata, "0", "0", "9")
+
+		// install hook that removes the dir right before readdirnames()
+		debug.Hook("pipe.readdirnames", func(context interface{}) {
+			path := context.(string)
+
+			if path != testdir {
+				return
+			}
+
+			t.Logf("in hook, removing test file %v", testdir)
+			ranHook = true
+
+			OK(t, os.RemoveAll(testdir))
+		})
+
+		cmdBackup(t, global, []string{filepath.Join(env.testdata, "0", "0")}, nil)
+		cmdCheck(t, global)
+
+		Assert(t, ranHook, "hook did not run")
+		debug.RemoveHook("pipe.walk2")
+
+		snapshots := cmdList(t, global, "snapshots")
+		Assert(t, len(snapshots) > 0,
+			"no snapshots found in repo (%v)", datafile)
+
+		files := cmdLs(t, global, snapshots[0].String())
+
+		Assert(t, len(files) > 1, "snapshot is empty")
+	})
+}
+
+func includes(haystack []string, needle string) bool {
+	for _, s := range haystack {
+		if s == needle {
+			return true
+		}
+	}
+
+	return false
+}
+
+func loadSnapshotMap(t testing.TB, global GlobalOptions) map[string]struct{} {
+	snapshotIDs := cmdList(t, global, "snapshots")
+
+	m := make(map[string]struct{})
+	for _, id := range snapshotIDs {
+		m[id.String()] = struct{}{}
+	}
+
+	return m
+}
+
+func lastSnapshot(old, new map[string]struct{}) (map[string]struct{}, string) {
+	for k := range new {
+		if _, ok := old[k]; !ok {
+			old[k] = struct{}{}
+			return old, k
+		}
+	}
+
+	return old, ""
+}
+
+var backupExcludeFilenames = []string{
+	"testfile1",
+	"foo.tar.gz",
+	"private/secret/passwords.txt",
+	"work/source/test.c",
+}
+
+func TestBackupExclude(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+		cmdInit(t, global)
+
+		datadir := filepath.Join(env.base, "testdata")
+
+		for _, filename := range backupExcludeFilenames {
+			fp := filepath.Join(datadir, filename)
+			OK(t, os.MkdirAll(filepath.Dir(fp), 0755))
+
+			f, err := os.Create(fp)
+			OK(t, err)
+
+			fmt.Fprintf(f, filename)
+			OK(t, f.Close())
+		}
+
+		snapshots := make(map[string]struct{})
+
+		cmdBackup(t, global, []string{datadir}, nil)
+		snapshots, snapshotID := lastSnapshot(snapshots, loadSnapshotMap(t, global))
+		files := cmdLs(t, global, snapshotID)
+		Assert(t, includes(files, filepath.Join("testdata", "foo.tar.gz")),
+			"expected file %q in first snapshot, but it's not included", "foo.tar.gz")
+
+		cmdBackupExcludes(t, global, []string{datadir}, nil, []string{"*.tar.gz"})
+		snapshots, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, global))
+		files = cmdLs(t, global, snapshotID)
+		Assert(t, !includes(files, filepath.Join("testdata", "foo.tar.gz")),
+			"expected file %q not in first snapshot, but it's included", "foo.tar.gz")
+
+		cmdBackupExcludes(t, global, []string{datadir}, nil, []string{"*.tar.gz", "private/secret"})
+		snapshots, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, global))
+		files = cmdLs(t, global, snapshotID)
+		Assert(t, !includes(files, filepath.Join("testdata", "foo.tar.gz")),
+			"expected file %q not in first snapshot, but it's included", "foo.tar.gz")
+		Assert(t, !includes(files, filepath.Join("testdata", "private", "secret", "passwords.txt")),
+			"expected file %q not in first snapshot, but it's included", "passwords.txt")
+	})
+}
+
+const (
+	incrementalFirstWrite  = 20 * 1042 * 1024
+	incrementalSecondWrite = 12 * 1042 * 1024
+	incrementalThirdWrite  = 4 * 1042 * 1024
+)
+
+func appendRandomData(filename string, bytes uint) error {
+	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0666)
+	if err != nil {
+		fmt.Fprint(os.Stderr, err)
+		return err
+	}
+
+	_, err = f.Seek(0, 2)
+	if err != nil {
+		fmt.Fprint(os.Stderr, err)
+		return err
+	}
+
+	_, err = io.Copy(f, io.LimitReader(rand.Reader, int64(bytes)))
+	if err != nil {
+		fmt.Fprint(os.Stderr, err)
+		return err
+	}
+
+	return f.Close()
+}
+
+func TestIncrementalBackup(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+		cmdInit(t, global)
+
+		datadir := filepath.Join(env.base, "testdata")
+		testfile := filepath.Join(datadir, "testfile")
+
+		OK(t, appendRandomData(testfile, incrementalFirstWrite))
+
+		cmdBackup(t, global, []string{datadir}, nil)
+		cmdCheck(t, global)
+		stat1 := dirStats(env.repo)
+
+		OK(t, appendRandomData(testfile, incrementalSecondWrite))
+
+		cmdBackup(t, global, []string{datadir}, nil)
+		cmdCheck(t, global)
+		stat2 := dirStats(env.repo)
+		if stat2.size-stat1.size > incrementalFirstWrite {
+			t.Errorf("repository size has grown by more than %d bytes", incrementalFirstWrite)
+		}
+		t.Logf("repository grown by %d bytes", stat2.size-stat1.size)
+
+		OK(t, appendRandomData(testfile, incrementalThirdWrite))
+
+		cmdBackup(t, global, []string{datadir}, nil)
+		cmdCheck(t, global)
+		stat3 := dirStats(env.repo)
+		if stat3.size-stat2.size > incrementalFirstWrite {
+			t.Errorf("repository size has grown by more than %d bytes", incrementalFirstWrite)
+		}
+		t.Logf("repository grown by %d bytes", stat3.size-stat2.size)
+	})
+}
+
+func cmdKey(t testing.TB, global GlobalOptions, args ...string) string {
+	var buf bytes.Buffer
+
+	global.stdout = &buf
+	cmd := &CmdKey{global: &global}
+	OK(t, cmd.Execute(args))
+
+	return buf.String()
+}
+
+func cmdKeyListOtherIDs(t testing.TB, global GlobalOptions) []string {
+	var buf bytes.Buffer
+
+	global.stdout = &buf
+	cmd := &CmdKey{global: &global}
+	OK(t, cmd.Execute([]string{"list"}))
+
+	scanner := bufio.NewScanner(&buf)
+	exp := regexp.MustCompile(`^ ([a-f0-9]+) `)
+
+	IDs := []string{}
+	for scanner.Scan() {
+		if id := exp.FindStringSubmatch(scanner.Text()); id != nil {
+			IDs = append(IDs, id[1])
+		}
+	}
+
+	return IDs
+}
+
+func cmdKeyAddNewKey(t testing.TB, global GlobalOptions, newPassword string) {
+	cmd := &CmdKey{global: &global, newPassword: newPassword}
+	OK(t, cmd.Execute([]string{"add"}))
+}
+
+func cmdKeyPasswd(t testing.TB, global GlobalOptions, newPassword string) {
+	cmd := &CmdKey{global: &global, newPassword: newPassword}
+	OK(t, cmd.Execute([]string{"passwd"}))
+}
+
+func cmdKeyRemove(t testing.TB, global GlobalOptions, IDs []string) {
+	cmd := &CmdKey{global: &global}
+	t.Logf("remove %d keys: %q\n", len(IDs), IDs)
+	for _, id := range IDs {
+		OK(t, cmd.Execute([]string{"rm", id}))
+	}
+}
+
+func TestKeyAddRemove(t *testing.T) {
+	passwordList := []string{
+		"OnnyiasyatvodsEvVodyawit",
+		"raicneirvOjEfEigonOmLasOd",
+	}
+
+	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+		cmdInit(t, global)
+
+		cmdKeyPasswd(t, global, "geheim2")
+		global.password = "geheim2"
+		t.Logf("changed password to %q", global.password)
+
+		for _, newPassword := range passwordList {
+			cmdKeyAddNewKey(t, global, newPassword)
+			t.Logf("added new password %q", newPassword)
+			global.password = newPassword
+			cmdKeyRemove(t, global, cmdKeyListOtherIDs(t, global))
+		}
+
+		global.password = passwordList[len(passwordList)-1]
+		t.Logf("testing access with last password %q\n", global.password)
+		cmdKey(t, global, "list")
+
+		cmdCheck(t, global)
+	})
+}
+
+func testFileSize(filename string, size int64) error {
+	fi, err := os.Stat(filename)
+	if err != nil {
+		return err
+	}
+
+	if fi.Size() != size {
+		return fmt.Errorf("wrong file size for %v: expected %v, got %v", filename, size, fi.Size())
+	}
+
+	return nil
+}
+
+func TestRestoreFilter(t *testing.T) {
+	testfiles := []struct {
+		name string
+		size uint
+	}{
+		{"testfile1.c", 100},
+		{"testfile2.exe", 101},
+		{"subdir1/subdir2/testfile3.docx", 102},
+		{"subdir1/subdir2/testfile4.c", 102},
+	}
+
+	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+		cmdInit(t, global)
+
+		for _, test := range testfiles {
+			p := filepath.Join(env.testdata, test.name)
+			OK(t, os.MkdirAll(filepath.Dir(p), 0755))
+			OK(t, appendRandomData(p, test.size))
+		}
+
+		cmdBackup(t, global, []string{env.testdata}, nil)
+		cmdCheck(t, global)
+
+		snapshotID := cmdList(t, global, "snapshots")[0]
+
+		// no restore filter should restore all files
+		cmdRestore(t, global, filepath.Join(env.base, "restore0"), snapshotID)
+		for _, test := range testfiles {
+			OK(t, testFileSize(filepath.Join(env.base, "restore0", "testdata", test.name), int64(test.size)))
+		}
+
+		for i, pat := range []string{"*.c", "*.exe", "*", "*file3*"} {
+			base := filepath.Join(env.base, fmt.Sprintf("restore%d", i+1))
+			cmdRestoreExcludes(t, global, base, snapshotID, []string{pat})
+			for _, test := range testfiles {
+				err := testFileSize(filepath.Join(base, "testdata", test.name), int64(test.size))
+				if ok, _ := filter.Match(pat, filepath.Base(test.name)); !ok {
+					OK(t, err)
+				} else {
+					Assert(t, os.IsNotExist(err),
+						"expected %v to not exist in restore step %v, but it exists, err %v", test.name, i+1, err)
+				}
+			}
+		}
+
+	})
+}
+
+func TestRestoreWithPermissionFailure(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+		datafile := filepath.Join("testdata", "repo-restore-permissions-test.tar.gz")
+		SetupTarTestFixture(t, env.base, datafile)
+
+		snapshots := cmdList(t, global, "snapshots")
+		Assert(t, len(snapshots) > 0,
+			"no snapshots found in repo (%v)", datafile)
+
+		global.stderr = ioutil.Discard
+		cmdRestore(t, global, filepath.Join(env.base, "restore"), snapshots[0])
+
+		// make sure that all files have been restored, regardeless of any
+		// permission errors
+		files := cmdLs(t, global, snapshots[0].String())
+		for _, filename := range files {
+			fi, err := os.Lstat(filepath.Join(env.base, "restore", filename))
+			OK(t, err)
+
+			Assert(t, !isFile(fi) || fi.Size() > 0,
+				"file %v restored, but filesize is 0", filename)
+		}
+	})
+}
+
+func setZeroModTime(filename string) error {
+	var utimes = []syscall.Timespec{
+		syscall.NsecToTimespec(0),
+		syscall.NsecToTimespec(0),
+	}
+
+	return syscall.UtimesNano(filename, utimes)
+}
+
+func TestRestoreNoMetadataOnIgnoredIntermediateDirs(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+		cmdInit(t, global)
+
+		p := filepath.Join(env.testdata, "subdir1", "subdir2", "subdir3", "file.ext")
+		OK(t, os.MkdirAll(filepath.Dir(p), 0755))
+		OK(t, appendRandomData(p, 200))
+		OK(t, setZeroModTime(filepath.Join(env.testdata, "subdir1", "subdir2")))
+
+		cmdBackup(t, global, []string{env.testdata}, nil)
+		cmdCheck(t, global)
+
+		snapshotID := cmdList(t, global, "snapshots")[0]
+
+		// restore with filter "*.ext", this should restore "file.ext", but
+		// since the directories are ignored and only created because of
+		// "file.ext", no meta data should be restored for them.
+		cmdRestoreIncludes(t, global, filepath.Join(env.base, "restore0"), snapshotID, []string{"*.ext"})
+
+		f1 := filepath.Join(env.base, "restore0", "testdata", "subdir1", "subdir2")
+		fi, err := os.Stat(f1)
+		OK(t, err)
+
+		Assert(t, fi.ModTime() != time.Unix(0, 0),
+			"meta data of intermediate directory has been restore although it was ignored")
+
+		// restore with filter "*", this should restore meta data on everything.
+		cmdRestoreIncludes(t, global, filepath.Join(env.base, "restore1"), snapshotID, []string{"*"})
+
+		f2 := filepath.Join(env.base, "restore1", "testdata", "subdir1", "subdir2")
+		fi, err = os.Stat(f2)
+		OK(t, err)
+
+		Assert(t, fi.ModTime() == time.Unix(0, 0),
+			"meta data of intermediate directory hasn't been restore")
+	})
+}
+
+func TestFind(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		cmdInit(t, global)
+		SetupTarTestFixture(t, env.testdata, datafile)
+		cmdBackup(t, global, []string{env.testdata}, nil)
+		cmdCheck(t, global)
+
+		results := cmdFind(t, global, "unexistingfile")
+		Assert(t, len(results) != 0, "unexisting file found in repo (%v)", datafile)
+
+		results = cmdFind(t, global, "testfile")
+		Assert(t, len(results) != 1, "file not found in repo (%v)", datafile)
+
+		results = cmdFind(t, global, "test")
+		Assert(t, len(results) < 2, "less than two file found in repo (%v)", datafile)
+	})
+}
+
+func TestRebuildIndex(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+		datafile := filepath.Join("..", "..", "checker", "testdata", "duplicate-packs-in-index-test-repo.tar.gz")
+		SetupTarTestFixture(t, env.base, datafile)
+
+		out := cmdCheckOutput(t, global)
+		if !strings.Contains(out, "contained in several indexes") {
+			t.Fatalf("did not find checker hint for packs in several indexes")
+		}
+
+		if !strings.Contains(out, "restic rebuild-index") {
+			t.Fatalf("did not find hint for rebuild-index comman")
+		}
+
+		cmdRebuildIndex(t, global)
+
+		out = cmdCheckOutput(t, global)
+		if len(out) != 0 {
+			t.Fatalf("expected no output from the checker, got: %v", out)
+		}
+	})
+}
+
+func TestRebuildIndexAlwaysFull(t *testing.T) {
+	repository.IndexFull = func(*repository.Index) bool { return true }
+	TestRebuildIndex(t)
+}
+
+var optimizeTests = []struct {
+	testFilename string
+	snapshots    backend.IDSet
+}{
+	{
+		filepath.Join("..", "..", "checker", "testdata", "checker-test-repo.tar.gz"),
+		backend.NewIDSet(ParseID("a13c11e582b77a693dd75ab4e3a3ba96538a056594a4b9076e4cacebe6e06d43")),
+	},
+	{
+		filepath.Join("testdata", "old-index-repo.tar.gz"),
+		nil,
+	},
+	{
+		filepath.Join("testdata", "old-index-repo.tar.gz"),
+		backend.NewIDSet(
+			ParseID("f7d83db709977178c9d1a09e4009355e534cde1a135b8186b8b118a3fc4fcd41"),
+			ParseID("51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02"),
+		),
+	},
+}
+
+func TestOptimizeRemoveUnusedBlobs(t *testing.T) {
+	for i, test := range optimizeTests {
+		withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+			SetupTarTestFixture(t, env.base, test.testFilename)
+
+			for id := range test.snapshots {
+				OK(t, removeFile(filepath.Join(env.repo, "snapshots", id.String())))
+			}
+
+			cmdOptimize(t, global)
+			output := cmdCheckOutput(t, global)
+
+			if len(output) > 0 {
+				t.Errorf("expected no output for check in test %d, got:\n%v", i, output)
+			}
+		})
+	}
+}
+
+func TestCheckRestoreNoLock(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
+		datafile := filepath.Join("testdata", "small-repo.tar.gz")
+		SetupTarTestFixture(t, env.base, datafile)
+
+		err := filepath.Walk(env.repo, func(p string, fi os.FileInfo, e error) error {
+			if e != nil {
+				return e
+			}
+			return os.Chmod(p, fi.Mode() & ^(os.FileMode(0222)))
+		})
+		OK(t, err)
+
+		global.NoLock = true
+		cmdCheck(t, global)
+
+		snapshotIDs := cmdList(t, global, "snapshots")
+		if len(snapshotIDs) == 0 {
+			t.Fatalf("found no snapshots")
+		}
+
+		cmdRestore(t, global, filepath.Join(env.base, "restore"), snapshotIDs[0])
+	})
+}

src/cmds/restic/lock.go

@@ -0,0 +1,125 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"sync"
+	"time"
+
+	"restic"
+	"restic/debug"
+	"restic/repository"
+)
+
+var globalLocks struct {
+	locks         []*restic.Lock
+	cancelRefresh chan struct{}
+	refreshWG     sync.WaitGroup
+	sync.Mutex
+}
+
+func lockRepo(repo *repository.Repository) (*restic.Lock, error) {
+	return lockRepository(repo, false)
+}
+
+func lockRepoExclusive(repo *repository.Repository) (*restic.Lock, error) {
+	return lockRepository(repo, true)
+}
+
+func lockRepository(repo *repository.Repository, exclusive bool) (*restic.Lock, error) {
+	lockFn := restic.NewLock
+	if exclusive {
+		lockFn = restic.NewExclusiveLock
+	}
+
+	lock, err := lockFn(repo)
+	if err != nil {
+		return nil, err
+	}
+
+	globalLocks.Lock()
+	if globalLocks.cancelRefresh == nil {
+		debug.Log("main.lockRepository", "start goroutine for lock refresh")
+		globalLocks.cancelRefresh = make(chan struct{})
+		globalLocks.refreshWG = sync.WaitGroup{}
+		globalLocks.refreshWG.Add(1)
+		go refreshLocks(&globalLocks.refreshWG, globalLocks.cancelRefresh)
+	}
+
+	globalLocks.locks = append(globalLocks.locks, lock)
+	globalLocks.Unlock()
+
+	return lock, err
+}
+
+var refreshInterval = 5 * time.Minute
+
+func refreshLocks(wg *sync.WaitGroup, done <-chan struct{}) {
+	debug.Log("main.refreshLocks", "start")
+	defer func() {
+		wg.Done()
+		globalLocks.Lock()
+		globalLocks.cancelRefresh = nil
+		globalLocks.Unlock()
+	}()
+
+	ticker := time.NewTicker(refreshInterval)
+
+	for {
+		select {
+		case <-done:
+			debug.Log("main.refreshLocks", "terminate")
+			return
+		case <-ticker.C:
+			debug.Log("main.refreshLocks", "refreshing locks")
+			globalLocks.Lock()
+			for _, lock := range globalLocks.locks {
+				err := lock.Refresh()
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "unable to refresh lock: %v\n", err)
+				}
+			}
+			globalLocks.Unlock()
+		}
+	}
+}
+
+func unlockRepo(lock *restic.Lock) error {
+	globalLocks.Lock()
+	defer globalLocks.Unlock()
+
+	debug.Log("unlockRepo", "unlocking repository")
+	if err := lock.Unlock(); err != nil {
+		debug.Log("unlockRepo", "error while unlocking: %v", err)
+		return err
+	}
+
+	for i := 0; i < len(globalLocks.locks); i++ {
+		if lock == globalLocks.locks[i] {
+			globalLocks.locks = append(globalLocks.locks[:i], globalLocks.locks[i+1:]...)
+			return nil
+		}
+	}
+
+	return nil
+}
+
+func unlockAll() error {
+	globalLocks.Lock()
+	defer globalLocks.Unlock()
+
+	debug.Log("unlockAll", "unlocking %d locks", len(globalLocks.locks))
+	for _, lock := range globalLocks.locks {
+		if err := lock.Unlock(); err != nil {
+			debug.Log("unlockAll", "error while unlocking: %v", err)
+			return err
+		}
+		debug.Log("unlockAll", "successfully removed lock")
+	}
+
+	return nil
+}
+
+func init() {
+	AddCleanupHandler(unlockAll)
+}

src/cmds/restic/main.go

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+
+	"github.com/jessevdk/go-flags"
+	"restic"
+	"restic/debug"
+)
+
+func init() {
+	// set GOMAXPROCS to number of CPUs
+	runtime.GOMAXPROCS(runtime.NumCPU())
+}
+
+func main() {
+	// defer profile.Start(profile.MemProfileRate(100000), profile.ProfilePath(".")).Stop()
+	// defer profile.Start(profile.CPUProfile, profile.ProfilePath(".")).Stop()
+	globalOpts.Repo = os.Getenv("RESTIC_REPOSITORY")
+	globalOpts.password = os.Getenv("RESTIC_PASSWORD")
+
+	debug.Log("restic", "main %#v", os.Args)
+
+	_, err := parser.Parse()
+	if e, ok := err.(*flags.Error); ok && e.Type == flags.ErrHelp {
+		parser.WriteHelp(os.Stdout)
+		os.Exit(0)
+	}
+
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+	}
+
+	if restic.IsAlreadyLocked(err) {
+		fmt.Fprintf(os.Stderr, "\nthe `unlock` command can be used to remove stale locks\n")
+	}
+
+	RunCleanupHandlers()
+
+	if err != nil {
+		os.Exit(1)
+	}
+}