Add VERSION file for 0.6.1

In addition, the build script isn't needed any more.

.travis.yml

@@ -2,8 +2,8 @@ language: go
 sudo: false
 
 go:
-  - 1.7.5
-  - 1.8.1
+  - 1.7.6
+  - 1.8.3
   - tip
 
 os:
@@ -17,14 +17,14 @@ env:
 matrix:
   exclude:
     - os: osx
-      go: 1.7.5
+      go: 1.7.6
     - os: osx
       go: tip
     - os: linux
-      go: 1.8.1
+      go: 1.8.3
   include:
     - os: linux
-      go: 1.8.1
+      go: 1.8.3
       sudo: true
       env:
         RESTIC_TEST_FUSE=1

CHANGELOG.md

@@ -1,6 +1,30 @@
 This file describes changes relevant to all users that are made in each
 released version of restic from the perspective of the user.
 
+Important Changes in 0.6.1
+==========================
+
+This is mostly a bugfix release and only contains small changes:
+
+ * We've fixed a bug where `rebuild-index` would corrupt the index when used
+   with the s3 backend together with the `default` layout. This is not the
+   default setting.
+
+ * Backends based on HTTP now allow several idle connections in parallel. This
+   is especially important for the REST backend, which (when used with a local
+   server) may create a lot connections and exhaust available ports quickly.
+   https://github.com/restic/restic/issues/985
+   https://github.com/restic/restic/pull/986
+
+ * Regular status report: We've removed the status report that was printed
+   every 10 seconds when restic is run non-interactively. You can still force
+   reporting the current status by sending a `USR1` signal to the process.
+   https://github.com/restic/restic/pull/974
+
+ * The `build.go` now strips the temporary directory used for compilation from
+   the binary. This is the first step in enabling reproducible builds.
+   https://github.com/restic/restic/pull/981
+
 Important Changes in 0.6.0
 ==========================
 

Dockerfile

@@ -18,7 +18,7 @@
 
 FROM ubuntu:14.04
 
-ARG GOVERSION=1.7.5
+ARG GOVERSION=1.8.3
 ARG GOARCH=amd64
 
 # install dependencies

README.rst

@@ -68,6 +68,15 @@ following principles in mind:
    data should be de-duplicated before it is actually written to the
    storage back end to save precious backup space.
 
+Reproducible Builds
+-------------------
+
+The binaries released with each restic version starting at 0.6.1 are
+[reproducible](https://reproducible-builds.org/), which means that you can
+easily reproduce a byte identical version from the source code for that
+release. Instructions on how to do that are contained in the
+[build repository](https://github.com/restic/build).
+
 News
 ----
 

VERSION

@@ -1 +1 @@
-0.6.0
+0.6.1

build.go

@@ -195,8 +195,11 @@ func cleanEnv() (env []string) {
 
 // build runs "go build args..." with GOPATH set to gopath.
 func build(cwd, goos, goarch, gopath string, args ...string) error {
-	args = append([]string{"build"}, args...)
-	cmd := exec.Command("go", args...)
+	a := []string{"build"}
+	a = append(a, "-asmflags", fmt.Sprintf("-trimpath=%s", gopath))
+	a = append(a, "-gcflags", fmt.Sprintf("-trimpath=%s", gopath))
+	a = append(a, args...)
+	cmd := exec.Command("go", a...)
 	cmd.Env = append(cleanEnv(), "GOPATH="+gopath, "GOARCH="+goarch, "GOOS="+goos)
 	if !enableCGO {
 		cmd.Env = append(cmd.Env, "CGO_ENABLED=0")

build_release_binaries.sh

@@ -1,64 +0,0 @@
-#!/bin/bash
-
-set -e
-
-if [[ -z "$VERSION" ]]; then
-    echo '$VERSION unset'
-    exit 1
-fi
-
-dir=$(mktemp -d --tmpdir restic-release-XXXXXX)
-echo "path is ${dir}"
-
-for R in       \
-    darwin/386     \
-    darwin/amd64   \
-    freebsd/386    \
-    freebsd/amd64  \
-    freebsd/arm    \
-    linux/386      \
-    linux/amd64    \
-    linux/arm      \
-    linux/arm64    \
-    openbsd/386   \
-    openbsd/amd64 \
-    windows/386    \
-    windows/amd64  \
-    ; do \
-
-    OS=$(dirname $R)
-    ARCH=$(basename $R)
-    filename=restic_${VERSION}_${OS}_${ARCH}
-
-    if [[ "$OS" == "windows" ]]; then
-        filename="${filename}.exe"
-    fi
-
-    echo $filename
-
-    go run build.go --goos $OS --goarch $ARCH --output ${filename}
-    if [[ "$OS" == "windows" ]]; then
-        zip ${filename%.exe}.zip ${filename}
-        rm ${filename}
-        mv ${filename%.exe}.zip ${dir}
-    else
-        bzip2 ${filename}
-        mv ${filename}.bz2 ${dir}
-    fi
-done
-
-echo "packing sources"
-git archive --format=tar --prefix=restic-$VERSION/ v$VERSION | gzip -n > restic-$VERSION.tar.gz
-mv restic-$VERSION.tar.gz ${dir}
-
-echo "creating checksums"
-pushd ${dir}
-sha256sum restic_*.{zip,bz2} restic-$VERSION.tar.gz > SHA256SUMS
-gpg --armor --detach-sign SHA256SUMS
-popd
-
-echo "creating source signature file"
-gpg --armor --detach-sign ${dir}/restic-$VERSION.tar.gz
-
-echo
-echo "done, path is ${dir}"

doc/manual.rst

@@ -820,8 +820,10 @@ Restic can write out a bash compatible autocompletion script:
     NOTE: The current version supports Bash only.
           This should work for *nix systems with Bash installed.
 
-    By default, the file is written directly to /etc/bash_completion.d
-    for convenience, and the command may need superuser rights, e.g.:
+By default, the file is written directly to ``/etc/bash_completion.d/``
+for convenience, and the command may need superuser rights, e.g.
+
+.. code-block:: console
 
     $ sudo restic autocomplete
 

src/restic/backend/http_transport.go

@@ -18,6 +18,7 @@ func Transport() http.RoundTripper {
 			DualStack: true,
 		}).DialContext,
 		MaxIdleConns:          100,
+		MaxIdleConnsPerHost:   100,
 		IdleConnTimeout:       90 * time.Second,
 		TLSHandshakeTimeout:   10 * time.Second,
 		ExpectContinueTimeout: 1 * time.Second,

src/restic/backend/layout_default.go

@@ -34,7 +34,7 @@ func (l *DefaultLayout) Dirname(h restic.Handle) string {
 func (l *DefaultLayout) Filename(h restic.Handle) string {
 	name := h.Name
 	if h.Type == restic.ConfigFile {
-		name = "config"
+		return l.Join(l.Path, "config")
 	}
 
 	return l.Join(l.Dirname(h), name)

src/restic/backend/layout_test.go

@@ -12,53 +12,103 @@ import (
 )
 
 func TestDefaultLayout(t *testing.T) {
-	path, cleanup := TempDir(t)
+	tempdir, cleanup := TempDir(t)
 	defer cleanup()
 
 	var tests = []struct {
+		path string
+		join func(...string) string
 		restic.Handle
 		filename string
 	}{
 		{
+			tempdir,
+			filepath.Join,
 			restic.Handle{Type: restic.DataFile, Name: "0123456"},
-			filepath.Join(path, "data", "01", "0123456"),
+			filepath.Join(tempdir, "data", "01", "0123456"),
 		},
 		{
+			tempdir,
+			filepath.Join,
 			restic.Handle{Type: restic.ConfigFile, Name: "CFG"},
-			filepath.Join(path, "config"),
+			filepath.Join(tempdir, "config"),
 		},
 		{
+			tempdir,
+			filepath.Join,
 			restic.Handle{Type: restic.SnapshotFile, Name: "123456"},
-			filepath.Join(path, "snapshots", "123456"),
+			filepath.Join(tempdir, "snapshots", "123456"),
 		},
 		{
+			tempdir,
+			filepath.Join,
 			restic.Handle{Type: restic.IndexFile, Name: "123456"},
-			filepath.Join(path, "index", "123456"),
+			filepath.Join(tempdir, "index", "123456"),
 		},
 		{
+			tempdir,
+			filepath.Join,
 			restic.Handle{Type: restic.LockFile, Name: "123456"},
-			filepath.Join(path, "locks", "123456"),
+			filepath.Join(tempdir, "locks", "123456"),
 		},
 		{
+			tempdir,
+			filepath.Join,
 			restic.Handle{Type: restic.KeyFile, Name: "123456"},
-			filepath.Join(path, "keys", "123456"),
+			filepath.Join(tempdir, "keys", "123456"),
+		},
+		{
+			"",
+			path.Join,
+			restic.Handle{Type: restic.DataFile, Name: "0123456"},
+			"data/01/0123456",
+		},
+		{
+			"",
+			path.Join,
+			restic.Handle{Type: restic.ConfigFile, Name: "CFG"},
+			"config",
+		},
+		{
+			"",
+			path.Join,
+			restic.Handle{Type: restic.SnapshotFile, Name: "123456"},
+			"snapshots/123456",
+		},
+		{
+			"",
+			path.Join,
+			restic.Handle{Type: restic.IndexFile, Name: "123456"},
+			"index/123456",
+		},
+		{
+			"",
+			path.Join,
+			restic.Handle{Type: restic.LockFile, Name: "123456"},
+			"locks/123456",
+		},
+		{
+			"",
+			path.Join,
+			restic.Handle{Type: restic.KeyFile, Name: "123456"},
+			"keys/123456",
 		},
-	}
-
-	l := &DefaultLayout{
-		Path: path,
-		Join: filepath.Join,
 	}
 
 	t.Run("Paths", func(t *testing.T) {
+		l := &DefaultLayout{
+			Path: tempdir,
+			Join: filepath.Join,
+		}
+
 		dirs := l.Paths()
 
 		want := []string{
-			filepath.Join(path, "data"),
-			filepath.Join(path, "snapshots"),
-			filepath.Join(path, "index"),
-			filepath.Join(path, "locks"),
-			filepath.Join(path, "keys"),
+			filepath.Join(tempdir, "data"),
+			filepath.Join(tempdir, "snapshots"),
+			filepath.Join(tempdir, "index"),
+			filepath.Join(tempdir, "locks"),
+			filepath.Join(tempdir, "keys"),
 		}
 
 		sort.Sort(sort.StringSlice(want))
@@ -71,6 +121,11 @@ func TestDefaultLayout(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(fmt.Sprintf("%v/%v", test.Type, test.Handle.Name), func(t *testing.T) {
+			l := &DefaultLayout{
+				Path: test.path,
+				Join: test.join,
+			}
+
 			filename := l.Filename(test.Handle)
 			if filename != test.filename {
 				t.Fatalf("wrong filename, want %v, got %v", test.filename, filename)

src/restic/backend/rest/rest.go

@@ -127,9 +127,8 @@ func (b *restBackend) Save(h restic.Handle, rd io.Reader) (err error) {
 		return errors.Wrap(err, "client.Post")
 	}
 
-	// fmt.Printf("status is %v (%v)\n", resp.Status, resp.StatusCode)
 	if resp.StatusCode != 200 {
-		return errors.Errorf("unexpected HTTP response code %v", resp.StatusCode)
+		return errors.Errorf("server response unexpected: %v (%v)", resp.Status, resp.StatusCode)
 	}
 
 	return nil
@@ -179,7 +178,7 @@ func (b *restBackend) Load(h restic.Handle, length int, offset int64) (io.ReadCl
 	if resp.StatusCode != 200 && resp.StatusCode != 206 {
 		io.Copy(ioutil.Discard, resp.Body)
 		resp.Body.Close()
-		return nil, errors.Errorf("unexpected HTTP response code %v", resp.StatusCode)
+		return nil, errors.Errorf("unexpected HTTP response (%v): %v", resp.StatusCode, resp.Status)
 	}
 
 	return resp.Body, nil
@@ -204,7 +203,7 @@ func (b *restBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
 	}
 
 	if resp.StatusCode != 200 {
-		return restic.FileInfo{}, errors.Errorf("unexpected HTTP response code %v", resp.StatusCode)
+		return restic.FileInfo{}, errors.Errorf("unexpected HTTP response (%v): %v", resp.StatusCode, resp.Status)
 	}
 
 	if resp.ContentLength < 0 {

src/restic/backend/s3/s3.go

@@ -88,11 +88,7 @@ func (be *s3) createConnections() {
 // IsNotExist returns true if the error is caused by a not existing file.
 func (be *s3) IsNotExist(err error) bool {
 	debug.Log("IsNotExist(%T, %#v)", err, err)
-	if os.IsNotExist(err) {
-		return true
-	}
-
-	return false
+	return os.IsNotExist(err)
 }
 
 // Join combines path components with slashes.
@@ -386,7 +382,7 @@ func (be *s3) List(t restic.FileType, done <-chan struct{}) <-chan string {
 			}
 
 			select {
-			case ch <- m:
+			case ch <- path.Base(m):
 			case <-done:
 				return
 			}

src/restic/backend/sftp/sftp.go

@@ -7,7 +7,6 @@ import (
 	"os"
 	"os/exec"
 	"path"
-	"path/filepath"
 	"restic"
 	"strings"
 	"time"
@@ -416,7 +415,7 @@ func (r *SFTP) List(t restic.FileType, done <-chan struct{}) <-chan string {
 			}
 
 			select {
-			case ch <- filepath.Base(walker.Path()):
+			case ch <- path.Base(walker.Path()):
 			case <-done:
 				return
 			}

src/restic/backend/test/tests.go

@@ -13,12 +13,19 @@ import (
 	"sort"
 	"strings"
 	"testing"
+	"time"
 
 	"restic/test"
 
 	"restic/backend"
 )
 
+func seedRand(t testing.TB) {
+	seed := time.Now().UnixNano()
+	rand.Seed(seed)
+	t.Logf("rand initialized with seed %d", seed)
+}
+
 // TestCreateWithConfig tests that creating a backend in a location which already
 // has a config file fails.
 func (s *Suite) TestCreateWithConfig(t *testing.T) {
@@ -101,15 +108,20 @@ func (s *Suite) TestConfig(t *testing.T) {
 
 // TestLoad tests the backend's Load function.
 func (s *Suite) TestLoad(t *testing.T) {
+	seedRand(t)
+
 	b := s.open(t)
 	defer s.close(t, b)
 
-	_, err := b.Load(restic.Handle{}, 0, 0)
+	rd, err := b.Load(restic.Handle{}, 0, 0)
 	if err == nil {
 		t.Fatalf("Load() did not return an error for invalid handle")
 	}
+	if rd != nil {
+		rd.Close()
+	}
 
-	_, err = b.Load(restic.Handle{Type: restic.DataFile, Name: "foobar"}, 0, 0)
+	err = testLoad(b, restic.Handle{Type: restic.DataFile, Name: "foobar"}, 0, 0)
 	if err == nil {
 		t.Fatalf("Load() did not return an error for non-existing blob")
 	}
@@ -125,7 +137,9 @@ func (s *Suite) TestLoad(t *testing.T) {
 		t.Fatalf("Save() error: %+v", err)
 	}
 
-	rd, err := b.Load(handle, 100, -1)
+	t.Logf("saved %d bytes as %v", length, handle)
+
+	rd, err = b.Load(handle, 100, -1)
 	if err == nil {
 		t.Fatalf("Load() returned no error for negative offset!")
 	}
@@ -147,8 +161,8 @@ func (s *Suite) TestLoad(t *testing.T) {
 		if o < len(d) {
 			d = d[o:]
 		} else {
-			o = len(d)
-			d = d[:0]
+			t.Logf("offset == length, skipping test")
+			continue
 		}
 
 		getlen := l
@@ -162,12 +176,14 @@ func (s *Suite) TestLoad(t *testing.T) {
 
 		rd, err := b.Load(handle, getlen, int64(o))
 		if err != nil {
+			t.Logf("Load, l %v, o %v, len(d) %v, getlen %v", l, o, len(d), getlen)
 			t.Errorf("Load(%d, %d) returned unexpected error: %+v", l, o, err)
 			continue
 		}
 
 		buf, err := ioutil.ReadAll(rd)
 		if err != nil {
+			t.Logf("Load, l %v, o %v, len(d) %v, getlen %v", l, o, len(d), getlen)
 			t.Errorf("Load(%d, %d) ReadAll() returned unexpected error: %+v", l, o, err)
 			if err = rd.Close(); err != nil {
 				t.Errorf("Load(%d, %d) rd.Close() returned error: %+v", l, o, err)
@@ -176,6 +192,7 @@ func (s *Suite) TestLoad(t *testing.T) {
 		}
 
 		if l == 0 && len(buf) != len(d) {
+			t.Logf("Load, l %v, o %v, len(d) %v, getlen %v", l, o, len(d), getlen)
 			t.Errorf("Load(%d, %d) wrong number of bytes read: want %d, got %d", l, o, len(d), len(buf))
 			if err = rd.Close(); err != nil {
 				t.Errorf("Load(%d, %d) rd.Close() returned error: %+v", l, o, err)
@@ -184,6 +201,7 @@ func (s *Suite) TestLoad(t *testing.T) {
 		}
 
 		if l > 0 && l <= len(d) && len(buf) != l {
+			t.Logf("Load, l %v, o %v, len(d) %v, getlen %v", l, o, len(d), getlen)
 			t.Errorf("Load(%d, %d) wrong number of bytes read: want %d, got %d", l, o, l, len(buf))
 			if err = rd.Close(); err != nil {
 				t.Errorf("Load(%d, %d) rd.Close() returned error: %+v", l, o, err)
@@ -192,6 +210,7 @@ func (s *Suite) TestLoad(t *testing.T) {
 		}
 
 		if l > len(d) && len(buf) != len(d) {
+			t.Logf("Load, l %v, o %v, len(d) %v, getlen %v", l, o, len(d), getlen)
 			t.Errorf("Load(%d, %d) wrong number of bytes read for overlong read: want %d, got %d", l, o, l, len(buf))
 			if err = rd.Close(); err != nil {
 				t.Errorf("Load(%d, %d) rd.Close() returned error: %+v", l, o, err)
@@ -200,6 +219,7 @@ func (s *Suite) TestLoad(t *testing.T) {
 		}
 
 		if !bytes.Equal(buf, d) {
+			t.Logf("Load, l %v, o %v, len(d) %v, getlen %v", l, o, len(d), getlen)
 			t.Errorf("Load(%d, %d) returned wrong bytes", l, o)
 			if err = rd.Close(); err != nil {
 				t.Errorf("Load(%d, %d) rd.Close() returned error: %+v", l, o, err)
@@ -209,6 +229,7 @@ func (s *Suite) TestLoad(t *testing.T) {
 
 		err = rd.Close()
 		if err != nil {
+			t.Logf("Load, l %v, o %v, len(d) %v, getlen %v", l, o, len(d), getlen)
 			t.Errorf("Load(%d, %d) rd.Close() returned unexpected error: %+v", l, o, err)
 			continue
 		}
@@ -234,6 +255,8 @@ func (ec errorCloser) Size() int64 {
 
 // TestSave tests saving data in the backend.
 func (s *Suite) TestSave(t *testing.T) {
+	seedRand(t)
+
 	b := s.open(t)
 	defer s.close(t, b)
 	var id restic.ID
@@ -396,6 +419,21 @@ func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) res
 	return h
 }
 
+// testLoad loads a blob (but discards its contents).
+func testLoad(b restic.Backend, h restic.Handle, length int, offset int64) error {
+	rd, err := b.Load(h, 0, 0)
+	if err != nil {
+		return err
+	}
+
+	_, err = io.Copy(ioutil.Discard, rd)
+	cerr := rd.Close()
+	if err == nil {
+		err = cerr
+	}
+	return err
+}
+
 // TestBackend tests all functions of the backend.
 func (s *Suite) TestBackend(t *testing.T) {
 	b := s.open(t)
@@ -421,8 +459,8 @@ func (s *Suite) TestBackend(t *testing.T) {
 			test.Assert(t, err != nil, "blob data could be extracted before creation")
 
 			// try to read not existing blob
-			_, err = b.Load(h, 0, 0)
-			test.Assert(t, err != nil, "blob reader could be obtained before creation")
+			err = testLoad(b, h, 0, 0)
+			test.Assert(t, err != nil, "blob could be read before creation")
 
 			// try to get string out, should fail
 			ret, err = b.Test(h)

src/restic/progress.go

@@ -14,6 +14,7 @@ const minTickerTime = time.Second / 60
 var isTerminal = terminal.IsTerminal(int(os.Stdout.Fd()))
 var forceUpdateProgress = make(chan bool)
 
+// Progress reports progress on an operation.
 type Progress struct {
 	OnStart  func()
 	OnUpdate ProgressFunc
@@ -32,6 +33,7 @@ type Progress struct {
 	running bool
 }
 
+// Stat captures newly done parts of the operation.
 type Stat struct {
 	Files  uint64
 	Dirs   uint64
@@ -41,6 +43,7 @@ type Stat struct {
 	Errors uint64
 }
 
+// ProgressFunc is used to report progress back to the user.
 type ProgressFunc func(s Stat, runtime time.Duration, ticker bool)
 
 // NewProgress returns a new progress reporter. When Start() is called, the
@@ -50,10 +53,7 @@ type ProgressFunc func(s Stat, runtime time.Duration, ticker bool)
 // synchronously and can use shared state.
 func NewProgress() *Progress {
 	var d time.Duration
-	if !isTerminal {
-		// TODO: make the duration for non-terminal progress (user) configurable
-		d = time.Duration(10) * time.Second
-	} else {
+	if isTerminal {
 		d = time.Second
 	}
 	return &Progress{d: d}
@@ -70,7 +70,10 @@ func (p *Progress) Start() {
 	p.running = true
 	p.Reset()
 	p.start = time.Now()
-	p.c = time.NewTicker(p.d)
+	p.c = nil
+	if p.d != 0 {
+		p.c = time.NewTicker(p.d)
+	}
 
 	if p.OnStart != nil {
 		p.OnStart()
@@ -143,14 +146,21 @@ func (p *Progress) reporter() {
 		p.updateProgress(cur, true)
 	}
 
+	var ticker <-chan time.Time
+	if p.c != nil {
+		ticker = p.c.C
+	}
+
 	for {
 		select {
-		case <-p.c.C:
+		case <-ticker:
 			updateProgress()
 		case <-forceUpdateProgress:
 			updateProgress()
 		case <-p.cancel:
-			p.c.Stop()
+			if p.c != nil {
+				p.c.Stop()
+			}
 			return
 		}
 	}

src/restic/testing.go

@@ -205,7 +205,7 @@ func TestCreateSnapshot(t testing.TB, repo Repository, at time.Time, depth int,
 func TestParseID(s string) ID {
 	id, err := ParseID(s)
 	if err != nil {
-		panic(err)
+		panic(fmt.Sprintf("unable to parse string %q as ID: %v", s, err))
 	}
 
 	return id