Merge pull request #1043 from restic/fix-gcs

s3: Fix GCS

src/restic/backend/s3/s3.go

@@ -4,8 +4,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
-	"net/url"
 	"os"
 	"path"
 	"restic"
@@ -16,7 +14,6 @@ import (
 	"restic/errors"
 
 	"github.com/minio/minio-go"
-	"github.com/minio/minio-go/pkg/s3utils"
 
 	"restic/debug"
 )
@@ -88,7 +85,15 @@ func Open(cfg Config) (restic.Backend, error) {
 // IsNotExist returns true if the error is caused by a not existing file.
 func (be *Backend) IsNotExist(err error) bool {
 	debug.Log("IsNotExist(%T, %#v)", err, err)
-	return os.IsNotExist(err)
+	if os.IsNotExist(errors.Cause(err)) {
+		return true
+	}
+
+	if e, ok := errors.Cause(err).(minio.ErrorResponse); ok && e.Code == "NoSuchKey" {
+		return true
+	}
+
+	return false
 }
 
 // Join combines path components with slashes.
@@ -162,17 +167,51 @@ func (be *Backend) Path() string {
 	return be.cfg.Prefix
 }
 
-func (be *Backend) isGoogleCloudStorage() bool {
-	scheme := "https://"
-	if be.cfg.UseHTTP {
-		scheme = "http://"
-	}
-	url, err := url.Parse(scheme + be.cfg.Endpoint)
+// nopCloserFile wraps *os.File and overwrites the Close() method with method
+// that does nothing. In addition, the method Len() is implemented, which
+// returns the size of the file (filesize - current offset).
+type nopCloserFile struct {
+	*os.File
+}
+
+func (f nopCloserFile) Close() error {
+	debug.Log("prevented Close()")
+	return nil
+}
+
+// Len returns the remaining length of the file (filesize - current offset).
+func (f nopCloserFile) Len() int {
+	debug.Log("Len() called")
+	fi, err := f.Stat()
 	if err != nil {
 		panic(err)
 	}
 
-	return s3utils.IsGoogleEndpoint(*url)
+	pos, err := f.Seek(0, io.SeekCurrent)
+	if err != nil {
+		panic(err)
+	}
+
+	size := fi.Size() - pos
+	debug.Log("returning file size %v", size)
+	return int(size)
+}
+
+type lenner interface {
+	Len() int
+	io.Reader
+}
+
+// nopCloserLenner wraps a lenner and overwrites the Close() method with method
+// that does nothing. In addition, the method Size() is implemented, which
+// returns the size of the file (filesize - current offset).
+type nopCloserLenner struct {
+	lenner
+}
+
+func (f *nopCloserLenner) Close() error {
+	debug.Log("prevented Close()")
+	return nil
 }
 
 // Save stores data in the backend at the handle.
@@ -192,9 +231,15 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err
 		return errors.New("key already exists")
 	}
 
-	// prevent GCS from closing the file
-	if be.isGoogleCloudStorage() {
-		rd = ioutil.NopCloser(rd)
+	// prevent the HTTP client from closing a file
+	if f, ok := rd.(*os.File); ok {
+		debug.Log("reader is %#T, using nopCloserFile{}", rd)
+		rd = nopCloserFile{f}
+	} else if l, ok := rd.(lenner); ok {
+		debug.Log("reader is %#T, using nopCloserLenner{}", rd)
+		rd = nopCloserLenner{l}
+	} else {
+		debug.Log("reader is %#T, no specific workaround enabled", rd)
 	}
 
 	be.sem.GetToken()

src/restic/backend/test/tests.go

@@ -241,8 +241,8 @@ func (s *Suite) TestLoad(t *testing.T) {
 
 type errorCloser struct {
 	io.Reader
-	size int64
-	t    testing.TB
+	l int
+	t testing.TB
 }
 
 func (ec errorCloser) Close() error {
@@ -250,8 +250,8 @@ func (ec errorCloser) Close() error {
 	return errors.New("forbidden method close was called")
 }
 
-func (ec errorCloser) Size() int64 {
-	return ec.size
+func (ec errorCloser) Len() int {
+	return ec.l
 }
 
 // TestSave tests saving data in the backend.
@@ -325,7 +325,7 @@ func (s *Suite) TestSave(t *testing.T) {
 
 	// wrap the tempfile in an errorCloser, so we can detect if the backend
 	// closes the reader
-	err = b.Save(context.TODO(), h, errorCloser{t: t, size: int64(length), Reader: tmpfile})
+	err = b.Save(context.TODO(), h, errorCloser{t: t, l: length, Reader: tmpfile})
 	if err != nil {
 		t.Fatal(err)
 	}