Update github.com/kurin/blazer to 0.5.0

This includes support for the upcoming B2 application keys feature.

vendor/github.com/kurin/blazer/b2/b2.go

@@ -32,6 +32,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"regexp"
 	"strconv"
 	"sync"
@@ -46,6 +47,7 @@ type Client struct {
 	sWriters map[string]*Writer
 	sReaders map[string]*Reader
 	sMethods []methodCounter
+	opts     clientOptions
 }
 
 // NewClient creates and returns a new Client with valid B2 service account
@@ -63,7 +65,10 @@ func NewClient(ctx context.Context, account, key string, opts ...ClientOption) (
 		},
 	}
 	opts = append(opts, client(c))
-	if err := c.backend.authorizeAccount(ctx, account, key, opts...); err != nil {
+	for _, f := range opts {
+		f(&c.opts)
+	}
+	if err := c.backend.authorizeAccount(ctx, account, key, c.opts); err != nil {
 		return nil, err
 	}
 	return c, nil
@@ -75,27 +80,9 @@ type clientOptions struct {
 	failSomeUploads bool
 	expireTokens    bool
 	capExceeded     bool
+	apiBase         string
 	userAgents      []string
-}
-
-// for testing
-func (c clientOptions) eq(o clientOptions) bool {
-	if c.client != o.client ||
-		c.transport != o.transport ||
-		c.failSomeUploads != o.failSomeUploads ||
-		c.expireTokens != o.expireTokens ||
-		c.capExceeded != o.capExceeded {
-		return false
-	}
-	if len(c.userAgents) != len(o.userAgents) {
-		return false
-	}
-	for i := range c.userAgents {
-		if c.userAgents[i] != o.userAgents[i] {
-			return false
-		}
-	}
-	return true
+	writerOpts      []WriterOption
 }
 
 // A ClientOption allows callers to adjust various per-client settings.
@@ -112,6 +99,13 @@ func UserAgent(agent string) ClientOption {
 	}
 }
 
+// APIBase returns a ClientOption specifying the URL root of API requests.
+func APIBase(url string) ClientOption {
+	return func(o *clientOptions) {
+		o.apiBase = url
+	}
+}
+
 // Transport sets the underlying HTTP transport mechanism.  If unset,
 // http.DefaultTransport is used.
 func Transport(rt http.RoundTripper) ClientOption {
@@ -434,7 +428,7 @@ type Attrs struct {
 	ContentType     string            // Used on upload, default is "application/octet-stream".
 	Status          ObjectState       // Not used on upload.
 	UploadTimestamp time.Time         // Not used on upload.
-	SHA1            string            // Not used on upload. Can be "none" for large files.
+	SHA1            string            // Can be "none" for large files.  If set on upload, will be used for large files.
 	LastModified    time.Time         // If present, and there are fewer than 10 keys in the Info field, this is saved on upload.
 	Info            map[string]string // Save arbitrary metadata on upload, but limited to 10 keys.
 }
@@ -474,6 +468,9 @@ func (o *Object) Attrs(ctx context.Context) (*Attrs, error) {
 		mtime = time.Unix(ms/1e3, (ms%1e3)*1e6)
 		delete(info, "src_last_modified_millis")
 	}
+	if v, ok := info["large_file_sha1"]; ok {
+		sha = v
+	}
 	return &Attrs{
 		Name:            name,
 		Size:            size,
@@ -524,14 +521,21 @@ func (o *Object) URL() string {
 // overwritten are not deleted, but are "hidden".
 //
 // Callers must close the writer when finished and check the error status.
-func (o *Object) NewWriter(ctx context.Context) *Writer {
+func (o *Object) NewWriter(ctx context.Context, opts ...WriterOption) *Writer {
 	ctx, cancel := context.WithCancel(ctx)
-	return &Writer{
+	w := &Writer{
 		o:      o,
 		name:   o.name,
 		ctx:    ctx,
 		cancel: cancel,
 	}
+	for _, f := range o.b.c.opts.writerOpts {
+		f(w)
+	}
+	for _, f := range opts {
+		f(w)
+	}
+	return w
 }
 
 // NewRangeReader returns a reader for the given object, reading up to length
@@ -760,5 +764,24 @@ func (b *Bucket) getObject(ctx context.Context, name string) (*Object, error) {
 // in a private bucket.  Only objects that begin with prefix can be accessed.
 // The token expires after the given duration.
 func (b *Bucket) AuthToken(ctx context.Context, prefix string, valid time.Duration) (string, error) {
-	return b.b.getDownloadAuthorization(ctx, prefix, valid)
+	return b.b.getDownloadAuthorization(ctx, prefix, valid, "")
+}
+
+// AuthURL returns a URL for the given object with embedded token and,
+// possibly, b2ContentDisposition arguments.  Leave b2cd blank for no content
+// disposition.
+func (o *Object) AuthURL(ctx context.Context, valid time.Duration, b2cd string) (*url.URL, error) {
+	token, err := o.b.b.getDownloadAuthorization(ctx, o.name, valid, b2cd)
+	if err != nil {
+		return nil, err
+	}
+	urlString := fmt.Sprintf("%s?Authorization=%s", o.URL(), url.QueryEscape(token))
+	if b2cd != "" {
+		urlString = fmt.Sprintf("%s&b2ContentDisposition=%s", urlString, url.QueryEscape(b2cd))
+	}
+	u, err := url.Parse(urlString)
+	if err != nil {
+		return nil, err
+	}
+	return u, nil
 }

vendor/github.com/kurin/blazer/b2/b2_test.go

@@ -71,7 +71,7 @@ type testRoot struct {
 	bucketMap map[string]map[string]string
 }
 
-func (t *testRoot) authorizeAccount(context.Context, string, string, ...ClientOption) error {
+func (t *testRoot) authorizeAccount(context.Context, string, string, clientOptions) error {
 	t.auths++
 	return nil
 }
@@ -108,6 +108,13 @@ func (t *testRoot) transient(err error) bool {
 	return e.retry || e.reupload || e.backoff > 0
 }
 
+func (t *testRoot) createKey(context.Context, string, []string, time.Duration, string, string) (b2KeyInterface, error) {
+	return nil, nil
+}
+func (t *testRoot) listKeys(context.Context, int, string) ([]b2KeyInterface, string, error) {
+	return nil, "", nil
+}
+
 func (t *testRoot) createBucket(_ context.Context, name, _ string, _ map[string]string, _ []LifecycleRule) (b2BucketInterface, error) {
 	if err := t.errs.getError("createBucket"); err != nil {
 		return nil, err
@@ -147,6 +154,7 @@ func (t *testBucket) btype() string                                    { return
 func (t *testBucket) attrs() *BucketAttrs                              { return nil }
 func (t *testBucket) deleteBucket(context.Context) error               { return nil }
 func (t *testBucket) updateBucket(context.Context, *BucketAttrs) error { return nil }
+func (t *testBucket) id() string                                       { return "" }
 
 func (t *testBucket) getUploadURL(context.Context) (b2URLInterface, error) {
 	if err := t.errs.getError("getUploadURL"); err != nil {
@@ -221,7 +229,7 @@ func (t *testBucket) downloadFileByName(_ context.Context, name string, offset,
 }
 
 func (t *testBucket) hideFile(context.Context, string) (b2FileInterface, error) { return nil, nil }
-func (t *testBucket) getDownloadAuthorization(context.Context, string, time.Duration) (string, error) {
+func (t *testBucket) getDownloadAuthorization(context.Context, string, time.Duration, string) (string, error) {
 	return "", nil
 }
 func (t *testBucket) baseURL() string                      { return "" }

vendor/github.com/kurin/blazer/b2/backend.go

@@ -28,22 +28,25 @@ type beRootInterface interface {
 	reauth(error) bool
 	transient(error) bool
 	reupload(error) bool
-	authorizeAccount(context.Context, string, string, ...ClientOption) error
+	authorizeAccount(context.Context, string, string, clientOptions) error
 	reauthorizeAccount(context.Context) error
 	createBucket(ctx context.Context, name, btype string, info map[string]string, rules []LifecycleRule) (beBucketInterface, error)
 	listBuckets(context.Context) ([]beBucketInterface, error)
+	createKey(context.Context, string, []string, time.Duration, string, string) (beKeyInterface, error)
+	listKeys(context.Context, int, string) ([]beKeyInterface, string, error)
 }
 
 type beRoot struct {
 	account, key string
 	b2i          b2RootInterface
-	options      []ClientOption
+	options      clientOptions
 }
 
 type beBucketInterface interface {
 	name() string
 	btype() BucketType
 	attrs() *BucketAttrs
+	id() string
 	updateBucket(context.Context, *BucketAttrs) error
 	deleteBucket(context.Context) error
 	getUploadURL(context.Context) (beURLInterface, error)
@@ -53,7 +56,7 @@ type beBucketInterface interface {
 	listUnfinishedLargeFiles(context.Context, int, string) ([]beFileInterface, string, error)
 	downloadFileByName(context.Context, string, int64, int64) (beFileReaderInterface, error)
 	hideFile(context.Context, string) (beFileInterface, error)
-	getDownloadAuthorization(context.Context, string, time.Duration) (string, error)
+	getDownloadAuthorization(context.Context, string, time.Duration, string) (string, error)
 	baseURL() string
 	file(string, string) beFileInterface
 }
@@ -145,26 +148,39 @@ type beFileInfo struct {
 	stamp  time.Time
 }
 
+type beKeyInterface interface {
+	del(context.Context) error
+	caps() []string
+	name() string
+	expires() time.Time
+	secret() string
+}
+
+type beKey struct {
+	b2i beRootInterface
+	k   b2KeyInterface
+}
+
 func (r *beRoot) backoff(err error) time.Duration { return r.b2i.backoff(err) }
 func (r *beRoot) reauth(err error) bool           { return r.b2i.reauth(err) }
 func (r *beRoot) reupload(err error) bool         { return r.b2i.reupload(err) }
 func (r *beRoot) transient(err error) bool        { return r.b2i.transient(err) }
 
-func (r *beRoot) authorizeAccount(ctx context.Context, account, key string, opts ...ClientOption) error {
+func (r *beRoot) authorizeAccount(ctx context.Context, account, key string, c clientOptions) error {
 	f := func() error {
-		if err := r.b2i.authorizeAccount(ctx, account, key, opts...); err != nil {
+		if err := r.b2i.authorizeAccount(ctx, account, key, c); err != nil {
 			return err
 		}
 		r.account = account
 		r.key = key
-		r.options = opts
+		r.options = c
 		return nil
 	}
 	return withBackoff(ctx, r, f)
 }
 
 func (r *beRoot) reauthorizeAccount(ctx context.Context) error {
-	return r.authorizeAccount(ctx, r.account, r.key, r.options...)
+	return r.authorizeAccount(ctx, r.account, r.key, r.options)
 }
 
 func (r *beRoot) createBucket(ctx context.Context, name, btype string, info map[string]string, rules []LifecycleRule) (beBucketInterface, error) {
@@ -213,17 +229,58 @@ func (r *beRoot) listBuckets(ctx context.Context) ([]beBucketInterface, error) {
 	return buckets, nil
 }
 
-func (b *beBucket) name() string {
-	return b.b2bucket.name()
+func (r *beRoot) createKey(ctx context.Context, name string, caps []string, valid time.Duration, bucketID string, prefix string) (beKeyInterface, error) {
+	var k *beKey
+	f := func() error {
+		g := func() error {
+			got, err := r.b2i.createKey(ctx, name, caps, valid, bucketID, prefix)
+			if err != nil {
+				return err
+			}
+			k = &beKey{
+				b2i: r,
+				k:   got,
+			}
+			return nil
+		}
+		return withReauth(ctx, r, g)
+	}
+	if err := withBackoff(ctx, r, f); err != nil {
+		return nil, err
+	}
+	return k, nil
 }
 
-func (b *beBucket) btype() BucketType {
-	return BucketType(b.b2bucket.btype())
+func (r *beRoot) listKeys(ctx context.Context, max int, next string) ([]beKeyInterface, string, error) {
+	var keys []beKeyInterface
+	var cur string
+	f := func() error {
+		g := func() error {
+			got, n, err := r.b2i.listKeys(ctx, max, next)
+			if err != nil {
+				return err
+			}
+			cur = n
+			for _, g := range got {
+				keys = append(keys, &beKey{
+					b2i: r,
+					k:   g,
+				})
+			}
+			return nil
+		}
+		return withReauth(ctx, r, g)
+	}
+	if err := withBackoff(ctx, r, f); err != nil {
+		return nil, "", err
+	}
+	return keys, cur, nil
 }
 
-func (b *beBucket) attrs() *BucketAttrs {
-	return b.b2bucket.attrs()
-}
+func (b *beBucket) name() string        { return b.b2bucket.name() }
+func (b *beBucket) btype() BucketType   { return BucketType(b.b2bucket.btype()) }
+func (b *beBucket) attrs() *BucketAttrs { return b.b2bucket.attrs() }
+func (b *beBucket) id() string          { return b.b2bucket.id() }
 
 func (b *beBucket) updateBucket(ctx context.Context, attrs *BucketAttrs) error {
 	f := func() error {
@@ -412,11 +469,11 @@ func (b *beBucket) hideFile(ctx context.Context, name string) (beFileInterface,
 	return file, nil
 }
 
-func (b *beBucket) getDownloadAuthorization(ctx context.Context, p string, v time.Duration) (string, error) {
+func (b *beBucket) getDownloadAuthorization(ctx context.Context, p string, v time.Duration, s string) (string, error) {
 	var tok string
 	f := func() error {
 		g := func() error {
-			t, err := b.b2bucket.getDownloadAuthorization(ctx, p, v)
+			t, err := b.b2bucket.getDownloadAuthorization(ctx, p, v, s)
 			if err != nil {
 				return err
 			}
@@ -649,6 +706,12 @@ func (b *beFilePart) number() int  { return b.b2filePart.number() }
 func (b *beFilePart) sha1() string { return b.b2filePart.sha1() }
 func (b *beFilePart) size() int64  { return b.b2filePart.size() }
 
+func (b *beKey) del(ctx context.Context) error { return b.k.del(ctx) }
+func (b *beKey) caps() []string                { return b.k.caps() }
+func (b *beKey) name() string                  { return b.k.name() }
+func (b *beKey) expires() time.Time            { return b.k.expires() }
+func (b *beKey) secret() string                { return b.k.secret() }
+
 func jitter(d time.Duration) time.Duration {
 	f := float64(d)
 	f /= 50
@@ -657,8 +720,8 @@ func jitter(d time.Duration) time.Duration {
 }
 
 func getBackoff(d time.Duration) time.Duration {
-	if d > 15*time.Second {
-		return d + jitter(d)
+	if d > 30*time.Second {
+		return 30*time.Second + jitter(d)
 	}
 	return d*2 + jitter(d*2)
 }

vendor/github.com/kurin/blazer/b2/baseline.go

@@ -27,19 +27,22 @@ import (
 // the only file in b2 that imports base.
 
 type b2RootInterface interface {
-	authorizeAccount(context.Context, string, string, ...ClientOption) error
+	authorizeAccount(context.Context, string, string, clientOptions) error
 	transient(error) bool
 	backoff(error) time.Duration
 	reauth(error) bool
 	reupload(error) bool
 	createBucket(context.Context, string, string, map[string]string, []LifecycleRule) (b2BucketInterface, error)
 	listBuckets(context.Context) ([]b2BucketInterface, error)
+	createKey(context.Context, string, []string, time.Duration, string, string) (b2KeyInterface, error)
+	listKeys(context.Context, int, string) ([]b2KeyInterface, string, error)
 }
 
 type b2BucketInterface interface {
 	name() string
 	btype() string
 	attrs() *BucketAttrs
+	id() string
 	updateBucket(context.Context, *BucketAttrs) error
 	deleteBucket(context.Context) error
 	getUploadURL(context.Context) (b2URLInterface, error)
@@ -49,7 +52,7 @@ type b2BucketInterface interface {
 	listUnfinishedLargeFiles(context.Context, int, string) ([]b2FileInterface, string, error)
 	downloadFileByName(context.Context, string, int64, int64) (b2FileReaderInterface, error)
 	hideFile(context.Context, string) (b2FileInterface, error)
-	getDownloadAuthorization(context.Context, string, time.Duration) (string, error)
+	getDownloadAuthorization(context.Context, string, time.Duration, string) (string, error)
 	baseURL() string
 	file(string, string) b2FileInterface
 }
@@ -96,6 +99,14 @@ type b2FilePartInterface interface {
 	size() int64
 }
 
+type b2KeyInterface interface {
+	del(context.Context) error
+	caps() []string
+	name() string
+	expires() time.Time
+	secret() string
+}
+
 type b2Root struct {
 	b *base.B2
 }
@@ -132,11 +143,11 @@ type b2FilePart struct {
 	b *base.FilePart
 }
 
-func (b *b2Root) authorizeAccount(ctx context.Context, account, key string, opts ...ClientOption) error {
-	c := &clientOptions{}
-	for _, f := range opts {
-		f(c)
-	}
+type b2Key struct {
+	b *base.Key
+}
+
+func (b *b2Root) authorizeAccount(ctx context.Context, account, key string, c clientOptions) error {
 	var aopts []base.AuthOption
 	ct := &clientTransport{client: c.client}
 	if c.transport != nil {
@@ -152,6 +163,9 @@ func (b *b2Root) authorizeAccount(ctx context.Context, account, key string, opts
 	if c.capExceeded {
 		aopts = append(aopts, base.ForceCapExceeded())
 	}
+	if c.apiBase != "" {
+		aopts = append(aopts, base.SetAPIBase(c.apiBase))
+	}
 	for _, agent := range c.userAgents {
 		aopts = append(aopts, base.UserAgent(agent))
 	}
@@ -249,6 +263,26 @@ func (b *b2Bucket) updateBucket(ctx context.Context, attrs *BucketAttrs) error {
 	return err
 }
 
+func (b *b2Root) createKey(ctx context.Context, name string, caps []string, valid time.Duration, bucketID string, prefix string) (b2KeyInterface, error) {
+	k, err := b.b.CreateKey(ctx, name, caps, valid, bucketID, prefix)
+	if err != nil {
+		return nil, err
+	}
+	return &b2Key{k}, nil
+}
+
+func (b *b2Root) listKeys(ctx context.Context, max int, next string) ([]b2KeyInterface, string, error) {
+	keys, next, err := b.b.ListKeys(ctx, max, next)
+	if err != nil {
+		return nil, "", err
+	}
+	var k []b2KeyInterface
+	for _, key := range keys {
+		k = append(k, &b2Key{key})
+	}
+	return k, next, nil
+}
+
 func (b *b2Bucket) deleteBucket(ctx context.Context) error {
 	return b.b.DeleteBucket(ctx)
 }
@@ -277,6 +311,8 @@ func (b *b2Bucket) attrs() *BucketAttrs {
 	}
 }
 
+func (b *b2Bucket) id() string { return b.b.ID }
+
 func (b *b2Bucket) getUploadURL(ctx context.Context) (b2URLInterface, error) {
 	url, err := b.b.GetUploadURL(ctx)
 	if err != nil {
@@ -352,8 +388,8 @@ func (b *b2Bucket) hideFile(ctx context.Context, name string) (b2FileInterface,
 	return &b2File{f}, nil
 }
 
-func (b *b2Bucket) getDownloadAuthorization(ctx context.Context, p string, v time.Duration) (string, error) {
-	return b.b.GetDownloadAuthorization(ctx, p, v)
+func (b *b2Bucket) getDownloadAuthorization(ctx context.Context, p string, v time.Duration, s string) (string, error) {
+	return b.b.GetDownloadAuthorization(ctx, p, v, s)
 }
 
 func (b *b2Bucket) baseURL() string {
@@ -466,3 +502,9 @@ func (b *b2FileInfo) stats() (string, string, int64, string, map[string]string,
 func (b *b2FilePart) number() int  { return b.b.Number }
 func (b *b2FilePart) sha1() string { return b.b.SHA1 }
 func (b *b2FilePart) size() int64  { return b.b.Size }
+
+func (b *b2Key) del(ctx context.Context) error { return b.b.Delete(ctx) }
+func (b *b2Key) caps() []string                { return b.b.Capabilities }
+func (b *b2Key) name() string                  { return b.b.Name }
+func (b *b2Key) expires() time.Time            { return b.b.Expires }
+func (b *b2Key) secret() string                { return b.b.Secret }

vendor/github.com/kurin/blazer/b2/integration_test.go

@@ -22,6 +22,7 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"os"
 	"reflect"
@@ -29,6 +30,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/kurin/blazer/internal/blog"
 	"github.com/kurin/blazer/x/transport"
 )
 
@@ -137,6 +139,9 @@ func TestReaderFromLive(t *testing.T) {
 		if rn != n {
 			t.Errorf("Read from B2: got %d bytes, want %d bytes", rn, n)
 		}
+		if err, ok := r.Verify(); ok && err != nil {
+			t.Errorf("Read from B2: %v", err)
+		}
 		if err := r.Close(); err != nil {
 			t.Errorf("r.Close(): %v", err)
 		}
@@ -323,6 +328,7 @@ func TestAttrs(t *testing.T) {
 		for _, attrs := range attrlist {
 			o := bucket.Object(e.name)
 			w := o.NewWriter(ctx).WithAttrs(attrs)
+			w.ChunkSize = 5e6
 			if _, err := io.Copy(w, io.LimitReader(zReader{}, e.size)); err != nil {
 				t.Error(err)
 				continue
@@ -429,6 +435,57 @@ func TestAuthTokLive(t *testing.T) {
 	}
 }
 
+func TestObjAuthTokLive(t *testing.T) {
+	ctx := context.Background()
+	ctx, cancel := context.WithTimeout(ctx, time.Minute)
+	defer cancel()
+	bucket, done := startLiveTest(ctx, t)
+	defer done()
+
+	table := []struct {
+		obj  string
+		d    time.Duration
+		b2cd string
+	}{
+		{
+			obj: "foo/bar",
+			d:   time.Minute,
+		},
+		{
+			obj:  "foo2/thing.pdf",
+			d:    time.Minute,
+			b2cd: "attachment",
+		},
+		{
+			obj:  "foo2/thing.pdf",
+			d:    time.Minute,
+			b2cd: `attachment; filename="what.png"`,
+		},
+	}
+
+	for _, e := range table {
+		fw := bucket.Object(e.obj).NewWriter(ctx)
+		io.Copy(fw, io.LimitReader(zReader{}, 1e5))
+		if err := fw.Close(); err != nil {
+			t.Fatal(err)
+		}
+
+		url, err := bucket.Object(e.obj).AuthURL(ctx, e.d, e.b2cd)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		blog.V(2).Infof("downloading %s", url.String())
+		frsp, err := http.Get(url.String())
+		if err != nil {
+			t.Fatal(err)
+		}
+		if frsp.StatusCode != 200 {
+			t.Fatalf("%s: got %s, want 200", url.String(), frsp.Status)
+		}
+	}
+}
+
 func TestRangeReaderLive(t *testing.T) {
 	ctx := context.Background()
 	ctx, cancel := context.WithTimeout(ctx, time.Minute)
@@ -510,6 +567,9 @@ func TestRangeReaderLive(t *testing.T) {
 		if got != want {
 			t.Errorf("NewRangeReader(_, %d, %d): got %q, want %q", e.offset, e.length, got, want)
 		}
+		if err, ok := r.Verify(); ok && err != nil {
+			t.Errorf("NewRangeReader(_, %d, %d): %v", e.offset, e.length, err)
+		}
 	}
 }
 
@@ -863,29 +923,161 @@ func TestReauthPreservesOptions(t *testing.T) {
 	bucket, done := startLiveTest(ctx, t)
 	defer done()
 
-	var first []ClientOption
-	opts := bucket.r.(*beRoot).options
-	for _, o := range opts {
-		first = append(first, o)
-	}
-
+	first := bucket.r.(*beRoot).options
 	if err := bucket.r.reauthorizeAccount(ctx); err != nil {
 		t.Fatalf("reauthorizeAccount: %v", err)
 	}
-
 	second := bucket.r.(*beRoot).options
-	if len(second) != len(first) {
-		t.Fatalf("options mismatch: got %d options, wanted %d", len(second), len(first))
+	if !reflect.DeepEqual(first, second) {
+		// Test that they are literally the same set of options, which is an
+		// implementation detail but is fine for now.
+		t.Errorf("options mismatch: got %v, want %v", second, first)
+	}
+}
+
+func TestVerifyReader(t *testing.T) {
+	ctx := context.Background()
+	bucket, done := startLiveTest(ctx, t)
+	defer done()
+
+	table := []struct {
+		name     string
+		fakeSHA  string
+		size     int64
+		off, len int64
+		valid    bool
+	}{
+		{
+			name:  "first",
+			size:  100,
+			off:   0,
+			len:   -1,
+			valid: true,
+		},
+		{
+			name:  "second",
+			size:  100,
+			off:   0,
+			len:   100,
+			valid: true,
+		},
+		{
+			name:  "third",
+			size:  100,
+			off:   0,
+			len:   99,
+			valid: false,
+		},
+		{
+			name:  "fourth",
+			size:  5e6 + 100,
+			off:   0,
+			len:   -1,
+			valid: false,
+		},
+		{
+			name:    "fifth",
+			size:    5e6 + 100,
+			off:     0,
+			len:     -1,
+			fakeSHA: "fbc815f2d6518858dec83ccb46263875fc894d88",
+			valid:   true,
+		},
 	}
 
-	var f, s clientOptions
-	for i := range first {
-		first[i](&f)
-		second[i](&s)
+	for _, e := range table {
+		o := bucket.Object(e.name)
+		w := o.NewWriter(ctx)
+		if e.fakeSHA != "" {
+			w = w.WithAttrs(&Attrs{SHA1: e.fakeSHA})
+		}
+		w.ChunkSize = 5e6
+		if _, err := io.Copy(w, io.LimitReader(zReader{}, e.size)); err != nil {
+			t.Error(err)
+			continue
+		}
+		if err := w.Close(); err != nil {
+			t.Error(err)
+			continue
+		}
+		r := o.NewRangeReader(ctx, e.off, e.len)
+		if _, err := io.Copy(ioutil.Discard, r); err != nil {
+			t.Error(err)
+		}
+		err, ok := r.Verify()
+		if ok != e.valid {
+			t.Errorf("%s: bad validity: got %v, want %v", e.name, ok, e.valid)
+		}
+		if e.valid && err != nil {
+			t.Errorf("%s does not verify: %v", e.name, err)
+		}
+	}
+}
+
+func TestCreateDeleteKey(t *testing.T) {
+	ctx := context.Background()
+	bucket, done := startLiveTest(ctx, t)
+	defer done()
+
+	table := []struct {
+		d      time.Duration
+		e      time.Time
+		bucket bool
+		cap    []string
+		pfx    string
+	}{
+		{
+			cap: []string{"deleteKeys"},
+		},
+		{
+			d:   time.Minute,
+			cap: []string{"deleteKeys"},
+			pfx: "prefox",
+		},
+		{
+			e:      time.Now().Add(time.Minute), // <shrug emojis>
+			cap:    []string{"writeFiles", "listFiles"},
+			bucket: true,
+		},
+		{
+			d:      time.Minute,
+			cap:    []string{"writeFiles", "listFiles"},
+			pfx:    "prefox",
+			bucket: true,
+		},
 	}
 
-	if !f.eq(s) {
-		t.Errorf("options mismatch: got %v, want %v", s, f)
+	for _, e := range table {
+		var opts []KeyOption
+		for _, cap := range e.cap {
+			opts = append(opts, Capability(cap))
+		}
+		if e.d != 0 {
+			opts = append(opts, Lifetime(e.d))
+		}
+		if !e.e.IsZero() {
+			opts = append(opts, Deadline(e.e))
+		}
+		var key *Key
+		if e.bucket {
+			opts = append(opts, Prefix(e.pfx))
+			bkey, err := bucket.CreateKey(ctx, "whee", opts...)
+			if err != nil {
+				t.Errorf("Bucket.CreateKey(%v, %v): %v", bucket.Name(), e, err)
+				continue
+			}
+			key = bkey
+		} else {
+			gkey, err := bucket.c.CreateKey(ctx, "whee", opts...)
+			if err != nil {
+				t.Errorf("Client.CreateKey(%v): %v", e, err)
+				continue
+			}
+			key = gkey
+		}
+		if err := key.Delete(ctx); err != nil {
+			t.Errorf("key.Delete(): %v", err)
+		}
 	}
 }
 

vendor/github.com/kurin/blazer/b2/key.go

@@ -0,0 +1,148 @@
+// Copyright 2018, Google
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package b2
+
+import (
+	"context"
+	"errors"
+	"io"
+	"time"
+)
+
+// Key is a B2 application key.  A Key grants limited access on a global or
+// per-bucket basis.
+type Key struct {
+	c *Client
+	k beKeyInterface
+}
+
+// Capabilities returns the list of capabilites granted by this application
+// key.
+func (k *Key) Capabilities() []string { return k.k.caps() }
+
+// Name returns the user-supplied name of this application key.  Key names are
+// useless.
+func (k *Key) Name() string { return k.k.name() }
+
+// Expires returns the expiration date of this application key.
+func (k *Key) Expires() time.Time { return k.k.expires() }
+
+// Delete removes the key from B2.
+func (k *Key) Delete(ctx context.Context) error { return k.k.del(ctx) }
+
+// Secret returns the value that should be passed into NewClient().  It is only
+// available on newly created keys; it is not available from ListKey
+// operations.
+func (k *Key) Secret() string { return k.k.secret() }
+
+type keyOptions struct {
+	caps     []string
+	prefix   string
+	lifetime time.Duration
+}
+
+// KeyOption specifies desired properties for application keys.
+type KeyOption func(*keyOptions)
+
+// Lifetime requests a key with the given lifetime.
+func Lifetime(d time.Duration) KeyOption {
+	return func(k *keyOptions) {
+		k.lifetime = d
+	}
+}
+
+// Deadline requests a key that expires after the given date.
+func Deadline(t time.Time) KeyOption {
+	d := t.Sub(time.Now())
+	return Lifetime(d)
+}
+
+// Capability requests a key with the given capability.
+func Capability(cap string) KeyOption {
+	return func(k *keyOptions) {
+		k.caps = append(k.caps, cap)
+	}
+}
+
+// Prefix limits the requested application key to be valid only for objects
+// that begin with prefix.  This can only be used when requesting an
+// application key within a specific bucket.
+func Prefix(prefix string) KeyOption {
+	return func(k *keyOptions) {
+		k.prefix = prefix
+	}
+}
+
+// CreateKey creates a global application key that is valid for all buckets in
+// this project.  The key's secret will only be accessible on the object
+// returned from this call.
+func (c *Client) CreateKey(ctx context.Context, name string, opts ...KeyOption) (*Key, error) {
+	var ko keyOptions
+	for _, o := range opts {
+		o(&ko)
+	}
+	if ko.prefix != "" {
+		return nil, errors.New("Prefix is not a valid option for global application keys")
+	}
+	ki, err := c.backend.createKey(ctx, name, ko.caps, ko.lifetime, "", "")
+	if err != nil {
+		return nil, err
+	}
+	return &Key{
+		c: c,
+		k: ki,
+	}, nil
+}
+
+// ListKeys lists all the keys associated with this project.  It takes the
+// maximum number of keys it should return in a call, as well as a cursor
+// (which should be empty for the initial call).  It will return up to count
+// keys, as well as the cursor for the next invocation.
+//
+// ListKeys returns io.EOF when there are no more keys, although it may do so
+// concurrently with the final set of keys.
+func (c *Client) ListKeys(ctx context.Context, count int, cursor string) ([]*Key, string, error) {
+	ks, next, err := c.backend.listKeys(ctx, count, cursor)
+	if err != nil {
+		return nil, "", err
+	}
+	if len(ks) == 0 {
+		return nil, "", io.EOF
+	}
+	var keys []*Key
+	for _, k := range ks {
+		keys = append(keys, &Key{
+			c: c,
+			k: k,
+		})
+	}
+	return keys, next, nil
+}
+
+// CreateKey creates a scoped application key that is valid only for this bucket.
+func (b *Bucket) CreateKey(ctx context.Context, name string, opts ...KeyOption) (*Key, error) {
+	var ko keyOptions
+	for _, o := range opts {
+		o(&ko)
+	}
+	ki, err := b.r.createKey(ctx, name, ko.caps, ko.lifetime, b.b.id(), ko.prefix)
+	if err != nil {
+		return nil, err
+	}
+	return &Key{
+		c: b.c,
+		k: ki,
+	}, nil
+}

vendor/github.com/kurin/blazer/b2/reader.go

@@ -17,9 +17,13 @@ package b2
 import (
 	"bytes"
 	"context"
+	"crypto/sha1"
 	"errors"
+	"fmt"
+	"hash"
 	"io"
 	"sync"
+	"time"
 
 	"github.com/kurin/blazer/internal/blog"
 )
@@ -38,21 +42,25 @@ type Reader struct {
 	// 10MB.
 	ChunkSize int
 
-	ctx    context.Context
-	cancel context.CancelFunc // cancels ctx
-	o      *Object
-	name   string
-	offset int64 // the start of the file
-	length int64 // the length to read, or -1
-	csize  int   // chunk size
-	read   int   // amount read
-	chwid  int   // chunks written
-	chrid  int   // chunks read
-	chbuf  chan *rchunk
-	init   sync.Once
-	rmux   sync.Mutex // guards rcond
-	rcond  *sync.Cond
-	chunks map[int]*rchunk
+	ctx        context.Context
+	cancel     context.CancelFunc // cancels ctx
+	o          *Object
+	name       string
+	offset     int64 // the start of the file
+	length     int64 // the length to read, or -1
+	csize      int   // chunk size
+	read       int   // amount read
+	chwid      int   // chunks written
+	chrid      int   // chunks read
+	chbuf      chan *rchunk
+	init       sync.Once
+	chunks     map[int]*rchunk
+	vrfy       hash.Hash
+	readOffEnd bool
+	sha1       string
+
+	rmux  sync.Mutex // guards rcond
+	rcond *sync.Cond
 
 	emux sync.RWMutex // guards err, believe it or not
 	err  error
@@ -122,10 +130,12 @@ func (r *Reader) thread() {
 				}
 				r.length -= size
 			}
+			var b backoff
 		redo:
 			fr, err := r.o.b.b.downloadFileByName(r.ctx, r.name, offset, size)
 			if err == errNoMoreContent {
 				// this read generated a 416 so we are entirely past the end of the object
+				r.readOffEnd = true
 				buf.final = true
 				r.rmux.Lock()
 				r.chunks[chunkID] = buf
@@ -138,7 +148,10 @@ func (r *Reader) thread() {
 				r.rcond.Broadcast()
 				return
 			}
-			rsize, _, _, _ := fr.stats()
+			rsize, _, sha1, _ := fr.stats()
+			if len(sha1) == 40 && r.sha1 != sha1 {
+				r.sha1 = sha1
+			}
 			mr := &meteredReader{r: noopResetter{fr}, size: int(rsize)}
 			r.smux.Lock()
 			r.smap[chunkID] = mr
@@ -150,7 +163,12 @@ func (r *Reader) thread() {
 			r.smux.Unlock()
 			if i < int64(rsize) || err == io.ErrUnexpectedEOF {
 				// Probably the network connection was closed early.  Retry.
-				blog.V(1).Infof("b2 reader %d: got %dB of %dB; retrying", chunkID, i, rsize)
+				blog.V(1).Infof("b2 reader %d: got %dB of %dB; retrying after %v", chunkID, i, rsize, b)
+				if err := b.wait(r.ctx); err != nil {
+					r.setErr(err)
+					r.rcond.Broadcast()
+					return
+				}
 				buf.Reset()
 				goto redo
 			}
@@ -211,13 +229,13 @@ func (r *Reader) initFunc() {
 		r.thread()
 		r.chbuf <- &rchunk{}
 	}
+	r.vrfy = sha1.New()
 }
 
 func (r *Reader) Read(p []byte) (int, error) {
 	if err := r.getErr(); err != nil {
 		return 0, err
 	}
-	// TODO: check the SHA1 hash here and verify it on Close.
 	r.init.Do(r.initFunc)
 	chunk, err := r.curChunk()
 	if err != nil {
@@ -225,6 +243,7 @@ func (r *Reader) Read(p []byte) (int, error) {
 		return 0, err
 	}
 	n, err := chunk.Read(p)
+	r.vrfy.Write(p[:n]) // Hash.Write never returns an error.
 	r.read += n
 	if err == io.EOF {
 		if chunk.final {
@@ -256,38 +275,49 @@ func (r *Reader) status() *ReaderStatus {
 	return rs
 }
 
-// copied from io.Copy, basically.
-func copyContext(ctx context.Context, dst io.Writer, src io.Reader) (written int64, err error) {
-	buf := make([]byte, 32*1024)
-	for {
-		if ctx.Err() != nil {
-			err = ctx.Err()
-			return
+// Verify checks the SHA1 hash on download and compares it to the SHA1 hash
+// submitted on upload.  If the two differ, this returns an error.  If the
+// correct hash could not be calculated (if, for example, the entire object was
+// not read, or if the object was uploaded as a "large file" and thus the SHA1
+// hash was not sent), this returns (nil, false).
+func (r *Reader) Verify() (error, bool) {
+	got := fmt.Sprintf("%x", r.vrfy.Sum(nil))
+	if r.sha1 == got {
+		return nil, true
+	}
+	// TODO: if the exact length of the file is requested AND the checksum is
+	// bad, this will return (nil, false) instead of (an error, true).  This is
+	// because there's no good way that I can tell to determine that we've hit
+	// the end of the file without reading off the end.  Consider reading N+1
+	// bytes at the very end to close this hole.
+	if r.offset > 0 || !r.readOffEnd || len(r.sha1) != 40 {
+		return nil, false
+	}
+	return fmt.Errorf("bad hash: got %v, want %v", got, r.sha1), true
+}
+
+// strip a writer of any non-Write methods
+type onlyWriter struct{ w io.Writer }
+
+func (ow onlyWriter) Write(p []byte) (int, error) { return ow.w.Write(p) }
+
+func copyContext(ctx context.Context, w io.Writer, r io.Reader) (int64, error) {
+	var n int64
+	var err error
+	done := make(chan struct{})
+	go func() {
+		if _, ok := w.(*Writer); ok {
+			w = onlyWriter{w}
 		}
-		nr, er := src.Read(buf)
-		if nr > 0 {
-			nw, ew := dst.Write(buf[0:nr])
-			if nw > 0 {
-				written += int64(nw)
-			}
-			if ew != nil {
-				err = ew
-				break
-			}
-			if nr != nw {
-				err = io.ErrShortWrite
-				break
-			}
-		}
-		if er == io.EOF {
-			break
-		}
-		if er != nil {
-			err = er
-			break
-		}
+		n, err = io.Copy(w, r)
+		close(done)
+	}()
+	select {
+	case <-done:
+		return n, err
+	case <-ctx.Done():
+		return 0, ctx.Err()
 	}
-	return written, err
 }
 
 type noopResetter struct {
@@ -295,3 +325,24 @@ type noopResetter struct {
 }
 
 func (noopResetter) Reset() error { return nil }
+
+type backoff time.Duration
+
+func (b *backoff) wait(ctx context.Context) error {
+	if *b == 0 {
+		*b = backoff(time.Millisecond)
+	}
+	select {
+	case <-time.After(time.Duration(*b)):
+		if time.Duration(*b) < time.Second*10 {
+			*b <<= 1
+		}
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+func (b backoff) String() string {
+	return time.Duration(b).String()
+}

vendor/github.com/kurin/blazer/b2/writer.go

@@ -476,18 +476,42 @@ func (w *Writer) Close() error {
 
 // WithAttrs sets the writable attributes of the resulting file to given
 // values.  WithAttrs must be called before the first call to Write.
+//
+// DEPRECATED: Use WithAttrsOption instead.
 func (w *Writer) WithAttrs(attrs *Attrs) *Writer {
 	w.contentType = attrs.ContentType
 	w.info = make(map[string]string)
 	for k, v := range attrs.Info {
 		w.info[k] = v
 	}
+	if len(w.info) < 10 && attrs.SHA1 != "" {
+		w.info["large_file_sha1"] = attrs.SHA1
+	}
 	if len(w.info) < 10 && !attrs.LastModified.IsZero() {
 		w.info["src_last_modified_millis"] = fmt.Sprintf("%d", attrs.LastModified.UnixNano()/1e6)
 	}
 	return w
 }
 
+// A WriterOption sets Writer-specific behavior.
+type WriterOption func(*Writer)
+
+// WithAttrs attaches the given Attrs to the writer.
+func WithAttrsOption(attrs *Attrs) WriterOption {
+	return func(w *Writer) {
+		w.WithAttrs(attrs)
+	}
+}
+
+// DefaultWriterOptions returns a ClientOption that will apply the given
+// WriterOptions to every Writer.  These options can be overridden by passing
+// new options to NewWriter.
+func DefaultWriterOptions(opts ...WriterOption) ClientOption {
+	return func(c *clientOptions) {
+		c.writerOpts = opts
+	}
+}
+
 func (w *Writer) status() *WriterStatus {
 	w.smux.RLock()
 	defer w.smux.RUnlock()