Update dependencies

Closes #2129
This commit is contained in:
Alexander Neumann
2019-02-10 13:24:37 +01:00
parent aa7043151a
commit b9f0f031b6
672 changed files with 116946 additions and 13086 deletions
+8
View File
@@ -0,0 +1,8 @@
# This is the list of Blazer authors for copyright purposes.
#
# This does not necessarily list everyone who has contributed code, since in
# some cases, their employer may be the copyright holder. To see the full list
# of contributors, see the revision history in source control.
#
# Tag yourself.
Google LLC
+1 -1
View File
@@ -1,4 +1,4 @@
Copyright 2016, Google
Copyright 2016, the Blazer authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
+14 -148
View File
@@ -1,4 +1,4 @@
// Copyright 2016, Google
// Copyright 2016, the Blazer authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -577,139 +577,6 @@ func (o *Object) Delete(ctx context.Context) error {
return o.f.deleteFileVersion(ctx)
}
// Cursor is passed to ListObjects to return subsequent pages.
//
// DEPRECATED. Will be removed in a future release.
type Cursor struct {
// Prefix limits the listed objects to those that begin with this string.
Prefix string
// Delimiter denotes the path separator. If set, object listings will be
// truncated at this character.
//
// For example, if the bucket contains objects foo/bar, foo/baz, and foo,
// then a delimiter of "/" will cause the listing to return "foo" and "foo/".
// Otherwise, the listing would have returned all object names.
//
// Note that objects returned that end in the delimiter may not be actual
// objects, e.g. you cannot read from (or write to, or delete) an object "foo/",
// both because no actual object exists and because B2 disallows object names
// that end with "/". If you want to ensure that all objects returned by
// ListObjects and ListCurrentObjects are actual objects, leave this unset.
Delimiter string
name string
id string
}
// ListObjects returns all objects in the bucket, including multiple versions
// of the same object. Cursor may be nil; when passed to a subsequent query,
// it will continue the listing.
//
// ListObjects will return io.EOF when there are no objects left in the bucket,
// however it may do so concurrently with the last objects.
//
// DEPRECATED. Will be removed in a future release.
func (b *Bucket) ListObjects(ctx context.Context, count int, c *Cursor) ([]*Object, *Cursor, error) {
if c == nil {
c = &Cursor{}
}
fs, name, id, err := b.b.listFileVersions(ctx, count, c.name, c.id, c.Prefix, c.Delimiter)
if err != nil {
return nil, nil, err
}
var next *Cursor
if name != "" && id != "" {
next = &Cursor{
Prefix: c.Prefix,
Delimiter: c.Delimiter,
name: name,
id: id,
}
}
var objects []*Object
for _, f := range fs {
objects = append(objects, &Object{
name: f.name(),
f: f,
b: b,
})
}
var rtnErr error
if len(objects) == 0 || next == nil {
rtnErr = io.EOF
}
return objects, next, rtnErr
}
// ListCurrentObjects is similar to ListObjects, except that it returns only
// current, unhidden objects in the bucket.
//
// DEPRECATED. Will be removed in a future release.
func (b *Bucket) ListCurrentObjects(ctx context.Context, count int, c *Cursor) ([]*Object, *Cursor, error) {
if c == nil {
c = &Cursor{}
}
fs, name, err := b.b.listFileNames(ctx, count, c.name, c.Prefix, c.Delimiter)
if err != nil {
return nil, nil, err
}
var next *Cursor
if name != "" {
next = &Cursor{
Prefix: c.Prefix,
Delimiter: c.Delimiter,
name: name,
}
}
var objects []*Object
for _, f := range fs {
objects = append(objects, &Object{
name: f.name(),
f: f,
b: b,
})
}
var rtnErr error
if len(objects) == 0 || next == nil {
rtnErr = io.EOF
}
return objects, next, rtnErr
}
// ListUnfinishedLargeFiles lists any objects that correspond to large file uploads that haven't been completed.
// This can happen for example when an upload is interrupted.
//
// DEPRECATED. Will be removed in a future release.
func (b *Bucket) ListUnfinishedLargeFiles(ctx context.Context, count int, c *Cursor) ([]*Object, *Cursor, error) {
if c == nil {
c = &Cursor{}
}
fs, name, err := b.b.listUnfinishedLargeFiles(ctx, count, c.name)
if err != nil {
return nil, nil, err
}
var next *Cursor
if name != "" {
next = &Cursor{
name: name,
}
}
var objects []*Object
for _, f := range fs {
objects = append(objects, &Object{
name: f.name(),
f: f,
b: b,
})
}
var rtnErr error
if len(objects) == 0 || next == nil {
rtnErr = io.EOF
}
return objects, next, rtnErr
}
// Hide hides the object from name-based listing.
func (o *Object) Hide(ctx context.Context) error {
if err := o.ensure(ctx); err != nil {
@@ -722,21 +589,20 @@ func (o *Object) Hide(ctx context.Context) error {
// Reveal unhides (if hidden) the named object. If there are multiple objects
// of a given name, it will reveal the most recent.
func (b *Bucket) Reveal(ctx context.Context, name string) error {
cur := &Cursor{
name: name,
iter := b.List(ctx, ListPrefix(name), ListHidden())
for iter.Next() {
obj := iter.Object()
if obj.Name() == name {
if obj.f.status() == "hide" {
return obj.Delete(ctx)
}
return nil
}
if obj.Name() > name {
break
}
}
objs, _, err := b.ListObjects(ctx, 1, cur)
if err != nil && err != io.EOF {
return err
}
if len(objs) < 1 || objs[0].name != name {
return b2err{err: fmt.Errorf("%s: not found", name), notFoundErr: true}
}
obj := objs[0]
if obj.f.status() != "hide" {
return nil
}
return obj.Delete(ctx)
return b2err{err: fmt.Errorf("%s: not found", name), notFoundErr: true}
}
// I don't want to import all of ioutil for this.
+12 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2016, Google
// Copyright 2016, the Blazer authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -95,6 +95,7 @@ type beFile struct {
type beLargeFileInterface interface {
finishLargeFile(context.Context) (beFileInterface, error)
getUploadPartURL(context.Context) (beFileChunkInterface, error)
cancel(context.Context) error
}
type beLargeFile struct {
@@ -654,6 +655,16 @@ func (b *beLargeFile) finishLargeFile(ctx context.Context) (beFileInterface, err
return file, nil
}
func (b *beLargeFile) cancel(ctx context.Context) error {
f := func() error {
g := func() error {
return b.b2largeFile.cancel(ctx)
}
return withReauth(ctx, b.ri, g)
}
return withBackoff(ctx, b.ri, f)
}
func (b *beFileChunk) reload(ctx context.Context) error {
f := func() error {
g := func() error {
+6 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2016, Google
// Copyright 2016, the Blazer authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -76,6 +76,7 @@ type b2FileInterface interface {
type b2LargeFileInterface interface {
finishLargeFile(context.Context) (b2FileInterface, error)
getUploadPartURL(context.Context) (b2FileChunkInterface, error)
cancel(context.Context) error
}
type b2FileChunkInterface interface {
@@ -474,6 +475,10 @@ func (b *b2LargeFile) getUploadPartURL(ctx context.Context) (b2FileChunkInterfac
return &b2FileChunk{c}, nil
}
func (b *b2LargeFile) cancel(ctx context.Context) error {
return b.b.CancelLargeFile(ctx)
}
func (b *b2FileChunk) reload(ctx context.Context) error {
return b.b.Reload(ctx)
}
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2017, Google
// Copyright 2017, the Blazer authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
+123 -9
View File
@@ -1,4 +1,4 @@
// Copyright 2018, Google
// Copyright 2018, the Blazer authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -50,7 +50,7 @@ type ObjectIterator struct {
final bool
err error
idx int
c *Cursor
c *cursor
opts objectIteratorOptions
objs []*Object
init sync.Once
@@ -58,7 +58,7 @@ type ObjectIterator struct {
count int
}
type lister func(context.Context, int, *Cursor) ([]*Object, *Cursor, error)
type lister func(context.Context, int, *cursor) ([]*Object, *cursor, error)
func (o *ObjectIterator) page(ctx context.Context) error {
if o.opts.locker != nil {
@@ -96,18 +96,18 @@ func (o *ObjectIterator) Next() bool {
}
switch {
case o.opts.unfinished:
o.l = o.bucket.ListUnfinishedLargeFiles
o.l = o.bucket.listUnfinishedLargeFiles
if o.count > 100 {
o.count = 100
}
case o.opts.hidden:
o.l = o.bucket.ListObjects
o.l = o.bucket.listObjects
default:
o.l = o.bucket.ListCurrentObjects
o.l = o.bucket.listCurrentObjects
}
o.c = &Cursor{
Prefix: o.opts.prefix,
Delimiter: o.opts.delimiter,
o.c = &cursor{
prefix: o.opts.prefix,
delimiter: o.opts.delimiter,
}
})
if o.err != nil {
@@ -215,3 +215,117 @@ func ListLocker(l sync.Locker) ListOption {
o.locker = l
}
}
type cursor struct {
// Prefix limits the listed objects to those that begin with this string.
prefix string
// Delimiter denotes the path separator. If set, object listings will be
// truncated at this character.
//
// For example, if the bucket contains objects foo/bar, foo/baz, and foo,
// then a delimiter of "/" will cause the listing to return "foo" and "foo/".
// Otherwise, the listing would have returned all object names.
//
// Note that objects returned that end in the delimiter may not be actual
// objects, e.g. you cannot read from (or write to, or delete) an object "foo/",
// both because no actual object exists and because B2 disallows object names
// that end with "/". If you want to ensure that all objects returned by
// ListObjects and ListCurrentObjects are actual objects, leave this unset.
delimiter string
name string
id string
}
func (b *Bucket) listObjects(ctx context.Context, count int, c *cursor) ([]*Object, *cursor, error) {
if c == nil {
c = &cursor{}
}
fs, name, id, err := b.b.listFileVersions(ctx, count, c.name, c.id, c.prefix, c.delimiter)
if err != nil {
return nil, nil, err
}
var next *cursor
if name != "" && id != "" {
next = &cursor{
prefix: c.prefix,
delimiter: c.delimiter,
name: name,
id: id,
}
}
var objects []*Object
for _, f := range fs {
objects = append(objects, &Object{
name: f.name(),
f: f,
b: b,
})
}
var rtnErr error
if len(objects) == 0 || next == nil {
rtnErr = io.EOF
}
return objects, next, rtnErr
}
func (b *Bucket) listCurrentObjects(ctx context.Context, count int, c *cursor) ([]*Object, *cursor, error) {
if c == nil {
c = &cursor{}
}
fs, name, err := b.b.listFileNames(ctx, count, c.name, c.prefix, c.delimiter)
if err != nil {
return nil, nil, err
}
var next *cursor
if name != "" {
next = &cursor{
prefix: c.prefix,
delimiter: c.delimiter,
name: name,
}
}
var objects []*Object
for _, f := range fs {
objects = append(objects, &Object{
name: f.name(),
f: f,
b: b,
})
}
var rtnErr error
if len(objects) == 0 || next == nil {
rtnErr = io.EOF
}
return objects, next, rtnErr
}
func (b *Bucket) listUnfinishedLargeFiles(ctx context.Context, count int, c *cursor) ([]*Object, *cursor, error) {
if c == nil {
c = &cursor{}
}
fs, name, err := b.b.listUnfinishedLargeFiles(ctx, count, c.name)
if err != nil {
return nil, nil, err
}
var next *cursor
if name != "" {
next = &cursor{
name: name,
}
}
var objects []*Object
for _, f := range fs {
objects = append(objects, &Object{
name: f.name(),
f: f,
b: b,
})
}
var rtnErr error
if len(objects) == 0 || next == nil {
rtnErr = io.EOF
}
return objects, next, rtnErr
}
+6 -2
View File
@@ -1,4 +1,4 @@
// Copyright 2018, Google
// Copyright 2018, the Blazer authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -132,7 +132,11 @@ func (c *Client) ListKeys(ctx context.Context, count int, cursor string) ([]*Key
k: k,
})
}
return keys, next, nil
var rerr error
if next == "" {
rerr = io.EOF
}
return keys, next, rerr
}
// CreateKey creates a scoped application key that is valid only for this bucket.
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2017, Google
// Copyright 2017, the Blazer authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2016, Google
// Copyright 2016, the Blazer authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2017, Google
// Copyright 2017, the Blazer authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
+95 -40
View File
@@ -1,4 +1,4 @@
// Copyright 2016, Google
// Copyright 2016, the Blazer authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -65,8 +65,11 @@ type Writer struct {
csize int
ctx context.Context
cancel context.CancelFunc
cancel context.CancelFunc // cancels ctx
ctxf func() context.Context
errf func(error)
ready chan chunk
cdone chan struct{}
wg sync.WaitGroup
start sync.Once
once sync.Once
@@ -100,11 +103,19 @@ func (w *Writer) setErr(err error) {
}
w.emux.Lock()
defer w.emux.Unlock()
if w.err == nil {
blog.V(1).Infof("error writing %s: %v", w.name, err)
w.err = err
w.cancel()
if w.err != nil {
return
}
blog.V(1).Infof("error writing %s: %v", w.name, err)
w.err = err
w.cancel()
if w.ctxf == nil {
return
}
if w.errf == nil {
w.errf = func(error) {}
}
w.errf(w.file.cancel(w.ctxf()))
}
func (w *Writer) getErr() error {
@@ -127,6 +138,15 @@ func (w *Writer) completeChunk(id int) {
var gid int32
func sleepCtx(ctx context.Context, d time.Duration) error {
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(d):
return nil
}
}
func (w *Writer) thread() {
w.wg.Add(1)
go func() {
@@ -138,57 +158,63 @@ func (w *Writer) thread() {
return
}
for {
chunk, ok := <-w.ready
if !ok {
var cnk chunk
select {
case cnk = <-w.ready:
case <-w.cdone:
return
}
if sha, ok := w.seen[chunk.id]; ok {
if sha != chunk.buf.Hash() {
if sha, ok := w.seen[cnk.id]; ok {
if sha != cnk.buf.Hash() {
w.setErr(errors.New("resumable upload was requested, but chunks don't match"))
return
}
chunk.buf.Close()
w.completeChunk(chunk.id)
blog.V(2).Infof("skipping chunk %d", chunk.id)
cnk.buf.Close()
w.completeChunk(cnk.id)
blog.V(2).Infof("skipping chunk %d", cnk.id)
continue
}
blog.V(2).Infof("thread %d handling chunk %d", id, chunk.id)
r, err := chunk.buf.Reader()
blog.V(2).Infof("thread %d handling chunk %d", id, cnk.id)
r, err := cnk.buf.Reader()
if err != nil {
w.setErr(err)
return
}
mr := &meteredReader{r: r, size: chunk.buf.Len()}
w.registerChunk(chunk.id, mr)
mr := &meteredReader{r: r, size: cnk.buf.Len()}
w.registerChunk(cnk.id, mr)
sleep := time.Millisecond * 15
redo:
n, err := fc.uploadPart(w.ctx, mr, chunk.buf.Hash(), chunk.buf.Len(), chunk.id)
if n != chunk.buf.Len() || err != nil {
n, err := fc.uploadPart(w.ctx, mr, cnk.buf.Hash(), cnk.buf.Len(), cnk.id)
if n != cnk.buf.Len() || err != nil {
if w.o.b.r.reupload(err) {
time.Sleep(sleep)
if err := sleepCtx(w.ctx, sleep); err != nil {
w.setErr(err)
w.completeChunk(cnk.id)
cnk.buf.Close() // TODO: log error
}
sleep *= 2
if sleep > time.Second*15 {
sleep = time.Second * 15
}
blog.V(1).Infof("b2 writer: wrote %d of %d: error: %v; retrying", n, chunk.buf.Len(), err)
blog.V(1).Infof("b2 writer: wrote %d of %d: error: %v; retrying", n, cnk.buf.Len(), err)
f, err := w.file.getUploadPartURL(w.ctx)
if err != nil {
w.setErr(err)
w.completeChunk(chunk.id)
chunk.buf.Close() // TODO: log error
w.completeChunk(cnk.id)
cnk.buf.Close() // TODO: log error
return
}
fc = f
goto redo
}
w.setErr(err)
w.completeChunk(chunk.id)
chunk.buf.Close() // TODO: log error
w.completeChunk(cnk.id)
cnk.buf.Close() // TODO: log error
return
}
w.completeChunk(chunk.id)
chunk.buf.Close() // TODO: log error
blog.V(2).Infof("chunk %d handled", chunk.id)
w.completeChunk(cnk.id)
cnk.buf.Close() // TODO: log error
blog.V(2).Infof("chunk %d handled", cnk.id)
}
}()
}
@@ -221,6 +247,9 @@ func (w *Writer) init() {
// Write satisfies the io.Writer interface.
func (w *Writer) Write(p []byte) (int, error) {
if len(p) == 0 {
return 0, nil
}
w.init()
if err := w.getErr(); err != nil {
return 0, err
@@ -300,21 +329,28 @@ func (w *Writer) getLargeFile() (beLargeFileInterface, error) {
}
return w.o.b.b.startLargeFile(w.ctx, w.name, ctype, w.info)
}
var got bool
iter := w.o.b.List(w.ctx, ListPrefix(w.name), ListUnfinished())
var fi beFileInterface
for iter.Next() {
obj := iter.Object()
if obj.Name() == w.name {
got = true
fi = obj.f
}
}
if iter.Err() != nil {
return nil, iter.Err()
}
if !got {
w.Resume = false
return w.getLargeFile()
}
next := 1
seen := make(map[int]string)
var size int64
var fi beFileInterface
for {
cur := &Cursor{name: w.name}
objs, _, err := w.o.b.ListObjects(w.ctx, 1, cur)
if err != nil {
return nil, err
}
if len(objs) < 1 || objs[0].name != w.name {
w.Resume = false
return w.getLargeFile()
}
fi = objs[0].f
parts, n, err := fi.listParts(w.ctx, next, 100)
if err != nil {
return nil, err
@@ -348,6 +384,7 @@ func (w *Writer) sendChunk() error {
}
w.file = lf
w.ready = make(chan chunk)
w.cdone = make(chan struct{})
if w.ConcurrentUploads < 1 {
w.ConcurrentUploads = 1
}
@@ -359,6 +396,8 @@ func (w *Writer) sendChunk() error {
return err
}
select {
case <-w.cdone:
return nil
case w.ready <- chunk{
id: w.cidx + 1,
buf: w.w,
@@ -443,6 +482,8 @@ func (w *Writer) ReadFrom(r io.Reader) (int64, error) {
func (w *Writer) Close() error {
w.done.Do(func() {
if !w.everStarted {
w.init()
w.setErr(w.simpleWriteFile())
return
}
defer w.o.b.c.removeWriter(w)
@@ -462,7 +503,9 @@ func (w *Writer) Close() error {
return
}
}
close(w.ready)
// See https://github.com/kurin/blazer/issues/60 for why we use a special
// channel for this.
close(w.cdone)
w.wg.Wait()
f, err := w.file.finishLargeFile(w.ctx)
if err != nil {
@@ -503,6 +546,18 @@ func WithAttrsOption(attrs *Attrs) WriterOption {
}
}
// WithCancelOnError requests the writer, if it has started a large file
// upload, to call b2_cancel_large_file on any permanent error. It calls ctxf
// to obtain a context with which to cancel the file; this is to allow callers
// to set specific timeouts. If errf is non-nil, then it is called with the
// (possibly nil) output of b2_cancel_large_file.
func WithCancelOnError(ctxf func() context.Context, errf func(error)) WriterOption {
return func(w *Writer) {
w.ctxf = ctxf
w.errf = errf
}
}
// DefaultWriterOptions returns a ClientOption that will apply the given
// WriterOptions to every Writer. These options can be overridden by passing
// new options to NewWriter.
+10 -3
View File
@@ -1,4 +1,4 @@
// Copyright 2016, Google
// Copyright 2016, the Blazer authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -42,7 +42,7 @@ import (
const (
APIBase = "https://api.backblazeb2.com"
DefaultUserAgent = "blazer/0.5.1"
DefaultUserAgent = "blazer/0.5.3"
)
type b2err struct {
@@ -321,6 +321,13 @@ func (rb *requestBody) getBody() io.Reader {
if rb == nil {
return nil
}
if rb.getSize() == 0 {
// https://github.com/kurin/blazer/issues/57
// When body is non-nil, but the request's ContentLength is 0, it is
// replaced with -1, which causes the client to send a chunked encoding,
// which confuses B2.
return http.NoBody
}
return rb.body
}
@@ -1274,7 +1281,7 @@ func (b *B2) ListKeys(ctx context.Context, max int, next string) ([]*Key, string
"Authorization": b.authToken,
}
b2resp := &b2types.ListKeysResponse{}
if err := b.opts.makeRequest(ctx, "b2_create_key", "POST", b.apiURI+b2types.V1api+"b2_create_key", b2req, b2resp, headers, nil); err != nil {
if err := b.opts.makeRequest(ctx, "b2_list_keys", "POST", b.apiURI+b2types.V1api+"b2_list_keys", b2req, b2resp, headers, nil); err != nil {
return nil, "", err
}
var keys []*Key
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2017, Google
// Copyright 2017, the Blazer authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2018, Google
// Copyright 2018, the Blazer authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2016, Google
// Copyright 2016, the Blazer authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2017, Google
// Copyright 2017, the Blazer authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2018, Google
// Copyright 2018, the Blazer authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.