mirror of
https://github.com/restic/restic.git
synced 2026-07-14 19:24:53 +00:00
+8
@@ -0,0 +1,8 @@
|
||||
# This is the list of Blazer authors for copyright purposes.
|
||||
#
|
||||
# This does not necessarily list everyone who has contributed code, since in
|
||||
# some cases, their employer may be the copyright holder. To see the full list
|
||||
# of contributors, see the revision history in source control.
|
||||
#
|
||||
# Tag yourself.
|
||||
Google LLC
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
Copyright 2016, Google
|
||||
Copyright 2016, the Blazer authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
||||
+14
-148
@@ -1,4 +1,4 @@
|
||||
// Copyright 2016, Google
|
||||
// Copyright 2016, the Blazer authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
@@ -577,139 +577,6 @@ func (o *Object) Delete(ctx context.Context) error {
|
||||
return o.f.deleteFileVersion(ctx)
|
||||
}
|
||||
|
||||
// Cursor is passed to ListObjects to return subsequent pages.
|
||||
//
|
||||
// DEPRECATED. Will be removed in a future release.
|
||||
type Cursor struct {
|
||||
// Prefix limits the listed objects to those that begin with this string.
|
||||
Prefix string
|
||||
|
||||
// Delimiter denotes the path separator. If set, object listings will be
|
||||
// truncated at this character.
|
||||
//
|
||||
// For example, if the bucket contains objects foo/bar, foo/baz, and foo,
|
||||
// then a delimiter of "/" will cause the listing to return "foo" and "foo/".
|
||||
// Otherwise, the listing would have returned all object names.
|
||||
//
|
||||
// Note that objects returned that end in the delimiter may not be actual
|
||||
// objects, e.g. you cannot read from (or write to, or delete) an object "foo/",
|
||||
// both because no actual object exists and because B2 disallows object names
|
||||
// that end with "/". If you want to ensure that all objects returned by
|
||||
// ListObjects and ListCurrentObjects are actual objects, leave this unset.
|
||||
Delimiter string
|
||||
|
||||
name string
|
||||
id string
|
||||
}
|
||||
|
||||
// ListObjects returns all objects in the bucket, including multiple versions
|
||||
// of the same object. Cursor may be nil; when passed to a subsequent query,
|
||||
// it will continue the listing.
|
||||
//
|
||||
// ListObjects will return io.EOF when there are no objects left in the bucket,
|
||||
// however it may do so concurrently with the last objects.
|
||||
//
|
||||
// DEPRECATED. Will be removed in a future release.
|
||||
func (b *Bucket) ListObjects(ctx context.Context, count int, c *Cursor) ([]*Object, *Cursor, error) {
|
||||
if c == nil {
|
||||
c = &Cursor{}
|
||||
}
|
||||
fs, name, id, err := b.b.listFileVersions(ctx, count, c.name, c.id, c.Prefix, c.Delimiter)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var next *Cursor
|
||||
if name != "" && id != "" {
|
||||
next = &Cursor{
|
||||
Prefix: c.Prefix,
|
||||
Delimiter: c.Delimiter,
|
||||
name: name,
|
||||
id: id,
|
||||
}
|
||||
}
|
||||
var objects []*Object
|
||||
for _, f := range fs {
|
||||
objects = append(objects, &Object{
|
||||
name: f.name(),
|
||||
f: f,
|
||||
b: b,
|
||||
})
|
||||
}
|
||||
var rtnErr error
|
||||
if len(objects) == 0 || next == nil {
|
||||
rtnErr = io.EOF
|
||||
}
|
||||
return objects, next, rtnErr
|
||||
}
|
||||
|
||||
// ListCurrentObjects is similar to ListObjects, except that it returns only
|
||||
// current, unhidden objects in the bucket.
|
||||
//
|
||||
// DEPRECATED. Will be removed in a future release.
|
||||
func (b *Bucket) ListCurrentObjects(ctx context.Context, count int, c *Cursor) ([]*Object, *Cursor, error) {
|
||||
if c == nil {
|
||||
c = &Cursor{}
|
||||
}
|
||||
fs, name, err := b.b.listFileNames(ctx, count, c.name, c.Prefix, c.Delimiter)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var next *Cursor
|
||||
if name != "" {
|
||||
next = &Cursor{
|
||||
Prefix: c.Prefix,
|
||||
Delimiter: c.Delimiter,
|
||||
name: name,
|
||||
}
|
||||
}
|
||||
var objects []*Object
|
||||
for _, f := range fs {
|
||||
objects = append(objects, &Object{
|
||||
name: f.name(),
|
||||
f: f,
|
||||
b: b,
|
||||
})
|
||||
}
|
||||
var rtnErr error
|
||||
if len(objects) == 0 || next == nil {
|
||||
rtnErr = io.EOF
|
||||
}
|
||||
return objects, next, rtnErr
|
||||
}
|
||||
|
||||
// ListUnfinishedLargeFiles lists any objects that correspond to large file uploads that haven't been completed.
|
||||
// This can happen for example when an upload is interrupted.
|
||||
//
|
||||
// DEPRECATED. Will be removed in a future release.
|
||||
func (b *Bucket) ListUnfinishedLargeFiles(ctx context.Context, count int, c *Cursor) ([]*Object, *Cursor, error) {
|
||||
if c == nil {
|
||||
c = &Cursor{}
|
||||
}
|
||||
fs, name, err := b.b.listUnfinishedLargeFiles(ctx, count, c.name)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var next *Cursor
|
||||
if name != "" {
|
||||
next = &Cursor{
|
||||
name: name,
|
||||
}
|
||||
}
|
||||
var objects []*Object
|
||||
for _, f := range fs {
|
||||
objects = append(objects, &Object{
|
||||
name: f.name(),
|
||||
f: f,
|
||||
b: b,
|
||||
})
|
||||
}
|
||||
var rtnErr error
|
||||
if len(objects) == 0 || next == nil {
|
||||
rtnErr = io.EOF
|
||||
}
|
||||
return objects, next, rtnErr
|
||||
}
|
||||
|
||||
// Hide hides the object from name-based listing.
|
||||
func (o *Object) Hide(ctx context.Context) error {
|
||||
if err := o.ensure(ctx); err != nil {
|
||||
@@ -722,21 +589,20 @@ func (o *Object) Hide(ctx context.Context) error {
|
||||
// Reveal unhides (if hidden) the named object. If there are multiple objects
|
||||
// of a given name, it will reveal the most recent.
|
||||
func (b *Bucket) Reveal(ctx context.Context, name string) error {
|
||||
cur := &Cursor{
|
||||
name: name,
|
||||
iter := b.List(ctx, ListPrefix(name), ListHidden())
|
||||
for iter.Next() {
|
||||
obj := iter.Object()
|
||||
if obj.Name() == name {
|
||||
if obj.f.status() == "hide" {
|
||||
return obj.Delete(ctx)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if obj.Name() > name {
|
||||
break
|
||||
}
|
||||
}
|
||||
objs, _, err := b.ListObjects(ctx, 1, cur)
|
||||
if err != nil && err != io.EOF {
|
||||
return err
|
||||
}
|
||||
if len(objs) < 1 || objs[0].name != name {
|
||||
return b2err{err: fmt.Errorf("%s: not found", name), notFoundErr: true}
|
||||
}
|
||||
obj := objs[0]
|
||||
if obj.f.status() != "hide" {
|
||||
return nil
|
||||
}
|
||||
return obj.Delete(ctx)
|
||||
return b2err{err: fmt.Errorf("%s: not found", name), notFoundErr: true}
|
||||
}
|
||||
|
||||
// I don't want to import all of ioutil for this.
|
||||
|
||||
+12
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright 2016, Google
|
||||
// Copyright 2016, the Blazer authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
@@ -95,6 +95,7 @@ type beFile struct {
|
||||
type beLargeFileInterface interface {
|
||||
finishLargeFile(context.Context) (beFileInterface, error)
|
||||
getUploadPartURL(context.Context) (beFileChunkInterface, error)
|
||||
cancel(context.Context) error
|
||||
}
|
||||
|
||||
type beLargeFile struct {
|
||||
@@ -654,6 +655,16 @@ func (b *beLargeFile) finishLargeFile(ctx context.Context) (beFileInterface, err
|
||||
return file, nil
|
||||
}
|
||||
|
||||
func (b *beLargeFile) cancel(ctx context.Context) error {
|
||||
f := func() error {
|
||||
g := func() error {
|
||||
return b.b2largeFile.cancel(ctx)
|
||||
}
|
||||
return withReauth(ctx, b.ri, g)
|
||||
}
|
||||
return withBackoff(ctx, b.ri, f)
|
||||
}
|
||||
|
||||
func (b *beFileChunk) reload(ctx context.Context) error {
|
||||
f := func() error {
|
||||
g := func() error {
|
||||
|
||||
+6
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright 2016, Google
|
||||
// Copyright 2016, the Blazer authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
@@ -76,6 +76,7 @@ type b2FileInterface interface {
|
||||
type b2LargeFileInterface interface {
|
||||
finishLargeFile(context.Context) (b2FileInterface, error)
|
||||
getUploadPartURL(context.Context) (b2FileChunkInterface, error)
|
||||
cancel(context.Context) error
|
||||
}
|
||||
|
||||
type b2FileChunkInterface interface {
|
||||
@@ -474,6 +475,10 @@ func (b *b2LargeFile) getUploadPartURL(ctx context.Context) (b2FileChunkInterfac
|
||||
return &b2FileChunk{c}, nil
|
||||
}
|
||||
|
||||
func (b *b2LargeFile) cancel(ctx context.Context) error {
|
||||
return b.b.CancelLargeFile(ctx)
|
||||
}
|
||||
|
||||
func (b *b2FileChunk) reload(ctx context.Context) error {
|
||||
return b.b.Reload(ctx)
|
||||
}
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright 2017, Google
|
||||
// Copyright 2017, the Blazer authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
||||
+123
-9
@@ -1,4 +1,4 @@
|
||||
// Copyright 2018, Google
|
||||
// Copyright 2018, the Blazer authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
@@ -50,7 +50,7 @@ type ObjectIterator struct {
|
||||
final bool
|
||||
err error
|
||||
idx int
|
||||
c *Cursor
|
||||
c *cursor
|
||||
opts objectIteratorOptions
|
||||
objs []*Object
|
||||
init sync.Once
|
||||
@@ -58,7 +58,7 @@ type ObjectIterator struct {
|
||||
count int
|
||||
}
|
||||
|
||||
type lister func(context.Context, int, *Cursor) ([]*Object, *Cursor, error)
|
||||
type lister func(context.Context, int, *cursor) ([]*Object, *cursor, error)
|
||||
|
||||
func (o *ObjectIterator) page(ctx context.Context) error {
|
||||
if o.opts.locker != nil {
|
||||
@@ -96,18 +96,18 @@ func (o *ObjectIterator) Next() bool {
|
||||
}
|
||||
switch {
|
||||
case o.opts.unfinished:
|
||||
o.l = o.bucket.ListUnfinishedLargeFiles
|
||||
o.l = o.bucket.listUnfinishedLargeFiles
|
||||
if o.count > 100 {
|
||||
o.count = 100
|
||||
}
|
||||
case o.opts.hidden:
|
||||
o.l = o.bucket.ListObjects
|
||||
o.l = o.bucket.listObjects
|
||||
default:
|
||||
o.l = o.bucket.ListCurrentObjects
|
||||
o.l = o.bucket.listCurrentObjects
|
||||
}
|
||||
o.c = &Cursor{
|
||||
Prefix: o.opts.prefix,
|
||||
Delimiter: o.opts.delimiter,
|
||||
o.c = &cursor{
|
||||
prefix: o.opts.prefix,
|
||||
delimiter: o.opts.delimiter,
|
||||
}
|
||||
})
|
||||
if o.err != nil {
|
||||
@@ -215,3 +215,117 @@ func ListLocker(l sync.Locker) ListOption {
|
||||
o.locker = l
|
||||
}
|
||||
}
|
||||
|
||||
type cursor struct {
|
||||
// Prefix limits the listed objects to those that begin with this string.
|
||||
prefix string
|
||||
|
||||
// Delimiter denotes the path separator. If set, object listings will be
|
||||
// truncated at this character.
|
||||
//
|
||||
// For example, if the bucket contains objects foo/bar, foo/baz, and foo,
|
||||
// then a delimiter of "/" will cause the listing to return "foo" and "foo/".
|
||||
// Otherwise, the listing would have returned all object names.
|
||||
//
|
||||
// Note that objects returned that end in the delimiter may not be actual
|
||||
// objects, e.g. you cannot read from (or write to, or delete) an object "foo/",
|
||||
// both because no actual object exists and because B2 disallows object names
|
||||
// that end with "/". If you want to ensure that all objects returned by
|
||||
// ListObjects and ListCurrentObjects are actual objects, leave this unset.
|
||||
delimiter string
|
||||
|
||||
name string
|
||||
id string
|
||||
}
|
||||
|
||||
func (b *Bucket) listObjects(ctx context.Context, count int, c *cursor) ([]*Object, *cursor, error) {
|
||||
if c == nil {
|
||||
c = &cursor{}
|
||||
}
|
||||
fs, name, id, err := b.b.listFileVersions(ctx, count, c.name, c.id, c.prefix, c.delimiter)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var next *cursor
|
||||
if name != "" && id != "" {
|
||||
next = &cursor{
|
||||
prefix: c.prefix,
|
||||
delimiter: c.delimiter,
|
||||
name: name,
|
||||
id: id,
|
||||
}
|
||||
}
|
||||
var objects []*Object
|
||||
for _, f := range fs {
|
||||
objects = append(objects, &Object{
|
||||
name: f.name(),
|
||||
f: f,
|
||||
b: b,
|
||||
})
|
||||
}
|
||||
var rtnErr error
|
||||
if len(objects) == 0 || next == nil {
|
||||
rtnErr = io.EOF
|
||||
}
|
||||
return objects, next, rtnErr
|
||||
}
|
||||
|
||||
func (b *Bucket) listCurrentObjects(ctx context.Context, count int, c *cursor) ([]*Object, *cursor, error) {
|
||||
if c == nil {
|
||||
c = &cursor{}
|
||||
}
|
||||
fs, name, err := b.b.listFileNames(ctx, count, c.name, c.prefix, c.delimiter)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var next *cursor
|
||||
if name != "" {
|
||||
next = &cursor{
|
||||
prefix: c.prefix,
|
||||
delimiter: c.delimiter,
|
||||
name: name,
|
||||
}
|
||||
}
|
||||
var objects []*Object
|
||||
for _, f := range fs {
|
||||
objects = append(objects, &Object{
|
||||
name: f.name(),
|
||||
f: f,
|
||||
b: b,
|
||||
})
|
||||
}
|
||||
var rtnErr error
|
||||
if len(objects) == 0 || next == nil {
|
||||
rtnErr = io.EOF
|
||||
}
|
||||
return objects, next, rtnErr
|
||||
}
|
||||
|
||||
func (b *Bucket) listUnfinishedLargeFiles(ctx context.Context, count int, c *cursor) ([]*Object, *cursor, error) {
|
||||
if c == nil {
|
||||
c = &cursor{}
|
||||
}
|
||||
fs, name, err := b.b.listUnfinishedLargeFiles(ctx, count, c.name)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var next *cursor
|
||||
if name != "" {
|
||||
next = &cursor{
|
||||
name: name,
|
||||
}
|
||||
}
|
||||
var objects []*Object
|
||||
for _, f := range fs {
|
||||
objects = append(objects, &Object{
|
||||
name: f.name(),
|
||||
f: f,
|
||||
b: b,
|
||||
})
|
||||
}
|
||||
var rtnErr error
|
||||
if len(objects) == 0 || next == nil {
|
||||
rtnErr = io.EOF
|
||||
}
|
||||
return objects, next, rtnErr
|
||||
}
|
||||
|
||||
+6
-2
@@ -1,4 +1,4 @@
|
||||
// Copyright 2018, Google
|
||||
// Copyright 2018, the Blazer authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
@@ -132,7 +132,11 @@ func (c *Client) ListKeys(ctx context.Context, count int, cursor string) ([]*Key
|
||||
k: k,
|
||||
})
|
||||
}
|
||||
return keys, next, nil
|
||||
var rerr error
|
||||
if next == "" {
|
||||
rerr = io.EOF
|
||||
}
|
||||
return keys, next, rerr
|
||||
}
|
||||
|
||||
// CreateKey creates a scoped application key that is valid only for this bucket.
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright 2017, Google
|
||||
// Copyright 2017, the Blazer authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright 2016, Google
|
||||
// Copyright 2016, the Blazer authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright 2017, Google
|
||||
// Copyright 2017, the Blazer authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
||||
+95
-40
@@ -1,4 +1,4 @@
|
||||
// Copyright 2016, Google
|
||||
// Copyright 2016, the Blazer authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
@@ -65,8 +65,11 @@ type Writer struct {
|
||||
|
||||
csize int
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
cancel context.CancelFunc // cancels ctx
|
||||
ctxf func() context.Context
|
||||
errf func(error)
|
||||
ready chan chunk
|
||||
cdone chan struct{}
|
||||
wg sync.WaitGroup
|
||||
start sync.Once
|
||||
once sync.Once
|
||||
@@ -100,11 +103,19 @@ func (w *Writer) setErr(err error) {
|
||||
}
|
||||
w.emux.Lock()
|
||||
defer w.emux.Unlock()
|
||||
if w.err == nil {
|
||||
blog.V(1).Infof("error writing %s: %v", w.name, err)
|
||||
w.err = err
|
||||
w.cancel()
|
||||
if w.err != nil {
|
||||
return
|
||||
}
|
||||
blog.V(1).Infof("error writing %s: %v", w.name, err)
|
||||
w.err = err
|
||||
w.cancel()
|
||||
if w.ctxf == nil {
|
||||
return
|
||||
}
|
||||
if w.errf == nil {
|
||||
w.errf = func(error) {}
|
||||
}
|
||||
w.errf(w.file.cancel(w.ctxf()))
|
||||
}
|
||||
|
||||
func (w *Writer) getErr() error {
|
||||
@@ -127,6 +138,15 @@ func (w *Writer) completeChunk(id int) {
|
||||
|
||||
var gid int32
|
||||
|
||||
func sleepCtx(ctx context.Context, d time.Duration) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(d):
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Writer) thread() {
|
||||
w.wg.Add(1)
|
||||
go func() {
|
||||
@@ -138,57 +158,63 @@ func (w *Writer) thread() {
|
||||
return
|
||||
}
|
||||
for {
|
||||
chunk, ok := <-w.ready
|
||||
if !ok {
|
||||
var cnk chunk
|
||||
select {
|
||||
case cnk = <-w.ready:
|
||||
case <-w.cdone:
|
||||
return
|
||||
}
|
||||
if sha, ok := w.seen[chunk.id]; ok {
|
||||
if sha != chunk.buf.Hash() {
|
||||
if sha, ok := w.seen[cnk.id]; ok {
|
||||
if sha != cnk.buf.Hash() {
|
||||
w.setErr(errors.New("resumable upload was requested, but chunks don't match"))
|
||||
return
|
||||
}
|
||||
chunk.buf.Close()
|
||||
w.completeChunk(chunk.id)
|
||||
blog.V(2).Infof("skipping chunk %d", chunk.id)
|
||||
cnk.buf.Close()
|
||||
w.completeChunk(cnk.id)
|
||||
blog.V(2).Infof("skipping chunk %d", cnk.id)
|
||||
continue
|
||||
}
|
||||
blog.V(2).Infof("thread %d handling chunk %d", id, chunk.id)
|
||||
r, err := chunk.buf.Reader()
|
||||
blog.V(2).Infof("thread %d handling chunk %d", id, cnk.id)
|
||||
r, err := cnk.buf.Reader()
|
||||
if err != nil {
|
||||
w.setErr(err)
|
||||
return
|
||||
}
|
||||
mr := &meteredReader{r: r, size: chunk.buf.Len()}
|
||||
w.registerChunk(chunk.id, mr)
|
||||
mr := &meteredReader{r: r, size: cnk.buf.Len()}
|
||||
w.registerChunk(cnk.id, mr)
|
||||
sleep := time.Millisecond * 15
|
||||
redo:
|
||||
n, err := fc.uploadPart(w.ctx, mr, chunk.buf.Hash(), chunk.buf.Len(), chunk.id)
|
||||
if n != chunk.buf.Len() || err != nil {
|
||||
n, err := fc.uploadPart(w.ctx, mr, cnk.buf.Hash(), cnk.buf.Len(), cnk.id)
|
||||
if n != cnk.buf.Len() || err != nil {
|
||||
if w.o.b.r.reupload(err) {
|
||||
time.Sleep(sleep)
|
||||
if err := sleepCtx(w.ctx, sleep); err != nil {
|
||||
w.setErr(err)
|
||||
w.completeChunk(cnk.id)
|
||||
cnk.buf.Close() // TODO: log error
|
||||
}
|
||||
sleep *= 2
|
||||
if sleep > time.Second*15 {
|
||||
sleep = time.Second * 15
|
||||
}
|
||||
blog.V(1).Infof("b2 writer: wrote %d of %d: error: %v; retrying", n, chunk.buf.Len(), err)
|
||||
blog.V(1).Infof("b2 writer: wrote %d of %d: error: %v; retrying", n, cnk.buf.Len(), err)
|
||||
f, err := w.file.getUploadPartURL(w.ctx)
|
||||
if err != nil {
|
||||
w.setErr(err)
|
||||
w.completeChunk(chunk.id)
|
||||
chunk.buf.Close() // TODO: log error
|
||||
w.completeChunk(cnk.id)
|
||||
cnk.buf.Close() // TODO: log error
|
||||
return
|
||||
}
|
||||
fc = f
|
||||
goto redo
|
||||
}
|
||||
w.setErr(err)
|
||||
w.completeChunk(chunk.id)
|
||||
chunk.buf.Close() // TODO: log error
|
||||
w.completeChunk(cnk.id)
|
||||
cnk.buf.Close() // TODO: log error
|
||||
return
|
||||
}
|
||||
w.completeChunk(chunk.id)
|
||||
chunk.buf.Close() // TODO: log error
|
||||
blog.V(2).Infof("chunk %d handled", chunk.id)
|
||||
w.completeChunk(cnk.id)
|
||||
cnk.buf.Close() // TODO: log error
|
||||
blog.V(2).Infof("chunk %d handled", cnk.id)
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -221,6 +247,9 @@ func (w *Writer) init() {
|
||||
|
||||
// Write satisfies the io.Writer interface.
|
||||
func (w *Writer) Write(p []byte) (int, error) {
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
w.init()
|
||||
if err := w.getErr(); err != nil {
|
||||
return 0, err
|
||||
@@ -300,21 +329,28 @@ func (w *Writer) getLargeFile() (beLargeFileInterface, error) {
|
||||
}
|
||||
return w.o.b.b.startLargeFile(w.ctx, w.name, ctype, w.info)
|
||||
}
|
||||
var got bool
|
||||
iter := w.o.b.List(w.ctx, ListPrefix(w.name), ListUnfinished())
|
||||
var fi beFileInterface
|
||||
for iter.Next() {
|
||||
obj := iter.Object()
|
||||
if obj.Name() == w.name {
|
||||
got = true
|
||||
fi = obj.f
|
||||
}
|
||||
}
|
||||
if iter.Err() != nil {
|
||||
return nil, iter.Err()
|
||||
}
|
||||
if !got {
|
||||
w.Resume = false
|
||||
return w.getLargeFile()
|
||||
}
|
||||
|
||||
next := 1
|
||||
seen := make(map[int]string)
|
||||
var size int64
|
||||
var fi beFileInterface
|
||||
for {
|
||||
cur := &Cursor{name: w.name}
|
||||
objs, _, err := w.o.b.ListObjects(w.ctx, 1, cur)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(objs) < 1 || objs[0].name != w.name {
|
||||
w.Resume = false
|
||||
return w.getLargeFile()
|
||||
}
|
||||
fi = objs[0].f
|
||||
parts, n, err := fi.listParts(w.ctx, next, 100)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -348,6 +384,7 @@ func (w *Writer) sendChunk() error {
|
||||
}
|
||||
w.file = lf
|
||||
w.ready = make(chan chunk)
|
||||
w.cdone = make(chan struct{})
|
||||
if w.ConcurrentUploads < 1 {
|
||||
w.ConcurrentUploads = 1
|
||||
}
|
||||
@@ -359,6 +396,8 @@ func (w *Writer) sendChunk() error {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-w.cdone:
|
||||
return nil
|
||||
case w.ready <- chunk{
|
||||
id: w.cidx + 1,
|
||||
buf: w.w,
|
||||
@@ -443,6 +482,8 @@ func (w *Writer) ReadFrom(r io.Reader) (int64, error) {
|
||||
func (w *Writer) Close() error {
|
||||
w.done.Do(func() {
|
||||
if !w.everStarted {
|
||||
w.init()
|
||||
w.setErr(w.simpleWriteFile())
|
||||
return
|
||||
}
|
||||
defer w.o.b.c.removeWriter(w)
|
||||
@@ -462,7 +503,9 @@ func (w *Writer) Close() error {
|
||||
return
|
||||
}
|
||||
}
|
||||
close(w.ready)
|
||||
// See https://github.com/kurin/blazer/issues/60 for why we use a special
|
||||
// channel for this.
|
||||
close(w.cdone)
|
||||
w.wg.Wait()
|
||||
f, err := w.file.finishLargeFile(w.ctx)
|
||||
if err != nil {
|
||||
@@ -503,6 +546,18 @@ func WithAttrsOption(attrs *Attrs) WriterOption {
|
||||
}
|
||||
}
|
||||
|
||||
// WithCancelOnError requests the writer, if it has started a large file
|
||||
// upload, to call b2_cancel_large_file on any permanent error. It calls ctxf
|
||||
// to obtain a context with which to cancel the file; this is to allow callers
|
||||
// to set specific timeouts. If errf is non-nil, then it is called with the
|
||||
// (possibly nil) output of b2_cancel_large_file.
|
||||
func WithCancelOnError(ctxf func() context.Context, errf func(error)) WriterOption {
|
||||
return func(w *Writer) {
|
||||
w.ctxf = ctxf
|
||||
w.errf = errf
|
||||
}
|
||||
}
|
||||
|
||||
// DefaultWriterOptions returns a ClientOption that will apply the given
|
||||
// WriterOptions to every Writer. These options can be overridden by passing
|
||||
// new options to NewWriter.
|
||||
|
||||
+10
-3
@@ -1,4 +1,4 @@
|
||||
// Copyright 2016, Google
|
||||
// Copyright 2016, the Blazer authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
@@ -42,7 +42,7 @@ import (
|
||||
|
||||
const (
|
||||
APIBase = "https://api.backblazeb2.com"
|
||||
DefaultUserAgent = "blazer/0.5.1"
|
||||
DefaultUserAgent = "blazer/0.5.3"
|
||||
)
|
||||
|
||||
type b2err struct {
|
||||
@@ -321,6 +321,13 @@ func (rb *requestBody) getBody() io.Reader {
|
||||
if rb == nil {
|
||||
return nil
|
||||
}
|
||||
if rb.getSize() == 0 {
|
||||
// https://github.com/kurin/blazer/issues/57
|
||||
// When body is non-nil, but the request's ContentLength is 0, it is
|
||||
// replaced with -1, which causes the client to send a chunked encoding,
|
||||
// which confuses B2.
|
||||
return http.NoBody
|
||||
}
|
||||
return rb.body
|
||||
}
|
||||
|
||||
@@ -1274,7 +1281,7 @@ func (b *B2) ListKeys(ctx context.Context, max int, next string) ([]*Key, string
|
||||
"Authorization": b.authToken,
|
||||
}
|
||||
b2resp := &b2types.ListKeysResponse{}
|
||||
if err := b.opts.makeRequest(ctx, "b2_create_key", "POST", b.apiURI+b2types.V1api+"b2_create_key", b2req, b2resp, headers, nil); err != nil {
|
||||
if err := b.opts.makeRequest(ctx, "b2_list_keys", "POST", b.apiURI+b2types.V1api+"b2_list_keys", b2req, b2resp, headers, nil); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
var keys []*Key
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright 2017, Google
|
||||
// Copyright 2017, the Blazer authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright 2018, Google
|
||||
// Copyright 2018, the Blazer authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright 2016, Google
|
||||
// Copyright 2016, the Blazer authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright 2017, Google
|
||||
// Copyright 2017, the Blazer authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright 2018, Google
|
||||
// Copyright 2018, the Blazer authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
|
||||
Reference in New Issue
Block a user