mirror of
https://github.com/restic/restic.git
synced 2026-11-21 13:02:42 +00:00
Update vendored dependencies
This includes github.com/kurin/blazer 0.2.0, which resolves #1291
This commit is contained in:
1 parent
ba23d24dd1
commit
61cb1cc6f8
1044 files changed
+203017
-97704
No files matched your search
+22
-2
@@ -28,6 +28,7 @@
|
||||
package b2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -35,8 +36,6 @@ import (
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
// Client is a Backblaze B2 client.
|
||||
@@ -67,11 +66,23 @@ type clientOptions struct {
|
||||
failSomeUploads bool
|
||||
expireTokens bool
|
||||
capExceeded bool
|
||||
userAgents []string
|
||||
}
|
||||
|
||||
// A ClientOption allows callers to adjust various per-client settings.
|
||||
type ClientOption func(*clientOptions)
|
||||
|
||||
// UserAgent sets the User-Agent HTTP header. The default header is
|
||||
// "blazer/<version>"; the value set here will be prepended to that. This can
|
||||
// be set multiple times.
|
||||
//
|
||||
// A user agent is generally of the form "<product>/<version> (<comments>)".
|
||||
func UserAgent(agent string) ClientOption {
|
||||
return func(o *clientOptions) {
|
||||
o.userAgents = append(o.userAgents, agent)
|
||||
}
|
||||
}
|
||||
|
||||
// Transport sets the underlying HTTP transport mechanism. If unset,
|
||||
// http.DefaultTransport is used.
|
||||
func Transport(rt http.RoundTripper) ClientOption {
|
||||
@@ -118,6 +129,7 @@ const (
|
||||
UnknownType BucketType = ""
|
||||
Private = "allPrivate"
|
||||
Public = "allPublic"
|
||||
Snapshot = "snapshot"
|
||||
)
|
||||
|
||||
// BucketAttrs holds a bucket's metadata attributes.
|
||||
@@ -582,11 +594,19 @@ func (b *Bucket) Reveal(ctx context.Context, name string) error {
|
||||
return obj.Delete(ctx)
|
||||
}
|
||||
|
||||
// I don't want to import all of ioutil for this.
|
||||
type discard struct{}
|
||||
|
||||
func (discard) Write(p []byte) (int, error) {
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func (b *Bucket) getObject(ctx context.Context, name string) (*Object, error) {
|
||||
fr, err := b.b.downloadFileByName(ctx, name, 0, 1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
io.Copy(discard{}, fr)
|
||||
fr.Close()
|
||||
return &Object{
|
||||
name: name,
|
||||
|
||||
+128
-3
@@ -16,6 +16,7 @@ package b2
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -26,8 +27,6 @@ import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -352,6 +351,82 @@ func (zReader) Read(p []byte) (int, error) {
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
type zReadSeeker struct {
|
||||
size int64
|
||||
pos int64
|
||||
}
|
||||
|
||||
func (rs *zReadSeeker) Read(p []byte) (int, error) {
|
||||
for i := rs.pos; ; i++ {
|
||||
j := int(i - rs.pos)
|
||||
if j >= len(p) || i >= rs.size {
|
||||
var rtn error
|
||||
if i >= rs.size {
|
||||
rtn = io.EOF
|
||||
}
|
||||
rs.pos = i
|
||||
return j, rtn
|
||||
}
|
||||
f := int(i) % len(pattern)
|
||||
p[j] = pattern[f]
|
||||
}
|
||||
}
|
||||
|
||||
func (rs *zReadSeeker) Seek(offset int64, whence int) (int64, error) {
|
||||
switch whence {
|
||||
case io.SeekStart:
|
||||
rs.pos = offset
|
||||
case io.SeekEnd:
|
||||
rs.pos = rs.size + offset
|
||||
}
|
||||
return rs.pos, nil
|
||||
}
|
||||
|
||||
func TestReaderFrom(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
table := []struct {
|
||||
size, pos int64
|
||||
}{
|
||||
{
|
||||
size: 10,
|
||||
},
|
||||
}
|
||||
|
||||
for _, e := range table {
|
||||
client := &Client{
|
||||
backend: &beRoot{
|
||||
b2i: &testRoot{
|
||||
bucketMap: make(map[string]map[string]string),
|
||||
errs: &errCont{},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
bucket, err := client.NewBucket(ctx, bucketName, &BucketAttrs{Type: Private})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := bucket.Delete(ctx); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
r := &zReadSeeker{pos: e.pos, size: e.size}
|
||||
w := bucket.Object("writer").NewWriter(ctx)
|
||||
n, err := w.ReadFrom(r)
|
||||
if err != nil {
|
||||
t.Errorf("ReadFrom(): %v", err)
|
||||
}
|
||||
if n != e.size {
|
||||
t.Errorf("ReadFrom(): got %d bytes, wanted %d bytes", n, e.size)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReauth(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
@@ -696,6 +771,52 @@ func TestFileBuffer(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNonBuffer(t *testing.T) {
|
||||
table := []struct {
|
||||
str string
|
||||
off int64
|
||||
len int64
|
||||
want string
|
||||
}{
|
||||
{
|
||||
str: "a string",
|
||||
off: 0,
|
||||
len: 3,
|
||||
want: "a s",
|
||||
},
|
||||
{
|
||||
str: "a string",
|
||||
off: 3,
|
||||
len: 1,
|
||||
want: "t",
|
||||
},
|
||||
{
|
||||
str: "a string",
|
||||
off: 3,
|
||||
len: 5,
|
||||
want: "tring",
|
||||
},
|
||||
}
|
||||
|
||||
for _, e := range table {
|
||||
nb := newNonBuffer(strings.NewReader(e.str), e.off, e.len)
|
||||
want := fmt.Sprintf("%s%x", e.want, sha1.Sum([]byte(e.str[int(e.off):int(e.off+e.len)])))
|
||||
r, err := nb.Reader()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
continue
|
||||
}
|
||||
got, err := ioutil.ReadAll(r)
|
||||
if err != nil {
|
||||
t.Errorf("ioutil.ReadAll(%#v): %v", e, err)
|
||||
continue
|
||||
}
|
||||
if want != string(got) {
|
||||
t.Errorf("ioutil.ReadAll(%#v): got %q, want %q", e, string(got), want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func writeFile(ctx context.Context, bucket *Bucket, name string, size int64, csize int) (*Object, string, error) {
|
||||
r := io.LimitReader(zReader{}, size)
|
||||
o := bucket.Object(name)
|
||||
@@ -704,9 +825,13 @@ func writeFile(ctx context.Context, bucket *Bucket, name string, size int64, csi
|
||||
w := io.MultiWriter(f, h)
|
||||
f.ConcurrentUploads = 5
|
||||
f.ChunkSize = csize
|
||||
if _, err := io.Copy(w, r); err != nil {
|
||||
n, err := io.Copy(w, r)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if n != size {
|
||||
return nil, "", fmt.Errorf("io.Copy(): wrote %d bytes; wanted %d bytes", n, size)
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
+7
-8
@@ -15,11 +15,10 @@
|
||||
package b2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"math/rand"
|
||||
"time"
|
||||
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
// This file wraps the baseline interfaces with backoff and retry semantics.
|
||||
@@ -63,7 +62,7 @@ type beBucket struct {
|
||||
}
|
||||
|
||||
type beURLInterface interface {
|
||||
uploadFile(context.Context, io.ReadSeeker, int, string, string, string, map[string]string) (beFileInterface, error)
|
||||
uploadFile(context.Context, readResetter, int, string, string, string, map[string]string) (beFileInterface, error)
|
||||
}
|
||||
|
||||
type beURL struct {
|
||||
@@ -100,7 +99,7 @@ type beLargeFile struct {
|
||||
|
||||
type beFileChunkInterface interface {
|
||||
reload(context.Context) error
|
||||
uploadPart(context.Context, io.ReadSeeker, string, int, int) (int, error)
|
||||
uploadPart(context.Context, readResetter, string, int, int) (int, error)
|
||||
}
|
||||
|
||||
type beFileChunk struct {
|
||||
@@ -414,10 +413,10 @@ func (b *beBucket) file(id, name string) beFileInterface {
|
||||
}
|
||||
}
|
||||
|
||||
func (b *beURL) uploadFile(ctx context.Context, r io.ReadSeeker, size int, name, ct, sha1 string, info map[string]string) (beFileInterface, error) {
|
||||
func (b *beURL) uploadFile(ctx context.Context, r readResetter, size int, name, ct, sha1 string, info map[string]string) (beFileInterface, error) {
|
||||
var file beFileInterface
|
||||
f := func() error {
|
||||
if _, err := r.Seek(0, 0); err != nil {
|
||||
if err := r.Reset(); err != nil {
|
||||
return err
|
||||
}
|
||||
f, err := b.b2url.uploadFile(ctx, r, size, name, ct, sha1, info)
|
||||
@@ -578,12 +577,12 @@ func (b *beFileChunk) reload(ctx context.Context) error {
|
||||
return withBackoff(ctx, b.ri, f)
|
||||
}
|
||||
|
||||
func (b *beFileChunk) uploadPart(ctx context.Context, r io.ReadSeeker, sha1 string, size, index int) (int, error) {
|
||||
func (b *beFileChunk) uploadPart(ctx context.Context, r readResetter, sha1 string, size, index int) (int, error) {
|
||||
// no re-auth; pass it back up to the caller so they can get an new upload URI and token
|
||||
// TODO: we should handle that here probably
|
||||
var i int
|
||||
f := func() error {
|
||||
if _, err := r.Seek(0, 0); err != nil {
|
||||
if err := r.Reset(); err != nil {
|
||||
return err
|
||||
}
|
||||
j, err := b.b2fileChunk.uploadPart(ctx, r, sha1, size, index)
|
||||
|
||||
+4
-2
@@ -15,13 +15,12 @@
|
||||
package b2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/kurin/blazer/base"
|
||||
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
// This file wraps the base package in a thin layer, for testing. It should be
|
||||
@@ -150,6 +149,9 @@ func (b *b2Root) authorizeAccount(ctx context.Context, account, key string, opts
|
||||
if c.capExceeded {
|
||||
aopts = append(aopts, base.ForceCapExceeded())
|
||||
}
|
||||
for _, agent := range c.userAgents {
|
||||
aopts = append(aopts, base.UserAgent(agent))
|
||||
}
|
||||
nb, err := base.AuthorizeAccount(ctx, account, key, aopts...)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
+70
-13
@@ -17,22 +17,84 @@ package b2
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha1"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type readResetter interface {
|
||||
Read([]byte) (int, error)
|
||||
Reset() error
|
||||
}
|
||||
|
||||
type resetter struct {
|
||||
rs io.ReadSeeker
|
||||
}
|
||||
|
||||
func (r resetter) Read(p []byte) (int, error) { return r.rs.Read(p) }
|
||||
func (r resetter) Reset() error { _, err := r.rs.Seek(0, 0); return err }
|
||||
|
||||
func newResetter(p []byte) readResetter { return resetter{rs: bytes.NewReader(p)} }
|
||||
|
||||
type writeBuffer interface {
|
||||
io.Writer
|
||||
Len() int
|
||||
Reader() (io.ReadSeeker, error)
|
||||
Reader() (readResetter, error)
|
||||
Hash() string // sha1 or whatever it is
|
||||
Close() error
|
||||
}
|
||||
|
||||
// nonBuffer doesn't buffer anything, but passes values directly from the
|
||||
// source readseeker. Many nonBuffers can point at different parts of the same
|
||||
// underlying source, and be accessed by multiple goroutines simultaneously.
|
||||
func newNonBuffer(rs io.ReaderAt, offset, size int64) writeBuffer {
|
||||
return &nonBuffer{
|
||||
r: io.NewSectionReader(rs, offset, size),
|
||||
size: int(size),
|
||||
hsh: sha1.New(),
|
||||
}
|
||||
}
|
||||
|
||||
type nonBuffer struct {
|
||||
r *io.SectionReader
|
||||
size int
|
||||
hsh hash.Hash
|
||||
|
||||
isEOF bool
|
||||
buf *strings.Reader
|
||||
}
|
||||
|
||||
func (nb *nonBuffer) Len() int { return nb.size + 40 }
|
||||
func (nb *nonBuffer) Hash() string { return "hex_digits_at_end" }
|
||||
func (nb *nonBuffer) Close() error { return nil }
|
||||
func (nb *nonBuffer) Reader() (readResetter, error) { return nb, nil }
|
||||
func (nb *nonBuffer) Write([]byte) (int, error) { return 0, errors.New("writes not supported") }
|
||||
|
||||
func (nb *nonBuffer) Read(p []byte) (int, error) {
|
||||
if nb.isEOF {
|
||||
return nb.buf.Read(p)
|
||||
}
|
||||
n, err := io.TeeReader(nb.r, nb.hsh).Read(p)
|
||||
if err == io.EOF {
|
||||
err = nil
|
||||
nb.isEOF = true
|
||||
nb.buf = strings.NewReader(fmt.Sprintf("%x", nb.hsh.Sum(nil)))
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (nb *nonBuffer) Reset() error {
|
||||
nb.hsh.Reset()
|
||||
nb.isEOF = false
|
||||
_, err := nb.r.Seek(0, 0)
|
||||
return err
|
||||
}
|
||||
|
||||
type memoryBuffer struct {
|
||||
buf *bytes.Buffer
|
||||
hsh hash.Hash
|
||||
@@ -56,15 +118,10 @@ func newMemoryBuffer() *memoryBuffer {
|
||||
return mb
|
||||
}
|
||||
|
||||
type thing struct {
|
||||
rs io.ReadSeeker
|
||||
t int
|
||||
}
|
||||
|
||||
func (mb *memoryBuffer) Write(p []byte) (int, error) { return mb.w.Write(p) }
|
||||
func (mb *memoryBuffer) Len() int { return mb.buf.Len() }
|
||||
func (mb *memoryBuffer) Reader() (io.ReadSeeker, error) { return bytes.NewReader(mb.buf.Bytes()), nil }
|
||||
func (mb *memoryBuffer) Hash() string { return fmt.Sprintf("%x", mb.hsh.Sum(nil)) }
|
||||
func (mb *memoryBuffer) Write(p []byte) (int, error) { return mb.w.Write(p) }
|
||||
func (mb *memoryBuffer) Len() int { return mb.buf.Len() }
|
||||
func (mb *memoryBuffer) Reader() (readResetter, error) { return newResetter(mb.buf.Bytes()), nil }
|
||||
func (mb *memoryBuffer) Hash() string { return fmt.Sprintf("%x", mb.hsh.Sum(nil)) }
|
||||
|
||||
func (mb *memoryBuffer) Close() error {
|
||||
mb.mux.Lock()
|
||||
@@ -107,7 +164,7 @@ func (fb *fileBuffer) Write(p []byte) (int, error) {
|
||||
func (fb *fileBuffer) Len() int { return fb.s }
|
||||
func (fb *fileBuffer) Hash() string { return fmt.Sprintf("%x", fb.hsh.Sum(nil)) }
|
||||
|
||||
func (fb *fileBuffer) Reader() (io.ReadSeeker, error) {
|
||||
func (fb *fileBuffer) Reader() (readResetter, error) {
|
||||
if _, err := fb.f.Seek(0, 0); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -124,5 +181,5 @@ type fr struct {
|
||||
f *os.File
|
||||
}
|
||||
|
||||
func (r *fr) Read(p []byte) (int, error) { return r.f.Read(p) }
|
||||
func (r *fr) Seek(a int64, b int) (int64, error) { return r.f.Seek(a, b) }
|
||||
func (r *fr) Read(p []byte) (int, error) { return r.f.Read(p) }
|
||||
func (r *fr) Reset() error { _, err := r.f.Seek(0, 0); return err }
|
||||
+154
-4
@@ -16,6 +16,7 @@ package b2
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -23,10 +24,9 @@ import (
|
||||
"os"
|
||||
"reflect"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -79,6 +79,86 @@ func TestReadWriteLive(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReaderFromLive(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
bucket, done := startLiveTest(ctx, t)
|
||||
defer done()
|
||||
|
||||
table := []struct {
|
||||
size, pos int64
|
||||
csize, writers int
|
||||
}{
|
||||
{
|
||||
// that it works at all
|
||||
size: 10,
|
||||
},
|
||||
{
|
||||
// large uploads
|
||||
size: 15e6 + 10,
|
||||
csize: 5e6,
|
||||
writers: 2,
|
||||
},
|
||||
{
|
||||
// an excess of writers
|
||||
size: 50e6,
|
||||
csize: 5e6,
|
||||
writers: 12,
|
||||
},
|
||||
{
|
||||
// with offset, seeks back to start after turning it into a ReaderAt
|
||||
size: 250,
|
||||
pos: 50,
|
||||
},
|
||||
}
|
||||
|
||||
for i, e := range table {
|
||||
rs := &zReadSeeker{pos: e.pos, size: e.size}
|
||||
o := bucket.Object(fmt.Sprintf("writer.%d", i))
|
||||
w := o.NewWriter(ctx)
|
||||
w.ChunkSize = e.csize
|
||||
w.ConcurrentUploads = e.writers
|
||||
n, err := w.ReadFrom(rs)
|
||||
if err != nil {
|
||||
t.Errorf("ReadFrom(): %v", err)
|
||||
}
|
||||
if n != e.size {
|
||||
t.Errorf("ReadFrom(): got %d bytes, wanted %d bytes", n, e.size)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Errorf("w.Close(): %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
r := o.NewReader(ctx)
|
||||
h := sha1.New()
|
||||
rn, err := io.Copy(h, r)
|
||||
if err != nil {
|
||||
t.Errorf("Read from B2: %v", err)
|
||||
}
|
||||
if rn != n {
|
||||
t.Errorf("Read from B2: got %d bytes, want %d bytes", rn, n)
|
||||
}
|
||||
if err := r.Close(); err != nil {
|
||||
t.Errorf("r.Close(): %v", err)
|
||||
}
|
||||
|
||||
hex := fmt.Sprintf("%x", h.Sum(nil))
|
||||
attrs, err := o.Attrs(ctx)
|
||||
if err != nil {
|
||||
t.Errorf("Attrs(): %v", err)
|
||||
continue
|
||||
}
|
||||
if attrs.SHA1 == "none" {
|
||||
continue
|
||||
}
|
||||
if hex != attrs.SHA1 {
|
||||
t.Errorf("SHA1: got %q, want %q", hex, attrs.SHA1)
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestHideShowLive(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
|
||||
@@ -200,7 +280,6 @@ func TestResumeWriter(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestAttrs(t *testing.T) {
|
||||
// TODO: test is flaky
|
||||
ctx := context.Background()
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
|
||||
defer cancel()
|
||||
@@ -765,6 +844,74 @@ func listObjects(ctx context.Context, f func(context.Context, int, *Cursor) ([]*
|
||||
|
||||
var transport = http.DefaultTransport
|
||||
|
||||
type eofTripper struct {
|
||||
rt http.RoundTripper
|
||||
t *testing.T
|
||||
}
|
||||
|
||||
func (et eofTripper) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
resp, err := et.rt.RoundTrip(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp.Body = &eofReadCloser{rc: resp.Body, t: et.t}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
type eofReadCloser struct {
|
||||
rc io.ReadCloser
|
||||
eof bool
|
||||
t *testing.T
|
||||
}
|
||||
|
||||
func (eof *eofReadCloser) Read(p []byte) (int, error) {
|
||||
n, err := eof.rc.Read(p)
|
||||
if err == io.EOF {
|
||||
eof.eof = true
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (eof *eofReadCloser) Close() error {
|
||||
if !eof.eof {
|
||||
eof.t.Error("http body closed with bytes unread")
|
||||
}
|
||||
return eof.rc.Close()
|
||||
}
|
||||
|
||||
// Checks that close is called.
|
||||
type ccTripper struct {
|
||||
t *testing.T
|
||||
rt http.RoundTripper
|
||||
trips int64
|
||||
}
|
||||
|
||||
func (cc *ccTripper) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
resp, err := cc.rt.RoundTrip(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
atomic.AddInt64(&cc.trips, 1)
|
||||
resp.Body = &ccRC{ReadCloser: resp.Body, c: &cc.trips}
|
||||
return resp, err
|
||||
}
|
||||
|
||||
func (cc *ccTripper) done() {
|
||||
if cc.trips != 0 {
|
||||
cc.t.Errorf("failed to close %d HTTP bodies", cc.trips)
|
||||
}
|
||||
}
|
||||
|
||||
type ccRC struct {
|
||||
io.ReadCloser
|
||||
c *int64
|
||||
}
|
||||
|
||||
func (cc *ccRC) Close() error {
|
||||
atomic.AddInt64(cc.c, -1)
|
||||
return cc.ReadCloser.Close()
|
||||
}
|
||||
|
||||
func startLiveTest(ctx context.Context, t *testing.T) (*Bucket, func()) {
|
||||
id := os.Getenv(apiID)
|
||||
key := os.Getenv(apiKey)
|
||||
@@ -772,7 +919,9 @@ func startLiveTest(ctx context.Context, t *testing.T) (*Bucket, func()) {
|
||||
t.Skipf("B2_ACCOUNT_ID or B2_SECRET_KEY unset; skipping integration tests")
|
||||
return nil, nil
|
||||
}
|
||||
client, err := NewClient(ctx, id, key, FailSomeUploads(), ExpireSomeAuthTokens(), Transport(transport))
|
||||
ccport := &ccTripper{rt: transport, t: t}
|
||||
tport := eofTripper{rt: ccport, t: t}
|
||||
client, err := NewClient(ctx, id, key, FailSomeUploads(), ExpireSomeAuthTokens(), Transport(tport), UserAgent("b2-test"), UserAgent("integration-test"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
return nil, nil
|
||||
@@ -783,6 +932,7 @@ func startLiveTest(ctx context.Context, t *testing.T) (*Bucket, func()) {
|
||||
return nil, nil
|
||||
}
|
||||
f := func() {
|
||||
defer ccport.done()
|
||||
for c := range listObjects(ctx, bucket.ListObjects) {
|
||||
if c.err != nil {
|
||||
continue
|
||||
|
||||
+5
-8
@@ -16,13 +16,12 @@ package b2
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"sync"
|
||||
|
||||
"github.com/kurin/blazer/internal/blog"
|
||||
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
var errNoMoreContent = errors.New("416: out of content")
|
||||
@@ -140,11 +139,12 @@ func (r *Reader) thread() {
|
||||
return
|
||||
}
|
||||
rsize, _, _, _ := fr.stats()
|
||||
mr := &meteredReader{r: &fakeSeeker{fr}, size: int(rsize)}
|
||||
mr := &meteredReader{r: noopResetter{fr}, size: int(rsize)}
|
||||
r.smux.Lock()
|
||||
r.smap[chunkID] = mr
|
||||
r.smux.Unlock()
|
||||
i, err := copyContext(r.ctx, buf, mr)
|
||||
fr.Close()
|
||||
r.smux.Lock()
|
||||
r.smap[chunkID] = nil
|
||||
r.smux.Unlock()
|
||||
@@ -290,11 +290,8 @@ func copyContext(ctx context.Context, dst io.Writer, src io.Reader) (written int
|
||||
return written, err
|
||||
}
|
||||
|
||||
// fakeSeeker exists so that we can wrap the http response body (an io.Reader
|
||||
// but not an io.Seeker) into a meteredReader, which will allow us to keep tabs
|
||||
// on how much of the chunk we've read so far.
|
||||
type fakeSeeker struct {
|
||||
type noopResetter struct {
|
||||
io.Reader
|
||||
}
|
||||
|
||||
func (fs *fakeSeeker) Seek(int64, int) (int64, error) { return 0, nil }
|
||||
func (noopResetter) Reset() error { return nil }
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
// Copyright 2017, Google
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package b2
|
||||
|
||||
import (
|
||||
"io"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type readerAt struct {
|
||||
rs io.ReadSeeker
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func (r *readerAt) ReadAt(p []byte, off int64) (int, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
// ReadAt is supposed to preserve the offset.
|
||||
cur, err := r.rs.Seek(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() { r.rs.Seek(cur, io.SeekStart) }()
|
||||
|
||||
if _, err := r.rs.Seek(off, io.SeekStart); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return io.ReadFull(r.rs, p)
|
||||
}
|
||||
|
||||
// wraps a ReadSeeker in a mutex to provite a ReaderAt how is this not in the
|
||||
// io package?
|
||||
func enReaderAt(rs io.ReadSeeker) io.ReaderAt {
|
||||
return &readerAt{rs: rs}
|
||||
}
|
||||
+92
-21
@@ -15,6 +15,7 @@
|
||||
package b2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -23,8 +24,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/kurin/blazer/internal/blog"
|
||||
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
// Writer writes data into Backblaze. It automatically switches to the large
|
||||
@@ -75,6 +74,7 @@ type Writer struct {
|
||||
file beLargeFileInterface
|
||||
seen map[int]string
|
||||
everStarted bool
|
||||
newBuffer func() (writeBuffer, error)
|
||||
|
||||
o *Object
|
||||
name string
|
||||
@@ -94,15 +94,8 @@ type chunk struct {
|
||||
buf writeBuffer
|
||||
}
|
||||
|
||||
func (w *Writer) getBuffer() (writeBuffer, error) {
|
||||
if !w.UseFileBuffer {
|
||||
return newMemoryBuffer(), nil
|
||||
}
|
||||
return newFileBuffer(w.FileBufferDir)
|
||||
}
|
||||
|
||||
func (w *Writer) setErr(err error) {
|
||||
if err == nil {
|
||||
if err == nil || err == io.EOF {
|
||||
return
|
||||
}
|
||||
w.emux.Lock()
|
||||
@@ -200,8 +193,7 @@ func (w *Writer) thread() {
|
||||
}()
|
||||
}
|
||||
|
||||
// Write satisfies the io.Writer interface.
|
||||
func (w *Writer) Write(p []byte) (int, error) {
|
||||
func (w *Writer) init() {
|
||||
w.start.Do(func() {
|
||||
w.everStarted = true
|
||||
w.smux.Lock()
|
||||
@@ -212,13 +204,24 @@ func (w *Writer) Write(p []byte) (int, error) {
|
||||
if w.csize == 0 {
|
||||
w.csize = 1e8
|
||||
}
|
||||
v, err := w.getBuffer()
|
||||
if w.newBuffer == nil {
|
||||
w.newBuffer = func() (writeBuffer, error) { return newMemoryBuffer(), nil }
|
||||
if w.UseFileBuffer {
|
||||
w.newBuffer = func() (writeBuffer, error) { return newFileBuffer(w.FileBufferDir) }
|
||||
}
|
||||
}
|
||||
v, err := w.newBuffer()
|
||||
if err != nil {
|
||||
w.setErr(err)
|
||||
return
|
||||
}
|
||||
w.w = v
|
||||
})
|
||||
}
|
||||
|
||||
// Write satisfies the io.Writer interface.
|
||||
func (w *Writer) Write(p []byte) (int, error) {
|
||||
w.init()
|
||||
if err := w.getErr(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
@@ -263,7 +266,7 @@ redo:
|
||||
f, err := ue.uploadFile(w.ctx, mr, int(w.w.Len()), w.name, ctype, sha1, w.info)
|
||||
if err != nil {
|
||||
if w.o.b.r.reupload(err) {
|
||||
blog.V(1).Infof("b2 writer: %v; retrying", err)
|
||||
blog.V(2).Infof("b2 writer: %v; retrying", err)
|
||||
u, err := w.o.b.b.getUploadURL(w.ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -352,7 +355,7 @@ func (w *Writer) sendChunk() error {
|
||||
return w.ctx.Err()
|
||||
}
|
||||
w.cidx++
|
||||
v, err := w.getBuffer()
|
||||
v, err := w.newBuffer()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -360,15 +363,83 @@ func (w *Writer) sendChunk() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReadFrom reads all of r into w, returning the first error or no error if r
|
||||
// returns io.EOF. If r is also an io.Seeker, ReadFrom will stream r directly
|
||||
// over the wire instead of buffering it locally. This reduces memory usage.
|
||||
//
|
||||
// Do not issue multiple calls to ReadFrom, or mix ReadFrom and Write. If you
|
||||
// have multiple readers you want to concatenate into the same B2 object, use
|
||||
// an io.MultiReader.
|
||||
//
|
||||
// Note that io.Copy will automatically choose to use ReadFrom.
|
||||
//
|
||||
// ReadFrom currently doesn't handle w.Resume; if w.Resume is true, ReadFrom
|
||||
// will act as if r is not an io.Seeker.
|
||||
func (w *Writer) ReadFrom(r io.Reader) (int64, error) {
|
||||
rs, ok := r.(io.ReadSeeker)
|
||||
if !ok || w.Resume {
|
||||
return copyContext(w.ctx, w, r)
|
||||
}
|
||||
blog.V(2).Info("streaming without buffer")
|
||||
size, err := rs.Seek(0, io.SeekEnd)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var ra io.ReaderAt
|
||||
if rat, ok := r.(io.ReaderAt); ok {
|
||||
ra = rat
|
||||
} else {
|
||||
ra = enReaderAt(rs)
|
||||
}
|
||||
var offset int64
|
||||
var wrote int64
|
||||
w.newBuffer = func() (writeBuffer, error) {
|
||||
left := size - offset
|
||||
if left <= 0 {
|
||||
// We're done sending real chunks; send empty chunks from now on so that
|
||||
// Close() works.
|
||||
w.newBuffer = func() (writeBuffer, error) { return newMemoryBuffer(), nil }
|
||||
w.w = newMemoryBuffer()
|
||||
return nil, io.EOF
|
||||
}
|
||||
csize := int64(w.csize)
|
||||
if left < csize {
|
||||
csize = left
|
||||
}
|
||||
nb := newNonBuffer(ra, offset, csize)
|
||||
wrote += csize // TODO: this is kind of a total lie
|
||||
offset += csize
|
||||
return nb, nil
|
||||
}
|
||||
w.init()
|
||||
if size < int64(w.csize) {
|
||||
// the magic happens on w.Close()
|
||||
return size, nil
|
||||
}
|
||||
for {
|
||||
if err := w.sendChunk(); err != nil {
|
||||
if err != io.EOF {
|
||||
return wrote, err
|
||||
}
|
||||
return wrote, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Close satisfies the io.Closer interface. It is critical to check the return
|
||||
// value of Close on all writers.
|
||||
// value of Close for all writers.
|
||||
func (w *Writer) Close() error {
|
||||
w.done.Do(func() {
|
||||
if !w.everStarted {
|
||||
return
|
||||
}
|
||||
defer w.o.b.c.removeWriter(w)
|
||||
defer w.w.Close() // TODO: log error
|
||||
defer func() {
|
||||
if err := w.w.Close(); err != nil {
|
||||
// this is non-fatal, but alarming
|
||||
blog.V(1).Infof("close %s: %v", w.name, err)
|
||||
}
|
||||
}()
|
||||
if w.cidx == 0 {
|
||||
w.setErr(w.simpleWriteFile())
|
||||
return
|
||||
@@ -423,7 +494,7 @@ func (w *Writer) status() *WriterStatus {
|
||||
type meteredReader struct {
|
||||
read int64
|
||||
size int
|
||||
r io.ReadSeeker
|
||||
r readResetter
|
||||
mux sync.Mutex
|
||||
}
|
||||
|
||||
@@ -435,11 +506,11 @@ func (mr *meteredReader) Read(p []byte) (int, error) {
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (mr *meteredReader) Seek(offset int64, whence int) (int64, error) {
|
||||
func (mr *meteredReader) Reset() error {
|
||||
mr.mux.Lock()
|
||||
defer mr.mux.Unlock()
|
||||
mr.read = offset
|
||||
return mr.r.Seek(offset, whence)
|
||||
mr.read = 0
|
||||
return mr.r.Reset()
|
||||
}
|
||||
|
||||
func (mr *meteredReader) done() float64 {
|
||||
|
||||
Reference in new issue
Block a user