Update vendored dependencies

This includes github.com/kurin/blazer 0.2.0, which resolves #1291
This commit is contained in:
Alexander Neumann committed 2017-10-01 10:13:39 +02:00
1 parent ba23d24dd1
commit 61cb1cc6f8
1044 files changed
+203017 -97704

No files matched your search

+22 -2
View File
@@ -28,6 +28,7 @@
package b2
import (
"context"
"fmt"
"io"
"net/http"
@@ -35,8 +36,6 @@ import (
"strconv"
"sync"
"time"
"golang.org/x/net/context"
)
// Client is a Backblaze B2 client.
@@ -67,11 +66,23 @@ type clientOptions struct {
failSomeUploads bool
expireTokens bool
capExceeded bool
userAgents []string
}
// A ClientOption allows callers to adjust various per-client settings.
type ClientOption func(*clientOptions)
// UserAgent sets the User-Agent HTTP header. The default header is
// "blazer/<version>"; the value set here will be prepended to that. This can
// be set multiple times.
//
// A user agent is generally of the form "<product>/<version> (<comments>)".
func UserAgent(agent string) ClientOption {
return func(o *clientOptions) {
o.userAgents = append(o.userAgents, agent)
}
}
// Transport sets the underlying HTTP transport mechanism. If unset,
// http.DefaultTransport is used.
func Transport(rt http.RoundTripper) ClientOption {
@@ -118,6 +129,7 @@ const (
UnknownType BucketType = ""
Private = "allPrivate"
Public = "allPublic"
Snapshot = "snapshot"
)
// BucketAttrs holds a bucket's metadata attributes.
@@ -582,11 +594,19 @@ func (b *Bucket) Reveal(ctx context.Context, name string) error {
return obj.Delete(ctx)
}
// I don't want to import all of ioutil for this.
type discard struct{}
func (discard) Write(p []byte) (int, error) {
return len(p), nil
}
func (b *Bucket) getObject(ctx context.Context, name string) (*Object, error) {
fr, err := b.b.downloadFileByName(ctx, name, 0, 1)
if err != nil {
return nil, err
}
io.Copy(discard{}, fr)
fr.Close()
return &Object{
name: name,
+128 -3
View File
@@ -16,6 +16,7 @@ package b2
import (
"bytes"
"context"
"crypto/sha1"
"fmt"
"io"
@@ -26,8 +27,6 @@ import (
"sync"
"testing"
"time"
"golang.org/x/net/context"
)
const (
@@ -352,6 +351,82 @@ func (zReader) Read(p []byte) (int, error) {
return len(p), nil
}
type zReadSeeker struct {
size int64
pos int64
}
func (rs *zReadSeeker) Read(p []byte) (int, error) {
for i := rs.pos; ; i++ {
j := int(i - rs.pos)
if j >= len(p) || i >= rs.size {
var rtn error
if i >= rs.size {
rtn = io.EOF
}
rs.pos = i
return j, rtn
}
f := int(i) % len(pattern)
p[j] = pattern[f]
}
}
func (rs *zReadSeeker) Seek(offset int64, whence int) (int64, error) {
switch whence {
case io.SeekStart:
rs.pos = offset
case io.SeekEnd:
rs.pos = rs.size + offset
}
return rs.pos, nil
}
func TestReaderFrom(t *testing.T) {
ctx := context.Background()
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
table := []struct {
size, pos int64
}{
{
size: 10,
},
}
for _, e := range table {
client := &Client{
backend: &beRoot{
b2i: &testRoot{
bucketMap: make(map[string]map[string]string),
errs: &errCont{},
},
},
}
bucket, err := client.NewBucket(ctx, bucketName, &BucketAttrs{Type: Private})
if err != nil {
t.Fatal(err)
}
defer func() {
if err := bucket.Delete(ctx); err != nil {
t.Error(err)
}
}()
r := &zReadSeeker{pos: e.pos, size: e.size}
w := bucket.Object("writer").NewWriter(ctx)
n, err := w.ReadFrom(r)
if err != nil {
t.Errorf("ReadFrom(): %v", err)
}
if n != e.size {
t.Errorf("ReadFrom(): got %d bytes, wanted %d bytes", n, e.size)
}
}
}
func TestReauth(t *testing.T) {
ctx := context.Background()
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
@@ -696,6 +771,52 @@ func TestFileBuffer(t *testing.T) {
}
}
func TestNonBuffer(t *testing.T) {
table := []struct {
str string
off int64
len int64
want string
}{
{
str: "a string",
off: 0,
len: 3,
want: "a s",
},
{
str: "a string",
off: 3,
len: 1,
want: "t",
},
{
str: "a string",
off: 3,
len: 5,
want: "tring",
},
}
for _, e := range table {
nb := newNonBuffer(strings.NewReader(e.str), e.off, e.len)
want := fmt.Sprintf("%s%x", e.want, sha1.Sum([]byte(e.str[int(e.off):int(e.off+e.len)])))
r, err := nb.Reader()
if err != nil {
t.Error(err)
continue
}
got, err := ioutil.ReadAll(r)
if err != nil {
t.Errorf("ioutil.ReadAll(%#v): %v", e, err)
continue
}
if want != string(got) {
t.Errorf("ioutil.ReadAll(%#v): got %q, want %q", e, string(got), want)
}
}
}
func writeFile(ctx context.Context, bucket *Bucket, name string, size int64, csize int) (*Object, string, error) {
r := io.LimitReader(zReader{}, size)
o := bucket.Object(name)
@@ -704,9 +825,13 @@ func writeFile(ctx context.Context, bucket *Bucket, name string, size int64, csi
w := io.MultiWriter(f, h)
f.ConcurrentUploads = 5
f.ChunkSize = csize
if _, err := io.Copy(w, r); err != nil {
n, err := io.Copy(w, r)
if err != nil {
return nil, "", err
}
if n != size {
return nil, "", fmt.Errorf("io.Copy(): wrote %d bytes; wanted %d bytes", n, size)
}
if err := f.Close(); err != nil {
return nil, "", err
}
+7 -8
View File
@@ -15,11 +15,10 @@
package b2
import (
"context"
"io"
"math/rand"
"time"
"golang.org/x/net/context"
)
// This file wraps the baseline interfaces with backoff and retry semantics.
@@ -63,7 +62,7 @@ type beBucket struct {
}
type beURLInterface interface {
uploadFile(context.Context, io.ReadSeeker, int, string, string, string, map[string]string) (beFileInterface, error)
uploadFile(context.Context, readResetter, int, string, string, string, map[string]string) (beFileInterface, error)
}
type beURL struct {
@@ -100,7 +99,7 @@ type beLargeFile struct {
type beFileChunkInterface interface {
reload(context.Context) error
uploadPart(context.Context, io.ReadSeeker, string, int, int) (int, error)
uploadPart(context.Context, readResetter, string, int, int) (int, error)
}
type beFileChunk struct {
@@ -414,10 +413,10 @@ func (b *beBucket) file(id, name string) beFileInterface {
}
}
func (b *beURL) uploadFile(ctx context.Context, r io.ReadSeeker, size int, name, ct, sha1 string, info map[string]string) (beFileInterface, error) {
func (b *beURL) uploadFile(ctx context.Context, r readResetter, size int, name, ct, sha1 string, info map[string]string) (beFileInterface, error) {
var file beFileInterface
f := func() error {
if _, err := r.Seek(0, 0); err != nil {
if err := r.Reset(); err != nil {
return err
}
f, err := b.b2url.uploadFile(ctx, r, size, name, ct, sha1, info)
@@ -578,12 +577,12 @@ func (b *beFileChunk) reload(ctx context.Context) error {
return withBackoff(ctx, b.ri, f)
}
func (b *beFileChunk) uploadPart(ctx context.Context, r io.ReadSeeker, sha1 string, size, index int) (int, error) {
func (b *beFileChunk) uploadPart(ctx context.Context, r readResetter, sha1 string, size, index int) (int, error) {
// no re-auth; pass it back up to the caller so they can get an new upload URI and token
// TODO: we should handle that here probably
var i int
f := func() error {
if _, err := r.Seek(0, 0); err != nil {
if err := r.Reset(); err != nil {
return err
}
j, err := b.b2fileChunk.uploadPart(ctx, r, sha1, size, index)
+4 -2
View File
@@ -15,13 +15,12 @@
package b2
import (
"context"
"io"
"net/http"
"time"
"github.com/kurin/blazer/base"
"golang.org/x/net/context"
)
// This file wraps the base package in a thin layer, for testing. It should be
@@ -150,6 +149,9 @@ func (b *b2Root) authorizeAccount(ctx context.Context, account, key string, opts
if c.capExceeded {
aopts = append(aopts, base.ForceCapExceeded())
}
for _, agent := range c.userAgents {
aopts = append(aopts, base.UserAgent(agent))
}
nb, err := base.AuthorizeAccount(ctx, account, key, aopts...)
if err != nil {
return err
+70 -13
View File
@@ -17,22 +17,84 @@ package b2
import (
"bytes"
"crypto/sha1"
"errors"
"fmt"
"hash"
"io"
"io/ioutil"
"os"
"strings"
"sync"
)
type readResetter interface {
Read([]byte) (int, error)
Reset() error
}
type resetter struct {
rs io.ReadSeeker
}
func (r resetter) Read(p []byte) (int, error) { return r.rs.Read(p) }
func (r resetter) Reset() error { _, err := r.rs.Seek(0, 0); return err }
func newResetter(p []byte) readResetter { return resetter{rs: bytes.NewReader(p)} }
type writeBuffer interface {
io.Writer
Len() int
Reader() (io.ReadSeeker, error)
Reader() (readResetter, error)
Hash() string // sha1 or whatever it is
Close() error
}
// nonBuffer doesn't buffer anything, but passes values directly from the
// source readseeker. Many nonBuffers can point at different parts of the same
// underlying source, and be accessed by multiple goroutines simultaneously.
func newNonBuffer(rs io.ReaderAt, offset, size int64) writeBuffer {
return &nonBuffer{
r: io.NewSectionReader(rs, offset, size),
size: int(size),
hsh: sha1.New(),
}
}
type nonBuffer struct {
r *io.SectionReader
size int
hsh hash.Hash
isEOF bool
buf *strings.Reader
}
func (nb *nonBuffer) Len() int { return nb.size + 40 }
func (nb *nonBuffer) Hash() string { return "hex_digits_at_end" }
func (nb *nonBuffer) Close() error { return nil }
func (nb *nonBuffer) Reader() (readResetter, error) { return nb, nil }
func (nb *nonBuffer) Write([]byte) (int, error) { return 0, errors.New("writes not supported") }
func (nb *nonBuffer) Read(p []byte) (int, error) {
if nb.isEOF {
return nb.buf.Read(p)
}
n, err := io.TeeReader(nb.r, nb.hsh).Read(p)
if err == io.EOF {
err = nil
nb.isEOF = true
nb.buf = strings.NewReader(fmt.Sprintf("%x", nb.hsh.Sum(nil)))
}
return n, err
}
func (nb *nonBuffer) Reset() error {
nb.hsh.Reset()
nb.isEOF = false
_, err := nb.r.Seek(0, 0)
return err
}
type memoryBuffer struct {
buf *bytes.Buffer
hsh hash.Hash
@@ -56,15 +118,10 @@ func newMemoryBuffer() *memoryBuffer {
return mb
}
type thing struct {
rs io.ReadSeeker
t int
}
func (mb *memoryBuffer) Write(p []byte) (int, error) { return mb.w.Write(p) }
func (mb *memoryBuffer) Len() int { return mb.buf.Len() }
func (mb *memoryBuffer) Reader() (io.ReadSeeker, error) { return bytes.NewReader(mb.buf.Bytes()), nil }
func (mb *memoryBuffer) Hash() string { return fmt.Sprintf("%x", mb.hsh.Sum(nil)) }
func (mb *memoryBuffer) Write(p []byte) (int, error) { return mb.w.Write(p) }
func (mb *memoryBuffer) Len() int { return mb.buf.Len() }
func (mb *memoryBuffer) Reader() (readResetter, error) { return newResetter(mb.buf.Bytes()), nil }
func (mb *memoryBuffer) Hash() string { return fmt.Sprintf("%x", mb.hsh.Sum(nil)) }
func (mb *memoryBuffer) Close() error {
mb.mux.Lock()
@@ -107,7 +164,7 @@ func (fb *fileBuffer) Write(p []byte) (int, error) {
func (fb *fileBuffer) Len() int { return fb.s }
func (fb *fileBuffer) Hash() string { return fmt.Sprintf("%x", fb.hsh.Sum(nil)) }
func (fb *fileBuffer) Reader() (io.ReadSeeker, error) {
func (fb *fileBuffer) Reader() (readResetter, error) {
if _, err := fb.f.Seek(0, 0); err != nil {
return nil, err
}
@@ -124,5 +181,5 @@ type fr struct {
f *os.File
}
func (r *fr) Read(p []byte) (int, error) { return r.f.Read(p) }
func (r *fr) Seek(a int64, b int) (int64, error) { return r.f.Seek(a, b) }
func (r *fr) Read(p []byte) (int, error) { return r.f.Read(p) }
func (r *fr) Reset() error { _, err := r.f.Seek(0, 0); return err }
+154 -4
View File
@@ -16,6 +16,7 @@ package b2
import (
"bytes"
"context"
"crypto/sha1"
"fmt"
"io"
@@ -23,10 +24,9 @@ import (
"os"
"reflect"
"sync"
"sync/atomic"
"testing"
"time"
"golang.org/x/net/context"
)
const (
@@ -79,6 +79,86 @@ func TestReadWriteLive(t *testing.T) {
}
}
func TestReaderFromLive(t *testing.T) {
ctx := context.Background()
ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
defer cancel()
bucket, done := startLiveTest(ctx, t)
defer done()
table := []struct {
size, pos int64
csize, writers int
}{
{
// that it works at all
size: 10,
},
{
// large uploads
size: 15e6 + 10,
csize: 5e6,
writers: 2,
},
{
// an excess of writers
size: 50e6,
csize: 5e6,
writers: 12,
},
{
// with offset, seeks back to start after turning it into a ReaderAt
size: 250,
pos: 50,
},
}
for i, e := range table {
rs := &zReadSeeker{pos: e.pos, size: e.size}
o := bucket.Object(fmt.Sprintf("writer.%d", i))
w := o.NewWriter(ctx)
w.ChunkSize = e.csize
w.ConcurrentUploads = e.writers
n, err := w.ReadFrom(rs)
if err != nil {
t.Errorf("ReadFrom(): %v", err)
}
if n != e.size {
t.Errorf("ReadFrom(): got %d bytes, wanted %d bytes", n, e.size)
}
if err := w.Close(); err != nil {
t.Errorf("w.Close(): %v", err)
continue
}
r := o.NewReader(ctx)
h := sha1.New()
rn, err := io.Copy(h, r)
if err != nil {
t.Errorf("Read from B2: %v", err)
}
if rn != n {
t.Errorf("Read from B2: got %d bytes, want %d bytes", rn, n)
}
if err := r.Close(); err != nil {
t.Errorf("r.Close(): %v", err)
}
hex := fmt.Sprintf("%x", h.Sum(nil))
attrs, err := o.Attrs(ctx)
if err != nil {
t.Errorf("Attrs(): %v", err)
continue
}
if attrs.SHA1 == "none" {
continue
}
if hex != attrs.SHA1 {
t.Errorf("SHA1: got %q, want %q", hex, attrs.SHA1)
}
}
}
func TestHideShowLive(t *testing.T) {
ctx := context.Background()
ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
@@ -200,7 +280,6 @@ func TestResumeWriter(t *testing.T) {
}
func TestAttrs(t *testing.T) {
// TODO: test is flaky
ctx := context.Background()
ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
defer cancel()
@@ -765,6 +844,74 @@ func listObjects(ctx context.Context, f func(context.Context, int, *Cursor) ([]*
var transport = http.DefaultTransport
type eofTripper struct {
rt http.RoundTripper
t *testing.T
}
func (et eofTripper) RoundTrip(req *http.Request) (*http.Response, error) {
resp, err := et.rt.RoundTrip(req)
if err != nil {
return nil, err
}
resp.Body = &eofReadCloser{rc: resp.Body, t: et.t}
return resp, nil
}
type eofReadCloser struct {
rc io.ReadCloser
eof bool
t *testing.T
}
func (eof *eofReadCloser) Read(p []byte) (int, error) {
n, err := eof.rc.Read(p)
if err == io.EOF {
eof.eof = true
}
return n, err
}
func (eof *eofReadCloser) Close() error {
if !eof.eof {
eof.t.Error("http body closed with bytes unread")
}
return eof.rc.Close()
}
// Checks that close is called.
type ccTripper struct {
t *testing.T
rt http.RoundTripper
trips int64
}
func (cc *ccTripper) RoundTrip(req *http.Request) (*http.Response, error) {
resp, err := cc.rt.RoundTrip(req)
if err != nil {
return nil, err
}
atomic.AddInt64(&cc.trips, 1)
resp.Body = &ccRC{ReadCloser: resp.Body, c: &cc.trips}
return resp, err
}
func (cc *ccTripper) done() {
if cc.trips != 0 {
cc.t.Errorf("failed to close %d HTTP bodies", cc.trips)
}
}
type ccRC struct {
io.ReadCloser
c *int64
}
func (cc *ccRC) Close() error {
atomic.AddInt64(cc.c, -1)
return cc.ReadCloser.Close()
}
func startLiveTest(ctx context.Context, t *testing.T) (*Bucket, func()) {
id := os.Getenv(apiID)
key := os.Getenv(apiKey)
@@ -772,7 +919,9 @@ func startLiveTest(ctx context.Context, t *testing.T) (*Bucket, func()) {
t.Skipf("B2_ACCOUNT_ID or B2_SECRET_KEY unset; skipping integration tests")
return nil, nil
}
client, err := NewClient(ctx, id, key, FailSomeUploads(), ExpireSomeAuthTokens(), Transport(transport))
ccport := &ccTripper{rt: transport, t: t}
tport := eofTripper{rt: ccport, t: t}
client, err := NewClient(ctx, id, key, FailSomeUploads(), ExpireSomeAuthTokens(), Transport(tport), UserAgent("b2-test"), UserAgent("integration-test"))
if err != nil {
t.Fatal(err)
return nil, nil
@@ -783,6 +932,7 @@ func startLiveTest(ctx context.Context, t *testing.T) (*Bucket, func()) {
return nil, nil
}
f := func() {
defer ccport.done()
for c := range listObjects(ctx, bucket.ListObjects) {
if c.err != nil {
continue
+5 -8
View File
@@ -16,13 +16,12 @@ package b2
import (
"bytes"
"context"
"errors"
"io"
"sync"
"github.com/kurin/blazer/internal/blog"
"golang.org/x/net/context"
)
var errNoMoreContent = errors.New("416: out of content")
@@ -140,11 +139,12 @@ func (r *Reader) thread() {
return
}
rsize, _, _, _ := fr.stats()
mr := &meteredReader{r: &fakeSeeker{fr}, size: int(rsize)}
mr := &meteredReader{r: noopResetter{fr}, size: int(rsize)}
r.smux.Lock()
r.smap[chunkID] = mr
r.smux.Unlock()
i, err := copyContext(r.ctx, buf, mr)
fr.Close()
r.smux.Lock()
r.smap[chunkID] = nil
r.smux.Unlock()
@@ -290,11 +290,8 @@ func copyContext(ctx context.Context, dst io.Writer, src io.Reader) (written int
return written, err
}
// fakeSeeker exists so that we can wrap the http response body (an io.Reader
// but not an io.Seeker) into a meteredReader, which will allow us to keep tabs
// on how much of the chunk we've read so far.
type fakeSeeker struct {
type noopResetter struct {
io.Reader
}
func (fs *fakeSeeker) Seek(int64, int) (int64, error) { return 0, nil }
func (noopResetter) Reset() error { return nil }
+48
View File
@@ -0,0 +1,48 @@
// Copyright 2017, Google
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package b2
import (
"io"
"sync"
)
type readerAt struct {
rs io.ReadSeeker
mu sync.Mutex
}
func (r *readerAt) ReadAt(p []byte, off int64) (int, error) {
r.mu.Lock()
defer r.mu.Unlock()
// ReadAt is supposed to preserve the offset.
cur, err := r.rs.Seek(0, io.SeekCurrent)
if err != nil {
return 0, err
}
defer func() { r.rs.Seek(cur, io.SeekStart) }()
if _, err := r.rs.Seek(off, io.SeekStart); err != nil {
return 0, err
}
return io.ReadFull(r.rs, p)
}
// wraps a ReadSeeker in a mutex to provite a ReaderAt how is this not in the
// io package?
func enReaderAt(rs io.ReadSeeker) io.ReaderAt {
return &readerAt{rs: rs}
}
+92 -21
View File
@@ -15,6 +15,7 @@
package b2
import (
"context"
"errors"
"fmt"
"io"
@@ -23,8 +24,6 @@ import (
"time"
"github.com/kurin/blazer/internal/blog"
"golang.org/x/net/context"
)
// Writer writes data into Backblaze. It automatically switches to the large
@@ -75,6 +74,7 @@ type Writer struct {
file beLargeFileInterface
seen map[int]string
everStarted bool
newBuffer func() (writeBuffer, error)
o *Object
name string
@@ -94,15 +94,8 @@ type chunk struct {
buf writeBuffer
}
func (w *Writer) getBuffer() (writeBuffer, error) {
if !w.UseFileBuffer {
return newMemoryBuffer(), nil
}
return newFileBuffer(w.FileBufferDir)
}
func (w *Writer) setErr(err error) {
if err == nil {
if err == nil || err == io.EOF {
return
}
w.emux.Lock()
@@ -200,8 +193,7 @@ func (w *Writer) thread() {
}()
}
// Write satisfies the io.Writer interface.
func (w *Writer) Write(p []byte) (int, error) {
func (w *Writer) init() {
w.start.Do(func() {
w.everStarted = true
w.smux.Lock()
@@ -212,13 +204,24 @@ func (w *Writer) Write(p []byte) (int, error) {
if w.csize == 0 {
w.csize = 1e8
}
v, err := w.getBuffer()
if w.newBuffer == nil {
w.newBuffer = func() (writeBuffer, error) { return newMemoryBuffer(), nil }
if w.UseFileBuffer {
w.newBuffer = func() (writeBuffer, error) { return newFileBuffer(w.FileBufferDir) }
}
}
v, err := w.newBuffer()
if err != nil {
w.setErr(err)
return
}
w.w = v
})
}
// Write satisfies the io.Writer interface.
func (w *Writer) Write(p []byte) (int, error) {
w.init()
if err := w.getErr(); err != nil {
return 0, err
}
@@ -263,7 +266,7 @@ redo:
f, err := ue.uploadFile(w.ctx, mr, int(w.w.Len()), w.name, ctype, sha1, w.info)
if err != nil {
if w.o.b.r.reupload(err) {
blog.V(1).Infof("b2 writer: %v; retrying", err)
blog.V(2).Infof("b2 writer: %v; retrying", err)
u, err := w.o.b.b.getUploadURL(w.ctx)
if err != nil {
return err
@@ -352,7 +355,7 @@ func (w *Writer) sendChunk() error {
return w.ctx.Err()
}
w.cidx++
v, err := w.getBuffer()
v, err := w.newBuffer()
if err != nil {
return err
}
@@ -360,15 +363,83 @@ func (w *Writer) sendChunk() error {
return nil
}
// ReadFrom reads all of r into w, returning the first error or no error if r
// returns io.EOF. If r is also an io.Seeker, ReadFrom will stream r directly
// over the wire instead of buffering it locally. This reduces memory usage.
//
// Do not issue multiple calls to ReadFrom, or mix ReadFrom and Write. If you
// have multiple readers you want to concatenate into the same B2 object, use
// an io.MultiReader.
//
// Note that io.Copy will automatically choose to use ReadFrom.
//
// ReadFrom currently doesn't handle w.Resume; if w.Resume is true, ReadFrom
// will act as if r is not an io.Seeker.
func (w *Writer) ReadFrom(r io.Reader) (int64, error) {
rs, ok := r.(io.ReadSeeker)
if !ok || w.Resume {
return copyContext(w.ctx, w, r)
}
blog.V(2).Info("streaming without buffer")
size, err := rs.Seek(0, io.SeekEnd)
if err != nil {
return 0, err
}
var ra io.ReaderAt
if rat, ok := r.(io.ReaderAt); ok {
ra = rat
} else {
ra = enReaderAt(rs)
}
var offset int64
var wrote int64
w.newBuffer = func() (writeBuffer, error) {
left := size - offset
if left <= 0 {
// We're done sending real chunks; send empty chunks from now on so that
// Close() works.
w.newBuffer = func() (writeBuffer, error) { return newMemoryBuffer(), nil }
w.w = newMemoryBuffer()
return nil, io.EOF
}
csize := int64(w.csize)
if left < csize {
csize = left
}
nb := newNonBuffer(ra, offset, csize)
wrote += csize // TODO: this is kind of a total lie
offset += csize
return nb, nil
}
w.init()
if size < int64(w.csize) {
// the magic happens on w.Close()
return size, nil
}
for {
if err := w.sendChunk(); err != nil {
if err != io.EOF {
return wrote, err
}
return wrote, nil
}
}
}
// Close satisfies the io.Closer interface. It is critical to check the return
// value of Close on all writers.
// value of Close for all writers.
func (w *Writer) Close() error {
w.done.Do(func() {
if !w.everStarted {
return
}
defer w.o.b.c.removeWriter(w)
defer w.w.Close() // TODO: log error
defer func() {
if err := w.w.Close(); err != nil {
// this is non-fatal, but alarming
blog.V(1).Infof("close %s: %v", w.name, err)
}
}()
if w.cidx == 0 {
w.setErr(w.simpleWriteFile())
return
@@ -423,7 +494,7 @@ func (w *Writer) status() *WriterStatus {
type meteredReader struct {
read int64
size int
r io.ReadSeeker
r readResetter
mux sync.Mutex
}
@@ -435,11 +506,11 @@ func (mr *meteredReader) Read(p []byte) (int, error) {
return n, err
}
func (mr *meteredReader) Seek(offset int64, whence int) (int64, error) {
func (mr *meteredReader) Reset() error {
mr.mux.Lock()
defer mr.mux.Unlock()
mr.read = offset
return mr.r.Seek(offset, whence)
mr.read = 0
return mr.r.Reset()
}
func (mr *meteredReader) done() float64 {