amorfo77/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2026-08-03 20:52:16 +00:00
Code Issues Projects Releases Wiki Activity

move Backend interface to backend package

Browse Source
This commit is contained in:
Michael Eischer
2023-10-25 23:00:18 +02:00
parent ceb0774af1
commit 1b8a67fe76
105 changed files with 822 additions and 775 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/backend/retry/backend_retry.go
+12 -12
View File
@@ -7,27 +7,27 @@ import (
"time"
"github.com/cenkalti/backoff/v4"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
)
// Backend retries operations on the backend in case of an error with a
// backoff.
type Backend struct {
restic.Backend
backend.Backend
MaxTries int
Report func(string, error, time.Duration)
Success func(string, int)
}
// statically ensure that RetryBackend implements restic.Backend.
var _ restic.Backend = &Backend{}
// statically ensure that RetryBackend implements backend.Backend.
var _ backend.Backend = &Backend{}
// New wraps be with a backend that retries operations after a
// backoff. report is called with a description and the error, if one occurred.
// success is called with the number of retries before a successful operation
// (it is not called if it succeeded on the first try)
func New(be restic.Backend, maxTries int, report func(string, error, time.Duration), success func(string, int)) *Backend {
func New(be backend.Backend, maxTries int, report func(string, error, time.Duration), success func(string, int)) *Backend {
return &Backend{
Backend: be,
MaxTries: maxTries,
@@ -92,7 +92,7 @@ func (be *Backend) retry(ctx context.Context, msg string, f func() error) error
}
// Save stores the data in the backend under the given handle.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
return be.retry(ctx, fmt.Sprintf("Save(%v)", h), func() error {
err := rd.Rewind()
if err != nil {
@@ -125,7 +125,7 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
// given offset. If length is larger than zero, only a portion of the file
// is returned. rd must be closed after use. If an error is returned, the
// ReadCloser must be nil.
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) (err error) {
func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (err error) {
return be.retry(ctx, fmt.Sprintf("Load(%v, %v, %v)", h, length, offset),
func() error {
err := be.Backend.Load(ctx, h, length, offset, consumer)
@@ -137,7 +137,7 @@ func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset
}
// Stat returns information about the File identified by h.
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (fi restic.FileInfo, err error) {
func (be *Backend) Stat(ctx context.Context, h backend.Handle) (fi backend.FileInfo, err error) {
err = be.retry(ctx, fmt.Sprintf("Stat(%v)", h),
func() error {
var innerError error
@@ -153,7 +153,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (fi restic.FileInf
}
// Remove removes a File with type t and name.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) (err error) {
func (be *Backend) Remove(ctx context.Context, h backend.Handle) (err error) {
return be.retry(ctx, fmt.Sprintf("Remove(%v)", h), func() error {
return be.Backend.Remove(ctx, h)
})
@@ -163,7 +163,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) (err error) {
// error is returned by the underlying backend, the request is retried. When fn
// returns an error, the operation is aborted and the error is returned to the
// caller.
func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
// create a new context that we can cancel when fn returns an error, so
// that listing is aborted
listCtx, cancel := context.WithCancel(ctx)
@@ -173,7 +173,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
var innerErr error // remember when fn returned an error, so we can return that to the caller
err := be.retry(listCtx, fmt.Sprintf("List(%v)", t), func() error {
return be.Backend.List(ctx, t, func(fi restic.FileInfo) error {
return be.Backend.List(ctx, t, func(fi backend.FileInfo) error {
if _, ok := listed[fi.Name]; ok {
return nil
}
@@ -196,6 +196,6 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
return err
}
func (be *Backend) Unwrap() restic.Backend {
func (be *Backend) Unwrap() backend.Backend {
return be.Backend
}
internal/backend/retry/backend_retry_test.go
+29 -28
View File
@@ -8,6 +8,7 @@ import (
"time"
"github.com/cenkalti/backoff/v4"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/mock"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
@@ -18,7 +19,7 @@ func TestBackendSaveRetry(t *testing.T) {
buf := bytes.NewBuffer(nil)
errcount := 0
be := &mock.Backend{
SaveFn: func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
SaveFn: func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if errcount == 0 {
errcount++
_, err := io.CopyN(io.Discard, rd, 120)
@@ -38,7 +39,7 @@ func TestBackendSaveRetry(t *testing.T) {
retryBackend := New(be, 10, nil, nil)
data := test.Random(23, 5*1024*1024+11241)
err := retryBackend.Save(context.TODO(), restic.Handle{}, restic.NewByteReader(data, be.Hasher()))
err := retryBackend.Save(context.TODO(), backend.Handle{}, backend.NewByteReader(data, be.Hasher()))
if err != nil {
t.Fatal(err)
}
@@ -56,14 +57,14 @@ func TestBackendSaveRetryAtomic(t *testing.T) {
errcount := 0
calledRemove := false
be := &mock.Backend{
SaveFn: func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
SaveFn: func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if errcount == 0 {
errcount++
return errors.New("injected error")
}
return nil
},
RemoveFn: func(ctx context.Context, h restic.Handle) error {
RemoveFn: func(ctx context.Context, h backend.Handle) error {
calledRemove = true
return nil
},
@@ -74,7 +75,7 @@ func TestBackendSaveRetryAtomic(t *testing.T) {
retryBackend := New(be, 10, nil, nil)
data := test.Random(23, 5*1024*1024+11241)
err := retryBackend.Save(context.TODO(), restic.Handle{}, restic.NewByteReader(data, be.Hasher()))
err := retryBackend.Save(context.TODO(), backend.Handle{}, backend.NewByteReader(data, be.Hasher()))
if err != nil {
t.Fatal(err)
}
@@ -91,15 +92,15 @@ func TestBackendListRetry(t *testing.T) {
retry := 0
be := &mock.Backend{
ListFn: func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
ListFn: func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
// fail during first retry, succeed during second
retry++
if retry == 1 {
_ = fn(restic.FileInfo{Name: ID1})
_ = fn(backend.FileInfo{Name: ID1})
return errors.New("test list error")
}
_ = fn(restic.FileInfo{Name: ID1})
_ = fn(restic.FileInfo{Name: ID2})
_ = fn(backend.FileInfo{Name: ID1})
_ = fn(backend.FileInfo{Name: ID2})
return nil
},
}
@@ -108,7 +109,7 @@ func TestBackendListRetry(t *testing.T) {
retryBackend := New(be, 10, nil, nil)
var listed []string
err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
err := retryBackend.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
listed = append(listed, fi.Name)
return nil
})
@@ -121,10 +122,10 @@ func TestBackendListRetryErrorFn(t *testing.T) {
var names = []string{"id1", "id2", "foo", "bar"}
be := &mock.Backend{
ListFn: func(ctx context.Context, tpe restic.FileType, fn func(restic.FileInfo) error) error {
ListFn: func(ctx context.Context, tpe backend.FileType, fn func(backend.FileInfo) error) error {
t.Logf("List called for %v", tpe)
for _, name := range names {
err := fn(restic.FileInfo{Name: name})
err := fn(backend.FileInfo{Name: name})
if err != nil {
return err
}
@@ -141,7 +142,7 @@ func TestBackendListRetryErrorFn(t *testing.T) {
var listed []string
run := 0
err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
err := retryBackend.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
t.Logf("fn called for %v", fi.Name)
run++
// return an error for the third item in the list
@@ -172,7 +173,7 @@ func TestBackendListRetryErrorBackend(t *testing.T) {
retries := 0
be := &mock.Backend{
ListFn: func(ctx context.Context, tpe restic.FileType, fn func(restic.FileInfo) error) error {
ListFn: func(ctx context.Context, tpe backend.FileType, fn func(backend.FileInfo) error) error {
t.Logf("List called for %v, retries %v", tpe, retries)
retries++
for i, name := range names {
@@ -180,7 +181,7 @@ func TestBackendListRetryErrorBackend(t *testing.T) {
return ErrBackendTest
}
err := fn(restic.FileInfo{Name: name})
err := fn(backend.FileInfo{Name: name})
if err != nil {
return err
}
@@ -195,7 +196,7 @@ func TestBackendListRetryErrorBackend(t *testing.T) {
retryBackend := New(be, maxRetries, nil, nil)
var listed []string
err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
err := retryBackend.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
t.Logf("fn called for %v", fi.Name)
listed = append(listed, fi.Name)
return nil
@@ -252,7 +253,7 @@ func TestBackendLoadRetry(t *testing.T) {
attempt := 0
be := mock.NewBackend()
be.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
// returns failing reader on first invocation, good reader on subsequent invocations
attempt++
if attempt > 1 {
@@ -265,7 +266,7 @@ func TestBackendLoadRetry(t *testing.T) {
retryBackend := New(be, 10, nil, nil)
var buf []byte
err := retryBackend.Load(context.TODO(), restic.Handle{}, 0, 0, func(rd io.Reader) (err error) {
err := retryBackend.Load(context.TODO(), backend.Handle{}, 0, 0, func(rd io.Reader) (err error) {
buf, err = io.ReadAll(rd)
return err
})
@@ -280,7 +281,7 @@ func TestBackendLoadNotExists(t *testing.T) {
attempt := 0
be := mock.NewBackend()
be.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
attempt++
if attempt > 1 {
t.Fail()
@@ -295,7 +296,7 @@ func TestBackendLoadNotExists(t *testing.T) {
TestFastRetries(t)
retryBackend := New(be, 10, nil, nil)
err := retryBackend.Load(context.TODO(), restic.Handle{}, 0, 0, func(rd io.Reader) (err error) {
err := retryBackend.Load(context.TODO(), backend.Handle{}, 0, 0, func(rd io.Reader) (err error) {
return nil
})
test.Assert(t, be.IsNotExistFn(err), "unexpected error %v", err)
@@ -308,13 +309,13 @@ func TestBackendStatNotExists(t *testing.T) {
attempt := 0
be := mock.NewBackend()
be.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
be.StatFn = func(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
attempt++
if attempt > 1 {
t.Fail()
return restic.FileInfo{}, errors.New("must not retry")
return backend.FileInfo{}, errors.New("must not retry")
}
return restic.FileInfo{}, notFound
return backend.FileInfo{}, notFound
}
be.IsNotExistFn = func(err error) bool {
return errors.Is(err, notFound)
@@ -323,7 +324,7 @@ func TestBackendStatNotExists(t *testing.T) {
TestFastRetries(t)
retryBackend := New(be, 10, nil, nil)
_, err := retryBackend.Stat(context.TODO(), restic.Handle{})
_, err := retryBackend.Stat(context.TODO(), backend.Handle{})
test.Assert(t, be.IsNotExistFn(err), "unexpected error %v", err)
test.Equals(t, 1, attempt)
}
@@ -337,7 +338,7 @@ func TestBackendCanceledContext(t *testing.T) {
// check that we received the expected context canceled error instead
TestFastRetries(t)
retryBackend := New(mock.NewBackend(), 2, nil, nil)
h := restic.Handle{Type: restic.PackFile, Name: restic.NewRandomID().String()}
h := backend.Handle{Type: backend.PackFile, Name: restic.NewRandomID().String()}
// create an already canceled context
ctx, cancel := context.WithCancel(context.Background())
@@ -346,15 +347,15 @@ func TestBackendCanceledContext(t *testing.T) {
_, err := retryBackend.Stat(ctx, h)
assertIsCanceled(t, err)
err = retryBackend.Save(ctx, h, restic.NewByteReader([]byte{}, nil))
err = retryBackend.Save(ctx, h, backend.NewByteReader([]byte{}, nil))
assertIsCanceled(t, err)
err = retryBackend.Remove(ctx, h)
assertIsCanceled(t, err)
err = retryBackend.Load(ctx, restic.Handle{}, 0, 0, func(rd io.Reader) (err error) {
err = retryBackend.Load(ctx, backend.Handle{}, 0, 0, func(rd io.Reader) (err error) {
return nil
})
assertIsCanceled(t, err)
err = retryBackend.List(ctx, restic.PackFile, func(restic.FileInfo) error {
err = retryBackend.List(ctx, backend.PackFile, func(backend.FileInfo) error {
return nil
})
assertIsCanceled(t, err)