amorfo77/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2026-05-28 12:55:23 +00:00
Code Issues Projects Releases Wiki Activity

move Backend interface to backend package

Browse Source
This commit is contained in:
Michael Eischer
2023-10-01 11:40:12 +02:00
parent ceb0774af1
commit 1b8a67fe76
105 changed files with 822 additions and 775 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/backend/azure/azure.go
+13 -13
View File
@@ -12,12 +12,12 @@ import (
"path"
"strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming"
@@ -43,7 +43,7 @@ const saveLargeSize = 256 * 1024 * 1024
const defaultListMaxItems = 5000
// make sure that *Backend implements backend.Backend
var _ restic.Backend = &Backend{}
var _ backend.Backend = &Backend{}
func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("azure", ParseConfig, location.NoPassword, Create, Open)
@@ -197,7 +197,7 @@ func (be *Backend) Path() string {
}
// Save stores data in the backend at the handle.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
objName := be.Filename(h)
debug.Log("InsertObject(%v, %v)", be.cfg.AccountName, objName)
@@ -214,7 +214,7 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
return err
}
func (be *Backend) saveSmall(ctx context.Context, objName string, rd restic.RewindReader) error {
func (be *Backend) saveSmall(ctx context.Context, objName string, rd backend.RewindReader) error {
blockBlobClient := be.container.NewBlockBlobClient(objName)
// upload it as a new "block", use the base64 hash for the ID
@@ -239,7 +239,7 @@ func (be *Backend) saveSmall(ctx context.Context, objName string, rd restic.Rewi
return errors.Wrap(err, "CommitBlockList")
}
func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.RewindReader) error {
func (be *Backend) saveLarge(ctx context.Context, objName string, rd backend.RewindReader) error {
blockBlobClient := be.container.NewBlockBlobClient(objName)
buf := make([]byte, 100*1024*1024)
@@ -294,11 +294,11 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
objName := be.Filename(h)
blockBlobClient := be.container.NewBlobClient(objName)
@@ -317,17 +317,17 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int,
}
// Stat returns information about a blob.
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
func (be *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
objName := be.Filename(h)
blobClient := be.container.NewBlobClient(objName)
props, err := blobClient.GetProperties(ctx, nil)
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "blob.GetProperties")
return backend.FileInfo{}, errors.Wrap(err, "blob.GetProperties")
}
fi := restic.FileInfo{
fi := backend.FileInfo{
Size: *props.ContentLength,
Name: h.Name,
}
@@ -335,7 +335,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo,
}
// Remove removes the blob with the given name and type.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
objName := be.Filename(h)
blob := be.container.NewBlobClient(objName)
@@ -350,7 +350,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
prefix, _ := be.Basedir(t)
// make sure prefix ends with a slash
@@ -381,7 +381,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
continue
}
fi := restic.FileInfo{
fi := backend.FileInfo{
Name: path.Base(m),
Size: *item.Properties.ContentLength,
}
internal/backend/azure/azure_test.go
+2 -2
View File
@@ -122,11 +122,11 @@ func TestUploadLargeFile(t *testing.T) {
data := rtest.Random(23, 300*1024*1024)
id := restic.Hash(data)
h := restic.Handle{Name: id.String(), Type: restic.PackFile}
h := backend.Handle{Name: id.String(), Type: backend.PackFile}
t.Logf("hash of %d bytes: %v", len(data), id)
err = be.Save(ctx, h, restic.NewByteReader(data, be.Hasher()))
err = be.Save(ctx, h, backend.NewByteReader(data, be.Hasher()))
if err != nil {
t.Fatal(err)
}
internal/backend/azure/config.go
+2 -2
View File
@@ -5,9 +5,9 @@ import (
"path"
"strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
)
// Config contains all configuration necessary to connect to an azure compatible
@@ -57,7 +57,7 @@ func ParseConfig(s string) (*Config, error) {
return &cfg, nil
}
var _ restic.ApplyEnvironmenter = &Config{}
var _ backend.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) {
internal/backend/b2/b2.go
+14 -14
View File
@@ -9,12 +9,12 @@ import (
"sync"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/Backblaze/blazer/b2"
"github.com/Backblaze/blazer/base"
@@ -34,8 +34,8 @@ type b2Backend struct {
// Billing happens in 1000 item granlarity, but we are more interested in reducing the number of network round trips
const defaultListMaxItems = 10 * 1000
// ensure statically that *b2Backend implements restic.Backend.
var _ restic.Backend = &b2Backend{}
// ensure statically that *b2Backend implements backend.Backend.
var _ backend.Backend = &b2Backend{}
func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("b2", ParseConfig, location.NoPassword, Create, Open)
@@ -85,7 +85,7 @@ func newClient(ctx context.Context, cfg Config, rt http.RoundTripper) (*b2.Clien
}
// Open opens a connection to the B2 service.
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
debug.Log("cfg %#v", cfg)
ctx, cancel := context.WithCancel(ctx)
@@ -118,7 +118,7 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend
// Create opens a connection to the B2 service. If the bucket does not exist yet,
// it is created.
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
debug.Log("cfg %#v", cfg)
ctx, cancel := context.WithCancel(ctx)
@@ -188,14 +188,14 @@ func (be *b2Backend) IsNotExist(err error) bool {
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *b2Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *b2Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *b2Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (be *b2Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
name := be.Layout.Filename(h)
obj := be.bucket.Object(name)
@@ -213,7 +213,7 @@ func (be *b2Backend) openReader(ctx context.Context, h restic.Handle, length int
}
// Save stores data in the backend at the handle.
func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *b2Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
@@ -237,18 +237,18 @@ func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.Rewind
}
// Stat returns information about a blob.
func (be *b2Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
func (be *b2Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
name := be.Filename(h)
obj := be.bucket.Object(name)
info, err := obj.Attrs(ctx)
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "Stat")
return backend.FileInfo{}, errors.Wrap(err, "Stat")
}
return restic.FileInfo{Size: info.Size, Name: h.Name}, nil
return backend.FileInfo{Size: info.Size, Name: h.Name}, nil
}
// Remove removes the blob with the given name and type.
func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error {
func (be *b2Backend) Remove(ctx context.Context, h backend.Handle) error {
// the retry backend will also repeat the remove method up to 10 times
for i := 0; i < 3; i++ {
obj := be.bucket.Object(be.Filename(h))
@@ -284,7 +284,7 @@ func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error {
}
// List returns a channel that yields all names of blobs of type t.
func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *b2Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
@@ -299,7 +299,7 @@ func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic
return err
}
fi := restic.FileInfo{
fi := backend.FileInfo{
Name: path.Base(obj.Name()),
Size: attrs.Size,
}
internal/backend/b2/config.go
+2 -2
View File
@@ -6,9 +6,9 @@ import (
"regexp"
"strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
)
// Config contains all configuration necessary to connect to an b2 compatible
@@ -82,7 +82,7 @@ func ParseConfig(s string) (*Config, error) {
return &cfg, nil
}
var _ restic.ApplyEnvironmenter = &Config{}
var _ backend.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) {
internal/backend/backend.go
+117
View File
@@ -0,0 +1,117 @@
package backend
import (
"context"
"hash"
"io"
)
// Backend is used to store and access data.
//
// Backend operations that return an error will be retried when a Backend is
// wrapped in a RetryBackend. To prevent that from happening, the operations
// should return a github.com/cenkalti/backoff/v4.PermanentError. Errors from
// the context package need not be wrapped, as context cancellation is checked
// separately by the retrying logic.
type Backend interface {
// Location returns a string that describes the type and location of the
// repository.
Location() string
// Connections returns the maxmimum number of concurrent backend operations.
Connections() uint
// Hasher may return a hash function for calculating a content hash for the backend
Hasher() hash.Hash
// HasAtomicReplace returns whether Save() can atomically replace files
HasAtomicReplace() bool
// Remove removes a File described by h.
Remove(ctx context.Context, h Handle) error
// Close the backend
Close() error
// Save stores the data from rd under the given handle.
Save(ctx context.Context, h Handle, rd RewindReader) error
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset. If length is larger than zero, only a portion of the file
// is read.
//
// The function fn may be called multiple times during the same Load invocation
// and therefore must be idempotent.
//
// Implementations are encouraged to use util.DefaultLoad
Load(ctx context.Context, h Handle, length int, offset int64, fn func(rd io.Reader) error) error
// Stat returns information about the File identified by h.
Stat(ctx context.Context, h Handle) (FileInfo, error)
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
//
// The function fn is called exactly once for each file during successful
// execution and at most once in case of an error.
//
// The function fn is called in the same Goroutine that List() is called
// from.
List(ctx context.Context, t FileType, fn func(FileInfo) error) error
// IsNotExist returns true if the error was caused by a non-existing file
// in the backend.
//
// The argument may be a wrapped error. The implementation is responsible
// for unwrapping it.
IsNotExist(err error) bool
// Delete removes all data in the backend.
Delete(ctx context.Context) error
}
type Unwrapper interface {
// Unwrap returns the underlying backend or nil if there is none.
Unwrap() Backend
}
func AsBackend[B Backend](b Backend) B {
for b != nil {
if be, ok := b.(B); ok {
return be
}
if be, ok := b.(Unwrapper); ok {
b = be.Unwrap()
} else {
// not the backend we're looking for
break
}
}
var be B
return be
}
type FreezeBackend interface {
Backend
// Freeze blocks all backend operations except those on lock files
Freeze()
// Unfreeze allows all backend operations to continue
Unfreeze()
}
// FileInfo is contains information about a file in the backend.
type FileInfo struct {
Size int64
Name string
}
// ApplyEnvironmenter fills in a backend configuration from the environment
type ApplyEnvironmenter interface {
ApplyEnvironment(prefix string)
}
// Lister allows listing files in a backend.
type Lister interface {
List(context.Context, FileType, func(FileInfo) error) error
}
internal/backend/backend_test.go
+38
View File
@@ -0,0 +1,38 @@
package backend_test
import (
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/test"
)
type testBackend struct {
backend.Backend
}
func (t *testBackend) Unwrap() backend.Backend {
return nil
}
type otherTestBackend struct {
backend.Backend
}
func (t *otherTestBackend) Unwrap() backend.Backend {
return t.Backend
}
func TestAsBackend(t *testing.T) {
other := otherTestBackend{}
test.Assert(t, backend.AsBackend[*testBackend](other) == nil, "otherTestBackend is not a testBackend backend")
testBe := &testBackend{}
test.Assert(t, backend.AsBackend[*testBackend](testBe) == testBe, "testBackend was not returned")
wrapper := &otherTestBackend{Backend: testBe}
test.Assert(t, backend.AsBackend[*testBackend](wrapper) == testBe, "failed to unwrap testBackend backend")
wrapper.Backend = other
test.Assert(t, backend.AsBackend[*testBackend](wrapper) == nil, "a wrapped otherTestBackend is not a testBackend")
}
internal/backend/dryrun/dry_backend.go
+10 -10
View File
@@ -5,8 +5,8 @@ import (
"hash"
"io"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
)
// Backend passes reads through to an underlying layer and accepts writes, but
@@ -15,20 +15,20 @@ import (
// the repo and does normal operations else.
// This is used for `backup --dry-run`.
type Backend struct {
b restic.Backend
b backend.Backend
}
// statically ensure that Backend implements restic.Backend.
var _ restic.Backend = &Backend{}
// statically ensure that Backend implements backend.Backend.
var _ backend.Backend = &Backend{}
func New(be restic.Backend) *Backend {
func New(be backend.Backend) *Backend {
b := &Backend{b: be}
debug.Log("created new dry backend")
return b
}
// Save adds new Data to the backend.
func (be *Backend) Save(_ context.Context, h restic.Handle, _ restic.RewindReader) error {
func (be *Backend) Save(_ context.Context, h backend.Handle, _ backend.RewindReader) error {
if err := h.Valid(); err != nil {
return err
}
@@ -38,7 +38,7 @@ func (be *Backend) Save(_ context.Context, h restic.Handle, _ restic.RewindReade
}
// Remove deletes a file from the backend.
func (be *Backend) Remove(_ context.Context, _ restic.Handle) error {
func (be *Backend) Remove(_ context.Context, _ backend.Handle) error {
return nil
}
@@ -72,14 +72,14 @@ func (be *Backend) IsNotExist(err error) bool {
return be.b.IsNotExist(err)
}
func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
return be.b.List(ctx, t, fn)
}
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(io.Reader) error) error {
func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(io.Reader) error) error {
return be.b.Load(ctx, h, length, offset, fn)
}
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
func (be *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
return be.b.Stat(ctx, h)
}
internal/backend/dryrun/dry_backend_test.go
+8 -8
View File
@@ -8,16 +8,16 @@ import (
"strings"
"testing"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/dryrun"
"github.com/restic/restic/internal/backend/mem"
)
// make sure that Backend implements backend.Backend
var _ restic.Backend = &dryrun.Backend{}
var _ backend.Backend = &dryrun.Backend{}
func newBackends() (*dryrun.Backend, restic.Backend) {
func newBackends() (*dryrun.Backend, backend.Backend) {
m := mem.New()
return dryrun.New(m), m
}
@@ -30,7 +30,7 @@ func TestDry(t *testing.T) {
// won't pass. Instead, perform a series of operations over the backend, testing the state
// at each step.
steps := []struct {
be restic.Backend
be backend.Backend
op string
fname string
content string
@@ -61,13 +61,13 @@ func TestDry(t *testing.T) {
for i, step := range steps {
var err error
handle := restic.Handle{Type: restic.PackFile, Name: step.fname}
handle := backend.Handle{Type: backend.PackFile, Name: step.fname}
switch step.op {
case "save":
err = step.be.Save(ctx, handle, restic.NewByteReader([]byte(step.content), step.be.Hasher()))
err = step.be.Save(ctx, handle, backend.NewByteReader([]byte(step.content), step.be.Hasher()))
case "list":
fileList := []string{}
err = step.be.List(ctx, restic.PackFile, func(fi restic.FileInfo) error {
err = step.be.List(ctx, backend.PackFile, func(fi backend.FileInfo) error {
fileList = append(fileList, fi.Name)
return nil
})
@@ -86,7 +86,7 @@ func TestDry(t *testing.T) {
case "remove":
err = step.be.Remove(ctx, handle)
case "stat":
var fi restic.FileInfo
var fi backend.FileInfo
fi, err = step.be.Stat(ctx, handle)
if err == nil {
fis := fmt.Sprintf("%s %d", fi.Name, fi.Size)
internal/backend/file.go
+79
View File
@@ -0,0 +1,79 @@
package backend
import (
"fmt"
"github.com/restic/restic/internal/errors"
)
// FileType is the type of a file in the backend.
type FileType uint8
// These are the different data types a backend can store.
const (
PackFile FileType = 1 + iota
KeyFile
LockFile
SnapshotFile
IndexFile
ConfigFile
)
func (t FileType) String() string {
s := "invalid"
switch t {
case PackFile:
// Spelled "data" instead of "pack" for historical reasons.
s = "data"
case KeyFile:
s = "key"
case LockFile:
s = "lock"
case SnapshotFile:
s = "snapshot"
case IndexFile:
s = "index"
case ConfigFile:
s = "config"
}
return s
}
// Handle is used to store and access data in a backend.
type Handle struct {
Type FileType
IsMetadata bool
Name string
}
func (h Handle) String() string {
name := h.Name
if len(name) > 10 {
name = name[:10]
}
return fmt.Sprintf("<%s/%s>", h.Type, name)
}
// Valid returns an error if h is not valid.
func (h Handle) Valid() error {
switch h.Type {
case PackFile:
case KeyFile:
case LockFile:
case SnapshotFile:
case IndexFile:
case ConfigFile:
default:
return errors.Errorf("invalid Type %d", h.Type)
}
if h.Type == ConfigFile {
return nil
}
if h.Name == "" {
return errors.New("invalid Name")
}
return nil
}
internal/backend/file_test.go
+36
View File
@@ -0,0 +1,36 @@
package backend
import (
"testing"
rtest "github.com/restic/restic/internal/test"
)
func TestHandleString(t *testing.T) {
rtest.Equals(t, "<data/foobar>", Handle{Type: PackFile, Name: "foobar"}.String())
rtest.Equals(t, "<lock/1>", Handle{Type: LockFile, Name: "1"}.String())
}
func TestHandleValid(t *testing.T) {
var handleTests = []struct {
h Handle
valid bool
}{
{Handle{Name: "foo"}, false},
{Handle{Type: 0}, false},
{Handle{Type: ConfigFile, Name: ""}, true},
{Handle{Type: PackFile, Name: ""}, false},
{Handle{Type: LockFile, Name: "010203040506"}, true},
}
for i, test := range handleTests {
err := test.h.Valid()
if err != nil && test.valid {
t.Errorf("test %v failed: error returned for valid handle: %v", i, err)
}
if !test.valid && err == nil {
t.Errorf("test %v failed: expected error for invalid handle not found", i)
}
}
}
internal/backend/gs/config.go
+2 -2
View File
@@ -5,9 +5,9 @@ import (
"path"
"strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
)
// Config contains all configuration necessary to connect to a Google Cloud Storage
@@ -59,7 +59,7 @@ func ParseConfig(s string) (*Config, error) {
return &cfg, nil
}
var _ restic.ApplyEnvironmenter = &Config{}
var _ backend.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) {
internal/backend/gs/gs.go
+14 -14
View File
@@ -13,11 +13,11 @@ import (
"cloud.google.com/go/storage"
"github.com/pkg/errors"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
@@ -45,8 +45,8 @@ type Backend struct {
layout.Layout
}
// Ensure that *Backend implements restic.Backend.
var _ restic.Backend = &Backend{}
// Ensure that *Backend implements backend.Backend.
var _ backend.Backend = &Backend{}
func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("gs", ParseConfig, location.NoPassword, Create, Open)
@@ -122,7 +122,7 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
}
// Open opens the gs backend at the specified bucket.
func Open(_ context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
func Open(_ context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
return open(cfg, rt)
}
@@ -131,7 +131,7 @@ func Open(_ context.Context, cfg Config, rt http.RoundTripper) (restic.Backend,
//
// The service account must have the "storage.buckets.create" permission to
// create a bucket the does not yet exist.
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
be, err := open(cfg, rt)
if err != nil {
return nil, errors.Wrap(err, "open")
@@ -203,7 +203,7 @@ func (be *Backend) Path() string {
}
// Save stores data in the backend at the handle.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
objName := be.Filename(h)
// Set chunk size to zero to disable resumable uploads.
@@ -253,14 +253,14 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
if length == 0 {
// negative length indicates read till end to GCS lib
length = -1
@@ -277,20 +277,20 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int,
}
// Stat returns information about a blob.
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
objName := be.Filename(h)
attr, err := be.bucket.Object(objName).Attrs(ctx)
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "service.Objects.Get")
return backend.FileInfo{}, errors.Wrap(err, "service.Objects.Get")
}
return restic.FileInfo{Size: attr.Size, Name: h.Name}, nil
return backend.FileInfo{Size: attr.Size, Name: h.Name}, nil
}
// Remove removes the blob with the given name and type.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
objName := be.Filename(h)
err := be.bucket.Object(objName).Delete(ctx)
@@ -304,7 +304,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
prefix, _ := be.Basedir(t)
// make sure prefix ends with a slash
@@ -330,7 +330,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
continue
}
fi := restic.FileInfo{
fi := backend.FileInfo{
Name: path.Base(m),
Size: int64(attrs.Size),
}
internal/backend/layout/layout.go
+6 -5
View File
@@ -7,6 +7,7 @@ import (
"path/filepath"
"regexp"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs"
@@ -15,9 +16,9 @@ import (
// Layout computes paths for file name storage.
type Layout interface {
Filename(restic.Handle) string
Dirname(restic.Handle) string
Basedir(restic.FileType) (dir string, subdirs bool)
Filename(backend.Handle) string
Dirname(backend.Handle) string
Basedir(backend.FileType) (dir string, subdirs bool)
Paths() []string
Name() string
}
@@ -102,13 +103,13 @@ func DetectLayout(ctx context.Context, repo Filesystem, dir string) (Layout, err
}
// key file in the "keys" dir (DefaultLayout)
foundKeysFile, err := hasBackendFile(ctx, repo, repo.Join(dir, defaultLayoutPaths[restic.KeyFile]))
foundKeysFile, err := hasBackendFile(ctx, repo, repo.Join(dir, defaultLayoutPaths[backend.KeyFile]))
if err != nil {
return nil, err
}
// key file in the "key" dir (S3LegacyLayout)
foundKeyFile, err := hasBackendFile(ctx, repo, repo.Join(dir, s3LayoutPaths[restic.KeyFile]))
foundKeyFile, err := hasBackendFile(ctx, repo, repo.Join(dir, s3LayoutPaths[backend.KeyFile]))
if err != nil {
return nil, err
}
internal/backend/layout/layout_default.go
+14 -14
View File
@@ -3,7 +3,7 @@ package layout
import (
"encoding/hex"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/backend"
)
// DefaultLayout implements the default layout for local and sftp backends, as
@@ -15,12 +15,12 @@ type DefaultLayout struct {
Join func(...string) string
}
var defaultLayoutPaths = map[restic.FileType]string{
restic.PackFile: "data",
restic.SnapshotFile: "snapshots",
restic.IndexFile: "index",
restic.LockFile: "locks",
restic.KeyFile: "keys",
var defaultLayoutPaths = map[backend.FileType]string{
backend.PackFile: "data",
backend.SnapshotFile: "snapshots",
backend.IndexFile: "index",
backend.LockFile: "locks",
backend.KeyFile: "keys",
}
func (l *DefaultLayout) String() string {
@@ -33,10 +33,10 @@ func (l *DefaultLayout) Name() string {
}
// Dirname returns the directory path for a given file type and name.
func (l *DefaultLayout) Dirname(h restic.Handle) string {
func (l *DefaultLayout) Dirname(h backend.Handle) string {
p := defaultLayoutPaths[h.Type]
if h.Type == restic.PackFile && len(h.Name) > 2 {
if h.Type == backend.PackFile && len(h.Name) > 2 {
p = l.Join(p, h.Name[:2]) + "/"
}
@@ -44,9 +44,9 @@ func (l *DefaultLayout) Dirname(h restic.Handle) string {
}
// Filename returns a path to a file, including its name.
func (l *DefaultLayout) Filename(h restic.Handle) string {
func (l *DefaultLayout) Filename(h backend.Handle) string {
name := h.Name
if h.Type == restic.ConfigFile {
if h.Type == backend.ConfigFile {
return l.Join(l.Path, "config")
}
@@ -62,15 +62,15 @@ func (l *DefaultLayout) Paths() (dirs []string) {
// also add subdirs
for i := 0; i < 256; i++ {
subdir := hex.EncodeToString([]byte{byte(i)})
dirs = append(dirs, l.Join(l.Path, defaultLayoutPaths[restic.PackFile], subdir))
dirs = append(dirs, l.Join(l.Path, defaultLayoutPaths[backend.PackFile], subdir))
}
return dirs
}
// Basedir returns the base dir name for type t.
func (l *DefaultLayout) Basedir(t restic.FileType) (dirname string, subdirs bool) {
if t == restic.PackFile {
func (l *DefaultLayout) Basedir(t backend.FileType) (dirname string, subdirs bool) {
if t == backend.PackFile {
subdirs = true
}
internal/backend/layout/layout_rest.go
+8 -6
View File
@@ -1,6 +1,8 @@
package layout
import "github.com/restic/restic/internal/restic"
import (
"github.com/restic/restic/internal/backend"
)
// RESTLayout implements the default layout for the REST protocol.
type RESTLayout struct {
@@ -21,8 +23,8 @@ func (l *RESTLayout) Name() string {
}
// Dirname returns the directory path for a given file type and name.
func (l *RESTLayout) Dirname(h restic.Handle) string {
if h.Type == restic.ConfigFile {
func (l *RESTLayout) Dirname(h backend.Handle) string {
if h.Type == backend.ConfigFile {
return l.URL + l.Join(l.Path, "/")
}
@@ -30,10 +32,10 @@ func (l *RESTLayout) Dirname(h restic.Handle) string {
}
// Filename returns a path to a file, including its name.
func (l *RESTLayout) Filename(h restic.Handle) string {
func (l *RESTLayout) Filename(h backend.Handle) string {
name := h.Name
if h.Type == restic.ConfigFile {
if h.Type == backend.ConfigFile {
name = "config"
}
@@ -49,6 +51,6 @@ func (l *RESTLayout) Paths() (dirs []string) {
}
// Basedir returns the base dir name for files of type t.
func (l *RESTLayout) Basedir(t restic.FileType) (dirname string, subdirs bool) {
func (l *RESTLayout) Basedir(t backend.FileType) (dirname string, subdirs bool) {
return l.URL + l.Join(l.Path, restLayoutPaths[t]), false
}
internal/backend/layout/layout_s3legacy.go
+14 -12
View File
@@ -1,6 +1,8 @@
package layout
import "github.com/restic/restic/internal/restic"
import (
"github.com/restic/restic/internal/backend"
)
// S3LegacyLayout implements the old layout used for s3 cloud storage backends, as
// described in the Design document.
@@ -10,12 +12,12 @@ type S3LegacyLayout struct {
Join func(...string) string
}
var s3LayoutPaths = map[restic.FileType]string{
restic.PackFile: "data",
restic.SnapshotFile: "snapshot",
restic.IndexFile: "index",
restic.LockFile: "lock",
restic.KeyFile: "key",
var s3LayoutPaths = map[backend.FileType]string{
backend.PackFile: "data",
backend.SnapshotFile: "snapshot",
backend.IndexFile: "index",
backend.LockFile: "lock",
backend.KeyFile: "key",
}
func (l *S3LegacyLayout) String() string {
@@ -44,8 +46,8 @@ func (l *S3LegacyLayout) join(url string, items ...string) string {
}
// Dirname returns the directory path for a given file type and name.
func (l *S3LegacyLayout) Dirname(h restic.Handle) string {
if h.Type == restic.ConfigFile {
func (l *S3LegacyLayout) Dirname(h backend.Handle) string {
if h.Type == backend.ConfigFile {
return l.URL + l.Join(l.Path, "/")
}
@@ -53,10 +55,10 @@ func (l *S3LegacyLayout) Dirname(h restic.Handle) string {
}
// Filename returns a path to a file, including its name.
func (l *S3LegacyLayout) Filename(h restic.Handle) string {
func (l *S3LegacyLayout) Filename(h backend.Handle) string {
name := h.Name
if h.Type == restic.ConfigFile {
if h.Type == backend.ConfigFile {
name = "config"
}
@@ -72,6 +74,6 @@ func (l *S3LegacyLayout) Paths() (dirs []string) {
}
// Basedir returns the base dir name for type t.
func (l *S3LegacyLayout) Basedir(t restic.FileType) (dirname string, subdirs bool) {
func (l *S3LegacyLayout) Basedir(t backend.FileType) (dirname string, subdirs bool) {
return l.Join(l.Path, s3LayoutPaths[t]), false
}
internal/backend/layout/layout_test.go
+40 -40
View File
@@ -9,7 +9,7 @@ import (
"sort"
"testing"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/backend"
rtest "github.com/restic/restic/internal/test"
)
@@ -19,79 +19,79 @@ func TestDefaultLayout(t *testing.T) {
var tests = []struct {
path string
join func(...string) string
restic.Handle
backend.Handle
filename string
}{
{
tempdir,
filepath.Join,
restic.Handle{Type: restic.PackFile, Name: "0123456"},
backend.Handle{Type: backend.PackFile, Name: "0123456"},
filepath.Join(tempdir, "data", "01", "0123456"),
},
{
tempdir,
filepath.Join,
restic.Handle{Type: restic.ConfigFile, Name: "CFG"},
backend.Handle{Type: backend.ConfigFile, Name: "CFG"},
filepath.Join(tempdir, "config"),
},
{
tempdir,
filepath.Join,
restic.Handle{Type: restic.SnapshotFile, Name: "123456"},
backend.Handle{Type: backend.SnapshotFile, Name: "123456"},
filepath.Join(tempdir, "snapshots", "123456"),
},
{
tempdir,
filepath.Join,
restic.Handle{Type: restic.IndexFile, Name: "123456"},
backend.Handle{Type: backend.IndexFile, Name: "123456"},
filepath.Join(tempdir, "index", "123456"),
},
{
tempdir,
filepath.Join,
restic.Handle{Type: restic.LockFile, Name: "123456"},
backend.Handle{Type: backend.LockFile, Name: "123456"},
filepath.Join(tempdir, "locks", "123456"),
},
{
tempdir,
filepath.Join,
restic.Handle{Type: restic.KeyFile, Name: "123456"},
backend.Handle{Type: backend.KeyFile, Name: "123456"},
filepath.Join(tempdir, "keys", "123456"),
},
{
"",
path.Join,
restic.Handle{Type: restic.PackFile, Name: "0123456"},
backend.Handle{Type: backend.PackFile, Name: "0123456"},
"data/01/0123456",
},
{
"",
path.Join,
restic.Handle{Type: restic.ConfigFile, Name: "CFG"},
backend.Handle{Type: backend.ConfigFile, Name: "CFG"},
"config",
},
{
"",
path.Join,
restic.Handle{Type: restic.SnapshotFile, Name: "123456"},
backend.Handle{Type: backend.SnapshotFile, Name: "123456"},
"snapshots/123456",
},
{
"",
path.Join,
restic.Handle{Type: restic.IndexFile, Name: "123456"},
backend.Handle{Type: backend.IndexFile, Name: "123456"},
"index/123456",
},
{
"",
path.Join,
restic.Handle{Type: restic.LockFile, Name: "123456"},
backend.Handle{Type: backend.LockFile, Name: "123456"},
"locks/123456",
},
{
"",
path.Join,
restic.Handle{Type: restic.KeyFile, Name: "123456"},
backend.Handle{Type: backend.KeyFile, Name: "123456"},
"keys/123456",
},
}
@@ -143,31 +143,31 @@ func TestRESTLayout(t *testing.T) {
path := rtest.TempDir(t)
var tests = []struct {
restic.Handle
backend.Handle
filename string
}{
{
restic.Handle{Type: restic.PackFile, Name: "0123456"},
backend.Handle{Type: backend.PackFile, Name: "0123456"},
filepath.Join(path, "data", "0123456"),
},
{
restic.Handle{Type: restic.ConfigFile, Name: "CFG"},
backend.Handle{Type: backend.ConfigFile, Name: "CFG"},
filepath.Join(path, "config"),
},
{
restic.Handle{Type: restic.SnapshotFile, Name: "123456"},
backend.Handle{Type: backend.SnapshotFile, Name: "123456"},
filepath.Join(path, "snapshots", "123456"),
},
{
restic.Handle{Type: restic.IndexFile, Name: "123456"},
backend.Handle{Type: backend.IndexFile, Name: "123456"},
filepath.Join(path, "index", "123456"),
},
{
restic.Handle{Type: restic.LockFile, Name: "123456"},
backend.Handle{Type: backend.LockFile, Name: "123456"},
filepath.Join(path, "locks", "123456"),
},
{
restic.Handle{Type: restic.KeyFile, Name: "123456"},
backend.Handle{Type: backend.KeyFile, Name: "123456"},
filepath.Join(path, "keys", "123456"),
},
}
@@ -209,61 +209,61 @@ func TestRESTLayout(t *testing.T) {
func TestRESTLayoutURLs(t *testing.T) {
var tests = []struct {
l Layout
h restic.Handle
h backend.Handle
fn string
dir string
}{
{
&RESTLayout{URL: "https://hostname.foo", Path: "", Join: path.Join},
restic.Handle{Type: restic.PackFile, Name: "foobar"},
backend.Handle{Type: backend.PackFile, Name: "foobar"},
"https://hostname.foo/data/foobar",
"https://hostname.foo/data/",
},
{
&RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.LockFile, Name: "foobar"},
backend.Handle{Type: backend.LockFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/locks/foobar",
"https://hostname.foo:1234/prefix/repo/locks/",
},
{
&RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.ConfigFile, Name: "foobar"},
backend.Handle{Type: backend.ConfigFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/config",
"https://hostname.foo:1234/prefix/repo/",
},
{
&S3LegacyLayout{URL: "https://hostname.foo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.PackFile, Name: "foobar"},
backend.Handle{Type: backend.PackFile, Name: "foobar"},
"https://hostname.foo/data/foobar",
"https://hostname.foo/data/",
},
{
&S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "", Join: path.Join},
restic.Handle{Type: restic.LockFile, Name: "foobar"},
backend.Handle{Type: backend.LockFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/lock/foobar",
"https://hostname.foo:1234/prefix/repo/lock/",
},
{
&S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join},
restic.Handle{Type: restic.ConfigFile, Name: "foobar"},
backend.Handle{Type: backend.ConfigFile, Name: "foobar"},
"https://hostname.foo:1234/prefix/repo/config",
"https://hostname.foo:1234/prefix/repo/",
},
{
&S3LegacyLayout{URL: "", Path: "", Join: path.Join},
restic.Handle{Type: restic.PackFile, Name: "foobar"},
backend.Handle{Type: backend.PackFile, Name: "foobar"},
"data/foobar",
"data/",
},
{
&S3LegacyLayout{URL: "", Path: "", Join: path.Join},
restic.Handle{Type: restic.LockFile, Name: "foobar"},
backend.Handle{Type: backend.LockFile, Name: "foobar"},
"lock/foobar",
"lock/",
},
{
&S3LegacyLayout{URL: "", Path: "/", Join: path.Join},
restic.Handle{Type: restic.ConfigFile, Name: "foobar"},
backend.Handle{Type: backend.ConfigFile, Name: "foobar"},
"/config",
"/",
},
@@ -288,31 +288,31 @@ func TestS3LegacyLayout(t *testing.T) {
path := rtest.TempDir(t)
var tests = []struct {
restic.Handle
backend.Handle
filename string
}{
{
restic.Handle{Type: restic.PackFile, Name: "0123456"},
backend.Handle{Type: backend.PackFile, Name: "0123456"},
filepath.Join(path, "data", "0123456"),
},
{
restic.Handle{Type: restic.ConfigFile, Name: "CFG"},
backend.Handle{Type: backend.ConfigFile, Name: "CFG"},
filepath.Join(path, "config"),
},
{
restic.Handle{Type: restic.SnapshotFile, Name: "123456"},
backend.Handle{Type: backend.SnapshotFile, Name: "123456"},
filepath.Join(path, "snapshot", "123456"),
},
{
restic.Handle{Type: restic.IndexFile, Name: "123456"},
backend.Handle{Type: backend.IndexFile, Name: "123456"},
filepath.Join(path, "index", "123456"),
},
{
restic.Handle{Type: restic.LockFile, Name: "123456"},
backend.Handle{Type: backend.LockFile, Name: "123456"},
filepath.Join(path, "lock", "123456"),
},
{
restic.Handle{Type: restic.KeyFile, Name: "123456"},
backend.Handle{Type: backend.KeyFile, Name: "123456"},
filepath.Join(path, "key", "123456"),
},
}
@@ -415,8 +415,8 @@ func TestParseLayout(t *testing.T) {
}
// test that the functions work (and don't panic)
_ = layout.Dirname(restic.Handle{Type: restic.PackFile})
_ = layout.Filename(restic.Handle{Type: restic.PackFile, Name: "1234"})
_ = layout.Dirname(backend.Handle{Type: backend.PackFile})
_ = layout.Filename(backend.Handle{Type: backend.PackFile, Name: "1234"})
_ = layout.Paths()
layoutName := fmt.Sprintf("%T", layout)
internal/backend/limiter/limiter_backend.go
+11 -11
View File
@@ -4,12 +4,12 @@ import (
"context"
"io"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/backend"
)
func WrapBackendConstructor[B restic.Backend, C any](constructor func(ctx context.Context, cfg C) (B, error)) func(ctx context.Context, cfg C, lim Limiter) (restic.Backend, error) {
return func(ctx context.Context, cfg C, lim Limiter) (restic.Backend, error) {
var be restic.Backend
func WrapBackendConstructor[B backend.Backend, C any](constructor func(ctx context.Context, cfg C) (B, error)) func(ctx context.Context, cfg C, lim Limiter) (backend.Backend, error) {
return func(ctx context.Context, cfg C, lim Limiter) (backend.Backend, error) {
var be backend.Backend
be, err := constructor(ctx, cfg)
if err != nil {
return nil, err
@@ -24,7 +24,7 @@ func WrapBackendConstructor[B restic.Backend, C any](constructor func(ctx contex
// LimitBackend wraps a Backend and applies rate limiting to Load() and Save()
// calls on the backend.
func LimitBackend(be restic.Backend, l Limiter) restic.Backend {
func LimitBackend(be backend.Backend, l Limiter) backend.Backend {
return rateLimitedBackend{
Backend: be,
limiter: l,
@@ -32,11 +32,11 @@ func LimitBackend(be restic.Backend, l Limiter) restic.Backend {
}
type rateLimitedBackend struct {
restic.Backend
backend.Backend
limiter Limiter
}
func (r rateLimitedBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (r rateLimitedBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
limited := limitedRewindReader{
RewindReader: rd,
limited: r.limiter.Upstream(rd),
@@ -46,7 +46,7 @@ func (r rateLimitedBackend) Save(ctx context.Context, h restic.Handle, rd restic
}
type limitedRewindReader struct {
restic.RewindReader
backend.RewindReader
limited io.Reader
}
@@ -55,13 +55,13 @@ func (l limitedRewindReader) Read(b []byte) (int, error) {
return l.limited.Read(b)
}
func (r rateLimitedBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
func (r rateLimitedBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
return r.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error {
return consumer(newDownstreamLimitedReader(rd, r.limiter))
})
}
func (r rateLimitedBackend) Unwrap() restic.Backend { return r.Backend }
func (r rateLimitedBackend) Unwrap() backend.Backend { return r.Backend }
type limitedReader struct {
io.Reader
@@ -85,4 +85,4 @@ func (l *limitedReader) WriteTo(w io.Writer) (int64, error) {
return l.writerTo.WriteTo(l.limiter.DownstreamWriter(w))
}
var _ restic.Backend = (*rateLimitedBackend)(nil)
var _ backend.Backend = (*rateLimitedBackend)(nil)
internal/backend/limiter/limiter_backend_test.go
+6 -6
View File
@@ -8,8 +8,8 @@ import (
"io"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/mock"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@@ -21,11 +21,11 @@ func randomBytes(t *testing.T, size int) []byte {
}
func TestLimitBackendSave(t *testing.T) {
testHandle := restic.Handle{Type: restic.PackFile, Name: "test"}
testHandle := backend.Handle{Type: backend.PackFile, Name: "test"}
data := randomBytes(t, 1234)
be := mock.NewBackend()
be.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
be.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
buf := new(bytes.Buffer)
_, err := io.Copy(buf, rd)
if err != nil {
@@ -39,7 +39,7 @@ func TestLimitBackendSave(t *testing.T) {
limiter := NewStaticLimiter(Limits{42 * 1024, 42 * 1024})
limbe := LimitBackend(be, limiter)
rd := restic.NewByteReader(data, nil)
rd := backend.NewByteReader(data, nil)
err := limbe.Save(context.TODO(), testHandle, rd)
rtest.OK(t, err)
}
@@ -64,7 +64,7 @@ func (r *tracedReadWriteToCloser) Close() error {
}
func TestLimitBackendLoad(t *testing.T) {
testHandle := restic.Handle{Type: restic.PackFile, Name: "test"}
testHandle := backend.Handle{Type: backend.PackFile, Name: "test"}
data := randomBytes(t, 1234)
for _, test := range []struct {
@@ -72,7 +72,7 @@ func TestLimitBackendLoad(t *testing.T) {
}{{false, false}, {false, true}, {true, false}, {true, true}} {
be := mock.NewBackend()
src := newTracedReadWriteToCloser(bytes.NewReader(data))
be.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
if length != 0 || offset != 0 {
return nil, fmt.Errorf("Not supported")
}
internal/backend/local/layout_test.go
+2 -2
View File
@@ -5,7 +5,7 @@ import (
"path/filepath"
"testing"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/backend"
rtest "github.com/restic/restic/internal/test"
)
@@ -49,7 +49,7 @@ func TestLayout(t *testing.T) {
}
packs := make(map[string]bool)
err = be.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
err = be.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
packs[fi.Name] = false
return nil
})
internal/backend/local/local.go
+15 -16
View File
@@ -16,7 +16,6 @@ import (
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic"
"github.com/cenkalti/backoff/v4"
)
@@ -28,8 +27,8 @@ type Local struct {
util.Modes
}
// ensure statically that *Local implements restic.Backend.
var _ restic.Backend = &Local{}
// ensure statically that *Local implements backend.Backend.
var _ backend.Backend = &Local{}
func NewFactory() location.Factory {
return location.NewLimitedBackendFactory("local", ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open))
@@ -43,7 +42,7 @@ func open(ctx context.Context, cfg Config) (*Local, error) {
return nil, err
}
fi, err := fs.Stat(l.Filename(restic.Handle{Type: restic.ConfigFile}))
fi, err := fs.Stat(l.Filename(backend.Handle{Type: backend.ConfigFile}))
m := util.DeriveModesFromFileInfo(fi, err)
debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir)
@@ -71,7 +70,7 @@ func Create(ctx context.Context, cfg Config) (*Local, error) {
}
// test if config file already exists
_, err = fs.Lstat(be.Filename(restic.Handle{Type: restic.ConfigFile}))
_, err = fs.Lstat(be.Filename(backend.Handle{Type: backend.ConfigFile}))
if err == nil {
return nil, errors.New("config file already exists")
}
@@ -112,7 +111,7 @@ func (b *Local) IsNotExist(err error) bool {
}
// Save stores data in the backend at the handle.
func (b *Local) Save(_ context.Context, h restic.Handle, rd restic.RewindReader) (err error) {
func (b *Local) Save(_ context.Context, h backend.Handle, rd backend.RewindReader) (err error) {
finalname := b.Filename(h)
dir := filepath.Dir(finalname)
@@ -210,11 +209,11 @@ var tempFile = os.CreateTemp // Overridden by test.
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (b *Local) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (b *Local) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return util.DefaultLoad(ctx, h, length, offset, b.openReader, fn)
}
func (b *Local) openReader(_ context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (b *Local) openReader(_ context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
f, err := fs.Open(b.Filename(h))
if err != nil {
return nil, err
@@ -236,17 +235,17 @@ func (b *Local) openReader(_ context.Context, h restic.Handle, length int, offse
}
// Stat returns information about a blob.
func (b *Local) Stat(_ context.Context, h restic.Handle) (restic.FileInfo, error) {
func (b *Local) Stat(_ context.Context, h backend.Handle) (backend.FileInfo, error) {
fi, err := fs.Stat(b.Filename(h))
if err != nil {
return restic.FileInfo{}, errors.WithStack(err)
return backend.FileInfo{}, errors.WithStack(err)
}
return restic.FileInfo{Size: fi.Size(), Name: h.Name}, nil
return backend.FileInfo{Size: fi.Size(), Name: h.Name}, nil
}
// Remove removes the blob with the given name and type.
func (b *Local) Remove(_ context.Context, h restic.Handle) error {
func (b *Local) Remove(_ context.Context, h backend.Handle) error {
fn := b.Filename(h)
// reset read-only flag
@@ -260,7 +259,7 @@ func (b *Local) Remove(_ context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (b *Local) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) (err error) {
func (b *Local) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) (err error) {
basedir, subdirs := b.Basedir(t)
if subdirs {
err = visitDirs(ctx, basedir, fn)
@@ -280,7 +279,7 @@ func (b *Local) List(ctx context.Context, t restic.FileType, fn func(restic.File
// two levels of directory structure (including dir itself as the first level).
// Also, visitDirs assumes it sees a directory full of directories, while
// visitFiles wants a directory full or regular files.
func visitDirs(ctx context.Context, dir string, fn func(restic.FileInfo) error) error {
func visitDirs(ctx context.Context, dir string, fn func(backend.FileInfo) error) error {
d, err := fs.Open(dir)
if err != nil {
return err
@@ -307,7 +306,7 @@ func visitDirs(ctx context.Context, dir string, fn func(restic.FileInfo) error)
return ctx.Err()
}
func visitFiles(ctx context.Context, dir string, fn func(restic.FileInfo) error, ignoreNotADirectory bool) error {
func visitFiles(ctx context.Context, dir string, fn func(backend.FileInfo) error, ignoreNotADirectory bool) error {
d, err := fs.Open(dir)
if err != nil {
return err
@@ -341,7 +340,7 @@ func visitFiles(ctx context.Context, dir string, fn func(restic.FileInfo) error,
default:
}
err := fn(restic.FileInfo{
err := fn(backend.FileInfo{
Name: fi.Name(),
Size: fi.Size(),
})
internal/backend/local/local_internal_test.go
+2 -2
View File
@@ -8,7 +8,7 @@ import (
"syscall"
"testing"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/backend"
rtest "github.com/restic/restic/internal/test"
"github.com/cenkalti/backoff/v4"
@@ -32,7 +32,7 @@ func TestNoSpacePermanent(t *testing.T) {
rtest.OK(t, be.Close())
}()
h := restic.Handle{Type: restic.ConfigFile}
h := backend.Handle{Type: backend.ConfigFile}
err = be.Save(context.Background(), h, nil)
_, ok := err.(*backoff.PermanentError)
rtest.Assert(t, ok,
internal/backend/location/display_location_test.go
+2 -2
View File
@@ -3,15 +3,15 @@ package location_test
import (
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
func TestStripPassword(t *testing.T) {
registry := location.NewRegistry()
registry.Register(
location.NewHTTPBackendFactory[any, restic.Backend]("test", nil,
location.NewHTTPBackendFactory[any, backend.Backend]("test", nil,
func(s string) string {
return "cleaned"
}, nil, nil,
internal/backend/location/location_test.go
+2 -2
View File
@@ -3,8 +3,8 @@ package location_test
import (
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
@@ -13,7 +13,7 @@ type testConfig struct {
}
func testFactory() location.Factory {
return location.NewHTTPBackendFactory[testConfig, restic.Backend](
return location.NewHTTPBackendFactory[testConfig, backend.Backend](
"local",
func(s string) (*testConfig, error) {
return &testConfig{loc: s}, nil
internal/backend/location/registry.go
+8 -8
View File
@@ -4,8 +4,8 @@ import (
"context"
"net/http"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/limiter"
"github.com/restic/restic/internal/restic"
)
type Registry struct {
@@ -33,11 +33,11 @@ type Factory interface {
Scheme() string
ParseConfig(s string) (interface{}, error)
StripPassword(s string) string
Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error)
Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error)
Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error)
Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error)
}
type genericBackendFactory[C any, T restic.Backend] struct {
type genericBackendFactory[C any, T backend.Backend] struct {
scheme string
parseConfigFn func(s string) (*C, error)
stripPasswordFn func(s string) string
@@ -58,14 +58,14 @@ func (f *genericBackendFactory[C, T]) StripPassword(s string) string {
}
return s
}
func (f *genericBackendFactory[C, T]) Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) {
func (f *genericBackendFactory[C, T]) Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error) {
return f.createFn(ctx, *cfg.(*C), rt, lim)
}
func (f *genericBackendFactory[C, T]) Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) {
func (f *genericBackendFactory[C, T]) Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error) {
return f.openFn(ctx, *cfg.(*C), rt, lim)
}
func NewHTTPBackendFactory[C any, T restic.Backend](
func NewHTTPBackendFactory[C any, T backend.Backend](
scheme string,
parseConfigFn func(s string) (*C, error),
stripPasswordFn func(s string) string,
@@ -85,7 +85,7 @@ func NewHTTPBackendFactory[C any, T restic.Backend](
}
}
func NewLimitedBackendFactory[C any, T restic.Backend](
func NewLimitedBackendFactory[C any, T backend.Backend](
scheme string,
parseConfigFn func(s string) (*C, error),
stripPasswordFn func(s string) string,
internal/backend/logger/log.go
+11 -11
View File
@@ -4,18 +4,18 @@ import (
"context"
"io"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
)
type Backend struct {
restic.Backend
backend.Backend
}
// statically ensure that Backend implements restic.Backend.
var _ restic.Backend = &Backend{}
// statically ensure that Backend implements backend.Backend.
var _ backend.Backend = &Backend{}
func New(be restic.Backend) *Backend {
func New(be backend.Backend) *Backend {
return &Backend{Backend: be}
}
@@ -26,7 +26,7 @@ func (be *Backend) IsNotExist(err error) bool {
}
// Save adds new Data to the backend.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
debug.Log("Save(%v, %v)", h, rd.Length())
err := be.Backend.Save(ctx, h, rd)
debug.Log(" save err %v", err)
@@ -34,28 +34,28 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
}
// Remove deletes a file from the backend.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
debug.Log("Remove(%v)", h)
err := be.Backend.Remove(ctx, h)
debug.Log(" remove err %v", err)
return err
}
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(io.Reader) error) error {
func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(io.Reader) error) error {
debug.Log("Load(%v, length %v, offset %v)", h, length, offset)
err := be.Backend.Load(ctx, h, length, offset, fn)
debug.Log(" load err %v", err)
return err
}
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
func (be *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
debug.Log("Stat(%v)", h)
fi, err := be.Backend.Stat(ctx, h)
debug.Log(" stat err %v", err)
return fi, err
}
func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
debug.Log("List(%v)", t)
err := be.Backend.List(ctx, t, fn)
debug.Log(" list err %v", err)
@@ -76,4 +76,4 @@ func (be *Backend) Close() error {
return err
}
func (be *Backend) Unwrap() restic.Backend { return be.Backend }
func (be *Backend) Unwrap() backend.Backend { return be.Backend }
internal/backend/mem/mem_backend.go
+15 -15
View File
@@ -10,17 +10,17 @@ import (
"sync"
"github.com/cespare/xxhash/v2"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
)
type memMap map[restic.Handle][]byte
type memMap map[backend.Handle][]byte
// make sure that MemoryBackend implements backend.Backend
var _ restic.Backend = &MemoryBackend{}
var _ backend.Backend = &MemoryBackend{}
// NewFactory creates a persistent mem backend
func NewFactory() location.Factory {
@@ -69,12 +69,12 @@ func (be *MemoryBackend) IsNotExist(err error) bool {
}
// Save adds new Data to the backend.
func (be *MemoryBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *MemoryBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
be.m.Lock()
defer be.m.Unlock()
h.IsMetadata = false
if h.Type == restic.ConfigFile {
if h.Type == backend.ConfigFile {
h.Name = ""
}
@@ -112,16 +112,16 @@ func (be *MemoryBackend) Save(ctx context.Context, h restic.Handle, rd restic.Re
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *MemoryBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *MemoryBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *MemoryBackend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (be *MemoryBackend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
be.m.Lock()
defer be.m.Unlock()
h.IsMetadata = false
if h.Type == restic.ConfigFile {
if h.Type == backend.ConfigFile {
h.Name = ""
}
@@ -143,25 +143,25 @@ func (be *MemoryBackend) openReader(ctx context.Context, h restic.Handle, length
}
// Stat returns information about a file in the backend.
func (be *MemoryBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
func (be *MemoryBackend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
be.m.Lock()
defer be.m.Unlock()
h.IsMetadata = false
if h.Type == restic.ConfigFile {
if h.Type == backend.ConfigFile {
h.Name = ""
}
e, ok := be.data[h]
if !ok {
return restic.FileInfo{}, errNotFound
return backend.FileInfo{}, errNotFound
}
return restic.FileInfo{Size: int64(len(e)), Name: h.Name}, ctx.Err()
return backend.FileInfo{Size: int64(len(e)), Name: h.Name}, ctx.Err()
}
// Remove deletes a file from the backend.
func (be *MemoryBackend) Remove(ctx context.Context, h restic.Handle) error {
func (be *MemoryBackend) Remove(ctx context.Context, h backend.Handle) error {
be.m.Lock()
defer be.m.Unlock()
@@ -176,7 +176,7 @@ func (be *MemoryBackend) Remove(ctx context.Context, h restic.Handle) error {
}
// List returns a channel which yields entries from the backend.
func (be *MemoryBackend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *MemoryBackend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
entries := make(map[string]int64)
be.m.Lock()
@@ -190,7 +190,7 @@ func (be *MemoryBackend) List(ctx context.Context, t restic.FileType, fn func(re
be.m.Unlock()
for name, size := range entries {
fi := restic.FileInfo{
fi := backend.FileInfo{
Name: name,
Size: size,
}
internal/backend/mock/backend.go
+14 -14
View File
@@ -5,19 +5,19 @@ import (
"hash"
"io"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
)
// Backend implements a mock backend.
type Backend struct {
CloseFn func() error
IsNotExistFn func(err error) bool
SaveFn func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error
OpenReaderFn func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error)
StatFn func(ctx context.Context, h restic.Handle) (restic.FileInfo, error)
ListFn func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error
RemoveFn func(ctx context.Context, h restic.Handle) error
SaveFn func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error
OpenReaderFn func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error)
StatFn func(ctx context.Context, h backend.Handle) (backend.FileInfo, error)
ListFn func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error
RemoveFn func(ctx context.Context, h backend.Handle) error
DeleteFn func(ctx context.Context) error
ConnectionsFn func() uint
LocationFn func() string
@@ -84,7 +84,7 @@ func (m *Backend) IsNotExist(err error) bool {
}
// Save data in the backend.
func (m *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (m *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if m.SaveFn == nil {
return errors.New("not implemented")
}
@@ -94,7 +94,7 @@ func (m *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (m *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (m *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
rd, err := m.openReader(ctx, h, length, offset)
if err != nil {
return err
@@ -107,7 +107,7 @@ func (m *Backend) Load(ctx context.Context, h restic.Handle, length int, offset
return rd.Close()
}
func (m *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (m *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
if m.OpenReaderFn == nil {
return nil, errors.New("not implemented")
}
@@ -116,16 +116,16 @@ func (m *Backend) openReader(ctx context.Context, h restic.Handle, length int, o
}
// Stat an object in the backend.
func (m *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
func (m *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
if m.StatFn == nil {
return restic.FileInfo{}, errors.New("not implemented")
return backend.FileInfo{}, errors.New("not implemented")
}
return m.StatFn(ctx, h)
}
// List items of type t.
func (m *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (m *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
if m.ListFn == nil {
return nil
}
@@ -134,7 +134,7 @@ func (m *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.Fi
}
// Remove data from the backend.
func (m *Backend) Remove(ctx context.Context, h restic.Handle) error {
func (m *Backend) Remove(ctx context.Context, h backend.Handle) error {
if m.RemoveFn == nil {
return errors.New("not implemented")
}
@@ -152,4 +152,4 @@ func (m *Backend) Delete(ctx context.Context) error {
}
// Make sure that Backend implements the backend interface.
var _ restic.Backend = &Backend{}
var _ backend.Backend = &Backend{}
internal/backend/rclone/internal_test.go
+3 -3
View File
@@ -5,8 +5,8 @@ import (
"os/exec"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@@ -32,9 +32,9 @@ func TestRcloneExit(t *testing.T) {
t.Log("killed rclone")
for i := 0; i < 10; i++ {
_, err = be.Stat(context.TODO(), restic.Handle{
_, err = be.Stat(context.TODO(), backend.Handle{
Name: "foo",
Type: restic.PackFile,
Type: backend.PackFile,
})
rtest.Assert(t, err != nil, "expected an error")
}
internal/backend/readerat.go
+4 -5
View File
@@ -6,13 +6,12 @@ import (
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
)
type backendReaderAt struct {
ctx context.Context
be restic.Backend
h restic.Handle
be Backend
h Handle
}
func (brd backendReaderAt) ReadAt(p []byte, offset int64) (n int, err error) {
@@ -22,12 +21,12 @@ func (brd backendReaderAt) ReadAt(p []byte, offset int64) (n int, err error) {
// ReaderAt returns an io.ReaderAt for a file in the backend. The returned reader
// should not escape the caller function to avoid unexpected interactions with the
// embedded context
func ReaderAt(ctx context.Context, be restic.Backend, h restic.Handle) io.ReaderAt {
func ReaderAt(ctx context.Context, be Backend, h Handle) io.ReaderAt {
return backendReaderAt{ctx: ctx, be: be, h: h}
}
// ReadAt reads from the backend handle h at the given position.
func ReadAt(ctx context.Context, be restic.Backend, h restic.Handle, offset int64, p []byte) (n int, err error) {
func ReadAt(ctx context.Context, be Backend, h Handle, offset int64, p []byte) (n int, err error) {
debug.Log("ReadAt(%v) at %v, len %v", h, offset, len(p))
err = be.Load(ctx, h, len(p), offset, func(rd io.Reader) (ierr error) {
internal/backend/rest/config.go
+2 -2
View File
@@ -5,9 +5,9 @@ import (
"os"
"strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
)
// Config contains all configuration necessary to connect to a REST server.
@@ -73,7 +73,7 @@ func prepareURL(s string) string {
return s
}
var _ restic.ApplyEnvironmenter = &Config{}
var _ backend.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) {
internal/backend/rest/rest.go
+23 -23
View File
@@ -11,16 +11,16 @@ import (
"path"
"strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
)
// make sure the rest backend implements restic.Backend
var _ restic.Backend = &Backend{}
// make sure the rest backend implements backend.Backend
var _ backend.Backend = &Backend{}
// Backend uses the REST protocol to access data stored on a server.
type Backend struct {
@@ -65,7 +65,7 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, er
return nil, err
}
_, err = be.Stat(ctx, restic.Handle{Type: restic.ConfigFile})
_, err = be.Stat(ctx, backend.Handle{Type: backend.ConfigFile})
if err == nil {
return nil, errors.New("config file already exists")
}
@@ -118,7 +118,7 @@ func (b *Backend) HasAtomicReplace() bool {
}
// Save stores data in the backend at the handle.
func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
@@ -157,7 +157,7 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea
// notExistError is returned whenever the requested file does not exist on the
// server.
type notExistError struct {
restic.Handle
backend.Handle
}
func (e *notExistError) Error() string {
@@ -172,7 +172,7 @@ func (b *Backend) IsNotExist(err error) bool {
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
r, err := b.openReader(ctx, h, length, offset)
if err != nil {
return err
@@ -201,7 +201,7 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset
return err
}
func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (b *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
req, err := http.NewRequestWithContext(ctx, "GET", b.Filename(h), nil)
if err != nil {
return nil, errors.WithStack(err)
@@ -238,37 +238,37 @@ func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, o
}
// Stat returns information about a blob.
func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodHead, b.Filename(h), nil)
if err != nil {
return restic.FileInfo{}, errors.WithStack(err)
return backend.FileInfo{}, errors.WithStack(err)
}
req.Header.Set("Accept", ContentTypeV2)
resp, err := b.client.Do(req)
if err != nil {
return restic.FileInfo{}, errors.WithStack(err)
return backend.FileInfo{}, errors.WithStack(err)
}
_, _ = io.Copy(io.Discard, resp.Body)
if err = resp.Body.Close(); err != nil {
return restic.FileInfo{}, errors.Wrap(err, "Close")
return backend.FileInfo{}, errors.Wrap(err, "Close")
}
if resp.StatusCode == http.StatusNotFound {
_ = resp.Body.Close()
return restic.FileInfo{}, &notExistError{h}
return backend.FileInfo{}, &notExistError{h}
}
if resp.StatusCode != http.StatusOK {
return restic.FileInfo{}, errors.Errorf("unexpected HTTP response (%v): %v", resp.StatusCode, resp.Status)
return backend.FileInfo{}, errors.Errorf("unexpected HTTP response (%v): %v", resp.StatusCode, resp.Status)
}
if resp.ContentLength < 0 {
return restic.FileInfo{}, errors.New("negative content length")
return backend.FileInfo{}, errors.New("negative content length")
}
bi := restic.FileInfo{
bi := backend.FileInfo{
Size: resp.ContentLength,
Name: h.Name,
}
@@ -277,7 +277,7 @@ func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, e
}
// Remove removes the blob with the given name and type.
func (b *Backend) Remove(ctx context.Context, h restic.Handle) error {
func (b *Backend) Remove(ctx context.Context, h backend.Handle) error {
req, err := http.NewRequestWithContext(ctx, "DELETE", b.Filename(h), nil)
if err != nil {
return errors.WithStack(err)
@@ -309,8 +309,8 @@ func (b *Backend) Remove(ctx context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
url := b.Dirname(restic.Handle{Type: t})
func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
url := b.Dirname(backend.Handle{Type: t})
if !strings.HasSuffix(url, "/") {
url += "/"
}
@@ -346,7 +346,7 @@ func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.Fi
// listv1 uses the REST protocol v1, where a list HTTP request (e.g. `GET
// /data/`) only returns the names of the files, so we need to issue an HTTP
// HEAD request for each file.
func (b *Backend) listv1(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error {
func (b *Backend) listv1(ctx context.Context, t backend.FileType, resp *http.Response, fn func(backend.FileInfo) error) error {
debug.Log("parsing API v1 response")
dec := json.NewDecoder(resp.Body)
var list []string
@@ -355,7 +355,7 @@ func (b *Backend) listv1(ctx context.Context, t restic.FileType, resp *http.Resp
}
for _, m := range list {
fi, err := b.Stat(ctx, restic.Handle{Name: m, Type: t})
fi, err := b.Stat(ctx, backend.Handle{Name: m, Type: t})
if err != nil {
return err
}
@@ -380,7 +380,7 @@ func (b *Backend) listv1(ctx context.Context, t restic.FileType, resp *http.Resp
// listv2 uses the REST protocol v2, where a list HTTP request (e.g. `GET
// /data/`) returns the names and sizes of all files.
func (b *Backend) listv2(ctx context.Context, resp *http.Response, fn func(restic.FileInfo) error) error {
func (b *Backend) listv2(ctx context.Context, resp *http.Response, fn func(backend.FileInfo) error) error {
debug.Log("parsing API v2 response")
dec := json.NewDecoder(resp.Body)
@@ -397,7 +397,7 @@ func (b *Backend) listv2(ctx context.Context, resp *http.Response, fn func(resti
return ctx.Err()
}
fi := restic.FileInfo{
fi := backend.FileInfo{
Name: item.Name,
Size: item.Size,
}
internal/backend/rest/rest_int_test.go
+7 -7
View File
@@ -10,8 +10,8 @@ import (
"strconv"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/rest"
"github.com/restic/restic/internal/restic"
)
func TestListAPI(t *testing.T) {
@@ -22,7 +22,7 @@ func TestListAPI(t *testing.T) {
Data string // response data
Requests int
Result []restic.FileInfo
Result []backend.FileInfo
}{
{
Name: "content-type-unknown",
@@ -32,7 +32,7 @@ func TestListAPI(t *testing.T) {
"3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352",
"8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b"
]`,
Result: []restic.FileInfo{
Result: []backend.FileInfo{
{Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 4386},
{Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 15214},
{Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 33393},
@@ -47,7 +47,7 @@ func TestListAPI(t *testing.T) {
"3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352",
"8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b"
]`,
Result: []restic.FileInfo{
Result: []backend.FileInfo{
{Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 4386},
{Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 15214},
{Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 33393},
@@ -62,7 +62,7 @@ func TestListAPI(t *testing.T) {
{"name": "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", "size": 1002},
{"name": "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", "size": 1003}
]`,
Result: []restic.FileInfo{
Result: []backend.FileInfo{
{Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 1001},
{Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 1002},
{Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 1003},
@@ -122,8 +122,8 @@ func TestListAPI(t *testing.T) {
t.Fatal(err)
}
var list []restic.FileInfo
err = be.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
var list []backend.FileInfo
err = be.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
list = append(list, fi)
return nil
})
internal/backend/retry/backend_retry.go
+12 -12
View File
@@ -7,27 +7,27 @@ import (
"time"
"github.com/cenkalti/backoff/v4"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
)
// Backend retries operations on the backend in case of an error with a
// backoff.
type Backend struct {
restic.Backend
backend.Backend
MaxTries int
Report func(string, error, time.Duration)
Success func(string, int)
}
// statically ensure that RetryBackend implements restic.Backend.
var _ restic.Backend = &Backend{}
// statically ensure that RetryBackend implements backend.Backend.
var _ backend.Backend = &Backend{}
// New wraps be with a backend that retries operations after a
// backoff. report is called with a description and the error, if one occurred.
// success is called with the number of retries before a successful operation
// (it is not called if it succeeded on the first try)
func New(be restic.Backend, maxTries int, report func(string, error, time.Duration), success func(string, int)) *Backend {
func New(be backend.Backend, maxTries int, report func(string, error, time.Duration), success func(string, int)) *Backend {
return &Backend{
Backend: be,
MaxTries: maxTries,
@@ -92,7 +92,7 @@ func (be *Backend) retry(ctx context.Context, msg string, f func() error) error
}
// Save stores the data in the backend under the given handle.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
return be.retry(ctx, fmt.Sprintf("Save(%v)", h), func() error {
err := rd.Rewind()
if err != nil {
@@ -125,7 +125,7 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
// given offset. If length is larger than zero, only a portion of the file
// is returned. rd must be closed after use. If an error is returned, the
// ReadCloser must be nil.
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) (err error) {
func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (err error) {
return be.retry(ctx, fmt.Sprintf("Load(%v, %v, %v)", h, length, offset),
func() error {
err := be.Backend.Load(ctx, h, length, offset, consumer)
@@ -137,7 +137,7 @@ func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset
}
// Stat returns information about the File identified by h.
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (fi restic.FileInfo, err error) {
func (be *Backend) Stat(ctx context.Context, h backend.Handle) (fi backend.FileInfo, err error) {
err = be.retry(ctx, fmt.Sprintf("Stat(%v)", h),
func() error {
var innerError error
@@ -153,7 +153,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (fi restic.FileInf
}
// Remove removes a File with type t and name.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) (err error) {
func (be *Backend) Remove(ctx context.Context, h backend.Handle) (err error) {
return be.retry(ctx, fmt.Sprintf("Remove(%v)", h), func() error {
return be.Backend.Remove(ctx, h)
})
@@ -163,7 +163,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) (err error) {
// error is returned by the underlying backend, the request is retried. When fn
// returns an error, the operation is aborted and the error is returned to the
// caller.
func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
// create a new context that we can cancel when fn returns an error, so
// that listing is aborted
listCtx, cancel := context.WithCancel(ctx)
@@ -173,7 +173,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
var innerErr error // remember when fn returned an error, so we can return that to the caller
err := be.retry(listCtx, fmt.Sprintf("List(%v)", t), func() error {
return be.Backend.List(ctx, t, func(fi restic.FileInfo) error {
return be.Backend.List(ctx, t, func(fi backend.FileInfo) error {
if _, ok := listed[fi.Name]; ok {
return nil
}
@@ -196,6 +196,6 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
return err
}
func (be *Backend) Unwrap() restic.Backend {
func (be *Backend) Unwrap() backend.Backend {
return be.Backend
}
internal/backend/retry/backend_retry_test.go
+29 -28
View File
@@ -8,6 +8,7 @@ import (
"time"
"github.com/cenkalti/backoff/v4"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/mock"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
@@ -18,7 +19,7 @@ func TestBackendSaveRetry(t *testing.T) {
buf := bytes.NewBuffer(nil)
errcount := 0
be := &mock.Backend{
SaveFn: func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
SaveFn: func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if errcount == 0 {
errcount++
_, err := io.CopyN(io.Discard, rd, 120)
@@ -38,7 +39,7 @@ func TestBackendSaveRetry(t *testing.T) {
retryBackend := New(be, 10, nil, nil)
data := test.Random(23, 5*1024*1024+11241)
err := retryBackend.Save(context.TODO(), restic.Handle{}, restic.NewByteReader(data, be.Hasher()))
err := retryBackend.Save(context.TODO(), backend.Handle{}, backend.NewByteReader(data, be.Hasher()))
if err != nil {
t.Fatal(err)
}
@@ -56,14 +57,14 @@ func TestBackendSaveRetryAtomic(t *testing.T) {
errcount := 0
calledRemove := false
be := &mock.Backend{
SaveFn: func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
SaveFn: func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if errcount == 0 {
errcount++
return errors.New("injected error")
}
return nil
},
RemoveFn: func(ctx context.Context, h restic.Handle) error {
RemoveFn: func(ctx context.Context, h backend.Handle) error {
calledRemove = true
return nil
},
@@ -74,7 +75,7 @@ func TestBackendSaveRetryAtomic(t *testing.T) {
retryBackend := New(be, 10, nil, nil)
data := test.Random(23, 5*1024*1024+11241)
err := retryBackend.Save(context.TODO(), restic.Handle{}, restic.NewByteReader(data, be.Hasher()))
err := retryBackend.Save(context.TODO(), backend.Handle{}, backend.NewByteReader(data, be.Hasher()))
if err != nil {
t.Fatal(err)
}
@@ -91,15 +92,15 @@ func TestBackendListRetry(t *testing.T) {
retry := 0
be := &mock.Backend{
ListFn: func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
ListFn: func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
// fail during first retry, succeed during second
retry++
if retry == 1 {
_ = fn(restic.FileInfo{Name: ID1})
_ = fn(backend.FileInfo{Name: ID1})
return errors.New("test list error")
}
_ = fn(restic.FileInfo{Name: ID1})
_ = fn(restic.FileInfo{Name: ID2})
_ = fn(backend.FileInfo{Name: ID1})
_ = fn(backend.FileInfo{Name: ID2})
return nil
},
}
@@ -108,7 +109,7 @@ func TestBackendListRetry(t *testing.T) {
retryBackend := New(be, 10, nil, nil)
var listed []string
err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
err := retryBackend.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
listed = append(listed, fi.Name)
return nil
})
@@ -121,10 +122,10 @@ func TestBackendListRetryErrorFn(t *testing.T) {
var names = []string{"id1", "id2", "foo", "bar"}
be := &mock.Backend{
ListFn: func(ctx context.Context, tpe restic.FileType, fn func(restic.FileInfo) error) error {
ListFn: func(ctx context.Context, tpe backend.FileType, fn func(backend.FileInfo) error) error {
t.Logf("List called for %v", tpe)
for _, name := range names {
err := fn(restic.FileInfo{Name: name})
err := fn(backend.FileInfo{Name: name})
if err != nil {
return err
}
@@ -141,7 +142,7 @@ func TestBackendListRetryErrorFn(t *testing.T) {
var listed []string
run := 0
err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
err := retryBackend.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
t.Logf("fn called for %v", fi.Name)
run++
// return an error for the third item in the list
@@ -172,7 +173,7 @@ func TestBackendListRetryErrorBackend(t *testing.T) {
retries := 0
be := &mock.Backend{
ListFn: func(ctx context.Context, tpe restic.FileType, fn func(restic.FileInfo) error) error {
ListFn: func(ctx context.Context, tpe backend.FileType, fn func(backend.FileInfo) error) error {
t.Logf("List called for %v, retries %v", tpe, retries)
retries++
for i, name := range names {
@@ -180,7 +181,7 @@ func TestBackendListRetryErrorBackend(t *testing.T) {
return ErrBackendTest
}
err := fn(restic.FileInfo{Name: name})
err := fn(backend.FileInfo{Name: name})
if err != nil {
return err
}
@@ -195,7 +196,7 @@ func TestBackendListRetryErrorBackend(t *testing.T) {
retryBackend := New(be, maxRetries, nil, nil)
var listed []string
err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
err := retryBackend.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
t.Logf("fn called for %v", fi.Name)
listed = append(listed, fi.Name)
return nil
@@ -252,7 +253,7 @@ func TestBackendLoadRetry(t *testing.T) {
attempt := 0
be := mock.NewBackend()
be.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
// returns failing reader on first invocation, good reader on subsequent invocations
attempt++
if attempt > 1 {
@@ -265,7 +266,7 @@ func TestBackendLoadRetry(t *testing.T) {
retryBackend := New(be, 10, nil, nil)
var buf []byte
err := retryBackend.Load(context.TODO(), restic.Handle{}, 0, 0, func(rd io.Reader) (err error) {
err := retryBackend.Load(context.TODO(), backend.Handle{}, 0, 0, func(rd io.Reader) (err error) {
buf, err = io.ReadAll(rd)
return err
})
@@ -280,7 +281,7 @@ func TestBackendLoadNotExists(t *testing.T) {
attempt := 0
be := mock.NewBackend()
be.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
attempt++
if attempt > 1 {
t.Fail()
@@ -295,7 +296,7 @@ func TestBackendLoadNotExists(t *testing.T) {
TestFastRetries(t)
retryBackend := New(be, 10, nil, nil)
err := retryBackend.Load(context.TODO(), restic.Handle{}, 0, 0, func(rd io.Reader) (err error) {
err := retryBackend.Load(context.TODO(), backend.Handle{}, 0, 0, func(rd io.Reader) (err error) {
return nil
})
test.Assert(t, be.IsNotExistFn(err), "unexpected error %v", err)
@@ -308,13 +309,13 @@ func TestBackendStatNotExists(t *testing.T) {
attempt := 0
be := mock.NewBackend()
be.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
be.StatFn = func(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
attempt++
if attempt > 1 {
t.Fail()
return restic.FileInfo{}, errors.New("must not retry")
return backend.FileInfo{}, errors.New("must not retry")
}
return restic.FileInfo{}, notFound
return backend.FileInfo{}, notFound
}
be.IsNotExistFn = func(err error) bool {
return errors.Is(err, notFound)
@@ -323,7 +324,7 @@ func TestBackendStatNotExists(t *testing.T) {
TestFastRetries(t)
retryBackend := New(be, 10, nil, nil)
_, err := retryBackend.Stat(context.TODO(), restic.Handle{})
_, err := retryBackend.Stat(context.TODO(), backend.Handle{})
test.Assert(t, be.IsNotExistFn(err), "unexpected error %v", err)
test.Equals(t, 1, attempt)
}
@@ -337,7 +338,7 @@ func TestBackendCanceledContext(t *testing.T) {
// check that we received the expected context canceled error instead
TestFastRetries(t)
retryBackend := New(mock.NewBackend(), 2, nil, nil)
h := restic.Handle{Type: restic.PackFile, Name: restic.NewRandomID().String()}
h := backend.Handle{Type: backend.PackFile, Name: restic.NewRandomID().String()}
// create an already canceled context
ctx, cancel := context.WithCancel(context.Background())
@@ -346,15 +347,15 @@ func TestBackendCanceledContext(t *testing.T) {
_, err := retryBackend.Stat(ctx, h)
assertIsCanceled(t, err)
err = retryBackend.Save(ctx, h, restic.NewByteReader([]byte{}, nil))
err = retryBackend.Save(ctx, h, backend.NewByteReader([]byte{}, nil))
assertIsCanceled(t, err)
err = retryBackend.Remove(ctx, h)
assertIsCanceled(t, err)
err = retryBackend.Load(ctx, restic.Handle{}, 0, 0, func(rd io.Reader) (err error) {
err = retryBackend.Load(ctx, backend.Handle{}, 0, 0, func(rd io.Reader) (err error) {
return nil
})
assertIsCanceled(t, err)
err = retryBackend.List(ctx, restic.PackFile, func(restic.FileInfo) error {
err = retryBackend.List(ctx, backend.PackFile, func(backend.FileInfo) error {
return nil
})
assertIsCanceled(t, err)
internal/backend/rewind_reader.go
+117
View File
@@ -0,0 +1,117 @@
package backend
import (
"bytes"
"hash"
"io"
"github.com/restic/restic/internal/errors"
)
// RewindReader allows resetting the Reader to the beginning of the data.
type RewindReader interface {
io.Reader
// Rewind rewinds the reader so the same data can be read again from the
// start.
Rewind() error
// Length returns the number of bytes that can be read from the Reader
// after calling Rewind.
Length() int64
// Hash return a hash of the data if requested by the backed.
Hash() []byte
}
// ByteReader implements a RewindReader for a byte slice.
type ByteReader struct {
*bytes.Reader
Len int64
hash []byte
}
// Rewind restarts the reader from the beginning of the data.
func (b *ByteReader) Rewind() error {
_, err := b.Reader.Seek(0, io.SeekStart)
return err
}
// Length returns the number of bytes read from the reader after Rewind is
// called.
func (b *ByteReader) Length() int64 {
return b.Len
}
// Hash return a hash of the data if requested by the backed.
func (b *ByteReader) Hash() []byte {
return b.hash
}
// statically ensure that *ByteReader implements RewindReader.
var _ RewindReader = &ByteReader{}
// NewByteReader prepares a ByteReader that can then be used to read buf.
func NewByteReader(buf []byte, hasher hash.Hash) *ByteReader {
var hash []byte
if hasher != nil {
// must never fail according to interface
_, err := hasher.Write(buf)
if err != nil {
panic(err)
}
hash = hasher.Sum(nil)
}
return &ByteReader{
Reader: bytes.NewReader(buf),
Len: int64(len(buf)),
hash: hash,
}
}
// statically ensure that *FileReader implements RewindReader.
var _ RewindReader = &FileReader{}
// FileReader implements a RewindReader for an open file.
type FileReader struct {
io.ReadSeeker
Len int64
hash []byte
}
// Rewind seeks to the beginning of the file.
func (f *FileReader) Rewind() error {
_, err := f.ReadSeeker.Seek(0, io.SeekStart)
return errors.Wrap(err, "Seek")
}
// Length returns the length of the file.
func (f *FileReader) Length() int64 {
return f.Len
}
// Hash return a hash of the data if requested by the backed.
func (f *FileReader) Hash() []byte {
return f.hash
}
// NewFileReader wraps f in a *FileReader.
func NewFileReader(f io.ReadSeeker, hash []byte) (*FileReader, error) {
pos, err := f.Seek(0, io.SeekEnd)
if err != nil {
return nil, errors.Wrap(err, "Seek")
}
fr := &FileReader{
ReadSeeker: f,
Len: pos,
hash: hash,
}
err = fr.Rewind()
if err != nil {
return nil, err
}
return fr, nil
}
internal/backend/rewind_reader_test.go
+175
View File
@@ -0,0 +1,175 @@
package backend
import (
"bytes"
"crypto/md5"
"hash"
"io"
"math/rand"
"os"
"path/filepath"
"testing"
"time"
"github.com/restic/restic/internal/test"
)
func TestByteReader(t *testing.T) {
buf := []byte("foobar")
for _, hasher := range []hash.Hash{nil, md5.New()} {
fn := func() RewindReader {
return NewByteReader(buf, hasher)
}
testRewindReader(t, fn, buf)
}
}
func TestFileReader(t *testing.T) {
buf := []byte("foobar")
d := test.TempDir(t)
filename := filepath.Join(d, "file-reader-test")
err := os.WriteFile(filename, buf, 0600)
if err != nil {
t.Fatal(err)
}
f, err := os.Open(filename)
if err != nil {
t.Fatal(err)
}
defer func() {
err := f.Close()
if err != nil {
t.Fatal(err)
}
}()
for _, hasher := range []hash.Hash{nil, md5.New()} {
fn := func() RewindReader {
var hash []byte
if hasher != nil {
// must never fail according to interface
_, err := hasher.Write(buf)
if err != nil {
panic(err)
}
hash = hasher.Sum(nil)
}
rd, err := NewFileReader(f, hash)
if err != nil {
t.Fatal(err)
}
return rd
}
testRewindReader(t, fn, buf)
}
}
func testRewindReader(t *testing.T, fn func() RewindReader, data []byte) {
seed := time.Now().UnixNano()
t.Logf("seed is %d", seed)
rnd := rand.New(rand.NewSource(seed))
type ReaderTestFunc func(t testing.TB, r RewindReader, data []byte)
var tests = []ReaderTestFunc{
func(t testing.TB, rd RewindReader, data []byte) {
if rd.Length() != int64(len(data)) {
t.Fatalf("wrong length returned, want %d, got %d", int64(len(data)), rd.Length())
}
buf := make([]byte, len(data))
_, err := io.ReadFull(rd, buf)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(buf, data) {
t.Fatalf("wrong data returned")
}
if rd.Length() != int64(len(data)) {
t.Fatalf("wrong length returned, want %d, got %d", int64(len(data)), rd.Length())
}
err = rd.Rewind()
if err != nil {
t.Fatal(err)
}
if rd.Length() != int64(len(data)) {
t.Fatalf("wrong length returned, want %d, got %d", int64(len(data)), rd.Length())
}
buf2 := make([]byte, int64(len(data)))
_, err = io.ReadFull(rd, buf2)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(buf2, data) {
t.Fatalf("wrong data returned")
}
if rd.Length() != int64(len(data)) {
t.Fatalf("wrong length returned, want %d, got %d", int64(len(data)), rd.Length())
}
if rd.Hash() != nil {
hasher := md5.New()
// must never fail according to interface
_, _ = hasher.Write(buf2)
if !bytes.Equal(rd.Hash(), hasher.Sum(nil)) {
t.Fatal("hash does not match data")
}
}
},
func(t testing.TB, rd RewindReader, data []byte) {
// read first bytes
buf := make([]byte, rnd.Intn(len(data)))
_, err := io.ReadFull(rd, buf)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(buf, data[:len(buf)]) {
t.Fatalf("wrong data returned")
}
err = rd.Rewind()
if err != nil {
t.Fatal(err)
}
buf2 := make([]byte, rnd.Intn(len(data)))
_, err = io.ReadFull(rd, buf2)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(buf2, data[:len(buf2)]) {
t.Fatalf("wrong data returned")
}
// read remainder
buf3 := make([]byte, len(data)-len(buf2))
_, err = io.ReadFull(rd, buf3)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(buf3, data[len(buf2):]) {
t.Fatalf("wrong data returned")
}
},
}
for _, test := range tests {
t.Run("", func(t *testing.T) {
rd := fn()
test(t, rd, data)
})
}
}
internal/backend/s3/config.go
+2 -2
View File
@@ -6,9 +6,9 @@ import (
"path"
"strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
)
// Config contains all configuration necessary to connect to an s3 compatible
@@ -94,7 +94,7 @@ func createConfig(endpoint, bucket, prefix string, useHTTP bool) (*Config, error
return &cfg, nil
}
var _ restic.ApplyEnvironmenter = &Config{}
var _ backend.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) {
internal/backend/s3/s3.go
+15 -15
View File
@@ -11,12 +11,12 @@ import (
"strings"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
@@ -30,7 +30,7 @@ type Backend struct {
}
// make sure that *Backend implements backend.Backend
var _ restic.Backend = &Backend{}
var _ backend.Backend = &Backend{}
func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("s3", ParseConfig, location.NoPassword, Create, Open)
@@ -127,13 +127,13 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro
// Open opens the S3 backend at bucket and region. The bucket is created if it
// does not exist yet.
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
return open(ctx, cfg, rt)
}
// Create opens the S3 backend at bucket and region and creates the bucket if
// it does not exist yet.
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
be, err := open(ctx, cfg, rt)
if err != nil {
return nil, errors.Wrap(err, "open")
@@ -272,7 +272,7 @@ func (be *Backend) Path() string {
}
// Save stores data in the backend at the handle.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
objName := be.Filename(h)
opts := minio.PutObjectOptions{StorageClass: be.cfg.StorageClass}
@@ -294,14 +294,14 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
objName := be.Filename(h)
opts := minio.GetObjectOptions{}
@@ -326,7 +326,7 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int,
}
// Stat returns information about a blob.
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
objName := be.Filename(h)
var obj *minio.Object
@@ -334,7 +334,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf
obj, err = be.client.GetObject(ctx, be.cfg.Bucket, objName, opts)
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "client.GetObject")
return backend.FileInfo{}, errors.Wrap(err, "client.GetObject")
}
// make sure that the object is closed properly.
@@ -347,14 +347,14 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf
fi, err := obj.Stat()
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "Stat")
return backend.FileInfo{}, errors.Wrap(err, "Stat")
}
return restic.FileInfo{Size: fi.Size, Name: h.Name}, nil
return backend.FileInfo{Size: fi.Size, Name: h.Name}, nil
}
// Remove removes the blob with the given name and type.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
objName := be.Filename(h)
err := be.client.RemoveObject(ctx, be.cfg.Bucket, objName, minio.RemoveObjectOptions{})
@@ -368,7 +368,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
prefix, recursive := be.Basedir(t)
// make sure prefix ends with a slash
@@ -400,7 +400,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
continue
}
fi := restic.FileInfo{
fi := backend.FileInfo{
Name: path.Base(m),
Size: obj.Size,
}
@@ -431,7 +431,7 @@ func (be *Backend) Delete(ctx context.Context) error {
func (be *Backend) Close() error { return nil }
// Rename moves a file based on the new layout l.
func (be *Backend) Rename(ctx context.Context, h restic.Handle, l layout.Layout) error {
func (be *Backend) Rename(ctx context.Context, h backend.Handle, l layout.Layout) error {
debug.Log("Rename %v to %v", h, l)
oldname := be.Filename(h)
newname := l.Filename(h)
internal/backend/s3/s3_test.go
+2 -2
View File
@@ -14,11 +14,11 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/s3"
"github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@@ -117,7 +117,7 @@ func newMinioTestSuite(t testing.TB) (*test.Suite[s3.Config], func()) {
return &cfg, nil
},
Factory: location.NewHTTPBackendFactory("s3", s3.ParseConfig, location.NoPassword, func(ctx context.Context, cfg s3.Config, rt http.RoundTripper) (be restic.Backend, err error) {
Factory: location.NewHTTPBackendFactory("s3", s3.ParseConfig, location.NoPassword, func(ctx context.Context, cfg s3.Config, rt http.RoundTripper) (be backend.Backend, err error) {
for i := 0; i < 10; i++ {
be, err = s3.Create(ctx, cfg, rt)
if err != nil {
internal/backend/sema/backend.go
+14 -14
View File
@@ -6,22 +6,22 @@ import (
"sync"
"github.com/cenkalti/backoff/v4"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
)
// make sure that connectionLimitedBackend implements restic.Backend
var _ restic.Backend = &connectionLimitedBackend{}
// make sure that connectionLimitedBackend implements backend.Backend
var _ backend.Backend = &connectionLimitedBackend{}
// connectionLimitedBackend limits the number of concurrent operations.
type connectionLimitedBackend struct {
restic.Backend
backend.Backend
sem semaphore
freezeLock sync.Mutex
}
// NewBackend creates a backend that limits the concurrent operations on the underlying backend
func NewBackend(be restic.Backend) restic.Backend {
func NewBackend(be backend.Backend) backend.Backend {
sem, err := newSemaphore(be.Connections())
if err != nil {
panic(err)
@@ -35,9 +35,9 @@ func NewBackend(be restic.Backend) restic.Backend {
// typeDependentLimit acquire a token unless the FileType is a lock file. The returned function
// must be called to release the token.
func (be *connectionLimitedBackend) typeDependentLimit(t restic.FileType) func() {
func (be *connectionLimitedBackend) typeDependentLimit(t backend.FileType) func() {
// allow concurrent lock file operations to ensure that the lock refresh is always possible
if t == restic.LockFile {
if t == backend.LockFile {
return func() {}
}
be.sem.GetToken()
@@ -59,7 +59,7 @@ func (be *connectionLimitedBackend) Unfreeze() {
}
// Save adds new Data to the backend.
func (be *connectionLimitedBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *connectionLimitedBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if err := h.Valid(); err != nil {
return backoff.Permanent(err)
}
@@ -75,7 +75,7 @@ func (be *connectionLimitedBackend) Save(ctx context.Context, h restic.Handle, r
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *connectionLimitedBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *connectionLimitedBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
if err := h.Valid(); err != nil {
return backoff.Permanent(err)
}
@@ -96,22 +96,22 @@ func (be *connectionLimitedBackend) Load(ctx context.Context, h restic.Handle, l
}
// Stat returns information about a file in the backend.
func (be *connectionLimitedBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
func (be *connectionLimitedBackend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
if err := h.Valid(); err != nil {
return restic.FileInfo{}, backoff.Permanent(err)
return backend.FileInfo{}, backoff.Permanent(err)
}
defer be.typeDependentLimit(h.Type)()
if ctx.Err() != nil {
return restic.FileInfo{}, ctx.Err()
return backend.FileInfo{}, ctx.Err()
}
return be.Backend.Stat(ctx, h)
}
// Remove deletes a file from the backend.
func (be *connectionLimitedBackend) Remove(ctx context.Context, h restic.Handle) error {
func (be *connectionLimitedBackend) Remove(ctx context.Context, h backend.Handle) error {
if err := h.Valid(); err != nil {
return backoff.Permanent(err)
}
@@ -125,6 +125,6 @@ func (be *connectionLimitedBackend) Remove(ctx context.Context, h restic.Handle)
return be.Backend.Remove(ctx, h)
}
func (be *connectionLimitedBackend) Unwrap() restic.Backend {
func (be *connectionLimitedBackend) Unwrap() backend.Backend {
return be.Backend
}
internal/backend/sema/backend_test.go
+32 -32
View File
@@ -8,37 +8,37 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/mock"
"github.com/restic/restic/internal/backend/sema"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
"golang.org/x/sync/errgroup"
)
func TestParameterValidationSave(t *testing.T) {
m := mock.NewBackend()
m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
return nil
}
be := sema.NewBackend(m)
err := be.Save(context.TODO(), restic.Handle{}, nil)
err := be.Save(context.TODO(), backend.Handle{}, nil)
test.Assert(t, err != nil, "Save() with invalid handle did not return an error")
}
func TestParameterValidationLoad(t *testing.T) {
m := mock.NewBackend()
m.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
m.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
return io.NopCloser(nil), nil
}
be := sema.NewBackend(m)
nilCb := func(rd io.Reader) error { return nil }
err := be.Load(context.TODO(), restic.Handle{}, 10, 0, nilCb)
err := be.Load(context.TODO(), backend.Handle{}, 10, 0, nilCb)
test.Assert(t, err != nil, "Load() with invalid handle did not return an error")
h := restic.Handle{Type: restic.PackFile, Name: "foobar"}
h := backend.Handle{Type: backend.PackFile, Name: "foobar"}
err = be.Load(context.TODO(), h, 10, -1, nilCb)
test.Assert(t, err != nil, "Save() with negative offset did not return an error")
err = be.Load(context.TODO(), h, -1, 0, nilCb)
@@ -47,23 +47,23 @@ func TestParameterValidationLoad(t *testing.T) {
func TestParameterValidationStat(t *testing.T) {
m := mock.NewBackend()
m.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
return restic.FileInfo{}, nil
m.StatFn = func(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
return backend.FileInfo{}, nil
}
be := sema.NewBackend(m)
_, err := be.Stat(context.TODO(), restic.Handle{})
_, err := be.Stat(context.TODO(), backend.Handle{})
test.Assert(t, err != nil, "Stat() with invalid handle did not return an error")
}
func TestParameterValidationRemove(t *testing.T) {
m := mock.NewBackend()
m.RemoveFn = func(ctx context.Context, h restic.Handle) error {
m.RemoveFn = func(ctx context.Context, h backend.Handle) error {
return nil
}
be := sema.NewBackend(m)
err := be.Remove(context.TODO(), restic.Handle{})
err := be.Remove(context.TODO(), backend.Handle{})
test.Assert(t, err != nil, "Remove() with invalid handle did not return an error")
}
@@ -71,7 +71,7 @@ func TestUnwrap(t *testing.T) {
m := mock.NewBackend()
be := sema.NewBackend(m)
unwrapper := be.(restic.BackendUnwrapper)
unwrapper := be.(backend.Unwrapper)
test.Assert(t, unwrapper.Unwrap() == m, "Unwrap() returned wrong backend")
}
@@ -100,7 +100,7 @@ func countingBlocker() (func(), func(int) int) {
return wait, unblock
}
func concurrencyTester(t *testing.T, setup func(m *mock.Backend), handler func(be restic.Backend) func() error, unblock func(int) int, isUnlimited bool) {
func concurrencyTester(t *testing.T, setup func(m *mock.Backend), handler func(be backend.Backend) func() error, unblock func(int) int, isUnlimited bool) {
expectBlocked := int(2)
workerCount := expectBlocked + 1
@@ -125,13 +125,13 @@ func concurrencyTester(t *testing.T, setup func(m *mock.Backend), handler func(b
func TestConcurrencyLimitSave(t *testing.T) {
wait, unblock := countingBlocker()
concurrencyTester(t, func(m *mock.Backend) {
m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
wait()
return nil
}
}, func(be restic.Backend) func() error {
}, func(be backend.Backend) func() error {
return func() error {
h := restic.Handle{Type: restic.PackFile, Name: "foobar"}
h := backend.Handle{Type: backend.PackFile, Name: "foobar"}
return be.Save(context.TODO(), h, nil)
}
}, unblock, false)
@@ -140,13 +140,13 @@ func TestConcurrencyLimitSave(t *testing.T) {
func TestConcurrencyLimitLoad(t *testing.T) {
wait, unblock := countingBlocker()
concurrencyTester(t, func(m *mock.Backend) {
m.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
m.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
wait()
return io.NopCloser(nil), nil
}
}, func(be restic.Backend) func() error {
}, func(be backend.Backend) func() error {
return func() error {
h := restic.Handle{Type: restic.PackFile, Name: "foobar"}
h := backend.Handle{Type: backend.PackFile, Name: "foobar"}
nilCb := func(rd io.Reader) error { return nil }
return be.Load(context.TODO(), h, 10, 0, nilCb)
}
@@ -156,13 +156,13 @@ func TestConcurrencyLimitLoad(t *testing.T) {
func TestConcurrencyLimitStat(t *testing.T) {
wait, unblock := countingBlocker()
concurrencyTester(t, func(m *mock.Backend) {
m.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
m.StatFn = func(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
wait()
return restic.FileInfo{}, nil
return backend.FileInfo{}, nil
}
}, func(be restic.Backend) func() error {
}, func(be backend.Backend) func() error {
return func() error {
h := restic.Handle{Type: restic.PackFile, Name: "foobar"}
h := backend.Handle{Type: backend.PackFile, Name: "foobar"}
_, err := be.Stat(context.TODO(), h)
return err
}
@@ -172,13 +172,13 @@ func TestConcurrencyLimitStat(t *testing.T) {
func TestConcurrencyLimitDelete(t *testing.T) {
wait, unblock := countingBlocker()
concurrencyTester(t, func(m *mock.Backend) {
m.RemoveFn = func(ctx context.Context, h restic.Handle) error {
m.RemoveFn = func(ctx context.Context, h backend.Handle) error {
wait()
return nil
}
}, func(be restic.Backend) func() error {
}, func(be backend.Backend) func() error {
return func() error {
h := restic.Handle{Type: restic.PackFile, Name: "foobar"}
h := backend.Handle{Type: backend.PackFile, Name: "foobar"}
return be.Remove(context.TODO(), h)
}
}, unblock, false)
@@ -187,13 +187,13 @@ func TestConcurrencyLimitDelete(t *testing.T) {
func TestConcurrencyUnlimitedLockSave(t *testing.T) {
wait, unblock := countingBlocker()
concurrencyTester(t, func(m *mock.Backend) {
m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
wait()
return nil
}
}, func(be restic.Backend) func() error {
}, func(be backend.Backend) func() error {
return func() error {
h := restic.Handle{Type: restic.LockFile, Name: "foobar"}
h := backend.Handle{Type: backend.LockFile, Name: "foobar"}
return be.Save(context.TODO(), h, nil)
}
}, unblock, true)
@@ -202,13 +202,13 @@ func TestConcurrencyUnlimitedLockSave(t *testing.T) {
func TestFreeze(t *testing.T) {
var counter int64
m := mock.NewBackend()
m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
atomic.AddInt64(&counter, 1)
return nil
}
m.ConnectionsFn = func() uint { return 2 }
be := sema.NewBackend(m)
fb := be.(restic.FreezeBackend)
fb := be.(backend.FreezeBackend)
// Freeze backend
fb.Freeze()
@@ -218,7 +218,7 @@ func TestFreeze(t *testing.T) {
wg.Add(1)
go func() {
defer wg.Done()
h := restic.Handle{Type: restic.PackFile, Name: "foobar"}
h := backend.Handle{Type: backend.PackFile, Name: "foobar"}
test.OK(t, be.Save(context.TODO(), h, nil))
}()
internal/backend/sftp/layout_test.go
+2 -2
View File
@@ -6,8 +6,8 @@ import (
"path/filepath"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/sftp"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@@ -56,7 +56,7 @@ func TestLayout(t *testing.T) {
}
packs := make(map[string]bool)
err = be.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
err = be.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
packs[fi.Name] = false
return nil
})
internal/backend/sftp/sftp.go
+13 -14
View File
@@ -20,7 +20,6 @@ import (
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/cenkalti/backoff/v4"
"github.com/pkg/sftp"
@@ -42,7 +41,7 @@ type SFTP struct {
util.Modes
}
var _ restic.Backend = &SFTP{}
var _ backend.Backend = &SFTP{}
func NewFactory() location.Factory {
return location.NewLimitedBackendFactory("sftp", ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open))
@@ -153,7 +152,7 @@ func open(ctx context.Context, sftp *SFTP, cfg Config) (*SFTP, error) {
debug.Log("layout: %v\n", sftp.Layout)
fi, err := sftp.c.Stat(sftp.Layout.Filename(restic.Handle{Type: restic.ConfigFile}))
fi, err := sftp.c.Stat(sftp.Layout.Filename(backend.Handle{Type: backend.ConfigFile}))
m := util.DeriveModesFromFileInfo(fi, err)
debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir)
@@ -263,7 +262,7 @@ func Create(ctx context.Context, cfg Config) (*SFTP, error) {
sftp.Modes = util.DefaultModes
// test if config file already exists
_, err = sftp.c.Lstat(sftp.Layout.Filename(restic.Handle{Type: restic.ConfigFile}))
_, err = sftp.c.Lstat(sftp.Layout.Filename(backend.Handle{Type: backend.ConfigFile}))
if err == nil {
return nil, errors.New("config file already exists")
}
@@ -314,7 +313,7 @@ func tempSuffix() string {
}
// Save stores data in the backend at the handle.
func (r *SFTP) Save(_ context.Context, h restic.Handle, rd restic.RewindReader) error {
func (r *SFTP) Save(_ context.Context, h backend.Handle, rd backend.RewindReader) error {
if err := r.clientError(); err != nil {
return err
}
@@ -414,11 +413,11 @@ func (r *SFTP) checkNoSpace(dir string, size int64, origErr error) error {
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (r *SFTP) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (r *SFTP) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return util.DefaultLoad(ctx, h, length, offset, r.openReader, fn)
}
func (r *SFTP) openReader(_ context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (r *SFTP) openReader(_ context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
f, err := r.c.Open(r.Filename(h))
if err != nil {
return nil, err
@@ -442,21 +441,21 @@ func (r *SFTP) openReader(_ context.Context, h restic.Handle, length int, offset
}
// Stat returns information about a blob.
func (r *SFTP) Stat(_ context.Context, h restic.Handle) (restic.FileInfo, error) {
func (r *SFTP) Stat(_ context.Context, h backend.Handle) (backend.FileInfo, error) {
if err := r.clientError(); err != nil {
return restic.FileInfo{}, err
return backend.FileInfo{}, err
}
fi, err := r.c.Lstat(r.Filename(h))
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "Lstat")
return backend.FileInfo{}, errors.Wrap(err, "Lstat")
}
return restic.FileInfo{Size: fi.Size(), Name: h.Name}, nil
return backend.FileInfo{Size: fi.Size(), Name: h.Name}, nil
}
// Remove removes the content stored at name.
func (r *SFTP) Remove(_ context.Context, h restic.Handle) error {
func (r *SFTP) Remove(_ context.Context, h backend.Handle) error {
if err := r.clientError(); err != nil {
return err
}
@@ -466,7 +465,7 @@ func (r *SFTP) Remove(_ context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (r *SFTP) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (r *SFTP) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
basedir, subdirs := r.Basedir(t)
walker := r.c.Walk(basedir)
for {
@@ -499,7 +498,7 @@ func (r *SFTP) List(ctx context.Context, t restic.FileType, fn func(restic.FileI
debug.Log("send %v\n", path.Base(walker.Path()))
rfi := restic.FileInfo{
rfi := backend.FileInfo{
Name: path.Base(walker.Path()),
Size: fi.Size(),
}
internal/backend/swift/config.go
+2 -2
View File
@@ -4,9 +4,9 @@ import (
"os"
"strings"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
)
// Config contains basic configuration needed to specify swift location for a swift server
@@ -74,7 +74,7 @@ func ParseConfig(s string) (*Config, error) {
return &cfg, nil
}
var _ restic.ApplyEnvironmenter = &Config{}
var _ backend.ApplyEnvironmenter = &Config{}
// ApplyEnvironment saves values from the environment to the config.
func (cfg *Config) ApplyEnvironment(prefix string) {
internal/backend/swift/swift.go
+13 -13
View File
@@ -13,12 +13,12 @@ import (
"strings"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/ncw/swift/v2"
)
@@ -32,8 +32,8 @@ type beSwift struct {
layout.Layout
}
// ensure statically that *beSwift implements restic.Backend.
var _ restic.Backend = &beSwift{}
// ensure statically that *beSwift implements backend.Backend.
var _ backend.Backend = &beSwift{}
func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("swift", ParseConfig, location.NoPassword, Open, Open)
@@ -41,7 +41,7 @@ func NewFactory() location.Factory {
// Open opens the swift backend at a container in region. The container is
// created if it does not exist yet.
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) {
debug.Log("config %#v", cfg)
be := &beSwift{
@@ -134,11 +134,11 @@ func (be *beSwift) HasAtomicReplace() bool {
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *beSwift) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *beSwift) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *beSwift) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
func (be *beSwift) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
objName := be.Filename(h)
@@ -160,7 +160,7 @@ func (be *beSwift) openReader(ctx context.Context, h restic.Handle, length int,
}
// Save stores data in the backend at the handle.
func (be *beSwift) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
func (be *beSwift) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
objName := be.Filename(h)
encoding := "binary/octet-stream"
@@ -174,19 +174,19 @@ func (be *beSwift) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
}
// Stat returns information about a blob.
func (be *beSwift) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
func (be *beSwift) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
objName := be.Filename(h)
obj, _, err := be.conn.Object(ctx, be.container, objName)
if err != nil {
return restic.FileInfo{}, errors.Wrap(err, "conn.Object")
return backend.FileInfo{}, errors.Wrap(err, "conn.Object")
}
return restic.FileInfo{Size: obj.Bytes, Name: h.Name}, nil
return backend.FileInfo{Size: obj.Bytes, Name: h.Name}, nil
}
// Remove removes the blob with the given name and type.
func (be *beSwift) Remove(ctx context.Context, h restic.Handle) error {
func (be *beSwift) Remove(ctx context.Context, h backend.Handle) error {
objName := be.Filename(h)
err := be.conn.ObjectDelete(ctx, be.container, objName)
@@ -195,7 +195,7 @@ func (be *beSwift) Remove(ctx context.Context, h restic.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (be *beSwift) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (be *beSwift) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
prefix, _ := be.Basedir(t)
prefix += "/"
@@ -212,7 +212,7 @@ func (be *beSwift) List(ctx context.Context, t restic.FileType, fn func(restic.F
continue
}
fi := restic.FileInfo{
fi := backend.FileInfo{
Name: m,
Size: obj.Bytes,
}
internal/backend/swift/swift_test.go
+2 -2
View File
@@ -6,9 +6,9 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/swift"
"github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@@ -20,7 +20,7 @@ func newSwiftTestSuite(t testing.TB) *test.Suite[swift.Config] {
// wait for removals for at least 5m
WaitForDelayedRemoval: 5 * time.Minute,
ErrorHandler: func(t testing.TB, be restic.Backend, err error) error {
ErrorHandler: func(t testing.TB, be backend.Backend, err error) error {
if err == nil {
return nil
}
internal/backend/test/benchmarks.go
+7 -6
View File
@@ -6,22 +6,23 @@ import (
"io"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
func saveRandomFile(t testing.TB, be restic.Backend, length int) ([]byte, restic.Handle) {
func saveRandomFile(t testing.TB, be backend.Backend, length int) ([]byte, backend.Handle) {
data := test.Random(23, length)
id := restic.Hash(data)
handle := restic.Handle{Type: restic.PackFile, Name: id.String()}
err := be.Save(context.TODO(), handle, restic.NewByteReader(data, be.Hasher()))
handle := backend.Handle{Type: backend.PackFile, Name: id.String()}
err := be.Save(context.TODO(), handle, backend.NewByteReader(data, be.Hasher()))
if err != nil {
t.Fatalf("Save() error: %+v", err)
}
return data, handle
}
func remove(t testing.TB, be restic.Backend, h restic.Handle) {
func remove(t testing.TB, be backend.Backend, h backend.Handle) {
if err := be.Remove(context.TODO(), h); err != nil {
t.Fatalf("Remove() returned error: %v", err)
}
@@ -146,9 +147,9 @@ func (s *Suite[C]) BenchmarkSave(t *testing.B) {
length := 1<<24 + 2123
data := test.Random(23, length)
id := restic.Hash(data)
handle := restic.Handle{Type: restic.PackFile, Name: id.String()}
handle := backend.Handle{Type: backend.PackFile, Name: id.String()}
rd := restic.NewByteReader(data, be.Hasher())
rd := backend.NewByteReader(data, be.Hasher())
t.SetBytes(int64(length))
t.ResetTimer()
internal/backend/test/doc.go
+1 -1
View File
@@ -17,7 +17,7 @@
//
// func newTestSuite(t testing.TB) *test.Suite {
// return &test.Suite{
// Create: func(cfg interface{}) (restic.Backend, error) {
// Create: func(cfg interface{}) (backend.Backend, error) {
// [...]
// },
// [...]
internal/backend/test/suite.go
+6 -7
View File
@@ -11,7 +11,6 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
@@ -35,7 +34,7 @@ type Suite[C any] struct {
WaitForDelayedRemoval time.Duration
// ErrorHandler allows ignoring certain errors.
ErrorHandler func(testing.TB, restic.Backend, error) error
ErrorHandler func(testing.TB, backend.Backend, error) error
}
// RunTests executes all defined tests as subtests of t.
@@ -156,7 +155,7 @@ func (s *Suite[C]) RunBenchmarks(b *testing.B) {
s.cleanup(b)
}
func (s *Suite[C]) createOrError() (restic.Backend, error) {
func (s *Suite[C]) createOrError() (backend.Backend, error) {
tr, err := backend.Transport(backend.TransportOptions{})
if err != nil {
return nil, fmt.Errorf("cannot create transport for tests: %v", err)
@@ -167,7 +166,7 @@ func (s *Suite[C]) createOrError() (restic.Backend, error) {
return nil, err
}
_, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
_, err = be.Stat(context.TODO(), backend.Handle{Type: backend.ConfigFile})
if err != nil && !be.IsNotExist(err) {
return nil, err
}
@@ -179,7 +178,7 @@ func (s *Suite[C]) createOrError() (restic.Backend, error) {
return be, nil
}
func (s *Suite[C]) create(t testing.TB) restic.Backend {
func (s *Suite[C]) create(t testing.TB) backend.Backend {
be, err := s.createOrError()
if err != nil {
t.Fatal(err)
@@ -187,7 +186,7 @@ func (s *Suite[C]) create(t testing.TB) restic.Backend {
return be
}
func (s *Suite[C]) open(t testing.TB) restic.Backend {
func (s *Suite[C]) open(t testing.TB) backend.Backend {
tr, err := backend.Transport(backend.TransportOptions{})
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
@@ -208,7 +207,7 @@ func (s *Suite[C]) cleanup(t testing.TB) {
s.close(t, be)
}
func (s *Suite[C]) close(t testing.TB, be restic.Backend) {
func (s *Suite[C]) close(t testing.TB, be backend.Backend) {
err := be.Close()
if err != nil {
t.Fatal(err)
internal/backend/test/tests.go
+50 -50
View File
@@ -27,7 +27,7 @@ func seedRand(t testing.TB) {
t.Logf("rand initialized with seed %d", seed)
}
func beTest(ctx context.Context, be restic.Backend, h restic.Handle) (bool, error) {
func beTest(ctx context.Context, be backend.Backend, h backend.Handle) (bool, error) {
_, err := be.Stat(ctx, h)
if err != nil && be.IsNotExist(err) {
return false, nil
@@ -49,7 +49,7 @@ func (s *Suite[C]) TestCreateWithConfig(t *testing.T) {
defer s.close(t, b)
// remove a config if present
cfgHandle := restic.Handle{Type: restic.ConfigFile}
cfgHandle := backend.Handle{Type: backend.ConfigFile}
cfgPresent, err := beTest(context.TODO(), b, cfgHandle)
if err != nil {
t.Fatalf("unable to test for config: %+v", err)
@@ -60,7 +60,7 @@ func (s *Suite[C]) TestCreateWithConfig(t *testing.T) {
}
// save a config
store(t, b, restic.ConfigFile, []byte("test config"))
store(t, b, backend.ConfigFile, []byte("test config"))
// now create the backend again, this must fail
_, err = s.createOrError()
@@ -69,7 +69,7 @@ func (s *Suite[C]) TestCreateWithConfig(t *testing.T) {
}
// remove config
err = b.Remove(context.TODO(), restic.Handle{Type: restic.ConfigFile, Name: ""})
err = b.Remove(context.TODO(), backend.Handle{Type: backend.ConfigFile, Name: ""})
if err != nil {
t.Fatalf("unexpected error removing config: %+v", err)
}
@@ -94,13 +94,13 @@ func (s *Suite[C]) TestConfig(t *testing.T) {
var testString = "Config"
// create config and read it back
_, err := backend.LoadAll(context.TODO(), nil, b, restic.Handle{Type: restic.ConfigFile})
_, err := backend.LoadAll(context.TODO(), nil, b, backend.Handle{Type: backend.ConfigFile})
if err == nil {
t.Fatalf("did not get expected error for non-existing config")
}
test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize error from LoadAll(): %v", err)
err = b.Save(context.TODO(), restic.Handle{Type: restic.ConfigFile}, restic.NewByteReader([]byte(testString), b.Hasher()))
err = b.Save(context.TODO(), backend.Handle{Type: backend.ConfigFile}, backend.NewByteReader([]byte(testString), b.Hasher()))
if err != nil {
t.Fatalf("Save() error: %+v", err)
}
@@ -108,7 +108,7 @@ func (s *Suite[C]) TestConfig(t *testing.T) {
// try accessing the config with different names, should all return the
// same config
for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} {
h := restic.Handle{Type: restic.ConfigFile, Name: name}
h := backend.Handle{Type: backend.ConfigFile, Name: name}
buf, err := backend.LoadAll(context.TODO(), nil, b, h)
if err != nil {
t.Fatalf("unable to read config with name %q: %+v", name, err)
@@ -120,7 +120,7 @@ func (s *Suite[C]) TestConfig(t *testing.T) {
}
// remove the config
remove(t, b, restic.Handle{Type: restic.ConfigFile})
remove(t, b, backend.Handle{Type: backend.ConfigFile})
}
// TestLoad tests the backend's Load function.
@@ -130,7 +130,7 @@ func (s *Suite[C]) TestLoad(t *testing.T) {
b := s.open(t)
defer s.close(t, b)
err := testLoad(b, restic.Handle{Type: restic.PackFile, Name: "foobar"})
err := testLoad(b, backend.Handle{Type: backend.PackFile, Name: "foobar"})
if err == nil {
t.Fatalf("Load() did not return an error for non-existing blob")
}
@@ -141,8 +141,8 @@ func (s *Suite[C]) TestLoad(t *testing.T) {
data := test.Random(23, length)
id := restic.Hash(data)
handle := restic.Handle{Type: restic.PackFile, Name: id.String()}
err = b.Save(context.TODO(), handle, restic.NewByteReader(data, b.Hasher()))
handle := backend.Handle{Type: backend.PackFile, Name: id.String()}
err = b.Save(context.TODO(), handle, backend.NewByteReader(data, b.Hasher()))
if err != nil {
t.Fatalf("Save() error: %+v", err)
}
@@ -243,7 +243,7 @@ func (s *Suite[C]) TestList(t *testing.T) {
// Check that the backend is empty to start with
var found []string
err := b.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
err := b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
found = append(found, fi.Name)
return nil
})
@@ -259,8 +259,8 @@ func (s *Suite[C]) TestList(t *testing.T) {
for i := 0; i < numTestFiles; i++ {
data := test.Random(rand.Int(), rand.Intn(100)+55)
id := restic.Hash(data)
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher()))
h := backend.Handle{Type: backend.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher()))
if err != nil {
t.Fatal(err)
}
@@ -284,7 +284,7 @@ func (s *Suite[C]) TestList(t *testing.T) {
s.SetListMaxItems(test.maxItems)
}
err := b.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
err := b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
id, err := restic.ParseID(fi.Name)
if err != nil {
t.Fatal(err)
@@ -320,9 +320,9 @@ func (s *Suite[C]) TestList(t *testing.T) {
}
t.Logf("remove %d files", numTestFiles)
handles := make([]restic.Handle, 0, len(list1))
handles := make([]backend.Handle, 0, len(list1))
for id := range list1 {
handles = append(handles, restic.Handle{Type: restic.PackFile, Name: id.String()})
handles = append(handles, backend.Handle{Type: backend.PackFile, Name: id.String()})
}
err = s.delayedRemove(t, b, handles...)
@@ -340,13 +340,13 @@ func (s *Suite[C]) TestListCancel(t *testing.T) {
b := s.open(t)
defer s.close(t, b)
testFiles := make([]restic.Handle, 0, numTestFiles)
testFiles := make([]backend.Handle, 0, numTestFiles)
for i := 0; i < numTestFiles; i++ {
data := []byte(fmt.Sprintf("random test blob %v", i))
id := restic.Hash(data)
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher()))
h := backend.Handle{Type: backend.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher()))
if err != nil {
t.Fatal(err)
}
@@ -358,7 +358,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) {
cancel()
// pass in a cancelled context
err := b.List(ctx, restic.PackFile, func(fi restic.FileInfo) error {
err := b.List(ctx, backend.PackFile, func(fi backend.FileInfo) error {
t.Errorf("got FileInfo %v for cancelled context", fi)
return nil
})
@@ -373,7 +373,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) {
defer cancel()
i := 0
err := b.List(ctx, restic.PackFile, func(fi restic.FileInfo) error {
err := b.List(ctx, backend.PackFile, func(fi backend.FileInfo) error {
i++
// cancel the context on the first file
if i == 1 {
@@ -396,7 +396,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) {
defer cancel()
i := 0
err := b.List(ctx, restic.PackFile, func(fi restic.FileInfo) error {
err := b.List(ctx, backend.PackFile, func(fi backend.FileInfo) error {
// cancel the context at the last file
i++
if i == numTestFiles {
@@ -423,7 +423,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) {
i := 0
// pass in a context with a timeout
err := b.List(ctxTimeout, restic.PackFile, func(fi restic.FileInfo) error {
err := b.List(ctxTimeout, backend.PackFile, func(fi backend.FileInfo) error {
i++
// wait until the context is cancelled
@@ -494,11 +494,11 @@ func (s *Suite[C]) TestSave(t *testing.T) {
data := test.Random(23, length)
id = sha256.Sum256(data)
h := restic.Handle{
Type: restic.PackFile,
h := backend.Handle{
Type: backend.PackFile,
Name: id.String(),
}
err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher()))
err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher()))
test.OK(t, err)
buf, err := backend.LoadAll(context.TODO(), nil, b, h)
@@ -546,7 +546,7 @@ func (s *Suite[C]) TestSave(t *testing.T) {
t.Fatal(err)
}
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
h := backend.Handle{Type: backend.PackFile, Name: id.String()}
// wrap the tempfile in an errorCloser, so we can detect if the backend
// closes the reader
@@ -585,7 +585,7 @@ func (s *Suite[C]) TestSave(t *testing.T) {
}
type incompleteByteReader struct {
restic.ByteReader
backend.ByteReader
}
func (r *incompleteByteReader) Length() int64 {
@@ -609,8 +609,8 @@ func (s *Suite[C]) TestSaveError(t *testing.T) {
copy(id[:], data)
// test that incomplete uploads fail
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, &incompleteByteReader{ByteReader: *restic.NewByteReader(data, b.Hasher())})
h := backend.Handle{Type: backend.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, &incompleteByteReader{ByteReader: *backend.NewByteReader(data, b.Hasher())})
// try to delete possible leftovers
_ = s.delayedRemove(t, b, h)
if err == nil {
@@ -619,7 +619,7 @@ func (s *Suite[C]) TestSaveError(t *testing.T) {
}
type wrongByteReader struct {
restic.ByteReader
backend.ByteReader
}
func (b *wrongByteReader) Hash() []byte {
@@ -648,8 +648,8 @@ func (s *Suite[C]) TestSaveWrongHash(t *testing.T) {
copy(id[:], data)
// test that upload with hash mismatch fails
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, &wrongByteReader{ByteReader: *restic.NewByteReader(data, b.Hasher())})
h := backend.Handle{Type: backend.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, &wrongByteReader{ByteReader: *backend.NewByteReader(data, b.Hasher())})
exists, err2 := beTest(context.TODO(), b, h)
if err2 != nil {
t.Fatal(err2)
@@ -674,23 +674,23 @@ var testStrings = []struct {
{"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"},
}
func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) restic.Handle {
func store(t testing.TB, b backend.Backend, tpe backend.FileType, data []byte) backend.Handle {
id := restic.Hash(data)
h := restic.Handle{Name: id.String(), Type: tpe}
err := b.Save(context.TODO(), h, restic.NewByteReader([]byte(data), b.Hasher()))
h := backend.Handle{Name: id.String(), Type: tpe}
err := b.Save(context.TODO(), h, backend.NewByteReader([]byte(data), b.Hasher()))
test.OK(t, err)
return h
}
// testLoad loads a blob (but discards its contents).
func testLoad(b restic.Backend, h restic.Handle) error {
func testLoad(b backend.Backend, h backend.Handle) error {
return b.Load(context.TODO(), h, 0, 0, func(rd io.Reader) (ierr error) {
_, ierr = io.Copy(io.Discard, rd)
return ierr
})
}
func (s *Suite[C]) delayedRemove(t testing.TB, be restic.Backend, handles ...restic.Handle) error {
func (s *Suite[C]) delayedRemove(t testing.TB, be backend.Backend, handles ...backend.Handle) error {
// Some backend (swift, I'm looking at you) may implement delayed
// removal of data. Let's wait a bit if this happens.
@@ -734,11 +734,11 @@ func (s *Suite[C]) delayedRemove(t testing.TB, be restic.Backend, handles ...res
return nil
}
func delayedList(t testing.TB, b restic.Backend, tpe restic.FileType, max int, maxwait time.Duration) restic.IDs {
func delayedList(t testing.TB, b backend.Backend, tpe backend.FileType, max int, maxwait time.Duration) restic.IDs {
list := restic.NewIDSet()
start := time.Now()
for i := 0; i < max; i++ {
err := b.List(context.TODO(), tpe, func(fi restic.FileInfo) error {
err := b.List(context.TODO(), tpe, func(fi backend.FileInfo) error {
id := restic.TestParseID(fi.Name)
list.Insert(id)
return nil
@@ -763,9 +763,9 @@ func (s *Suite[C]) TestBackend(t *testing.T) {
test.Assert(t, !b.IsNotExist(nil), "IsNotExist() recognized nil error")
for _, tpe := range []restic.FileType{
restic.PackFile, restic.KeyFile, restic.LockFile,
restic.SnapshotFile, restic.IndexFile,
for _, tpe := range []backend.FileType{
backend.PackFile, backend.KeyFile, backend.LockFile,
backend.SnapshotFile, backend.IndexFile,
} {
// detect non-existing files
for _, ts := range testStrings {
@@ -773,7 +773,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) {
test.OK(t, err)
// test if blob is already in repository
h := restic.Handle{Type: tpe, Name: id.String()}
h := backend.Handle{Type: tpe, Name: id.String()}
ret, err := beTest(context.TODO(), b, h)
test.OK(t, err)
test.Assert(t, !ret, "blob was found to exist before creating")
@@ -799,7 +799,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) {
store(t, b, tpe, []byte(ts.data))
// test Load()
h := restic.Handle{Type: tpe, Name: ts.id}
h := backend.Handle{Type: tpe, Name: ts.id}
buf, err := backend.LoadAll(context.TODO(), nil, b, h)
test.OK(t, err)
test.Equals(t, ts.data, string(buf))
@@ -823,7 +823,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) {
// test adding the first file again
ts := testStrings[0]
h := restic.Handle{Type: tpe, Name: ts.id}
h := backend.Handle{Type: tpe, Name: ts.id}
// remove and recreate
err := s.delayedRemove(t, b, h)
@@ -835,7 +835,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) {
test.Assert(t, !ok, "removed blob still present")
// create blob
err = b.Save(context.TODO(), h, restic.NewByteReader([]byte(ts.data), b.Hasher()))
err = b.Save(context.TODO(), h, backend.NewByteReader([]byte(ts.data), b.Hasher()))
test.OK(t, err)
// list items
@@ -859,12 +859,12 @@ func (s *Suite[C]) TestBackend(t *testing.T) {
t.Fatalf("lists aren't equal, want:\n %v\n got:\n%v\n", IDs, list)
}
var handles []restic.Handle
var handles []backend.Handle
for _, ts := range testStrings {
id, err := restic.ParseID(ts.id)
test.OK(t, err)
h := restic.Handle{Type: tpe, Name: id.String()}
h := backend.Handle{Type: tpe, Name: id.String()}
found, err := beTest(context.TODO(), b, h)
test.OK(t, err)
internal/backend/util/defaults.go
+13 -13
View File
@@ -4,12 +4,12 @@ import (
"context"
"io"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/backend"
)
// DefaultLoad implements Backend.Load using lower-level openReader func
func DefaultLoad(ctx context.Context, h restic.Handle, length int, offset int64,
openReader func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error),
func DefaultLoad(ctx context.Context, h backend.Handle, length int, offset int64,
openReader func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error),
fn func(rd io.Reader) error) error {
rd, err := openReader(ctx, h, length, offset)
@@ -25,23 +25,23 @@ func DefaultLoad(ctx context.Context, h restic.Handle, length int, offset int64,
}
// DefaultDelete removes all restic keys in the bucket. It will not remove the bucket itself.
func DefaultDelete(ctx context.Context, be restic.Backend) error {
alltypes := []restic.FileType{
restic.PackFile,
restic.KeyFile,
restic.LockFile,
restic.SnapshotFile,
restic.IndexFile}
func DefaultDelete(ctx context.Context, be backend.Backend) error {
alltypes := []backend.FileType{
backend.PackFile,
backend.KeyFile,
backend.LockFile,
backend.SnapshotFile,
backend.IndexFile}
for _, t := range alltypes {
err := be.List(ctx, t, func(fi restic.FileInfo) error {
return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name})
err := be.List(ctx, t, func(fi backend.FileInfo) error {
return be.Remove(ctx, backend.Handle{Type: t, Name: fi.Name})
})
if err != nil {
return nil
}
}
err := be.Remove(ctx, restic.Handle{Type: restic.ConfigFile})
err := be.Remove(ctx, backend.Handle{Type: backend.ConfigFile})
if err != nil && be.IsNotExist(err) {
err = nil
}
internal/backend/util/defaults_test.go
+5 -5
View File
@@ -5,9 +5,9 @@ import (
"io"
"testing"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/util"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@@ -26,11 +26,11 @@ func (rd *mockReader) Close() error {
func TestDefaultLoad(t *testing.T) {
h := restic.Handle{Name: "id", Type: restic.PackFile}
h := backend.Handle{Name: "id", Type: backend.PackFile}
rd := &mockReader{}
// happy case, assert correct parameters are passed around and content stream is closed
err := util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih restic.Handle, length int, offset int64) (io.ReadCloser, error) {
err := util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih backend.Handle, length int, offset int64) (io.ReadCloser, error) {
rtest.Equals(t, h, ih)
rtest.Equals(t, int(10), length)
rtest.Equals(t, int64(11), offset)
@@ -44,7 +44,7 @@ func TestDefaultLoad(t *testing.T) {
rtest.Equals(t, true, rd.closed)
// unhappy case, assert producer errors are handled correctly
err = util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih restic.Handle, length int, offset int64) (io.ReadCloser, error) {
err = util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih backend.Handle, length int, offset int64) (io.ReadCloser, error) {
return nil, errors.Errorf("producer error")
}, func(ird io.Reader) error {
t.Fatalf("unexpected consumer invocation")
@@ -54,7 +54,7 @@ func TestDefaultLoad(t *testing.T) {
// unhappy case, assert consumer errors are handled correctly
rd = &mockReader{}
err = util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih restic.Handle, length int, offset int64) (io.ReadCloser, error) {
err = util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih backend.Handle, length int, offset int64) (io.ReadCloser, error) {
return rd, nil
}, func(ird io.Reader) error {
return errors.Errorf("consumer error")
internal/backend/utils.go
+8 -9
View File
@@ -11,7 +11,6 @@ import (
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
)
func verifyContentMatchesName(s string, data []byte) (bool, error) {
@@ -33,7 +32,7 @@ func verifyContentMatchesName(s string, data []byte) (bool, error) {
// LoadAll reads all data stored in the backend for the handle into the given
// buffer, which is truncated. If the buffer is not large enough or nil, a new
// one is allocated.
func LoadAll(ctx context.Context, buf []byte, be restic.Backend, h restic.Handle) ([]byte, error) {
func LoadAll(ctx context.Context, buf []byte, be Backend, h Handle) ([]byte, error) {
retriedInvalidData := false
err := be.Load(ctx, h, 0, 0, func(rd io.Reader) error {
// make sure this is idempotent, in case an error occurs this function may be called multiple times!
@@ -47,7 +46,7 @@ func LoadAll(ctx context.Context, buf []byte, be restic.Backend, h restic.Handle
// retry loading damaged data only once. If a file fails to download correctly
// the second time, then it is likely corrupted at the backend. Return the data
// to the caller in that case to let it decide what to do with the data.
if !retriedInvalidData && h.Type != restic.ConfigFile {
if !retriedInvalidData && h.Type != ConfigFile {
if matches, err := verifyContentMatchesName(h.Name, buf); err == nil && !matches {
debug.Log("retry loading broken blob %v", h)
retriedInvalidData = true
@@ -77,11 +76,11 @@ func LimitReadCloser(r io.ReadCloser, n int64) *LimitedReadCloser {
}
type memorizedLister struct {
fileInfos []restic.FileInfo
tpe restic.FileType
fileInfos []FileInfo
tpe FileType
}
func (m *memorizedLister) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
func (m *memorizedLister) List(ctx context.Context, t FileType, fn func(FileInfo) error) error {
if t != m.tpe {
return fmt.Errorf("filetype mismatch, expected %s got %s", m.tpe, t)
}
@@ -97,13 +96,13 @@ func (m *memorizedLister) List(ctx context.Context, t restic.FileType, fn func(r
return ctx.Err()
}
func MemorizeList(ctx context.Context, be restic.Lister, t restic.FileType) (restic.Lister, error) {
func MemorizeList(ctx context.Context, be Lister, t FileType) (Lister, error) {
if _, ok := be.(*memorizedLister); ok {
return be, nil
}
var fileInfos []restic.FileInfo
err := be.List(ctx, t, func(fi restic.FileInfo) error {
var fileInfos []FileInfo
err := be.List(ctx, t, func(fi FileInfo) error {
fileInfos = append(fileInfos, fi)
return nil
})
internal/backend/utils_test.go
+20 -20
View File
@@ -26,11 +26,11 @@ func TestLoadAll(t *testing.T) {
data := rtest.Random(23+i, rand.Intn(MiB)+500*KiB)
id := restic.Hash(data)
h := restic.Handle{Name: id.String(), Type: restic.PackFile}
err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher()))
h := backend.Handle{Name: id.String(), Type: backend.PackFile}
err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher()))
rtest.OK(t, err)
buf, err := backend.LoadAll(context.TODO(), buf, b, restic.Handle{Type: restic.PackFile, Name: id.String()})
buf, err := backend.LoadAll(context.TODO(), buf, b, backend.Handle{Type: backend.PackFile, Name: id.String()})
rtest.OK(t, err)
if len(buf) != len(data) {
@@ -45,10 +45,10 @@ func TestLoadAll(t *testing.T) {
}
}
func save(t testing.TB, be restic.Backend, buf []byte) restic.Handle {
func save(t testing.TB, be backend.Backend, buf []byte) backend.Handle {
id := restic.Hash(buf)
h := restic.Handle{Name: id.String(), Type: restic.PackFile}
err := be.Save(context.TODO(), h, restic.NewByteReader(buf, be.Hasher()))
h := backend.Handle{Name: id.String(), Type: backend.PackFile}
err := be.Save(context.TODO(), h, backend.NewByteReader(buf, be.Hasher()))
if err != nil {
t.Fatal(err)
}
@@ -56,10 +56,10 @@ func save(t testing.TB, be restic.Backend, buf []byte) restic.Handle {
}
type quickRetryBackend struct {
restic.Backend
backend.Backend
}
func (be *quickRetryBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *quickRetryBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
err := be.Backend.Load(ctx, h, length, offset, fn)
if err != nil {
// retry
@@ -76,19 +76,19 @@ func TestLoadAllBroken(t *testing.T) {
// damage buffer
data[0] ^= 0xff
b.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
b.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader(data)), nil
}
// must fail on first try
_, err := backend.LoadAll(context.TODO(), nil, b, restic.Handle{Type: restic.PackFile, Name: id.String()})
_, err := backend.LoadAll(context.TODO(), nil, b, backend.Handle{Type: backend.PackFile, Name: id.String()})
if err == nil {
t.Fatalf("missing expected error")
}
// must return the broken data after a retry
be := &quickRetryBackend{Backend: b}
buf, err := backend.LoadAll(context.TODO(), nil, be, restic.Handle{Type: restic.PackFile, Name: id.String()})
buf, err := backend.LoadAll(context.TODO(), nil, be, backend.Handle{Type: backend.PackFile, Name: id.String()})
rtest.OK(t, err)
if !bytes.Equal(buf, data) {
@@ -104,7 +104,7 @@ func TestLoadAllAppend(t *testing.T) {
h2 := save(t, b, randomData)
var tests = []struct {
handle restic.Handle
handle backend.Handle
buf []byte
want []byte
}{
@@ -152,11 +152,11 @@ func TestLoadAllAppend(t *testing.T) {
func TestMemoizeList(t *testing.T) {
// setup backend to serve as data source for memoized list
be := mock.NewBackend()
files := []restic.FileInfo{
files := []backend.FileInfo{
{Size: 42, Name: restic.NewRandomID().String()},
{Size: 45, Name: restic.NewRandomID().String()},
}
be.ListFn = func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
be.ListFn = func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
for _, fi := range files {
if err := fn(fi); err != nil {
return err
@@ -165,17 +165,17 @@ func TestMemoizeList(t *testing.T) {
return nil
}
mem, err := backend.MemorizeList(context.TODO(), be, restic.SnapshotFile)
mem, err := backend.MemorizeList(context.TODO(), be, backend.SnapshotFile)
rtest.OK(t, err)
err = mem.List(context.TODO(), restic.IndexFile, func(fi restic.FileInfo) error {
err = mem.List(context.TODO(), backend.IndexFile, func(fi backend.FileInfo) error {
t.Fatal("file type mismatch")
return nil // the memoized lister must return an error by itself
})
rtest.Assert(t, err != nil, "missing error on file typ mismatch")
var memFiles []restic.FileInfo
err = mem.List(context.TODO(), restic.SnapshotFile, func(fi restic.FileInfo) error {
var memFiles []backend.FileInfo
err = mem.List(context.TODO(), backend.SnapshotFile, func(fi backend.FileInfo) error {
memFiles = append(memFiles, fi)
return nil
})
@@ -186,9 +186,9 @@ func TestMemoizeList(t *testing.T) {
func TestMemoizeListError(t *testing.T) {
// setup backend to serve as data source for memoized list
be := mock.NewBackend()
be.ListFn = func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
be.ListFn = func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
return fmt.Errorf("list error")
}
_, err := backend.MemorizeList(context.TODO(), be, restic.SnapshotFile)
_, err := backend.MemorizeList(context.TODO(), be, backend.SnapshotFile)
rtest.Assert(t, err != nil, "missing error on list error")
}