diff --git a/internal/backend/azure/azure.go b/internal/backend/azure/azure.go index 576985b5b..234e5ddd1 100644 --- a/internal/backend/azure/azure.go +++ b/internal/backend/azure/azure.go @@ -31,11 +31,9 @@ import ( // Backend stores data on an azure endpoint. type Backend struct { - cfg Config - container *azContainer.Client - connections uint - prefix string - listMaxItems int + cfg Config + container *azContainer.Client + connections uint layout.Layout accessTier blob.AccessTier @@ -145,12 +143,11 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) { } be := &Backend{ - container: client, - cfg: cfg, - connections: cfg.Connections, - Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join), - listMaxItems: defaultListMaxItems, - accessTier: accessTier, + container: client, + cfg: cfg, + connections: cfg.Connections, + Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join), + accessTier: accessTier, } return be, nil @@ -195,11 +192,6 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string return be, nil } -// SetListMaxItems sets the number of list items to load per request. -func (be *Backend) SetListMaxItems(i int) { - be.listMaxItems = i -} - // IsNotExist returns true if the error is caused by a not existing file. func (be *Backend) IsNotExist(err error) bool { return bloberror.HasCode(err, bloberror.BlobNotFound) @@ -231,11 +223,6 @@ func (be *Backend) Hasher() hash.Hash { return md5.New() } -// Path returns the path in the bucket that is used for this backend. -func (be *Backend) Path() string { - return be.prefix -} - // useAccessTier determines whether to apply the configured access tier to a given file. // For archive access tier, only data files are stored using that class; metadata // must remain instantly accessible. @@ -419,7 +406,7 @@ func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend prefix += "/" } - maxI := int32(be.listMaxItems) + maxI := int32(defaultListMaxItems) opts := &azContainer.ListBlobsFlatOptions{ MaxResults: &maxI, diff --git a/internal/backend/b2/b2.go b/internal/backend/b2/b2.go index 6e751479d..062c2b7fd 100644 --- a/internal/backend/b2/b2.go +++ b/internal/backend/b2/b2.go @@ -23,10 +23,9 @@ import ( // b2Backend is a backend which stores its data on Backblaze B2. type b2Backend struct { - client *b2.Client - bucket *b2.Bucket - cfg Config - listMaxItems int + client *b2.Client + bucket *b2.Bucket + cfg Config layout.Layout canDelete bool @@ -107,12 +106,11 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string, } be := &b2Backend{ - client: client, - bucket: bucket, - cfg: cfg, - Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join), - listMaxItems: defaultListMaxItems, - canDelete: true, + client: client, + bucket: bucket, + cfg: cfg, + Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join), + canDelete: true, } return be, nil @@ -140,20 +138,14 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string } be := &b2Backend{ - client: client, - bucket: bucket, - cfg: cfg, - Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join), - listMaxItems: defaultListMaxItems, + client: client, + bucket: bucket, + cfg: cfg, + Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join), } return be, nil } -// SetListMaxItems sets the number of list items to load per request. -func (be *b2Backend) SetListMaxItems(i int) { - be.listMaxItems = i -} - func (be *b2Backend) Properties() backend.Properties { return backend.Properties{ Connections: be.cfg.Connections, @@ -304,7 +296,7 @@ func (be *b2Backend) List(ctx context.Context, t backend.FileType, fn func(backe defer cancel() prefix, _ := be.Basedir(t) - iter := be.bucket.List(ctx, b2.ListPrefix(prefix), b2.ListPageSize(be.listMaxItems)) + iter := be.bucket.List(ctx, b2.ListPrefix(prefix), b2.ListPageSize(defaultListMaxItems)) for iter.Next() { obj := iter.Object() diff --git a/internal/backend/cache/backend.go b/internal/backend/cache/backend.go index 34dd07903..db47767a0 100644 --- a/internal/backend/cache/backend.go +++ b/internal/backend/cache/backend.go @@ -9,8 +9,8 @@ import ( "github.com/restic/restic/internal/debug" ) -// Backend wraps a restic.Backend and adds a cache. -type Backend struct { +// cacheBackend wraps a restic.cacheBackend and adds a cache. +type cacheBackend struct { backend.Backend *Cache @@ -23,10 +23,10 @@ type Backend struct { } // ensure Backend implements backend.Backend -var _ backend.Backend = &Backend{} +var _ backend.Backend = &cacheBackend{} -func newBackend(be backend.Backend, c *Cache, errorLog func(string, ...interface{})) *Backend { - return &Backend{ +func newBackend(be backend.Backend, c *Cache, errorLog func(string, ...interface{})) *cacheBackend { + return &cacheBackend{ Backend: be, Cache: c, inProgress: make(map[backend.Handle]chan struct{}), @@ -35,7 +35,7 @@ func newBackend(be backend.Backend, c *Cache, errorLog func(string, ...interface } // Remove deletes a file from the backend and the cache if it has been cached. -func (b *Backend) Remove(ctx context.Context, h backend.Handle) error { +func (b *cacheBackend) Remove(ctx context.Context, h backend.Handle) error { debug.Log("cache Remove(%v)", h) err := b.Backend.Remove(ctx, h) if err != nil { @@ -57,7 +57,7 @@ func autoCacheTypes(h backend.Handle) bool { } // Save stores a new file in the backend and the cache. -func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { +func (b *cacheBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { if !autoCacheTypes(h) { return b.Backend.Save(ctx, h, rd) } @@ -91,7 +91,7 @@ func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindR return nil } -func (b *Backend) cacheFile(ctx context.Context, h backend.Handle) error { +func (b *cacheBackend) cacheFile(ctx context.Context, h backend.Handle) error { finish := make(chan struct{}) b.inProgressMutex.Lock() @@ -135,7 +135,7 @@ func (b *Backend) cacheFile(ctx context.Context, h backend.Handle) error { } // loadFromCache will try to load the file from the cache. -func (b *Backend) loadFromCache(h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (bool, error) { +func (b *cacheBackend) loadFromCache(h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (bool, error) { rd, inCache, err := b.Cache.load(h, length, offset) if err != nil { return inCache, err @@ -150,7 +150,7 @@ func (b *Backend) loadFromCache(h backend.Handle, length int, offset int64, cons } // Load loads a file from the cache or the backend. -func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { +func (b *cacheBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { b.inProgressMutex.Lock() waitForFinish, inProgress := b.inProgress[h] b.inProgressMutex.Unlock() @@ -197,7 +197,7 @@ func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset // Stat tests whether the backend has a file. If it does not exist but still // exists in the cache, it is removed from the cache. -func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { +func (b *cacheBackend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { debug.Log("cache Stat(%v)", h) fi, err := b.Backend.Stat(ctx, h) @@ -210,15 +210,15 @@ func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, } // IsNotExist returns true if the error is caused by a non-existing file. -func (b *Backend) IsNotExist(err error) bool { +func (b *cacheBackend) IsNotExist(err error) bool { return b.Backend.IsNotExist(err) } -func (b *Backend) Unwrap() backend.Backend { +func (b *cacheBackend) Unwrap() backend.Backend { return b.Backend } -func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(f backend.FileInfo) error) error { +func (b *cacheBackend) List(ctx context.Context, t backend.FileType, fn func(f backend.FileInfo) error) error { if !b.Cache.canBeCached(t) { return b.Backend.List(ctx, t, fn) } @@ -252,11 +252,11 @@ func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(f backen } // Warmup delegates to wrapped backend. -func (b *Backend) Warmup(ctx context.Context, h []backend.Handle) ([]backend.Handle, error) { +func (b *cacheBackend) Warmup(ctx context.Context, h []backend.Handle) ([]backend.Handle, error) { return b.Backend.Warmup(ctx, h) } // WarmupWait delegates to wrapped backend. -func (b *Backend) WarmupWait(ctx context.Context, h []backend.Handle) error { +func (b *cacheBackend) WarmupWait(ctx context.Context, h []backend.Handle) error { return b.Backend.WarmupWait(ctx, h) } diff --git a/internal/backend/gs/gs.go b/internal/backend/gs/gs.go index aaf1ae33c..8ebc4902c 100644 --- a/internal/backend/gs/gs.go +++ b/internal/backend/gs/gs.go @@ -27,27 +27,25 @@ import ( "google.golang.org/api/option" ) -// Backend stores data in a GCS bucket. +// gs stores data in a GCS bucket. // // The service account used to access the bucket must have these permissions: // - storage.objects.create // - storage.objects.delete // - storage.objects.get // - storage.objects.list -type Backend struct { - gcsClient *storage.Client - projectID string - connections uint - bucketName string - region string - bucket *storage.BucketHandle - prefix string - listMaxItems int +type gs struct { + gcsClient *storage.Client + projectID string + connections uint + bucketName string + region string + bucket *storage.BucketHandle layout.Layout } // Ensure that *Backend implements backend.Backend. -var _ backend.Backend = &Backend{} +var _ backend.Backend = &gs{} func NewFactory() location.Factory { return location.NewHTTPBackendFactory("gs", ParseConfig, location.NoPassword, Create, Open) @@ -86,7 +84,7 @@ func getStorageClient(rt http.RoundTripper) (*storage.Client, error) { return gcsClient, nil } -func (be *Backend) bucketExists(ctx context.Context, bucket *storage.BucketHandle) (bool, error) { +func (be *gs) bucketExists(ctx context.Context, bucket *storage.BucketHandle) (bool, error) { _, err := bucket.Attrs(ctx) if err == storage.ErrBucketNotExist { return false, nil @@ -94,9 +92,7 @@ func (be *Backend) bucketExists(ctx context.Context, bucket *storage.BucketHandl return err == nil, err } -const defaultListMaxItems = 1000 - -func open(cfg Config, rt http.RoundTripper) (*Backend, error) { +func open(cfg Config, rt http.RoundTripper) (*gs, error) { debug.Log("open, config %#v", cfg) gcsClient, err := getStorageClient(rt) @@ -104,16 +100,14 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) { return nil, errors.Wrap(err, "getStorageClient") } - be := &Backend{ - gcsClient: gcsClient, - projectID: cfg.ProjectID, - connections: cfg.Connections, - bucketName: cfg.Bucket, - region: cfg.Region, - bucket: gcsClient.Bucket(cfg.Bucket), - prefix: cfg.Prefix, - Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join), - listMaxItems: defaultListMaxItems, + be := &gs{ + gcsClient: gcsClient, + projectID: cfg.ProjectID, + connections: cfg.Connections, + bucketName: cfg.Bucket, + region: cfg.Region, + bucket: gcsClient.Bucket(cfg.Bucket), + Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join), } return be, nil @@ -161,17 +155,12 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string return be, nil } -// SetListMaxItems sets the number of list items to load per request. -func (be *Backend) SetListMaxItems(i int) { - be.listMaxItems = i -} - // IsNotExist returns true if the error is caused by a not existing file. -func (be *Backend) IsNotExist(err error) bool { +func (be *gs) IsNotExist(err error) bool { return errors.Is(err, storage.ErrObjectNotExist) } -func (be *Backend) IsPermanentError(err error) bool { +func (be *gs) IsPermanentError(err error) bool { if be.IsNotExist(err) { return true } @@ -186,7 +175,7 @@ func (be *Backend) IsPermanentError(err error) bool { return false } -func (be *Backend) Properties() backend.Properties { +func (be *gs) Properties() backend.Properties { return backend.Properties{ Connections: be.connections, HasAtomicReplace: true, @@ -194,17 +183,12 @@ func (be *Backend) Properties() backend.Properties { } // Hasher may return a hash function for calculating a content hash for the backend -func (be *Backend) Hasher() hash.Hash { +func (be *gs) Hasher() hash.Hash { return md5.New() } -// Path returns the path in the bucket that is used for this backend. -func (be *Backend) Path() string { - return be.prefix -} - // Save stores data in the backend at the handle. -func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { +func (be *gs) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { objName := be.Filename(h) // Set chunk size to zero to disable resumable uploads. @@ -254,14 +238,14 @@ func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.Rewind // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (be *gs) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { ctx, cancel := context.WithCancel(ctx) defer cancel() return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn) } -func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { +func (be *gs) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { if length == 0 { // negative length indicates read till end to GCS lib length = -1 @@ -283,7 +267,7 @@ func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, } // Stat returns information about a blob. -func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) { +func (be *gs) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) { objName := be.Filename(h) attr, err := be.bucket.Object(objName).Attrs(ctx) @@ -296,7 +280,7 @@ func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileI } // Remove removes the blob with the given name and type. -func (be *Backend) Remove(ctx context.Context, h backend.Handle) error { +func (be *gs) Remove(ctx context.Context, h backend.Handle) error { objName := be.Filename(h) err := be.bucket.Object(objName).Delete(ctx) @@ -310,7 +294,7 @@ func (be *Backend) Remove(ctx context.Context, h backend.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. -func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { +func (be *gs) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { prefix, _ := be.Basedir(t) // make sure prefix ends with a slash @@ -355,15 +339,15 @@ func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend } // Delete removes all restic keys in the bucket. It will not remove the bucket itself. -func (be *Backend) Delete(ctx context.Context) error { +func (be *gs) Delete(ctx context.Context) error { return util.DefaultDelete(ctx, be) } // Close does nothing. -func (be *Backend) Close() error { return nil } +func (be *gs) Close() error { return nil } // Warmup not implemented -func (be *Backend) Warmup(_ context.Context, _ []backend.Handle) ([]backend.Handle, error) { +func (be *gs) Warmup(_ context.Context, _ []backend.Handle) ([]backend.Handle, error) { return []backend.Handle{}, nil } -func (be *Backend) WarmupWait(_ context.Context, _ []backend.Handle) error { return nil } +func (be *gs) WarmupWait(_ context.Context, _ []backend.Handle) error { return nil } diff --git a/internal/backend/rclone/backend.go b/internal/backend/rclone/backend.go index 8af00dfed..19d956b3f 100644 --- a/internal/backend/rclone/backend.go +++ b/internal/backend/rclone/backend.go @@ -28,8 +28,8 @@ import ( "golang.org/x/net/http2" ) -// Backend is used to access data stored somewhere via rclone. -type Backend struct { +// rclone is used to access data stored somewhere via rclone. +type rclone struct { *rest.Backend tr *http2.Transport cmd *exec.Cmd @@ -141,7 +141,7 @@ func wrapConn(c *StdioConn, lim limiter.Limiter) *wrappedConn { } // New initializes a Backend and starts the process. -func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*Backend, error) { +func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*rclone, error) { var ( args []string err error @@ -196,7 +196,7 @@ func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog f } cmd := stdioConn.cmd - be := &Backend{ + be := &rclone{ tr: tr, cmd: cmd, waitCh: waitCh, @@ -270,7 +270,7 @@ func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog f } // Open starts an rclone process with the given config. -func Open(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*Backend, error) { +func Open(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (backend.Backend, error) { be, err := newBackend(ctx, cfg, lim, errorLog) if err != nil { return nil, err @@ -297,7 +297,7 @@ func Open(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(st } // Create initializes a new restic repo with rclone. -func Create(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*Backend, error) { +func Create(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (backend.Backend, error) { be, err := newBackend(ctx, cfg, lim, errorLog) if err != nil { return nil, err @@ -328,7 +328,7 @@ func Create(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func( const waitForExit = 5 * time.Second // Close terminates the backend. -func (be *Backend) Close() error { +func (be *rclone) Close() error { debug.Log("exiting rclone") be.tr.CloseIdleConnections() @@ -348,7 +348,7 @@ func (be *Backend) Close() error { return be.waitResult } -func (be *Backend) Properties() backend.Properties { +func (be *rclone) Properties() backend.Properties { properties := be.Backend.Properties() properties.HasFlakyErrors = true return properties diff --git a/internal/backend/rclone/internal_test.go b/internal/backend/rclone/internal_test.go index adc251557..8da8dcd6f 100644 --- a/internal/backend/rclone/internal_test.go +++ b/internal/backend/rclone/internal_test.go @@ -27,7 +27,7 @@ func TestRcloneExit(t *testing.T) { _ = be.Close() }() - err = be.cmd.Process.Kill() + err = be.(*rclone).cmd.Process.Kill() rtest.OK(t, err) t.Log("killed rclone") diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index f447d74fd..2b249e705 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -25,15 +25,15 @@ import ( "github.com/minio/minio-go/v7/pkg/credentials" ) -// Backend stores data on an S3 endpoint. -type Backend struct { +// s3 stores data on an S3 endpoint. +type s3 struct { client *minio.Client cfg Config layout.Layout } // make sure that *Backend implements backend.Backend -var _ backend.Backend = &Backend{} +var _ backend.Backend = &s3{} var archiveClasses = []string{"GLACIER", "DEEP_ARCHIVE"} @@ -50,7 +50,7 @@ func NewFactory() location.Factory { return location.NewHTTPBackendFactory("s3", ParseConfig, location.NoPassword, Create, Open) } -func open(cfg Config, rt http.RoundTripper) (*Backend, error) { +func open(cfg Config, rt http.RoundTripper) (*s3, error) { debug.Log("open, config %#v", cfg) if cfg.EnableRestore && !feature.Flag.Enabled(feature.S3Restore) { @@ -89,7 +89,7 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) { return nil, errors.Wrap(err, "minio.New") } - be := &Backend{ + be := &s3{ client: client, cfg: cfg, Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join), @@ -240,12 +240,12 @@ func isAccessDenied(err error) bool { } // IsNotExist returns true if the error is caused by a not existing file. -func (be *Backend) IsNotExist(err error) bool { +func (be *s3) IsNotExist(err error) bool { var e minio.ErrorResponse return errors.As(err, &e) && e.Code == "NoSuchKey" } -func (be *Backend) IsPermanentError(err error) bool { +func (be *s3) IsPermanentError(err error) bool { if be.IsNotExist(err) { return true } @@ -260,7 +260,7 @@ func (be *Backend) IsPermanentError(err error) bool { return false } -func (be *Backend) Properties() backend.Properties { +func (be *s3) Properties() backend.Properties { return backend.Properties{ Connections: be.cfg.Connections, HasAtomicReplace: true, @@ -268,26 +268,21 @@ func (be *Backend) Properties() backend.Properties { } // Hasher may return a hash function for calculating a content hash for the backend -func (be *Backend) Hasher() hash.Hash { +func (be *s3) Hasher() hash.Hash { return nil } -// Path returns the path in the bucket that is used for this backend. -func (be *Backend) Path() string { - return be.cfg.Prefix -} - // useStorageClass returns whether file should be saved in the provided Storage Class // For archive storage classes, only data files are stored using that class; metadata // must remain instantly accessible. -func (be *Backend) useStorageClass(h backend.Handle) bool { +func (be *s3) useStorageClass(h backend.Handle) bool { isDataFile := h.Type == backend.PackFile && !h.IsMetadata isArchiveClass := slices.Contains(archiveClasses, be.cfg.StorageClass) return !isArchiveClass || isDataFile } // Save stores data in the backend at the handle. -func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { +func (be *s3) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { objName := be.Filename(h) opts := minio.PutObjectOptions{ @@ -313,14 +308,14 @@ func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.Rewind // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (be *s3) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { ctx, cancel := context.WithCancel(ctx) defer cancel() return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn) } -func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { +func (be *s3) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { objName := be.Filename(h) opts := minio.GetObjectOptions{} @@ -352,7 +347,7 @@ func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, } // Stat returns information about a blob. -func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) { +func (be *s3) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) { objName := be.Filename(h) var obj *minio.Object @@ -380,7 +375,7 @@ func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileI } // Remove removes the blob with the given name and type. -func (be *Backend) Remove(ctx context.Context, h backend.Handle) error { +func (be *s3) Remove(ctx context.Context, h backend.Handle) error { objName := be.Filename(h) err := be.client.RemoveObject(ctx, be.cfg.Bucket, objName, minio.RemoveObjectOptions{}) @@ -394,7 +389,7 @@ func (be *Backend) Remove(ctx context.Context, h backend.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. -func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { +func (be *s3) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { prefix, recursive := be.Basedir(t) // make sure prefix ends with a slash @@ -449,15 +444,15 @@ func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend } // Delete removes all restic keys in the bucket. It will not remove the bucket itself. -func (be *Backend) Delete(ctx context.Context) error { +func (be *s3) Delete(ctx context.Context) error { return util.DefaultDelete(ctx, be) } // Close does nothing -func (be *Backend) Close() error { return nil } +func (be *s3) Close() error { return nil } // Warmup transitions handles from cold to hot storage if needed. -func (be *Backend) Warmup(ctx context.Context, handles []backend.Handle) ([]backend.Handle, error) { +func (be *s3) Warmup(ctx context.Context, handles []backend.Handle) ([]backend.Handle, error) { handlesWarmingUp := []backend.Handle{} if be.cfg.EnableRestore { @@ -478,7 +473,7 @@ func (be *Backend) Warmup(ctx context.Context, handles []backend.Handle) ([]back } // requestRestore sends a glacier restore request on a given file. -func (be *Backend) requestRestore(ctx context.Context, filename string) (bool, error) { +func (be *s3) requestRestore(ctx context.Context, filename string) (bool, error) { objectInfo, err := be.client.StatObject(ctx, be.cfg.Bucket, filename, minio.StatObjectOptions{}) if err != nil { return false, err @@ -514,7 +509,7 @@ func (be *Backend) requestRestore(ctx context.Context, filename string) (bool, e } // getWarmupStatus returns the warmup status of the provided object. -func (be *Backend) getWarmupStatus(objectInfo minio.ObjectInfo) warmupStatus { +func (be *s3) getWarmupStatus(objectInfo minio.ObjectInfo) warmupStatus { // We can't use objectInfo.StorageClass to get the storage class of the // object because this field is only set during ListObjects operations. // The response header is the documented way to get the storage class @@ -545,7 +540,7 @@ func (be *Backend) getWarmupStatus(objectInfo minio.ObjectInfo) warmupStatus { } // WarmupWait waits until all handles are in hot storage. -func (be *Backend) WarmupWait(ctx context.Context, handles []backend.Handle) error { +func (be *s3) WarmupWait(ctx context.Context, handles []backend.Handle) error { timeoutCtx, timeoutCtxCancel := context.WithTimeout(ctx, be.cfg.RestoreTimeout) defer timeoutCtxCancel() @@ -564,7 +559,7 @@ func (be *Backend) WarmupWait(ctx context.Context, handles []backend.Handle) err } // waitForRestore waits for a given file to be restored. -func (be *Backend) waitForRestore(ctx context.Context, filename string) error { +func (be *s3) waitForRestore(ctx context.Context, filename string) error { for { var objectInfo minio.ObjectInfo diff --git a/internal/backend/test/tests.go b/internal/backend/test/tests.go index 5c223412b..00c678942 100644 --- a/internal/backend/test/tests.go +++ b/internal/backend/test/tests.go @@ -251,10 +251,6 @@ func (s *Suite[C]) TestLoad(t *testing.T) { test.OK(t, b.Remove(context.TODO(), handle)) } -type setter interface { - SetListMaxItems(int) -} - // TestList makes sure that the backend implements List() pagination correctly. func (s *Suite[C]) TestList(t *testing.T) { random := seedRand(t) @@ -302,54 +298,39 @@ func (s *Suite[C]) TestList(t *testing.T) { t.Logf("wrote %v files", len(list1)) - var tests = []struct { - maxItems int - }{ - {11}, {23}, {numTestFiles}, {numTestFiles + 10}, {numTestFiles + 1123}, + list2 := make(map[restic.ID]int64) + + err = b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error { + id, err := restic.ParseID(fi.Name) + if err != nil { + t.Fatal(err) + } + list2[id] = fi.Size + return nil + }) + + if err != nil { + t.Fatalf("List returned error %v", err) } - for _, test := range tests { - t.Run(fmt.Sprintf("max-%v", test.maxItems), func(t *testing.T) { - list2 := make(map[restic.ID]int64) + t.Logf("loaded %v IDs from backend", len(list2)) - if s, ok := b.(setter); ok { - t.Logf("setting max list items to %d", test.maxItems) - s.SetListMaxItems(test.maxItems) - } + for id, size := range list1 { + size2, ok := list2[id] + if !ok { + t.Errorf("id %v not returned by List()", id.Str()) + } - err := b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error { - id, err := restic.ParseID(fi.Name) - if err != nil { - t.Fatal(err) - } - list2[id] = fi.Size - return nil - }) + if size != size2 { + t.Errorf("wrong size for id %v returned: want %v, got %v", id.Str(), size, size2) + } + } - if err != nil { - t.Fatalf("List returned error %v", err) - } - - t.Logf("loaded %v IDs from backend", len(list2)) - - for id, size := range list1 { - size2, ok := list2[id] - if !ok { - t.Errorf("id %v not returned by List()", id.Str()) - } - - if size != size2 { - t.Errorf("wrong size for id %v returned: want %v, got %v", id.Str(), size, size2) - } - } - - for id := range list2 { - _, ok := list1[id] - if !ok { - t.Errorf("extra id %v returned by List()", id.Str()) - } - } - }) + for id := range list2 { + _, ok := list1[id] + if !ok { + t.Errorf("extra id %v returned by List()", id.Str()) + } } t.Logf("remove %d files", numTestFiles)