amorfo77/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2026-06-17 22:24:17 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #21858 from MichaelEischer/unexport-backends

Browse Source 
Unexport several backend types
This commit is contained in:
Michael Eischer
2026-06-10 22:51:32 +02:00
committed by GitHub
parent c745f810b3 721411e04c
commit 784812361e
8 changed files with 131 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/backend/azure/azure.go
+9 -22
View File
@@ -31,11 +31,9 @@ import (
// Backend stores data on an azure endpoint.
type Backend struct {
cfg Config
container *azContainer.Client
connections uint
prefix string
listMaxItems int
cfg Config
container *azContainer.Client
connections uint
layout.Layout
accessTier blob.AccessTier
@@ -145,12 +143,11 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
}
be := &Backend{
container: client,
cfg: cfg,
connections: cfg.Connections,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
listMaxItems: defaultListMaxItems,
accessTier: accessTier,
container: client,
cfg: cfg,
connections: cfg.Connections,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
accessTier: accessTier,
}
return be, nil
@@ -195,11 +192,6 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string
return be, nil
}
// SetListMaxItems sets the number of list items to load per request.
func (be *Backend) SetListMaxItems(i int) {
be.listMaxItems = i
}
// IsNotExist returns true if the error is caused by a not existing file.
func (be *Backend) IsNotExist(err error) bool {
return bloberror.HasCode(err, bloberror.BlobNotFound)
@@ -231,11 +223,6 @@ func (be *Backend) Hasher() hash.Hash {
return md5.New()
}
// Path returns the path in the bucket that is used for this backend.
func (be *Backend) Path() string {
return be.prefix
}
// useAccessTier determines whether to apply the configured access tier to a given file.
// For archive access tier, only data files are stored using that class; metadata
// must remain instantly accessible.
@@ -419,7 +406,7 @@ func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend
prefix += "/"
}
maxI := int32(be.listMaxItems)
maxI := int32(defaultListMaxItems)
opts := &azContainer.ListBlobsFlatOptions{
MaxResults: &maxI,
internal/backend/b2/b2.go
+13 -21
View File
@@ -23,10 +23,9 @@ import (
// b2Backend is a backend which stores its data on Backblaze B2.
type b2Backend struct {
client *b2.Client
bucket *b2.Bucket
cfg Config
listMaxItems int
client *b2.Client
bucket *b2.Bucket
cfg Config
layout.Layout
canDelete bool
@@ -107,12 +106,11 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string,
}
be := &b2Backend{
client: client,
bucket: bucket,
cfg: cfg,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
listMaxItems: defaultListMaxItems,
canDelete: true,
client: client,
bucket: bucket,
cfg: cfg,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
canDelete: true,
}
return be, nil
@@ -140,20 +138,14 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string
}
be := &b2Backend{
client: client,
bucket: bucket,
cfg: cfg,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
listMaxItems: defaultListMaxItems,
client: client,
bucket: bucket,
cfg: cfg,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
}
return be, nil
}
// SetListMaxItems sets the number of list items to load per request.
func (be *b2Backend) SetListMaxItems(i int) {
be.listMaxItems = i
}
func (be *b2Backend) Properties() backend.Properties {
return backend.Properties{
Connections: be.cfg.Connections,
@@ -304,7 +296,7 @@ func (be *b2Backend) List(ctx context.Context, t backend.FileType, fn func(backe
defer cancel()
prefix, _ := be.Basedir(t)
iter := be.bucket.List(ctx, b2.ListPrefix(prefix), b2.ListPageSize(be.listMaxItems))
iter := be.bucket.List(ctx, b2.ListPrefix(prefix), b2.ListPageSize(defaultListMaxItems))
for iter.Next() {
obj := iter.Object()
internal/backend/cache/backend.go
Vendored
+16 -16
View File
@@ -9,8 +9,8 @@ import (
"github.com/restic/restic/internal/debug"
)
// Backend wraps a restic.Backend and adds a cache.
type Backend struct {
// cacheBackend wraps a restic.cacheBackend and adds a cache.
type cacheBackend struct {
backend.Backend
*Cache
@@ -23,10 +23,10 @@ type Backend struct {
}
// ensure Backend implements backend.Backend
var _ backend.Backend = &Backend{}
var _ backend.Backend = &cacheBackend{}
func newBackend(be backend.Backend, c *Cache, errorLog func(string, ...interface{})) *Backend {
return &Backend{
func newBackend(be backend.Backend, c *Cache, errorLog func(string, ...interface{})) *cacheBackend {
return &cacheBackend{
Backend: be,
Cache: c,
inProgress: make(map[backend.Handle]chan struct{}),
@@ -35,7 +35,7 @@ func newBackend(be backend.Backend, c *Cache, errorLog func(string, ...interface
}
// Remove deletes a file from the backend and the cache if it has been cached.
func (b *Backend) Remove(ctx context.Context, h backend.Handle) error {
func (b *cacheBackend) Remove(ctx context.Context, h backend.Handle) error {
debug.Log("cache Remove(%v)", h)
err := b.Backend.Remove(ctx, h)
if err != nil {
@@ -57,7 +57,7 @@ func autoCacheTypes(h backend.Handle) bool {
}
// Save stores a new file in the backend and the cache.
func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
func (b *cacheBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
if !autoCacheTypes(h) {
return b.Backend.Save(ctx, h, rd)
}
@@ -91,7 +91,7 @@ func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindR
return nil
}
func (b *Backend) cacheFile(ctx context.Context, h backend.Handle) error {
func (b *cacheBackend) cacheFile(ctx context.Context, h backend.Handle) error {
finish := make(chan struct{})
b.inProgressMutex.Lock()
@@ -135,7 +135,7 @@ func (b *Backend) cacheFile(ctx context.Context, h backend.Handle) error {
}
// loadFromCache will try to load the file from the cache.
func (b *Backend) loadFromCache(h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (bool, error) {
func (b *cacheBackend) loadFromCache(h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (bool, error) {
rd, inCache, err := b.Cache.load(h, length, offset)
if err != nil {
return inCache, err
@@ -150,7 +150,7 @@ func (b *Backend) loadFromCache(h backend.Handle, length int, offset int64, cons
}
// Load loads a file from the cache or the backend.
func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
func (b *cacheBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
b.inProgressMutex.Lock()
waitForFinish, inProgress := b.inProgress[h]
b.inProgressMutex.Unlock()
@@ -197,7 +197,7 @@ func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset
// Stat tests whether the backend has a file. If it does not exist but still
// exists in the cache, it is removed from the cache.
func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
func (b *cacheBackend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
debug.Log("cache Stat(%v)", h)
fi, err := b.Backend.Stat(ctx, h)
@@ -210,15 +210,15 @@ func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo,
}
// IsNotExist returns true if the error is caused by a non-existing file.
func (b *Backend) IsNotExist(err error) bool {
func (b *cacheBackend) IsNotExist(err error) bool {
return b.Backend.IsNotExist(err)
}
func (b *Backend) Unwrap() backend.Backend {
func (b *cacheBackend) Unwrap() backend.Backend {
return b.Backend
}
func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(f backend.FileInfo) error) error {
func (b *cacheBackend) List(ctx context.Context, t backend.FileType, fn func(f backend.FileInfo) error) error {
if !b.Cache.canBeCached(t) {
return b.Backend.List(ctx, t, fn)
}
@@ -252,11 +252,11 @@ func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(f backen
}
// Warmup delegates to wrapped backend.
func (b *Backend) Warmup(ctx context.Context, h []backend.Handle) ([]backend.Handle, error) {
func (b *cacheBackend) Warmup(ctx context.Context, h []backend.Handle) ([]backend.Handle, error) {
return b.Backend.Warmup(ctx, h)
}
// WarmupWait delegates to wrapped backend.
func (b *Backend) WarmupWait(ctx context.Context, h []backend.Handle) error {
func (b *cacheBackend) WarmupWait(ctx context.Context, h []backend.Handle) error {
return b.Backend.WarmupWait(ctx, h)
}
internal/backend/gs/gs.go
+33 -49
View File
@@ -27,27 +27,25 @@ import (
"google.golang.org/api/option"
)
// Backend stores data in a GCS bucket.
// gs stores data in a GCS bucket.
//
// The service account used to access the bucket must have these permissions:
// - storage.objects.create
// - storage.objects.delete
// - storage.objects.get
// - storage.objects.list
type Backend struct {
gcsClient *storage.Client
projectID string
connections uint
bucketName string
region string
bucket *storage.BucketHandle
prefix string
listMaxItems int
type gs struct {
gcsClient *storage.Client
projectID string
connections uint
bucketName string
region string
bucket *storage.BucketHandle
layout.Layout
}
// Ensure that *Backend implements backend.Backend.
var _ backend.Backend = &Backend{}
var _ backend.Backend = &gs{}
func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("gs", ParseConfig, location.NoPassword, Create, Open)
@@ -86,7 +84,7 @@ func getStorageClient(rt http.RoundTripper) (*storage.Client, error) {
return gcsClient, nil
}
func (be *Backend) bucketExists(ctx context.Context, bucket *storage.BucketHandle) (bool, error) {
func (be *gs) bucketExists(ctx context.Context, bucket *storage.BucketHandle) (bool, error) {
_, err := bucket.Attrs(ctx)
if err == storage.ErrBucketNotExist {
return false, nil
@@ -94,9 +92,7 @@ func (be *Backend) bucketExists(ctx context.Context, bucket *storage.BucketHandl
return err == nil, err
}
const defaultListMaxItems = 1000
func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
func open(cfg Config, rt http.RoundTripper) (*gs, error) {
debug.Log("open, config %#v", cfg)
gcsClient, err := getStorageClient(rt)
@@ -104,16 +100,14 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
return nil, errors.Wrap(err, "getStorageClient")
}
be := &Backend{
gcsClient: gcsClient,
projectID: cfg.ProjectID,
connections: cfg.Connections,
bucketName: cfg.Bucket,
region: cfg.Region,
bucket: gcsClient.Bucket(cfg.Bucket),
prefix: cfg.Prefix,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
listMaxItems: defaultListMaxItems,
be := &gs{
gcsClient: gcsClient,
projectID: cfg.ProjectID,
connections: cfg.Connections,
bucketName: cfg.Bucket,
region: cfg.Region,
bucket: gcsClient.Bucket(cfg.Bucket),
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
}
return be, nil
@@ -161,17 +155,12 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper, _ func(string
return be, nil
}
// SetListMaxItems sets the number of list items to load per request.
func (be *Backend) SetListMaxItems(i int) {
be.listMaxItems = i
}
// IsNotExist returns true if the error is caused by a not existing file.
func (be *Backend) IsNotExist(err error) bool {
func (be *gs) IsNotExist(err error) bool {
return errors.Is(err, storage.ErrObjectNotExist)
}
func (be *Backend) IsPermanentError(err error) bool {
func (be *gs) IsPermanentError(err error) bool {
if be.IsNotExist(err) {
return true
}
@@ -186,7 +175,7 @@ func (be *Backend) IsPermanentError(err error) bool {
return false
}
func (be *Backend) Properties() backend.Properties {
func (be *gs) Properties() backend.Properties {
return backend.Properties{
Connections: be.connections,
HasAtomicReplace: true,
@@ -194,17 +183,12 @@ func (be *Backend) Properties() backend.Properties {
}
// Hasher may return a hash function for calculating a content hash for the backend
func (be *Backend) Hasher() hash.Hash {
func (be *gs) Hasher() hash.Hash {
return md5.New()
}
// Path returns the path in the bucket that is used for this backend.
func (be *Backend) Path() string {
return be.prefix
}
// Save stores data in the backend at the handle.
func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
func (be *gs) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
objName := be.Filename(h)
// Set chunk size to zero to disable resumable uploads.
@@ -254,14 +238,14 @@ func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.Rewind
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *gs) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
func (be *gs) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
if length == 0 {
// negative length indicates read till end to GCS lib
length = -1
@@ -283,7 +267,7 @@ func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int,
}
// Stat returns information about a blob.
func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
func (be *gs) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
objName := be.Filename(h)
attr, err := be.bucket.Object(objName).Attrs(ctx)
@@ -296,7 +280,7 @@ func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileI
}
// Remove removes the blob with the given name and type.
func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
func (be *gs) Remove(ctx context.Context, h backend.Handle) error {
objName := be.Filename(h)
err := be.bucket.Object(objName).Delete(ctx)
@@ -310,7 +294,7 @@ func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
func (be *gs) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
prefix, _ := be.Basedir(t)
// make sure prefix ends with a slash
@@ -355,15 +339,15 @@ func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend
}
// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
func (be *Backend) Delete(ctx context.Context) error {
func (be *gs) Delete(ctx context.Context) error {
return util.DefaultDelete(ctx, be)
}
// Close does nothing.
func (be *Backend) Close() error { return nil }
func (be *gs) Close() error { return nil }
// Warmup not implemented
func (be *Backend) Warmup(_ context.Context, _ []backend.Handle) ([]backend.Handle, error) {
func (be *gs) Warmup(_ context.Context, _ []backend.Handle) ([]backend.Handle, error) {
return []backend.Handle{}, nil
}
func (be *Backend) WarmupWait(_ context.Context, _ []backend.Handle) error { return nil }
func (be *gs) WarmupWait(_ context.Context, _ []backend.Handle) error { return nil }
internal/backend/rclone/backend.go
+8 -8
View File
@@ -28,8 +28,8 @@ import (
"golang.org/x/net/http2"
)
// Backend is used to access data stored somewhere via rclone.
type Backend struct {
// rclone is used to access data stored somewhere via rclone.
type rclone struct {
*rest.Backend
tr *http2.Transport
cmd *exec.Cmd
@@ -141,7 +141,7 @@ func wrapConn(c *StdioConn, lim limiter.Limiter) *wrappedConn {
}
// New initializes a Backend and starts the process.
func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*Backend, error) {
func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*rclone, error) {
var (
args []string
err error
@@ -196,7 +196,7 @@ func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog f
}
cmd := stdioConn.cmd
be := &Backend{
be := &rclone{
tr: tr,
cmd: cmd,
waitCh: waitCh,
@@ -270,7 +270,7 @@ func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog f
}
// Open starts an rclone process with the given config.
func Open(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*Backend, error) {
func Open(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (backend.Backend, error) {
be, err := newBackend(ctx, cfg, lim, errorLog)
if err != nil {
return nil, err
@@ -297,7 +297,7 @@ func Open(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(st
}
// Create initializes a new restic repo with rclone.
func Create(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (*Backend, error) {
func Create(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(string, ...interface{})) (backend.Backend, error) {
be, err := newBackend(ctx, cfg, lim, errorLog)
if err != nil {
return nil, err
@@ -328,7 +328,7 @@ func Create(ctx context.Context, cfg Config, lim limiter.Limiter, errorLog func(
const waitForExit = 5 * time.Second
// Close terminates the backend.
func (be *Backend) Close() error {
func (be *rclone) Close() error {
debug.Log("exiting rclone")
be.tr.CloseIdleConnections()
@@ -348,7 +348,7 @@ func (be *Backend) Close() error {
return be.waitResult
}
func (be *Backend) Properties() backend.Properties {
func (be *rclone) Properties() backend.Properties {
properties := be.Backend.Properties()
properties.HasFlakyErrors = true
return properties
internal/backend/rclone/internal_test.go
+1 -1
View File
@@ -27,7 +27,7 @@ func TestRcloneExit(t *testing.T) {
_ = be.Close()
}()
err = be.cmd.Process.Kill()
err = be.(*rclone).cmd.Process.Kill()
rtest.OK(t, err)
t.Log("killed rclone")
internal/backend/s3/s3.go
+23 -28
View File
@@ -25,15 +25,15 @@ import (
"github.com/minio/minio-go/v7/pkg/credentials"
)
// Backend stores data on an S3 endpoint.
type Backend struct {
// s3 stores data on an S3 endpoint.
type s3 struct {
client *minio.Client
cfg Config
layout.Layout
}
// make sure that *Backend implements backend.Backend
var _ backend.Backend = &Backend{}
var _ backend.Backend = &s3{}
var archiveClasses = []string{"GLACIER", "DEEP_ARCHIVE"}
@@ -50,7 +50,7 @@ func NewFactory() location.Factory {
return location.NewHTTPBackendFactory("s3", ParseConfig, location.NoPassword, Create, Open)
}
func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
func open(cfg Config, rt http.RoundTripper) (*s3, error) {
debug.Log("open, config %#v", cfg)
if cfg.EnableRestore && !feature.Flag.Enabled(feature.S3Restore) {
@@ -89,7 +89,7 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
return nil, errors.Wrap(err, "minio.New")
}
be := &Backend{
be := &s3{
client: client,
cfg: cfg,
Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join),
@@ -240,12 +240,12 @@ func isAccessDenied(err error) bool {
}
// IsNotExist returns true if the error is caused by a not existing file.
func (be *Backend) IsNotExist(err error) bool {
func (be *s3) IsNotExist(err error) bool {
var e minio.ErrorResponse
return errors.As(err, &e) && e.Code == "NoSuchKey"
}
func (be *Backend) IsPermanentError(err error) bool {
func (be *s3) IsPermanentError(err error) bool {
if be.IsNotExist(err) {
return true
}
@@ -260,7 +260,7 @@ func (be *Backend) IsPermanentError(err error) bool {
return false
}
func (be *Backend) Properties() backend.Properties {
func (be *s3) Properties() backend.Properties {
return backend.Properties{
Connections: be.cfg.Connections,
HasAtomicReplace: true,
@@ -268,26 +268,21 @@ func (be *Backend) Properties() backend.Properties {
}
// Hasher may return a hash function for calculating a content hash for the backend
func (be *Backend) Hasher() hash.Hash {
func (be *s3) Hasher() hash.Hash {
return nil
}
// Path returns the path in the bucket that is used for this backend.
func (be *Backend) Path() string {
return be.cfg.Prefix
}
// useStorageClass returns whether file should be saved in the provided Storage Class
// For archive storage classes, only data files are stored using that class; metadata
// must remain instantly accessible.
func (be *Backend) useStorageClass(h backend.Handle) bool {
func (be *s3) useStorageClass(h backend.Handle) bool {
isDataFile := h.Type == backend.PackFile && !h.IsMetadata
isArchiveClass := slices.Contains(archiveClasses, be.cfg.StorageClass)
return !isArchiveClass || isDataFile
}
// Save stores data in the backend at the handle.
func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
func (be *s3) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error {
objName := be.Filename(h)
opts := minio.PutObjectOptions{
@@ -313,14 +308,14 @@ func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.Rewind
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
func (be *s3) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn)
}
func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
func (be *s3) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
objName := be.Filename(h)
opts := minio.GetObjectOptions{}
@@ -352,7 +347,7 @@ func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int,
}
// Stat returns information about a blob.
func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
func (be *s3) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) {
objName := be.Filename(h)
var obj *minio.Object
@@ -380,7 +375,7 @@ func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileI
}
// Remove removes the blob with the given name and type.
func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
func (be *s3) Remove(ctx context.Context, h backend.Handle) error {
objName := be.Filename(h)
err := be.client.RemoveObject(ctx, be.cfg.Bucket, objName, minio.RemoveObjectOptions{})
@@ -394,7 +389,7 @@ func (be *Backend) Remove(ctx context.Context, h backend.Handle) error {
// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
func (be *s3) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
prefix, recursive := be.Basedir(t)
// make sure prefix ends with a slash
@@ -449,15 +444,15 @@ func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend
}
// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
func (be *Backend) Delete(ctx context.Context) error {
func (be *s3) Delete(ctx context.Context) error {
return util.DefaultDelete(ctx, be)
}
// Close does nothing
func (be *Backend) Close() error { return nil }
func (be *s3) Close() error { return nil }
// Warmup transitions handles from cold to hot storage if needed.
func (be *Backend) Warmup(ctx context.Context, handles []backend.Handle) ([]backend.Handle, error) {
func (be *s3) Warmup(ctx context.Context, handles []backend.Handle) ([]backend.Handle, error) {
handlesWarmingUp := []backend.Handle{}
if be.cfg.EnableRestore {
@@ -478,7 +473,7 @@ func (be *Backend) Warmup(ctx context.Context, handles []backend.Handle) ([]back
}
// requestRestore sends a glacier restore request on a given file.
func (be *Backend) requestRestore(ctx context.Context, filename string) (bool, error) {
func (be *s3) requestRestore(ctx context.Context, filename string) (bool, error) {
objectInfo, err := be.client.StatObject(ctx, be.cfg.Bucket, filename, minio.StatObjectOptions{})
if err != nil {
return false, err
@@ -514,7 +509,7 @@ func (be *Backend) requestRestore(ctx context.Context, filename string) (bool, e
}
// getWarmupStatus returns the warmup status of the provided object.
func (be *Backend) getWarmupStatus(objectInfo minio.ObjectInfo) warmupStatus {
func (be *s3) getWarmupStatus(objectInfo minio.ObjectInfo) warmupStatus {
// We can't use objectInfo.StorageClass to get the storage class of the
// object because this field is only set during ListObjects operations.
// The response header is the documented way to get the storage class
@@ -545,7 +540,7 @@ func (be *Backend) getWarmupStatus(objectInfo minio.ObjectInfo) warmupStatus {
}
// WarmupWait waits until all handles are in hot storage.
func (be *Backend) WarmupWait(ctx context.Context, handles []backend.Handle) error {
func (be *s3) WarmupWait(ctx context.Context, handles []backend.Handle) error {
timeoutCtx, timeoutCtxCancel := context.WithTimeout(ctx, be.cfg.RestoreTimeout)
defer timeoutCtxCancel()
@@ -564,7 +559,7 @@ func (be *Backend) WarmupWait(ctx context.Context, handles []backend.Handle) err
}
// waitForRestore waits for a given file to be restored.
func (be *Backend) waitForRestore(ctx context.Context, filename string) error {
func (be *s3) waitForRestore(ctx context.Context, filename string) error {
for {
var objectInfo minio.ObjectInfo
internal/backend/test/tests.go
+28 -47
View File
@@ -251,10 +251,6 @@ func (s *Suite[C]) TestLoad(t *testing.T) {
test.OK(t, b.Remove(context.TODO(), handle))
}
type setter interface {
SetListMaxItems(int)
}
// TestList makes sure that the backend implements List() pagination correctly.
func (s *Suite[C]) TestList(t *testing.T) {
random := seedRand(t)
@@ -302,54 +298,39 @@ func (s *Suite[C]) TestList(t *testing.T) {
t.Logf("wrote %v files", len(list1))
var tests = []struct {
maxItems int
}{
{11}, {23}, {numTestFiles}, {numTestFiles + 10}, {numTestFiles + 1123},
list2 := make(map[restic.ID]int64)
err = b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
id, err := restic.ParseID(fi.Name)
if err != nil {
t.Fatal(err)
}
list2[id] = fi.Size
return nil
})
if err != nil {
t.Fatalf("List returned error %v", err)
}
for _, test := range tests {
t.Run(fmt.Sprintf("max-%v", test.maxItems), func(t *testing.T) {
list2 := make(map[restic.ID]int64)
t.Logf("loaded %v IDs from backend", len(list2))
if s, ok := b.(setter); ok {
t.Logf("setting max list items to %d", test.maxItems)
s.SetListMaxItems(test.maxItems)
}
for id, size := range list1 {
size2, ok := list2[id]
if !ok {
t.Errorf("id %v not returned by List()", id.Str())
}
err := b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error {
id, err := restic.ParseID(fi.Name)
if err != nil {
t.Fatal(err)
}
list2[id] = fi.Size
return nil
})
if size != size2 {
t.Errorf("wrong size for id %v returned: want %v, got %v", id.Str(), size, size2)
}
}
if err != nil {
t.Fatalf("List returned error %v", err)
}
t.Logf("loaded %v IDs from backend", len(list2))
for id, size := range list1 {
size2, ok := list2[id]
if !ok {
t.Errorf("id %v not returned by List()", id.Str())
}
if size != size2 {
t.Errorf("wrong size for id %v returned: want %v, got %v", id.Str(), size, size2)
}
}
for id := range list2 {
_, ok := list1[id]
if !ok {
t.Errorf("extra id %v returned by List()", id.Str())
}
}
})
for id := range list2 {
_, ok := list1[id]
if !ok {
t.Errorf("extra id %v returned by List()", id.Str())
}
}
t.Logf("remove %d files", numTestFiles)