Merge pull request #3085 from LordGaav/s3-list-objects-v1-flag

Extended option to select V1 API for ListObjects on S3 backend

internal/backend/s3/config.go

@@ -20,16 +20,18 @@ type Config struct {
 	Layout        string `option:"layout" help:"use this backend layout (default: auto-detect)"`
 	StorageClass  string `option:"storage-class" help:"set S3 storage class (STANDARD, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING or REDUCED_REDUNDANCY)"`
 
-	Connections  uint   `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
-	MaxRetries   uint   `option:"retries" help:"set the number of retries attempted"`
-	Region       string `option:"region" help:"set region"`
-	BucketLookup string `option:"bucket-lookup" help:"bucket lookup style: 'auto', 'dns', or 'path'."`
+	Connections   uint   `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
+	MaxRetries    uint   `option:"retries" help:"set the number of retries attempted"`
+	Region        string `option:"region" help:"set region"`
+	BucketLookup  string `option:"bucket-lookup" help:"bucket lookup style: 'auto', 'dns', or 'path'"`
+	ListObjectsV1 bool   `option:"list-objects-v1" help:"use deprecated V1 api for ListObjects calls"`
 }
 
 // NewConfig returns a new Config with the default values filled in.
 func NewConfig() Config {
 	return Config{
-		Connections: 5,
+		Connections:   5,
+		ListObjectsV1: false,
 	}
 }
 

internal/backend/s3/s3.go

@@ -205,9 +205,12 @@ func (be *Backend) ReadDir(ctx context.Context, dir string) (list []os.FileInfo,
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
+	debug.Log("using ListObjectsV1(%v)", be.cfg.ListObjectsV1)
+
 	for obj := range be.client.ListObjects(ctx, be.cfg.Bucket, minio.ListObjectsOptions{
 		Prefix:    dir,
 		Recursive: false,
+		UseV1:     be.cfg.ListObjectsV1,
 	}) {
 		if obj.Err != nil {
 			return nil, err
@@ -427,12 +430,15 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
+	debug.Log("using ListObjectsV1(%v)", be.cfg.ListObjectsV1)
+
 	// NB: unfortunately we can't protect this with be.sem.GetToken() here.
 	// Doing so would enable a deadlock situation (gh-1399), as ListObjects()
 	// starts its own goroutine and returns results via a channel.
 	listresp := be.client.ListObjects(ctx, be.cfg.Bucket, minio.ListObjectsOptions{
 		Prefix:    prefix,
 		Recursive: recursive,
+		UseV1:     be.cfg.ListObjectsV1,
 	})
 
 	for obj := range listresp {

internal/options/options.go

@@ -199,6 +199,14 @@ func (o Options) Apply(ns string, dst interface{}) error {
 
 			v.Field(i).SetUint(vi)
 
+		case "bool":
+			vi, err := strconv.ParseBool(value)
+			if err != nil {
+				return err
+			}
+
+			v.Field(i).SetBool(vi)
+
 		case "Duration":
 			d, err := time.ParseDuration(value)
 			if err != nil {