mirror of
https://github.com/restic/restic.git
synced 2026-05-15 14:45:24 +00:00
Update github.com/minio/minio-go
This corrects a severe performance regression when the current minio server is used as a backend, see https://github.com/minio/minio/issues/4275
This commit is contained in:
+102
-18
@@ -20,6 +20,7 @@ import (
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio-go/pkg/encrypt"
|
||||
"github.com/minio/minio-go/pkg/s3utils"
|
||||
)
|
||||
|
||||
@@ -30,6 +31,29 @@ func (c Client) PutObjectWithProgress(bucketName, objectName string, reader io.R
|
||||
return c.PutObjectWithMetadata(bucketName, objectName, reader, metaData, progress)
|
||||
}
|
||||
|
||||
// PutEncryptedObject - Encrypt and store object.
|
||||
func (c Client) PutEncryptedObject(bucketName, objectName string, reader io.Reader, encryptMaterials encrypt.Materials, metaData map[string][]string, progress io.Reader) (n int64, err error) {
|
||||
|
||||
if encryptMaterials == nil {
|
||||
return 0, ErrInvalidArgument("Unable to recognize empty encryption properties")
|
||||
}
|
||||
|
||||
if err := encryptMaterials.SetupEncryptMode(reader); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if metaData == nil {
|
||||
metaData = make(map[string][]string)
|
||||
}
|
||||
|
||||
// Set the necessary encryption headers, for future decryption.
|
||||
metaData[amzHeaderIV] = []string{encryptMaterials.GetIV()}
|
||||
metaData[amzHeaderKey] = []string{encryptMaterials.GetKey()}
|
||||
metaData[amzHeaderMatDesc] = []string{encryptMaterials.GetDesc()}
|
||||
|
||||
return c.PutObjectWithMetadata(bucketName, objectName, encryptMaterials, metaData, progress)
|
||||
}
|
||||
|
||||
// PutObjectWithMetadata - with metadata.
|
||||
func (c Client) PutObjectWithMetadata(bucketName, objectName string, reader io.Reader, metaData map[string][]string, progress io.Reader) (n int64, err error) {
|
||||
// Input validation.
|
||||
@@ -75,24 +99,6 @@ func (c Client) PutObjectWithMetadata(bucketName, objectName string, reader io.R
|
||||
return c.putObjectNoChecksum(bucketName, objectName, reader, size, metaData, progress)
|
||||
}
|
||||
|
||||
// NOTE: S3 doesn't allow anonymous multipart requests.
|
||||
if s3utils.IsAmazonEndpoint(c.endpointURL) && c.anonymous {
|
||||
if size <= -1 {
|
||||
return 0, ErrorResponse{
|
||||
Code: "NotImplemented",
|
||||
Message: "Content-Length cannot be negative for anonymous requests.",
|
||||
Key: objectName,
|
||||
BucketName: bucketName,
|
||||
}
|
||||
}
|
||||
if size > maxSinglePutObjectSize {
|
||||
return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName)
|
||||
}
|
||||
// Do not compute MD5 for anonymous requests to Amazon
|
||||
// S3. Uploads up to 5GiB in size.
|
||||
return c.putObjectNoChecksum(bucketName, objectName, reader, size, metaData, progress)
|
||||
}
|
||||
|
||||
// putSmall object.
|
||||
if size < minPartSize && size >= 0 {
|
||||
return c.putObjectSingle(bucketName, objectName, reader, size, metaData, progress)
|
||||
@@ -115,3 +121,81 @@ func (c Client) PutObjectWithMetadata(bucketName, objectName string, reader io.R
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// PutObjectStreaming using AWS streaming signature V4
|
||||
func (c Client) PutObjectStreaming(bucketName, objectName string, reader io.Reader) (n int64, err error) {
|
||||
return c.PutObjectStreamingWithProgress(bucketName, objectName, reader, nil, nil)
|
||||
}
|
||||
|
||||
// PutObjectStreamingWithMetadata using AWS streaming signature V4
|
||||
func (c Client) PutObjectStreamingWithMetadata(bucketName, objectName string, reader io.Reader, metadata map[string][]string) (n int64, err error) {
|
||||
return c.PutObjectStreamingWithProgress(bucketName, objectName, reader, metadata, nil)
|
||||
}
|
||||
|
||||
// PutObjectStreamingWithProgress using AWS streaming signature V4
|
||||
func (c Client) PutObjectStreamingWithProgress(bucketName, objectName string, reader io.Reader, metadata map[string][]string, progress io.Reader) (n int64, err error) {
|
||||
// NOTE: Streaming signature is not supported by GCS.
|
||||
if s3utils.IsGoogleEndpoint(c.endpointURL) {
|
||||
return 0, ErrorResponse{
|
||||
Code: "NotImplemented",
|
||||
Message: "AWS streaming signature v4 is not supported with Google Cloud Storage",
|
||||
Key: objectName,
|
||||
BucketName: bucketName,
|
||||
}
|
||||
}
|
||||
// This method should return error with signature v2 minioClient.
|
||||
if c.signature.isV2() {
|
||||
return 0, ErrorResponse{
|
||||
Code: "NotImplemented",
|
||||
Message: "AWS streaming signature v4 is not supported with minio client initialized for AWS signature v2",
|
||||
Key: objectName,
|
||||
BucketName: bucketName,
|
||||
}
|
||||
}
|
||||
|
||||
// Size of the object.
|
||||
var size int64
|
||||
|
||||
// Get reader size.
|
||||
size, err = getReaderSize(reader)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Check for largest object size allowed.
|
||||
if size > int64(maxMultipartPutObjectSize) {
|
||||
return 0, ErrEntityTooLarge(size, maxMultipartPutObjectSize, bucketName, objectName)
|
||||
}
|
||||
|
||||
// If size cannot be found on a stream, it is not possible
|
||||
// to upload using streaming signature, fall back to multipart.
|
||||
if size < 0 {
|
||||
return c.putObjectMultipartStream(bucketName, objectName, reader, size, metadata, progress)
|
||||
}
|
||||
|
||||
// Set signature type to streaming signature v4.
|
||||
c.signature = SignatureV4Streaming
|
||||
|
||||
if size < minPartSize && size >= 0 {
|
||||
return c.putObjectNoChecksum(bucketName, objectName, reader, size, metadata, progress)
|
||||
}
|
||||
|
||||
// For all sizes greater than 64MiB do multipart.
|
||||
n, err = c.putObjectMultipartStreamNoChecksum(bucketName, objectName, reader, size, metadata, progress)
|
||||
if err != nil {
|
||||
errResp := ToErrorResponse(err)
|
||||
// Verify if multipart functionality is not available, if not
|
||||
// fall back to single PutObject operation.
|
||||
if errResp.Code == "AccessDenied" && strings.Contains(errResp.Message, "Access Denied") {
|
||||
// Verify if size of reader is greater than '5GiB'.
|
||||
if size > maxSinglePutObjectSize {
|
||||
return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName)
|
||||
}
|
||||
// Fall back to uploading as single PutObject operation.
|
||||
return c.putObjectNoChecksum(bucketName, objectName, reader, size, metadata, progress)
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
return n, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user