Skip to content

Commit

Permalink
requestMetadata should take the md5 and sha256 with expected encodings.
Browse files Browse the repository at this point in the history 
This is done for readability and also to better communicate
the requirements of an API.
  • Loading branch information
@harshavardhana @minio-trusted
harshavardhana authored and minio-trusted committed Oct 22, 2017
1 parent 9169d8a commit 3093a47
Show file tree
Hide file tree
Showing 18 changed files with 172 additions and 148 deletions.
2 changes: 1 addition & 1 deletion api-compose-object.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ func (s *SSEInfo) getSSEHeaders(isCopySource bool) map[string]string {
return map[string]string{
"x-amz-" + cs + "server-side-encryption-customer-algorithm": s.algo,
"x-amz-" + cs + "server-side-encryption-customer-key": base64.StdEncoding.EncodeToString(s.key),
"x-amz-" + cs + "server-side-encryption-customer-key-MD5": base64.StdEncoding.EncodeToString(sumMD5(s.key)),
"x-amz-" + cs + "server-side-encryption-customer-key-MD5": sumMD5Base64(s.key),
}
}

Expand Down
8 changes: 4 additions & 4 deletions api-get-object.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -612,10 +612,10 @@ func (c Client) getObject(ctx context.Context, bucketName, objectName string, op

// Execute GET on objectName.
resp, err := c.executeMethod(ctx, "GET", requestMetadata{
bucketName: bucketName,
objectName: objectName,
customHeader: opts.Header(),
contentSHA256Bytes: emptySHA256,
bucketName: bucketName,
objectName: objectName,
customHeader: opts.Header(),
contentSHA256Hex: emptySHA256Hex,
})
if err != nil {
return nil, ObjectInfo{}, err
Expand Down
6 changes: 3 additions & 3 deletions api-get-policy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,9 +81,9 @@ func (c Client) getBucketPolicy(bucketName string) (policy.BucketAccessPolicy, e

// Execute GET on bucket to list objects.
resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{
bucketName: bucketName,
queryValues: urlValues,
contentSHA256Bytes: emptySHA256,
bucketName: bucketName,
queryValues: urlValues,
contentSHA256Hex: emptySHA256Hex,
})

defer closeResponse(resp)
Expand Down
28 changes: 14 additions & 14 deletions api-list.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ import (
//
func (c Client) ListBuckets() ([]BucketInfo, error) {
// Execute GET on service.
resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{contentSHA256Bytes: emptySHA256})
resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{contentSHA256Hex: emptySHA256Hex})
defer closeResponse(resp)
if err != nil {
return nil, err
Expand Down Expand Up @@ -217,9 +217,9 @@ func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken s

// Execute GET on bucket to list objects.
resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{
bucketName: bucketName,
queryValues: urlValues,
contentSHA256Bytes: emptySHA256,
bucketName: bucketName,
queryValues: urlValues,
contentSHA256Hex: emptySHA256Hex,
})
defer closeResponse(resp)
if err != nil {
Expand Down Expand Up @@ -395,9 +395,9 @@ func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimit

// Execute GET on bucket to list objects.
resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{
bucketName: bucketName,
queryValues: urlValues,
contentSHA256Bytes: emptySHA256,
bucketName: bucketName,
queryValues: urlValues,
contentSHA256Hex: emptySHA256Hex,
})
defer closeResponse(resp)
if err != nil {
Expand Down Expand Up @@ -574,9 +574,9 @@ func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker,

// Execute GET on bucketName to list multipart uploads.
resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{
bucketName: bucketName,
queryValues: urlValues,
contentSHA256Bytes: emptySHA256,
bucketName: bucketName,
queryValues: urlValues,
contentSHA256Hex: emptySHA256Hex,
})
defer closeResponse(resp)
if err != nil {
Expand Down Expand Up @@ -692,10 +692,10 @@ func (c Client) listObjectPartsQuery(bucketName, objectName, uploadID string, pa

// Execute GET on objectName to get list of parts.
resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{
bucketName: bucketName,
objectName: objectName,
queryValues: urlValues,
contentSHA256Bytes: emptySHA256,
bucketName: bucketName,
objectName: objectName,
queryValues: urlValues,
contentSHA256Hex: emptySHA256Hex,
})
defer closeResponse(resp)
if err != nil {
Expand Down
12 changes: 6 additions & 6 deletions api-notification.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,9 @@ func (c Client) getBucketNotification(bucketName string) (BucketNotification, er

// Execute GET on bucket to list objects.
resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{
bucketName: bucketName,
queryValues: urlValues,
contentSHA256Bytes: emptySHA256,
bucketName: bucketName,
queryValues: urlValues,
contentSHA256Hex: emptySHA256Hex,
})

defer closeResponse(resp)
Expand Down Expand Up @@ -172,9 +172,9 @@ func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, even

// Execute GET on bucket to list objects.
resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{
bucketName: bucketName,
queryValues: urlValues,
contentSHA256Bytes: emptySHA256,
bucketName: bucketName,
queryValues: urlValues,
contentSHA256Hex: emptySHA256Hex,
})
if err != nil {
continue
Expand Down
34 changes: 17 additions & 17 deletions api-put-bucket.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,8 @@ func (c Client) MakeBucket(bucketName string, location string) (err error) {
if err != nil {
return err
}
reqMetadata.contentMD5Bytes = sumMD5(createBucketConfigBytes)
reqMetadata.contentSHA256Bytes = sum256(createBucketConfigBytes)
reqMetadata.contentMD5Base64 = sumMD5Base64(createBucketConfigBytes)
reqMetadata.contentSHA256Hex = sum256Hex(createBucketConfigBytes)
reqMetadata.contentBody = bytes.NewReader(createBucketConfigBytes)
reqMetadata.contentLength = int64(len(createBucketConfigBytes))
}
Expand Down Expand Up @@ -162,12 +162,12 @@ func (c Client) putBucketPolicy(bucketName string, policyInfo policy.BucketAcces

policyBuffer := bytes.NewReader(policyBytes)
reqMetadata := requestMetadata{
bucketName: bucketName,
queryValues: urlValues,
contentBody: policyBuffer,
contentLength: int64(len(policyBytes)),
contentMD5Bytes: sumMD5(policyBytes),
contentSHA256Bytes: sum256(policyBytes),
bucketName: bucketName,
queryValues: urlValues,
contentBody: policyBuffer,
contentLength: int64(len(policyBytes)),
contentMD5Base64: sumMD5Base64(policyBytes),
contentSHA256Hex: sum256Hex(policyBytes),
}

// Execute PUT to upload a new bucket policy.
Expand Down Expand Up @@ -197,9 +197,9 @@ func (c Client) removeBucketPolicy(bucketName string) error {

// Execute DELETE on objectName.
resp, err := c.executeMethod(context.Background(), "DELETE", requestMetadata{
bucketName: bucketName,
queryValues: urlValues,
contentSHA256Bytes: emptySHA256,
bucketName: bucketName,
queryValues: urlValues,
contentSHA256Hex: emptySHA256Hex,
})
defer closeResponse(resp)
if err != nil {
Expand Down Expand Up @@ -227,12 +227,12 @@ func (c Client) SetBucketNotification(bucketName string, bucketNotification Buck

notifBuffer := bytes.NewReader(notifBytes)
reqMetadata := requestMetadata{
bucketName: bucketName,
queryValues: urlValues,
contentBody: notifBuffer,
contentLength: int64(len(notifBytes)),
contentMD5Bytes: sumMD5(notifBytes),
contentSHA256Bytes: sum256(notifBytes),
bucketName: bucketName,
queryValues: urlValues,
contentBody: notifBuffer,
contentLength: int64(len(notifBytes)),
contentMD5Base64: sumMD5Base64(notifBytes),
contentSHA256Hex: sum256Hex(notifBytes),
}

// Execute PUT to upload a new bucket notification.
Expand Down
26 changes: 20 additions & 6 deletions api-put-object-common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,6 @@ import (
"github.com/minio/minio-go/pkg/s3utils"
)

// Verify if reader is *os.File
func isFile(reader io.Reader) (ok bool) {
_, ok = reader.(*os.File)
return
}

// Verify if reader is *minio.Object
func isObject(reader io.Reader) (ok bool) {
_, ok = reader.(*Object)
Expand All @@ -40,6 +34,26 @@ func isObject(reader io.Reader) (ok bool) {
// Verify if reader is a generic ReaderAt
func isReadAt(reader io.Reader) (ok bool) {
_, ok = reader.(io.ReaderAt)
if ok {
var v *os.File
v, ok = reader.(*os.File)
if ok {
// Stdin, Stdout and Stderr all have *os.File type
// which happen to also are io.ReaderAt compatible
// we need to add special conditions for them to
// be ignored.
for _, f := range []string{
"/dev/stdin",
"/dev/stdout",
"/dev/stderr",
} {
if f == v.Name() {
ok = false
break
}
}
}
}
return
}

Expand Down
46 changes: 30 additions & 16 deletions api-put-object-multipart.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ package minio
import (
"bytes"
"context"
"encoding/base64"
"encoding/hex"
"encoding/xml"
"fmt"
"io"
Expand Down Expand Up @@ -120,10 +122,22 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje
// as we read from the source.
rd := newHook(bytes.NewReader(buf[:length]), opts.Progress)

// Checksums..
var (
md5Base64 string
sha256Hex string
)
if hashSums["md5"] != nil {
md5Base64 = base64.StdEncoding.EncodeToString(hashSums["md5"])
}
if hashSums["sha256"] != nil {
sha256Hex = hex.EncodeToString(hashSums["sha256"])
}

// Proceed to upload the part.
var objPart ObjectPart
objPart, err = c.uploadPart(ctx, bucketName, objectName, uploadID, rd, partNumber,
hashSums["md5"], hashSums["sha256"], int64(length), opts.UserMetadata)
md5Base64, sha256Hex, int64(length), opts.UserMetadata)
if err != nil {
return totalUploadedSize, err
}
Expand Down Expand Up @@ -215,7 +229,7 @@ const serverEncryptionKeyPrefix = "x-amz-server-side-encryption"

// uploadPart - Uploads a part in a multipart upload.
func (c Client) uploadPart(ctx context.Context, bucketName, objectName, uploadID string, reader io.Reader,
partNumber int, md5Sum, sha256Sum []byte, size int64, metadata map[string]string) (ObjectPart, error) {
partNumber int, md5Base64, sha256Hex string, size int64, metadata map[string]string) (ObjectPart, error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return ObjectPart{}, err
Expand Down Expand Up @@ -254,14 +268,14 @@ func (c Client) uploadPart(ctx context.Context, bucketName, objectName, uploadID
}

reqMetadata := requestMetadata{
bucketName: bucketName,
objectName: objectName,
queryValues: urlValues,
customHeader: customHeader,
contentBody: reader,
contentLength: size,
contentMD5Bytes: md5Sum,
contentSHA256Bytes: sha256Sum,
bucketName: bucketName,
objectName: objectName,
queryValues: urlValues,
customHeader: customHeader,
contentBody: reader,
contentLength: size,
contentMD5Base64: md5Base64,
contentSHA256Hex: sha256Hex,
}

// Execute PUT on each part.
Expand Down Expand Up @@ -308,12 +322,12 @@ func (c Client) completeMultipartUpload(ctx context.Context, bucketName, objectN
// Instantiate all the complete multipart buffer.
completeMultipartUploadBuffer := bytes.NewReader(completeMultipartUploadBytes)
reqMetadata := requestMetadata{
bucketName: bucketName,
objectName: objectName,
queryValues: urlValues,
contentBody: completeMultipartUploadBuffer,
contentLength: int64(len(completeMultipartUploadBytes)),
contentSHA256Bytes: sum256(completeMultipartUploadBytes),
bucketName: bucketName,
objectName: objectName,
queryValues: urlValues,
contentBody: completeMultipartUploadBuffer,
contentLength: int64(len(completeMultipartUploadBytes)),
contentSHA256Hex: sum256Hex(completeMultipartUploadBytes),
}

// Execute POST to complete multipart upload for an objectName.
Expand Down
32 changes: 14 additions & 18 deletions api-put-object-streaming.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,19 +33,15 @@ import (
//
// Following code handles these types of readers.
//
// - *os.File
// - *minio.Object
// - Any reader which has a method 'ReadAt()'
//
func (c Client) putObjectMultipartStream(ctx context.Context, bucketName, objectName string,
reader io.Reader, size int64, opts PutObjectOptions) (n int64, err error) {

// Verify if reader is *minio.Object, *os.File or io.ReaderAt.
// NOTE: Verification of object is kept for a specific purpose
// while it is going to be duck typed similar to io.ReaderAt.
// It is to indicate that *minio.Object implements io.ReaderAt.
// and such a functionality is used in the subsequent code path.
if isFile(reader) || !isObject(reader) && isReadAt(reader) {
// Verify if reader is *minio.Object, if yes then use a sequential uploader.
if !isObject(reader) && isReadAt(reader) {
// Verify if the reader implements ReadAt then we will use parallel uploader.
n, err = c.putObjectMultipartStreamFromReadAt(ctx, bucketName, objectName, reader.(io.ReaderAt), size, opts)
} else {
n, err = c.putObjectMultipartStreamNoChecksum(ctx, bucketName, objectName, reader, size, opts)
Expand Down Expand Up @@ -171,7 +167,7 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa
var objPart ObjectPart
objPart, err = c.uploadPart(ctx, bucketName, objectName, uploadID,
sectionReader, uploadReq.PartNum,
nil, nil, partSize, opts.UserMetadata)
"", "", partSize, opts.UserMetadata)
if err != nil {
uploadedPartsCh <- uploadedPartRes{
Size: 0,
Expand Down Expand Up @@ -284,7 +280,7 @@ func (c Client) putObjectMultipartStreamNoChecksum(ctx context.Context, bucketNa
var objPart ObjectPart
objPart, err = c.uploadPart(ctx, bucketName, objectName, uploadID,
io.LimitReader(hookReader, partSize),
partNumber, nil, nil, partSize, opts.UserMetadata)
partNumber, "", "", partSize, opts.UserMetadata)
if err != nil {
return totalUploadedSize, err
}
Expand Down Expand Up @@ -358,7 +354,7 @@ func (c Client) putObjectNoChecksum(ctx context.Context, bucketName, objectName

// This function does not calculate sha256 and md5sum for payload.
// Execute put object.
st, err := c.putObjectDo(ctx, bucketName, objectName, readSeeker, nil, nil, size, opts)
st, err := c.putObjectDo(ctx, bucketName, objectName, readSeeker, "", "", size, opts)
if err != nil {
return 0, err
}
Expand All @@ -370,7 +366,7 @@ func (c Client) putObjectNoChecksum(ctx context.Context, bucketName, objectName

// putObjectDo - executes the put object http operation.
// NOTE: You must have WRITE permissions on a bucket to add an object to it.
func (c Client) putObjectDo(ctx context.Context, bucketName, objectName string, reader io.Reader, md5Sum []byte, sha256Sum []byte, size int64, opts PutObjectOptions) (ObjectInfo, error) {
func (c Client) putObjectDo(ctx context.Context, bucketName, objectName string, reader io.Reader, md5Base64, sha256Hex string, size int64, opts PutObjectOptions) (ObjectInfo, error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return ObjectInfo{}, err
Expand All @@ -383,13 +379,13 @@ func (c Client) putObjectDo(ctx context.Context, bucketName, objectName string,

// Populate request metadata.
reqMetadata := requestMetadata{
bucketName: bucketName,
objectName: objectName,
customHeader: customHeader,
contentBody: reader,
contentLength: size,
contentMD5Bytes: md5Sum,
contentSHA256Bytes: sha256Sum,
bucketName: bucketName,
objectName: objectName,
customHeader: customHeader,
contentBody: reader,
contentLength: size,
contentMD5Base64: md5Base64,
contentSHA256Hex: sha256Hex,
}

// Execute PUT an objectName.
Expand Down

0 comments on commit 3093a47

Please sign in to comment.