Skip to content

Commit

Permalink
refactor GET object operations
Browse files Browse the repository at this point in the history 
This change refactors get-object calls to take GetOptions.
This is required to use client and server side encryption within
minio-go.
  • Loading branch information
Andreas Auernhammer committed Sep 27, 2017
1 parent c61055a commit 5f80bfe
Show file tree
Hide file tree
Showing 19 changed files with 232 additions and 202 deletions.
1 change: 1 addition & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,7 @@ The full API Reference is available here.
### Full Examples : Encrypted Object Operations
* [put-encrypted-object.go](https://github.com/minio/minio-go/blob/master/examples/s3/put-encrypted-object.go)
* [get-encrypted-object.go](https://github.com/minio/minio-go/blob/master/examples/s3/get-encrypted-object.go)
* [fput-encrypted-object.go](https://github.com/minio/minio-go/blob/master/examples/s3/fputencrypted-object.go)

### Full Examples : Presigned Operations
* [presignedgetobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedgetobject.go)
Expand Down
6 changes: 3 additions & 3 deletions api-compose-object.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -244,11 +244,11 @@ func (s *SourceInfo) getProps(c Client) (size int64, etag string, userMeta map[s
// Get object info - need size and etag here. Also, decryption
// headers are added to the stat request if given.
var objInfo ObjectInfo
rh := NewGetReqHeaders()
opts := StatObjectOptions{}
for k, v := range s.decryptKey.getSSEHeaders(false) {
rh.Set(k, v)
opts.Set(k, v)
}
objInfo, err = c.statObject(s.bucket, s.object, rh)
objInfo, err = c.statObject(s.bucket, s.object, opts)
if err != nil {
err = fmt.Errorf("Could not stat object - %s/%s: %v", s.bucket, s.object, err)
} else {
Expand Down
5 changes: 3 additions & 2 deletions api-get-object-context.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package minio
import "context"

// GetObjectWithContext - returns an seekable, readable object.
func (c Client) GetObjectWithContext(ctx context.Context, bucketName, objectName string) (*Object, error) {
return c.getObjectWithContext(ctx, bucketName, objectName)
// The options can be used to specify the GET request further.
func (c Client) GetObjectWithContext(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (*Object, error) {
return c.getObjectWithContext(ctx, bucketName, objectName, opts)
}
28 changes: 19 additions & 9 deletions api-get-object-file.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,23 +21,34 @@ import (
"os"
"path/filepath"

"github.com/minio/minio-go/pkg/encrypt"

"context"

"github.com/minio/minio-go/pkg/s3utils"
)

// FGetObjectWithContext - download contents of an object to a local file.
func (c Client) FGetObjectWithContext(ctx context.Context, bucketName, objectName, filePath string) error {
return c.fGetObjectWithContext(ctx, bucketName, objectName, filePath)
// The options can be used to specify the GET request further.
func (c Client) FGetObjectWithContext(ctx context.Context, bucketName, objectName, filePath string, opts GetObjectOptions) error {
return c.fGetObjectWithContext(ctx, bucketName, objectName, filePath, opts)
}

// FGetObject - download contents of an object to a local file.
func (c Client) FGetObject(bucketName, objectName, filePath string) error {
return c.fGetObjectWithContext(context.Background(), bucketName, objectName, filePath)
func (c Client) FGetObject(bucketName, objectName, filePath string, opts GetObjectOptions) error {
return c.fGetObjectWithContext(context.Background(), bucketName, objectName, filePath, opts)
}

// FGetEncryptedObject - Decrypt and store an object at filePath.
func (c Client) FGetEncryptedObject(bucketName, objectName, filePath string, materials encrypt.Materials) error {
if materials == nil {
return ErrInvalidArgument("Unable to recognize empty encryption properties")
}
return c.FGetObject(bucketName, objectName, filePath, GetObjectOptions{Materials: materials})
}

// fGetObjectWithContext - fgetObject wrapper function with context
func (c Client) fGetObjectWithContext(ctx context.Context, bucketName, objectName, filePath string) error {
func (c Client) fGetObjectWithContext(ctx context.Context, bucketName, objectName, filePath string, opts GetObjectOptions) error {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return err
Expand Down Expand Up @@ -72,7 +83,7 @@ func (c Client) fGetObjectWithContext(ctx context.Context, bucketName, objectNam
}

// Gather md5sum.
objectStat, err := c.StatObject(bucketName, objectName)
objectStat, err := c.StatObject(bucketName, objectName, StatObjectOptions{opts})
if err != nil {
return err
}
Expand All @@ -94,13 +105,12 @@ func (c Client) fGetObjectWithContext(ctx context.Context, bucketName, objectNam

// Initialize get object request headers to set the
// appropriate range offsets to read from.
reqHeaders := NewGetReqHeaders()
if st.Size() > 0 {
reqHeaders.SetRange(st.Size(), 0)
opts.SetRange(st.Size(), 0)
}

// Seek to current position for incoming reader.
objectReader, objectStat, err := c.getObject(ctx, bucketName, objectName, reqHeaders)
objectReader, objectStat, err := c.getObject(ctx, bucketName, objectName, opts)
if err != nil {
return err
}
Expand Down
88 changes: 30 additions & 58 deletions api-get-object.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,32 +37,16 @@ func (c Client) GetEncryptedObject(bucketName, objectName string, encryptMateria
return nil, ErrInvalidArgument("Unable to recognize empty encryption properties")
}

// Fetch encrypted object
encReader, err := c.GetObject(bucketName, objectName)
if err != nil {
return nil, err
}
// Stat object to get its encryption metadata
st, err := encReader.Stat()
if err != nil {
return nil, err
}

// Setup object for decrytion, object is transparently
// decrypted as the consumer starts reading.
encryptMaterials.SetupDecryptMode(encReader, st.Metadata.Get(amzHeaderIV), st.Metadata.Get(amzHeaderKey))

// Success.
return encryptMaterials, nil
return c.GetObject(bucketName, objectName, GetObjectOptions{Materials: encryptMaterials})
}

// GetObject - returns an seekable, readable object.
func (c Client) GetObject(bucketName, objectName string) (*Object, error) {
return c.getObjectWithContext(context.Background(), bucketName, objectName)
func (c Client) GetObject(bucketName, objectName string, opts GetObjectOptions) (*Object, error) {
return c.getObjectWithContext(context.Background(), bucketName, objectName, opts)
}

// GetObject wrapper function that accepts a request context
func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName string) (*Object, error) {
func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (*Object, error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return nil, err
Expand Down Expand Up @@ -108,34 +92,26 @@ func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName
if req.isFirstReq {
// First request is a Read/ReadAt.
if req.isReadOp {
reqHeaders := NewGetReqHeaders()
// Differentiate between wanting the whole object and just a range.
if req.isReadAt {
// If this is a ReadAt request only get the specified range.
// Range is set with respect to the offset and length of the buffer requested.
// Do not set objectInfo from the first readAt request because it will not get
// the whole object.
reqHeaders.SetRange(req.Offset, req.Offset+int64(len(req.Buffer))-1)
httpReader, objectInfo, err = c.getObject(ctx, bucketName, objectName, reqHeaders)
} else {
if req.Offset > 0 {
reqHeaders.SetRange(req.Offset, 0)
}

// First request is a Read request.
httpReader, objectInfo, err = c.getObject(ctx, bucketName, objectName, reqHeaders)
opts.SetRange(req.Offset, req.Offset+int64(len(req.Buffer))-1)
} else if req.Offset > 0 {
opts.SetRange(req.Offset, 0)
}
httpReader, objectInfo, err = c.getObject(ctx, bucketName, objectName, opts)
if err != nil {
resCh <- getResponse{
Error: err,
}
resCh <- getResponse{Error: err}
return
}
etag = objectInfo.ETag
// Read at least firstReq.Buffer bytes, if not we have
// reached our EOF.
size, err := io.ReadFull(httpReader, req.Buffer)
if err == io.ErrUnexpectedEOF {
if size > 0 && err == io.ErrUnexpectedEOF {
// If an EOF happens after reading some but not
// all the bytes ReadFull returns ErrUnexpectedEOF
err = io.EOF
Expand All @@ -150,7 +126,7 @@ func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName
} else {
// First request is a Stat or Seek call.
// Only need to run a StatObject until an actual Read or ReadAt request comes through.
objectInfo, err = c.StatObject(bucketName, objectName)
objectInfo, err = c.statObject(bucketName, objectName, StatObjectOptions{opts})
if err != nil {
resCh <- getResponse{
Error: err,
Expand All @@ -165,11 +141,10 @@ func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName
}
}
} else if req.settingObjectInfo { // Request is just to get objectInfo.
reqHeaders := NewGetReqHeaders()
if etag != "" {
reqHeaders.SetMatchETag(etag)
opts.SetMatchETag(etag)
}
objectInfo, err := c.statObject(bucketName, objectName, reqHeaders)
objectInfo, err := c.statObject(bucketName, objectName, StatObjectOptions{opts})
if err != nil {
resCh <- getResponse{
Error: err,
Expand All @@ -189,9 +164,8 @@ func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName
// new ones when they haven't been already.
// All readAt requests are new requests.
if req.DidOffsetChange || !req.beenRead {
reqHeaders := NewGetReqHeaders()
if etag != "" {
reqHeaders.SetMatchETag(etag)
opts.SetMatchETag(etag)
}
if httpReader != nil {
// Close previously opened http reader.
Expand All @@ -200,16 +174,11 @@ func (c Client) getObjectWithContext(ctx context.Context, bucketName, objectName
// If this request is a readAt only get the specified range.
if req.isReadAt {
// Range is set with respect to the offset and length of the buffer requested.
reqHeaders.SetRange(req.Offset, req.Offset+int64(len(req.Buffer))-1)
httpReader, _, err = c.getObject(ctx, bucketName, objectName, reqHeaders)
} else {
// Range is set with respect to the offset.
if req.Offset > 0 {
reqHeaders.SetRange(req.Offset, 0)
}

httpReader, objectInfo, err = c.getObject(ctx, bucketName, objectName, reqHeaders)
opts.SetRange(req.Offset, req.Offset+int64(len(req.Buffer))-1)
} else if req.Offset > 0 { // Range is set with respect to the offset.
opts.SetRange(req.Offset, 0)
}
httpReader, objectInfo, err = c.getObject(ctx, bucketName, objectName, opts)
if err != nil {
resCh <- getResponse{
Error: err,
Expand Down Expand Up @@ -632,7 +601,7 @@ func newObject(reqCh chan<- getRequest, resCh <-chan getResponse, doneCh chan<-
//
// For more information about the HTTP Range header.
// go to http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35.
func (c Client) getObject(ctx context.Context, bucketName, objectName string, reqHeaders RequestHeaders) (io.ReadCloser, ObjectInfo, error) {
func (c Client) getObject(ctx context.Context, bucketName, objectName string, opts GetObjectOptions) (io.ReadCloser, ObjectInfo, error) {
// Validate input arguments.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return nil, ObjectInfo{}, err
Expand All @@ -641,17 +610,11 @@ func (c Client) getObject(ctx context.Context, bucketName, objectName string, re
return nil, ObjectInfo{}, err
}

// Set all the necessary reqHeaders.
customHeader := make(http.Header)
for key, value := range reqHeaders.Header {
customHeader[key] = value
}

// Execute GET on objectName.
resp, err := c.executeMethod(ctx, "GET", requestMetadata{
bucketName: bucketName,
objectName: objectName,
customHeader: customHeader,
customHeader: opts.Header(),
contentSHA256Bytes: emptySHA256,
})
if err != nil {
Expand Down Expand Up @@ -698,6 +661,15 @@ func (c Client) getObject(ctx context.Context, bucketName, objectName string, re
Metadata: extractObjMetadata(resp.Header),
}

reader := resp.Body
if opts.Materials != nil {
err = opts.Materials.SetupDecryptMode(reader, objectStat.Metadata.Get(amzHeaderIV), objectStat.Metadata.Get(amzHeaderKey))
if err != nil {
return nil, ObjectInfo{}, err
}
reader = opts.Materials
}

// do not close body here, caller will close
return resp.Body, objectStat, nil
return reader, objectStat, nil
}
64 changes: 39 additions & 25 deletions request-headers.go → api-get-options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,80 +20,94 @@ import (
"fmt"
"net/http"
"time"

"github.com/minio/minio-go/pkg/encrypt"
)

// RequestHeaders - implement methods for setting special
// request headers for GET, HEAD object operations.
// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html
type RequestHeaders struct {
http.Header
// GetObjectOptions are used to specify additional headers or options
// during GET requests.
type GetObjectOptions struct {
headers map[string]string

Materials encrypt.Materials
}

// StatObjectOptions are used to specify additional headers or options
// during GET info/stat requests.
type StatObjectOptions struct {
GetObjectOptions
}

// NewGetReqHeaders - initializes a new request headers for GET request.
func NewGetReqHeaders() RequestHeaders {
return RequestHeaders{
Header: make(http.Header),
// Header returns the http.Header representation of the GET options.
func (o GetObjectOptions) Header() http.Header {
headers := make(http.Header, len(o.headers))
for k, v := range o.headers {
headers.Set(k, v)
}
return headers
}

// NewHeadReqHeaders - initializes a new request headers for HEAD request.
func NewHeadReqHeaders() RequestHeaders {
return RequestHeaders{
Header: make(http.Header),
// Set adds a key value pair to the options. The
// key-value pair will be part of the HTTP GET request
// headers.
func (o *GetObjectOptions) Set(key, value string) {
if o.headers == nil {
o.headers = make(map[string]string)
}
o.headers[http.CanonicalHeaderKey(key)] = value
}

// SetMatchETag - set match etag.
func (c RequestHeaders) SetMatchETag(etag string) error {
func (o *GetObjectOptions) SetMatchETag(etag string) error {
if etag == "" {
return ErrInvalidArgument("ETag cannot be empty.")
}
c.Set("If-Match", "\""+etag+"\"")
o.Set("If-Match", "\""+etag+"\"")
return nil
}

// SetMatchETagExcept - set match etag except.
func (c RequestHeaders) SetMatchETagExcept(etag string) error {
func (o *GetObjectOptions) SetMatchETagExcept(etag string) error {
if etag == "" {
return ErrInvalidArgument("ETag cannot be empty.")
}
c.Set("If-None-Match", "\""+etag+"\"")
o.Set("If-None-Match", "\""+etag+"\"")
return nil
}

// SetUnmodified - set unmodified time since.
func (c RequestHeaders) SetUnmodified(modTime time.Time) error {
func (o *GetObjectOptions) SetUnmodified(modTime time.Time) error {
if modTime.IsZero() {
return ErrInvalidArgument("Modified since cannot be empty.")
}
c.Set("If-Unmodified-Since", modTime.Format(http.TimeFormat))
o.Set("If-Unmodified-Since", modTime.Format(http.TimeFormat))
return nil
}

// SetModified - set modified time since.
func (c RequestHeaders) SetModified(modTime time.Time) error {
func (o *GetObjectOptions) SetModified(modTime time.Time) error {
if modTime.IsZero() {
return ErrInvalidArgument("Modified since cannot be empty.")
}
c.Set("If-Modified-Since", modTime.Format(http.TimeFormat))
o.Set("If-Modified-Since", modTime.Format(http.TimeFormat))
return nil
}

// SetRange - set the start and end offset of the object to be read.
// See https://tools.ietf.org/html/rfc7233#section-3.1 for reference.
func (c RequestHeaders) SetRange(start, end int64) error {
func (o *GetObjectOptions) SetRange(start, end int64) error {
switch {
case start == 0 && end < 0:
// Read last '-end' bytes. `bytes=-N`.
c.Set("Range", fmt.Sprintf("bytes=%d", end))
o.Set("Range", fmt.Sprintf("bytes=%d", end))
case 0 < start && end == 0:
// Read everything starting from offset
// 'start'. `bytes=N-`.
c.Set("Range", fmt.Sprintf("bytes=%d-", start))
o.Set("Range", fmt.Sprintf("bytes=%d-", start))
case 0 <= start && start <= end:
// Read everything starting at 'start' till the
// 'end'. `bytes=N-M`
c.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end))
o.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end))
default:
// All other cases such as
// bytes=-3-
Expand Down

0 comments on commit 5f80bfe

Please sign in to comment.