lowlevelapi.go

/*
 * Minimal object storage library (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package minio

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"net/http"
	"strconv"
	"strings"
	"time"
)

// lowLevelAPI container to hold unexported internal functions
type lowLevelAPI struct {
	config *Config
}

// putBucketRequest wrapper creates a new putBucket request
func (a lowLevelAPI) putBucketRequest(bucket, acl, location string) (*request, error) {
	var r *request
	var err error
	op := &operation{
		HTTPServer: a.config.Endpoint,
		HTTPMethod: "PUT",
		HTTPPath:   "/" + bucket,
	}
	var createBucketConfigBuffer *bytes.Reader
	// If location is set use it and create proper bucket configuration
	switch {
	case location != "":
		createBucketConfig := new(createBucketConfiguration)
		createBucketConfig.Location = location
		var createBucketConfigBytes []byte
		switch {
		case a.config.AcceptType == "application/xml":
			createBucketConfigBytes, err = xml.Marshal(createBucketConfig)
		case a.config.AcceptType == "application/json":
			createBucketConfigBytes, err = json.Marshal(createBucketConfig)
		default:
			createBucketConfigBytes, err = xml.Marshal(createBucketConfig)
		}
		if err != nil {
			return nil, err
		}
		createBucketConfigBuffer = bytes.NewReader(createBucketConfigBytes)
	}
	switch {
	case createBucketConfigBuffer == nil:
		r, err = newRequest(op, a.config, nil)
		if err != nil {
			return nil, err
		}
	default:
		r, err = newRequest(op, a.config, createBucketConfigBuffer)
		if err != nil {
			return nil, err
		}
		r.req.ContentLength = int64(createBucketConfigBuffer.Len())
	}
	// by default bucket is private
	switch {
	case acl != "":
		r.Set("x-amz-acl", acl)
	default:
		r.Set("x-amz-acl", "private")
	}

	return r, nil
}

/// Bucket Write Operations

// putBucket create a new bucket
//
// Requires valid AWS Access Key ID to authenticate requests
// Anonymous requests are never allowed to create buckets
//
// optional arguments are acl and location - by default all buckets are created
// with ``private`` acl and location set to US Standard if one wishes to set
// different ACLs and Location one can set them properly.
//
// ACL valid values
// ------------------
// private - owner gets full access [DEFAULT]
// public-read - owner gets full access, others get read access
// public-read-write - owner gets full access, others get full access too
// authenticated-read - owner gets full access, authenticated users get read access
// ------------------
//
// Location valid values
// ------------------
// [ us-west-1 | us-west-2 | eu-west-1 | eu-central-1 | ap-southeast-1 | ap-northeast-1 | ap-southeast-2 | sa-east-1 ]
//
// Default - US standard
func (a lowLevelAPI) putBucket(bucket, acl, location string) error {
	req, err := a.putBucketRequest(bucket, acl, location)
	if err != nil {
		return err
	}
	resp, err := req.Do()
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp != nil {
		if resp.StatusCode != http.StatusOK {
			return a.responseToError(resp.Body)
		}
	}
	return nil
}

// putBucketRequestACL wrapper creates a new putBucketACL request
func (a lowLevelAPI) putBucketACLRequest(bucket, acl string) (*request, error) {
	op := &operation{
		HTTPServer: a.config.Endpoint,
		HTTPMethod: "PUT",
		HTTPPath:   "/" + bucket + "?acl",
	}
	req, err := newRequest(op, a.config, nil)
	if err != nil {
		return nil, err
	}
	req.Set("x-amz-acl", acl)
	return req, nil
}

// putBucketACL set the permissions on an existing bucket using Canned ACL's
func (a lowLevelAPI) putBucketACL(bucket, acl string) error {
	req, err := a.putBucketACLRequest(bucket, acl)
	if err != nil {
		return err
	}
	resp, err := req.Do()
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp != nil {
		if resp.StatusCode != http.StatusOK {
			return a.responseToError(resp.Body)
		}
	}
	return nil
}

// getBucketACLRequest wrapper creates a new getBucketACL request
func (a lowLevelAPI) getBucketACLRequest(bucket string) (*request, error) {
	op := &operation{
		HTTPServer: a.config.Endpoint,
		HTTPMethod: "GET",
		HTTPPath:   "/" + bucket + "?acl",
	}
	req, err := newRequest(op, a.config, nil)
	if err != nil {
		return nil, err
	}
	return req, nil
}

// getBucketACL get the acl information on an existing bucket
func (a lowLevelAPI) getBucketACL(bucket string) (accessControlPolicy, error) {
	req, err := a.getBucketACLRequest(bucket)
	if err != nil {
		return accessControlPolicy{}, err
	}
	resp, err := req.Do()
	if err != nil {
		return accessControlPolicy{}, err
	}
	defer resp.Body.Close()
	if resp != nil {
		if resp.StatusCode != http.StatusOK {
			return accessControlPolicy{}, a.responseToError(resp.Body)
		}
	}
	policy := accessControlPolicy{}
	err = acceptTypeDecoder(resp.Body, a.config.AcceptType, &policy)
	if err != nil {
		return accessControlPolicy{}, err
	}
	if policy.AccessControlList.Grant == nil {
		errorResponse := ErrorResponse{
			Code:      "InternalError",
			Message:   "Unexpected error please report this at https://github.com/minio/minio-go/issues",
			Resource:  "/" + bucket,
			RequestID: "minio",
		}
		return accessControlPolicy{}, errorResponse
	}
	return policy, nil
}

// getBucketLocationRequest wrapper creates a new getBucketLocation request
func (a lowLevelAPI) getBucketLocationRequest(bucket string) (*request, error) {
	op := &operation{
		HTTPServer: a.config.Endpoint,
		HTTPMethod: "GET",
		HTTPPath:   "/" + bucket + "?location",
	}
	req, err := newRequest(op, a.config, nil)
	if err != nil {
		return nil, err
	}
	return req, nil
}

// getBucketLocation uses location subresource to return a bucket's region
func (a lowLevelAPI) getBucketLocation(bucket string) (string, error) {
	req, err := a.getBucketLocationRequest(bucket)
	if err != nil {
		return "", err
	}
	resp, err := req.Do()
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	if resp != nil {
		if resp.StatusCode != http.StatusOK {
			return "", a.responseToError(resp.Body)
		}
	}
	var locationConstraint string
	err = acceptTypeDecoder(resp.Body, a.config.AcceptType, &locationConstraint)
	if err != nil {
		return "", err
	}
	return locationConstraint, nil
}

// listObjectsRequest wrapper creates a new listObjects request
func (a lowLevelAPI) listObjectsRequest(bucket, marker, prefix, delimiter string, maxkeys int) (*request, error) {
	// resourceQuery - get resources properly escaped and lined up before using them in http request
	resourceQuery := func() (*string, error) {
		var err error
		switch {
		case marker != "":
			marker, err = urlEncodeName(marker)
			if err != nil {
				return nil, err
			}
			marker = fmt.Sprintf("&marker=%s", marker)
			fallthrough
		case prefix != "":
			prefix, err = urlEncodeName(prefix)
			if err != nil {
				return nil, err
			}
			prefix = fmt.Sprintf("&prefix=%s", prefix)
			fallthrough
		case delimiter != "":
			delimiter, err = urlEncodeName(delimiter)
			if err != nil {
				return nil, err
			}
			delimiter = fmt.Sprintf("&delimiter=%s", delimiter)
		}
		query := fmt.Sprintf("?max-keys=%d", maxkeys) + marker + prefix + delimiter
		return &query, nil
	}
	query, err := resourceQuery()
	if err != nil {
		return nil, err
	}
	op := &operation{
		HTTPServer: a.config.Endpoint,
		HTTPMethod: "GET",
		HTTPPath:   "/" + bucket + *query,
	}
	r, err := newRequest(op, a.config, nil)
	if err != nil {
		return nil, err
	}
	return r, nil
}

/// Bucket Read Operations

// listObjects - (List Objects) - List some or all (up to 1000) of the objects in a bucket.
//
// You can use the request parameters as selection criteria to return a subset of the objects in a bucket.
// request paramters :-
// ---------
// ?marker - Specifies the key to start with when listing objects in a bucket.
// ?delimiter - A delimiter is a character you use to group keys.
// ?prefix - Limits the response to keys that begin with the specified prefix.
// ?max-keys - Sets the maximum number of keys returned in the response body.
func (a lowLevelAPI) listObjects(bucket, marker, prefix, delimiter string, maxkeys int) (listBucketResult, error) {
	if err := invalidBucketToError(bucket); err != nil {
		return listBucketResult{}, err
	}
	req, err := a.listObjectsRequest(bucket, marker, prefix, delimiter, maxkeys)
	if err != nil {
		return listBucketResult{}, err
	}
	resp, err := req.Do()
	if err != nil {
		return listBucketResult{}, err
	}
	defer resp.Body.Close()
	if resp != nil {
		if resp.StatusCode != http.StatusOK {
			return listBucketResult{}, a.responseToError(resp.Body)
		}
	}
	listBucketResult := listBucketResult{}
	err = acceptTypeDecoder(resp.Body, a.config.AcceptType, &listBucketResult)
	if err != nil {
		return listBucketResult, err
	}
	// close body while returning, along with any error
	return listBucketResult, nil
}

// headBucketRequest wrapper creates a new headBucket request
func (a lowLevelAPI) headBucketRequest(bucket string) (*request, error) {
	op := &operation{
		HTTPServer: a.config.Endpoint,
		HTTPMethod: "HEAD",
		HTTPPath:   "/" + bucket,
	}
	return newRequest(op, a.config, nil)
}

// headBucket useful to determine if a bucket exists and you have permission to access it.
func (a lowLevelAPI) headBucket(bucket string) error {
	if err := invalidBucketToError(bucket); err != nil {
		return err
	}
	req, err := a.headBucketRequest(bucket)
	if err != nil {
		return err
	}
	resp, err := req.Do()
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp != nil {
		if resp.StatusCode != http.StatusOK {
			// Head has no response body, handle it
			var errorResponse ErrorResponse
			switch {
			case resp.StatusCode == http.StatusNotFound:
				errorResponse = ErrorResponse{
					Code:      "NoSuchBucket",
					Message:   "The specified bucket does not exist.",
					Resource:  "/" + bucket,
					RequestID: resp.Header.Get("x-amz-request-id"),
				}
			case resp.StatusCode == http.StatusForbidden:
				errorResponse = ErrorResponse{
					Code:      "AccessDenied",
					Message:   "Access Denied",
					Resource:  "/" + bucket,
					RequestID: resp.Header.Get("x-amz-request-id"),
				}
			}
			return errorResponse
		}
	}
	return nil
}

// deleteBucketRequest wrapper creates a new deleteBucket request
func (a lowLevelAPI) deleteBucketRequest(bucket string) (*request, error) {
	op := &operation{
		HTTPServer: a.config.Endpoint,
		HTTPMethod: "DELETE",
		HTTPPath:   "/" + bucket,
	}
	return newRequest(op, a.config, nil)
}

// deleteBucket deletes the bucket named in the URI
//
// NOTE: -
//  All objects (including all object versions and delete markers)
//  in the bucket must be deleted before successfully attempting this request
func (a lowLevelAPI) deleteBucket(bucket string) error {
	if err := invalidBucketToError(bucket); err != nil {
		return err
	}
	req, err := a.deleteBucketRequest(bucket)
	if err != nil {
		return err
	}
	resp, err := req.Do()
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp != nil {
		if resp.StatusCode != http.StatusOK {
			var errorResponse ErrorResponse
			switch {
			case resp.StatusCode == http.StatusNotFound:
				errorResponse = ErrorResponse{
					Code:      "NoSuchBucket",
					Message:   "The specified bucket does not exist.",
					Resource:  "/" + bucket,
					RequestID: resp.Header.Get("x-amz-request-id"),
				}
			case resp.StatusCode == http.StatusForbidden:
				errorResponse = ErrorResponse{
					Code:      "AccessDenied",
					Message:   "Access Denied",
					Resource:  "/" + bucket,
					RequestID: resp.Header.Get("x-amz-request-id"),
				}
			}
			return errorResponse
		}
	}
	return nil
}

/// Object Read/Write/Stat Operations

// putObjectRequest wrapper creates a new PutObject request
func (a lowLevelAPI) putObjectRequest(bucket, object, contentType string, md5SumBytes []byte, size int64, body io.ReadSeeker) (*request, error) {
	if strings.TrimSpace(contentType) == "" {
		contentType = "application/octet-stream"
	}
	encodedObject, err := urlEncodeName(object)
	if err != nil {
		return nil, err
	}
	op := &operation{
		HTTPServer: a.config.Endpoint,
		HTTPMethod: "PUT",
		HTTPPath:   "/" + bucket + "/" + encodedObject,
	}
	r, err := newRequest(op, a.config, body)
	if err != nil {
		return nil, err
	}
	// set Content-MD5 as base64 encoded md5
	r.Set("Content-MD5", base64.StdEncoding.EncodeToString(md5SumBytes))
	r.Set("Content-Type", contentType)
	r.req.ContentLength = size
	return r, nil
}

// putObject - add an object to a bucket
// NOTE: You must have WRITE permissions on a bucket to add an object to it.
func (a lowLevelAPI) putObject(bucket, object, contentType string, md5SumBytes []byte, size int64, body io.ReadSeeker) (ObjectStat, error) {
	req, err := a.putObjectRequest(bucket, object, contentType, md5SumBytes, size, body)
	if err != nil {
		return ObjectStat{}, err
	}
	resp, err := req.Do()
	if err != nil {
		return ObjectStat{}, err
	}
	defer resp.Body.Close()
	if resp != nil {
		if resp.StatusCode != http.StatusOK {
			return ObjectStat{}, a.responseToError(resp.Body)
		}
	}
	var metadata ObjectStat
	metadata.ETag = strings.Trim(resp.Header.Get("ETag"), "\"") // trim off the odd double quotes
	return metadata, nil
}

// getObjectRequest wrapper creates a new getObject request
func (a lowLevelAPI) getObjectRequest(bucket, object string, offset, length int64) (*request, error) {
	encodedObject, err := urlEncodeName(object)
	if err != nil {
		return nil, err
	}
	op := &operation{
		HTTPServer: a.config.Endpoint,
		HTTPMethod: "GET",
		HTTPPath:   "/" + bucket + "/" + encodedObject,
	}
	r, err := newRequest(op, a.config, nil)
	if err != nil {
		return nil, err
	}
	switch {
	case length > 0 && offset > 0:
		r.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1))
	case offset > 0 && length == 0:
		r.Set("Range", fmt.Sprintf("bytes=%d-", offset))
	case length > 0 && offset == 0:
		r.Set("Range", fmt.Sprintf("bytes=-%d", length))
	}
	return r, nil
}

// getObject - retrieve object from Object Storage
//
// Additionally this function also takes range arguments to download the specified
// range bytes of an object. Setting offset and length = 0 will download the full object.
//
// For more information about the HTTP Range header, go to http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35.
func (a lowLevelAPI) getObject(bucket, object string, offset, length int64) (io.ReadCloser, ObjectStat, error) {
	if err := invalidArgumentToError(object); err != nil {
		return nil, ObjectStat{}, err
	}
	req, err := a.getObjectRequest(bucket, object, offset, length)
	if err != nil {
		return nil, ObjectStat{}, err
	}
	resp, err := req.Do()
	if err != nil {
		return nil, ObjectStat{}, err
	}
	if resp != nil {
		switch resp.StatusCode {
		case http.StatusOK:
		case http.StatusPartialContent:
		default:
			return nil, ObjectStat{}, a.responseToError(resp.Body)
		}
	}
	md5sum := strings.Trim(resp.Header.Get("ETag"), "\"") // trim off the odd double quotes
	if md5sum == "" {
		return nil, ObjectStat{}, errors.New("missing ETag")
	}
	date, err := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified"))
	if err != nil {
		return nil, ObjectStat{}, err
	}
	var objectstat ObjectStat
	objectstat.ETag = md5sum
	objectstat.Key = object
	objectstat.Size = resp.ContentLength
	objectstat.LastModified = date

	// do not close body here, caller will close
	return resp.Body, objectstat, nil
}

// deleteObjectRequest wrapper creates a new deleteObject request
func (a lowLevelAPI) deleteObjectRequest(bucket, object string) (*request, error) {
	encodedObject, err := urlEncodeName(object)
	if err != nil {
		return nil, err
	}
	op := &operation{
		HTTPServer: a.config.Endpoint,
		HTTPMethod: "DELETE",
		HTTPPath:   "/" + bucket + "/" + encodedObject,
	}
	return newRequest(op, a.config, nil)
}

// deleteObject deletes a given object from a bucket
func (a lowLevelAPI) deleteObject(bucket, object string) error {
	if err := invalidBucketToError(bucket); err != nil {
		return err
	}
	if err := invalidArgumentToError(object); err != nil {
		return err
	}
	req, err := a.deleteObjectRequest(bucket, object)
	if err != nil {
		return err
	}
	resp, err := req.Do()
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp != nil {
		if resp.StatusCode != http.StatusOK {
			var errorResponse ErrorResponse
			switch {
			case resp.StatusCode == http.StatusNotFound:
				errorResponse = ErrorResponse{
					Code:      "NoSuchKey",
					Message:   "The specified key does not exist.",
					Resource:  "/" + bucket + "/" + object,
					RequestID: resp.Header.Get("x-amz-request-id"),
				}
			case resp.StatusCode == http.StatusForbidden:
				errorResponse = ErrorResponse{
					Code:      "AccessDenied",
					Message:   "Access Denied",
					Resource:  "/" + bucket + "/" + object,
					RequestID: resp.Header.Get("x-amz-request-id"),
				}
			}
			return errorResponse
		}
	}
	return nil
}

// headObjectRequest wrapper creates a new headObject request
func (a lowLevelAPI) headObjectRequest(bucket, object string) (*request, error) {
	encodedObject, err := urlEncodeName(object)
	if err != nil {
		return nil, err
	}
	op := &operation{
		HTTPServer: a.config.Endpoint,
		HTTPMethod: "HEAD",
		HTTPPath:   "/" + bucket + "/" + encodedObject,
	}
	return newRequest(op, a.config, nil)
}

// headObject retrieves metadata from an object without returning the object itself
func (a lowLevelAPI) headObject(bucket, object string) (ObjectStat, error) {
	if err := invalidBucketToError(bucket); err != nil {
		return ObjectStat{}, err
	}
	if err := invalidArgumentToError(object); err != nil {
		return ObjectStat{}, err
	}
	req, err := a.headObjectRequest(bucket, object)
	if err != nil {
		return ObjectStat{}, err
	}
	resp, err := req.Do()
	if err != nil {
		return ObjectStat{}, err
	}
	defer resp.Body.Close()
	if resp != nil {
		if resp.StatusCode != http.StatusOK {
			var errorResponse ErrorResponse
			switch {
			case resp.StatusCode == http.StatusNotFound:
				errorResponse = ErrorResponse{
					Code:      "NoSuchKey",
					Message:   "The specified key does not exist.",
					Resource:  "/" + bucket + "/" + object,
					RequestID: resp.Header.Get("x-amz-request-id"),
				}
			case resp.StatusCode == http.StatusForbidden:
				errorResponse = ErrorResponse{
					Code:      "AccessDenied",
					Message:   "Access Denied",
					Resource:  "/" + bucket + "/" + object,
					RequestID: resp.Header.Get("x-amz-request-id"),
				}
			}
			return ObjectStat{}, errorResponse
		}
	}
	md5sum := strings.Trim(resp.Header.Get("ETag"), "\"") // trim off the odd double quotes
	if md5sum == "" {
		return ObjectStat{}, errors.New("missing ETag")
	}
	size, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64)
	if err != nil {
		return ObjectStat{}, err
	}
	date, err := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified"))
	if err != nil {
		return ObjectStat{}, err
	}
	var objectstat ObjectStat
	objectstat.ETag = md5sum
	objectstat.Key = object
	objectstat.Size = size
	objectstat.LastModified = date
	return objectstat, nil
}

/// Service Operations

// listBucketRequest wrapper creates a new listBuckets request
func (a lowLevelAPI) listBucketsRequest() (*request, error) {
	op := &operation{
		HTTPServer: a.config.Endpoint,
		HTTPMethod: "GET",
		HTTPPath:   "/",
	}
	return newRequest(op, a.config, nil)
}

// listBuckets list of all buckets owned by the authenticated sender of the request
func (a lowLevelAPI) listBuckets() (listAllMyBucketsResult, error) {
	req, err := a.listBucketsRequest()
	if err != nil {
		return listAllMyBucketsResult{}, err
	}
	resp, err := req.Do()
	if err != nil {
		return listAllMyBucketsResult{}, err
	}
	defer resp.Body.Close()
	if resp != nil {
		// for un-authenticated requests, amazon sends a redirect handle it
		if resp.StatusCode == http.StatusTemporaryRedirect {
			return listAllMyBucketsResult{}, errors.New(resp.Status)
		}
		if resp.StatusCode != http.StatusOK {
			return listAllMyBucketsResult{}, a.responseToError(resp.Body)
		}
	}
	listAllMyBucketsResult := listAllMyBucketsResult{}
	err = acceptTypeDecoder(resp.Body, a.config.AcceptType, &listAllMyBucketsResult)
	if err != nil {
		return listAllMyBucketsResult, err
	}
	return listAllMyBucketsResult, nil
}