Implement backblaze-b2 gateway support

Fixes #4072

appveyor.yml

@@ -37,20 +37,19 @@ build_script:
 test_script:
   # Unit tests
   - ps: Add-AppveyorTest "Unit Tests" -Outcome Running
-  - mkdir build\coverage
   - go test -v -timeout 17m -race github.com/minio/minio/cmd...
   - go test -v -race github.com/minio/minio/pkg...
-  - go test -v -timeout 17m -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
+  - go test -v -timeout 17m -coverprofile=c.txt -covermode=atomic github.com/minio/minio/cmd
   - ps: Update-AppveyorTest "Unit Tests" -Outcome Passed
 
 after_test:
-  - go tool cover -html=build\coverage\coverage.txt -o build\coverage\coverage.html
-  - ps: Push-AppveyorArtifact build\coverage\coverage.txt
-  - ps: Push-AppveyorArtifact build\coverage\coverage.html
+  - go tool cover-html=coverage.txt -o coverage.html
+  - ps: Push-AppveyorArtifact c.txt
+  - ps: Push-AppveyorArtifact coverage.html
   # Upload coverage report.
   - "SET PATH=C:\\Python34;C:\\Python34\\Scripts;%PATH%"
   - pip install codecov
-  - codecov -X gcov -f "build\coverage\coverage.txt"
+  - codecov -X gcov -f "c.txt"
 
 # to disable deployment
 deploy: off

cmd/api-headers.go

@@ -70,6 +70,14 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, contentRange *h
 		w.Header().Set("ETag", "\""+objInfo.ETag+"\"")
 	}
 
+	if objInfo.ContentType != "" {
+		w.Header().Set("Content-Type", objInfo.ContentType)
+	}
+
+	if objInfo.ContentEncoding != "" {
+		w.Header().Set("Content-Encoding", objInfo.ContentEncoding)
+	}
+
 	// Set all other user defined metadata.
 	for k, v := range objInfo.UserDefined {
 		w.Header().Set(k, v)

cmd/gateway-b2-anonymous.go

@@ -0,0 +1,145 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// AnonPutObject creates a new object anonymously with the incoming data,
+func (l *b2Objects) AnonPutObject(bucket string, object string, size int64, data io.Reader,
+	metadata map[string]string, sha256sum string) (objInfo ObjectInfo, err error) {
+	return objInfo, traceError(NotImplemented{})
+}
+
+// mkRange converts offset, size into Range header equivalent.
+func mkRange(offset, size int64) string {
+	if offset == 0 && size == 0 {
+		return ""
+	}
+	if size == 0 {
+		return fmt.Sprintf("%s%d-", byteRangePrefix, offset)
+	}
+	return fmt.Sprintf("%s%d-%d", byteRangePrefix, offset, offset+size-1)
+}
+
+// AnonGetObject - Get object anonymously
+func (l *b2Objects) AnonGetObject(bucket string, object string, startOffset int64, length int64, writer io.Writer) error {
+	uri := fmt.Sprintf("%s/file/%s/%s", l.b2Client.DownloadURI, bucket, object)
+	req, err := http.NewRequest("GET", uri, nil)
+	if err != nil {
+		return err
+	}
+	rng := mkRange(startOffset, length)
+	if rng != "" {
+		req.Header.Set("Range", rng)
+	}
+	resp, err := l.anonClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	_, err = io.Copy(writer, resp.Body)
+	return err
+}
+
+// Converts http Header into ObjectInfo. This function looks for all the
+// standard Backblaze B2 headers to convert into ObjectInfo.
+//
+// Content-Length is converted to Size.
+// X-Bz-Upload-Timestamp is converted to ModTime.
+// X-Bz-Info-<header>:<value> is converted to <header>:<value>
+// Content-Type is converted to ContentType.
+// X-Bz-Content-Sha1 is converted to ETag.
+func headerToObjectInfo(bucket, object string, header http.Header) (objInfo ObjectInfo, err error) {
+	clen, err := strconv.ParseInt(header.Get("Content-Length"), 10, 64)
+	if err != nil {
+		return objInfo, err
+	}
+	timeStamp, err := strconv.ParseInt(header.Get("X-Bz-Upload-Timestamp"), 10, 64)
+	if err != nil {
+		return objInfo, err
+	}
+
+	info := make(map[string]string)
+	for key := range header {
+		if !strings.HasPrefix(key, "X-Bz-Info-") {
+			continue
+		}
+		name, err := url.QueryUnescape(strings.TrimPrefix(key, "X-Bz-Info-"))
+		if err != nil {
+			return objInfo, err
+		}
+		val, err := url.QueryUnescape(header.Get(key))
+		if err != nil {
+			return objInfo, err
+		}
+		info[name] = val
+	}
+	objInfo = ObjectInfo{
+		Bucket:      bucket,
+		Name:        object,
+		ContentType: header.Get("Content-Type"),
+		ModTime:     time.Unix(timeStamp/1000, timeStamp%1000*1e6),
+		Size:        clen,
+		ETag:        header.Get("X-Bz-Content-Sha1"),
+		UserDefined: info,
+	}
+	return objInfo, nil
+}
+
+// AnonGetObjectInfo - Get object info anonymously
+func (l *b2Objects) AnonGetObjectInfo(bucket string, object string) (objInfo ObjectInfo, err error) {
+	uri := fmt.Sprintf("%s/file/%s/%s", l.b2Client.DownloadURI, bucket, object)
+	req, err := http.NewRequest("HEAD", uri, nil)
+	if err != nil {
+		return objInfo, err
+	}
+	resp, err := l.anonClient.Do(req)
+	if err != nil {
+		return objInfo, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 && resp.StatusCode != 206 {
+		return objInfo, errors.New(resp.Status)
+	}
+	return headerToObjectInfo(bucket, object, resp.Header)
+}
+
+// AnonListObjects - List objects anonymously
+func (l *b2Objects) AnonListObjects(bucket string, prefix string, marker string, delimiter string,
+	maxKeys int) (loi ListObjectsInfo, err error) {
+	return loi, traceError(NotImplemented{})
+}
+
+// AnonListObjectsV2 - List objects in V2 mode, anonymously
+func (l *b2Objects) AnonListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int,
+	fetchOwner bool, startAfter string) (loi ListObjectsV2Info, err error) {
+	return loi, traceError(NotImplemented{})
+}
+
+// AnonGetBucketInfo - Get bucket metadata anonymously.
+func (l *b2Objects) AnonGetBucketInfo(bucket string) (bi BucketInfo, err error) {
+	return bi, traceError(NotImplemented{})
+}

cmd/gateway-b2-anonymous_test.go

@@ -0,0 +1,104 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"net/http"
+	"testing"
+)
+
+// Tests headerToObjectInfo
+func TestHeaderToObjectInfo(t *testing.T) {
+	testCases := []struct {
+		bucket, object string
+		header         http.Header
+		objInfo        ObjectInfo
+	}{
+		{
+			bucket: "bucket",
+			object: "object",
+			header: http.Header{
+				"Content-Length":         []string{"10"},
+				"Content-Type":           []string{"application/javascript"},
+				"X-Bz-Upload-Timestamp":  []string{"1000"},
+				"X-Bz-Info-X-Amz-Meta-1": []string{"test1"},
+				"X-Bz-Content-Sha1":      []string{"xxxxx"},
+			},
+			objInfo: ObjectInfo{
+				Bucket:      "bucket",
+				Name:        "object",
+				ContentType: "application/javascript",
+				Size:        10,
+				UserDefined: map[string]string{
+					"X-Amz-Meta-1": "test1",
+				},
+				ETag: "xxxxx",
+			},
+		},
+	}
+	for i, testCase := range testCases {
+		gotObjInfo, err := headerToObjectInfo(testCase.bucket, testCase.object, testCase.header)
+		if err != nil {
+			t.Fatalf("Test %d: %s", i+1, err)
+		}
+		if gotObjInfo.Bucket != testCase.objInfo.Bucket {
+			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.objInfo.Bucket, gotObjInfo.Bucket)
+		}
+		if gotObjInfo.Name != testCase.objInfo.Name {
+			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.objInfo.Name, gotObjInfo.Name)
+		}
+		if gotObjInfo.ContentType != testCase.objInfo.ContentType {
+			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.objInfo.ContentType, gotObjInfo.ContentType)
+		}
+		if gotObjInfo.ETag != testCase.objInfo.ETag {
+			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.objInfo.ETag, gotObjInfo.ETag)
+		}
+	}
+}
+
+// Tests mkRange test.
+func TestMkRange(t *testing.T) {
+	testCases := []struct {
+		offset, size int64
+		expectedRng  string
+	}{
+		// No offset set, size not set.
+		{
+			offset:      0,
+			size:        0,
+			expectedRng: "",
+		},
+		// Offset set, size not set.
+		{
+			offset:      10,
+			size:        0,
+			expectedRng: "bytes=10-",
+		},
+		// Offset set, size set.
+		{
+			offset:      10,
+			size:        11,
+			expectedRng: "bytes=10-20",
+		},
+	}
+	for i, testCase := range testCases {
+		gotRng := mkRange(testCase.offset, testCase.size)
+		if gotRng != testCase.expectedRng {
+			t.Errorf("Test %d: expected %s, got %s", i+1, testCase.expectedRng, gotRng)
+		}
+	}
+}

cmd/gateway-b2-unsupported.go

@@ -0,0 +1,44 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+// HealBucket - Not relevant.
+func (l *b2Objects) HealBucket(bucket string) error {
+	return traceError(NotImplemented{})
+}
+
+// ListBucketsHeal - Not relevant.
+func (l *b2Objects) ListBucketsHeal() (buckets []BucketInfo, err error) {
+	return []BucketInfo{}, traceError(NotImplemented{})
+}
+
+// HealObject - Not relevant.
+func (l *b2Objects) HealObject(bucket string, object string) (int, int, error) {
+	return 0, 0, traceError(NotImplemented{})
+}
+
+// ListObjectsHeal - Not relevant.
+func (l *b2Objects) ListObjectsHeal(bucket string, prefix string, marker string, delimiter string,
+	maxKeys int) (loi ListObjectsInfo, e error) {
+	return loi, traceError(NotImplemented{})
+}
+
+// ListUploadsHeal - Not relevant.
+func (l *b2Objects) ListUploadsHeal(bucket string, prefix string, marker string, uploadIDMarker string,
+	delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
+	return lmi, traceError(NotImplemented{})
+}