Add aws-sdk-go presigned PUT test

Dockerfile.dev

@@ -30,6 +30,9 @@ COPY run /mint/run
 COPY build/awscli /mint/build/awscli
 RUN build/awscli/install.sh
 
+COPY build/aws-sdk-go /mint/build/aws-sdk-go
+RUN build/aws-sdk-go/install.sh
+
 COPY build/aws-sdk-php /mint/build/aws-sdk-php
 RUN build/aws-sdk-php/install.sh
 

README.md

@@ -3,6 +3,7 @@
 Mint is a testing framework for Minio object server, available as a docker image. It runs correctness, benchmarking and stress tests. Following are the SDKs/tools used in correctness tests.
 
 - awscli
+- aws-sdk-go
 - aws-sdk-php
 - aws-sdk-ruby
 - mc

build/aws-sdk-go/install.sh

@@ -0,0 +1,24 @@
+#!/bin/bash -e
+#
+#  Mint (C) 2017 Minio, Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+test_run_dir="$MINT_RUN_CORE_DIR/aws-sdk-go"
+go get -u github.com/aws/aws-sdk-go/aws/...
+go get -u github.com/aws/aws-sdk-go/aws/credentials/...
+go get -u github.com/aws/aws-sdk-go/aws/session/...
+go get -u github.com/aws/aws-sdk-go/service/s3/...
+go get -u github.com/sirupsen/logrus/...
+go build -o "$test_run_dir/aws-sdk-go" "$test_run_dir/quick-tests.go"

run/core/aws-sdk-go/quick-tests.go

@@ -0,0 +1,219 @@
+/*
+*
+*  Mint, (C) 2017 Minio, Inc.
+*
+*  Licensed under the Apache License, Version 2.0 (the "License");
+*  you may not use this file except in compliance with the License.
+*  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+*  Unless required by applicable law or agreed to in writing, software
+
+*  distributed under the License is distributed on an "AS IS" BASIS,
+*  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+*  See the License for the specific language governing permissions and
+*  limitations under the License.
+*
+ */
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	log "github.com/sirupsen/logrus"
+)
+
+const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569"
+const (
+	letterIdxBits = 6                    // 6 bits to represent a letter index
+	letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
+	letterIdxMax  = 63 / letterIdxBits   // # of letter indices fitting in 63 bits
+)
+
+type ErrorResponse struct {
+	XMLName    xml.Name `xml:"Error" json:"-"`
+	Code       string
+	Message    string
+	BucketName string
+	Key        string
+	RequestID  string `xml:"RequestId"`
+	HostID     string `xml:"HostId"`
+
+	// Region where the bucket is located. This header is returned
+	// only in HEAD bucket and ListObjects response.
+	Region string
+
+	// Headers of the returned S3 XML error
+	Headers http.Header `xml:"-" json:"-"`
+}
+
+type mintJSONFormatter struct {
+}
+
+func (f *mintJSONFormatter) Format(entry *log.Entry) ([]byte, error) {
+	data := make(log.Fields, len(entry.Data))
+	for k, v := range entry.Data {
+		switch v := v.(type) {
+		case error:
+			// Otherwise errors are ignored by `encoding/json`
+			// https://github.com/sirupsen/logrus/issues/137
+			data[k] = v.Error()
+		default:
+			data[k] = v
+		}
+	}
+
+	serialized, err := json.Marshal(data)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err)
+	}
+	return append(serialized, '\n'), nil
+}
+
+// log successful test runs
+func successLogger(function string, args map[string]interface{}, startTime time.Time) *log.Entry {
+	// calculate the test case duration
+	duration := time.Since(startTime)
+	// log with the fields as per mint
+	fields := log.Fields{"name": "aws-sdk-go", "function": function, "args": args, "duration": duration.Nanoseconds() / 1000000, "status": "pass"}
+	return log.WithFields(fields)
+}
+
+// log failed test runs
+func failureLog(function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) *log.Entry {
+	// calculate the test case duration
+	duration := time.Since(startTime)
+	var fields log.Fields
+	// log with the fields as per mint
+	if err != nil {
+		fields = log.Fields{"name": "aws-sdk-go", "function": function, "args": args,
+			"duration": duration.Nanoseconds() / 1000000, "status": "fail", "alert": alert, "message": message, "error": err}
+	} else {
+		fields = log.Fields{"name": "aws-sdk-go", "function": function, "args": args,
+			"duration": duration.Nanoseconds() / 1000000, "status": "fail", "alert": alert, "message": message}
+	}
+	return log.WithFields(fields)
+}
+
+func randString(n int, src rand.Source, prefix string) string {
+	b := make([]byte, n)
+	// A rand.Int63() generates 63 random bits, enough for letterIdxMax letters!
+	for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; {
+		if remain == 0 {
+			cache, remain = src.Int63(), letterIdxMax
+		}
+		if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
+			b[i] = letterBytes[idx]
+			i--
+		}
+		cache >>= letterIdxBits
+		remain--
+	}
+	return prefix + string(b[0:30-len(prefix)])
+}
+
+func testPresignedPut(s3Client *s3.S3) {
+	startTime := time.Now()
+	function := "PresignedPut"
+	bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test")
+	object := "presignedTest"
+	expiry := 1 * time.Minute
+	args := map[string]interface{}{
+		"bucketName": bucket,
+		"objectName": object,
+		"expiry":     expiry,
+	}
+
+	req, _ := s3Client.PutObjectRequest(&s3.PutObjectInput{
+		Bucket:      aws.String(bucket),
+		Key:         aws.String(object),
+		ContentType: aws.String("application/octet-stream"),
+	})
+
+	req.HTTPRequest.Header.Set("X-Amz-Content-Sha256", "invalid-sha256")
+	url, err := req.Presign(expiry)
+	if err != nil {
+		failureLog(function, args, startTime, "", "AWS SDK Go presigned Put request creation failed", err).Fatal()
+		return
+	}
+
+	rreq, err := http.NewRequest("PUT", url, bytes.NewReader([]byte("")))
+	rreq.Header.Add("X-Amz-Content-Sha256", "invalid-sha256")
+	rreq.Header.Add("Content-Type", "application/octet-stream")
+
+	resp, err := http.DefaultClient.Do(rreq)
+	if err != nil {
+		failureLog(function, args, startTime, "", "AWS SDK Go presigned put request failed", err).Fatal()
+		return
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	resp.Body.Close()
+	if err != nil {
+		failureLog(function, args, startTime, "", "AWS SDK Go reading response body failed", err).Fatal()
+		return
+	}
+
+	errResp := ErrorResponse{}
+	err = xml.Unmarshal(body, &errResp)
+	if err != nil {
+		failureLog(function, args, startTime, "", "AWS SDK Go unmarshalling xml failed", err).Fatal()
+		return
+	}
+
+	if errResp.Code != "XAmzContentSHA256Mismatch" {
+		failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go presigned PUT expected to fail with XAmzContentSHA256Mismatch but got %v", errResp.Code), errors.New("AWS S3 error code mismatch")).Fatal()
+		return
+	}
+	successLogger(function, args, startTime).Info()
+}
+
+func main() {
+	endpoint := os.Getenv("SERVER_ENDPOINT")
+	accessKey := os.Getenv("ACCESS_KEY")
+	secretKey := os.Getenv("SECRET_KEY")
+	secure := os.Getenv("ENABLE_HTTPS")
+	sdkEndpoint := "http://" + endpoint
+	if secure == "1" {
+		sdkEndpoint = "https://"
+	}
+
+	creds := credentials.NewStaticCredentials(accessKey, secretKey, "")
+	newSession := session.New()
+	s3Config := &aws.Config{
+		Credentials:      creds,
+		Endpoint:         aws.String(sdkEndpoint),
+		Region:           aws.String("us-east-1"),
+		S3ForcePathStyle: aws.Bool(true),
+	}
+
+	s3Config = s3Config.WithLogLevel(aws.LogDebug)
+
+	// Create an S3 service object in the default region.
+	s3Client := s3.New(newSession, s3Config)
+
+	// Output to stdout instead of the default stderr
+	log.SetOutput(os.Stdout)
+	// create custom formatter
+	mintFormatter := mintJSONFormatter{}
+	// set custom formatter
+	log.SetFormatter(&mintFormatter)
+	// log Info or above -- success cases are Info level, failures are Fatal level
+	log.SetLevel(log.InfoLevel)
+	// execute tests
+	testPresignedPut(s3Client)
+}

run/core/aws-sdk-go/run.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+#
+#  Mint (C) 2017 Minio, Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+# handle command line arguments
+if [ $# -ne 2 ]; then
+    echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>"
+    exit -1
+fi
+
+output_log_file="$1"
+error_log_file="$2"
+
+# run tests
+/mint/run/core/aws-sdk-go/aws-sdk-go 1>>"$output_log_file" 2>"$error_log_file"

run/core/aws-sdk-php/quick-tests.php

@@ -22,6 +22,7 @@
 use Aws\Credentials;
 use Aws\Exception\AwsException;
 use GuzzleHttp\Psr7;
+use GuzzleHttp\Psr7\Request;
 use GuzzleHttp\Client;
 
 // Constants
@@ -814,90 +815,6 @@ function testAnonDeleteObjects($s3Client, $params) {
     }
 }
 
- /**
-  * testPresignedPut tests presigned PUT S3 API under following conditions
-  * - Set content-sha256
-  * - No content-sha256
-  * - Invalid content-sha256
-  *
-  * @param $s3Client AWS\S3\S3Client object
-  *
-  * @param $params associative array containing bucket and object name
-  *
-  * @return void
-  */
-function testPresignedPut($s3Client, $params) {
-    // Test to validate the validate sha256 set to the value of the body.
-    // ------ 1 ------
-    $bucket = $params['Bucket'];
-    $object = $params['Object'];
-    $sha256 = "03675ac53ff9cd1535ccc7dfcdfa2c458c5218371f418dc136f2d19ac1fbe8a5";
-
-    $cmd = $s3Client->getCommand('PutObject', [
-            'Bucket' => $bucket,
-            'Key'    => $object,
-            'ContentSHA256' => $sha256,
-    ]);
-
-    $request = $s3Client->createPresignedRequest($cmd, '+20 minutes');
-    $presignedUrl = (string) $request->getUri();
-
-    $client = new Client();
-    $body = "Hello, World";
-    $res = $client->request('PUT', $presignedUrl, ['body' => $body]);
-
-    if ($res->getStatusCode() != HTTP_OK) {
-        throw new Exception('presignedPutObject API failed for ' .
-                            $bucket, $object);
-    }
-    // ------ 1 ------
-
-    // Test to simple presigned PUT with just body no validation.
-    // ------ 2 ------
-    $bucket = $params['Bucket'];
-    $object = $params['Object'];
-
-    $cmd = $s3Client->getCommand('PutObject', [
-            'Bucket' => $bucket,
-            'Key'    => $object,
-    ]);
-
-    $request = $s3Client->createPresignedRequest($cmd, '+20 minutes');
-    $presignedUrl = (string) $request->getUri();
-    $res = $client->request('PUT', $presignedUrl, ['body' => $body]);
-
-    if ($res->getStatusCode() != HTTP_OK) {
-        throw new Exception('presignedPutObject API failed for ' .
-                            $bucket, $object);
-    }
-    // ------ 2 ------
-
-    // Test to presigned PUT with invalid sha256.
-    // ------ 3 ------
-    $bucket = $params['Bucket'];
-    $object = $params['Object'];
-    $sha256 = "invalid-sha256";
-
-    $cmd = $s3Client->getCommand('PutObject', [
-            'Bucket' => $bucket,
-            'Key'    => $object,
-            'ContentSHA256' => $sha256,
-    ]);
-
-    $request = $s3Client->createPresignedRequest($cmd, '+20 minutes');
-    $presignedUrl = (string) $request->getUri();
-
-    $client = new Client();
-    $body = "Hello, World";
-    $res = $client->request('PUT', $presignedUrl, ['body' => $body]);
-
-    if ($res->getStatusCode() != HTTP_BADREQUEST) {
-        throw new Exception('presignedPutObject API failed for ' .
-                            $bucket, $object);
-    }
-    // ------ 3 ------
-}
-
  /**
   *  testBucketPolicy tests GET/PUT Bucket policy S3 APIs
   *
@@ -1170,7 +1087,6 @@ public function out($data) {
     runTest($s3Client, 'testMultipartUpload', "createMultipartUpload ( array \$params = [] )", $testParams);
     runTest($s3Client, 'testMultipartUploadFailure', "uploadPart ( array \$params = [] )", $testParams);
     runTest($s3Client, 'testAbortMultipartUpload', "abortMultipartupload ( array \$params = [] )", $testParams);
-    runTest($s3Client, 'testPresignedPut', "presignedPut ( array \$params = [] )", $testParams);
     runTest($s3Client, 'testBucketPolicy', "getBucketPolicy ( array \$params = [] )", ['Bucket' => $emptyBucket]);
 }
 finally {