From 53947958f0cf256f44d21d60e258087e661b4f60 Mon Sep 17 00:00:00 2001 From: Krishnan Parthasarathi Date: Wed, 15 Nov 2017 12:48:35 +0530 Subject: [PATCH] Add aws-sdk-go presigned PUT test - Add aws-sdk-go to mint - Add presigned PUT test in aws-sdk-go - Remove presigned PUT test in aws-sdk-php since the SDK doesn't support 'X-Amz-Content-Sha256' header with presigned requests --- Dockerfile.dev | 3 + build/aws-sdk-go/install.sh | 24 +++ run/core/aws-sdk-go/quick-tests.go | 219 +++++++++++++++++++++++++++ run/core/aws-sdk-go/run.sh | 28 ++++ run/core/aws-sdk-php/quick-tests.php | 25 +-- run/core/minio-go/README.md | 2 +- 6 files changed, 276 insertions(+), 25 deletions(-) create mode 100755 build/aws-sdk-go/install.sh create mode 100644 run/core/aws-sdk-go/quick-tests.go create mode 100755 run/core/aws-sdk-go/run.sh diff --git a/Dockerfile.dev b/Dockerfile.dev index b224ce81..3a261b2c 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -30,6 +30,9 @@ COPY run /mint/run COPY build/awscli /mint/build/awscli RUN build/awscli/install.sh +COPY build/aws-sdk-go /mint/build/aws-sdk-go +RUN build/aws-sdk-go/install.sh + COPY build/aws-sdk-php /mint/build/aws-sdk-php RUN build/aws-sdk-php/install.sh diff --git a/build/aws-sdk-go/install.sh b/build/aws-sdk-go/install.sh new file mode 100755 index 00000000..229b4daa --- /dev/null +++ b/build/aws-sdk-go/install.sh @@ -0,0 +1,24 @@ +#!/bin/bash -e +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +test_run_dir="$MINT_RUN_CORE_DIR/aws-sdk-go" +go get -u github.com/aws/aws-sdk-go/aws/... +go get -u github.com/aws/aws-sdk-go/aws/credentials/... +go get -u github.com/aws/aws-sdk-go/aws/session/... +go get -u github.com/aws/aws-sdk-go/service/s3/... +go get -u github.com/sirupsen/logrus/... +go build -o "$test_run_dir/aws-sdk-go" "$test_run_dir/quick-tests.go" diff --git a/run/core/aws-sdk-go/quick-tests.go b/run/core/aws-sdk-go/quick-tests.go new file mode 100644 index 00000000..32d9faa1 --- /dev/null +++ b/run/core/aws-sdk-go/quick-tests.go @@ -0,0 +1,219 @@ +/* +* +* Mint, (C) 2017 Minio, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software + +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* + */ + +package main + +import ( + "bytes" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "io/ioutil" + "math/rand" + "net/http" + "os" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + log "github.com/sirupsen/logrus" +) + +const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" +const ( + letterIdxBits = 6 // 6 bits to represent a letter index + letterIdxMask = 1<= 0; { + if remain == 0 { + cache, remain = src.Int63(), letterIdxMax + } + if idx := int(cache & letterIdxMask); idx < len(letterBytes) { + b[i] = letterBytes[idx] + i-- + } + cache >>= letterIdxBits + remain-- + } + return prefix + string(b[0:30-len(prefix)]) +} + +func testPresignedPut(s3Client *s3.S3) { + startTime := time.Now() + function := "PresignedPut" + bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test") + object := "presignedTest" + expiry := 1 * time.Minute + args := map[string]interface{}{ + "bucketName": bucket, + "objectName": object, + "expiry": expiry, + } + + req, _ := s3Client.PutObjectRequest(&s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(object), + ContentType: aws.String("application/octet-stream"), + }) + + req.HTTPRequest.Header.Set("X-Amz-Content-Sha256", "invalid-sha256") + url, err := req.Presign(expiry) + if err != nil { + failureLog(function, args, startTime, "", "AWS SDK Go presigned Put request creation failed", err).Fatal() + return + } + + rreq, err := http.NewRequest("PUT", url, bytes.NewReader([]byte(""))) + rreq.Header.Add("X-Amz-Content-Sha256", "invalid-sha256") + rreq.Header.Add("Content-Type", "application/octet-stream") + + resp, err := http.DefaultClient.Do(rreq) + if err != nil { + failureLog(function, args, startTime, "", "AWS SDK Go presigned put request failed", err).Fatal() + return + } + body, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + failureLog(function, args, startTime, "", "AWS SDK Go reading response body failed", err).Fatal() + return + } + + errResp := ErrorResponse{} + err = xml.Unmarshal(body, &errResp) + if err != nil { + failureLog(function, args, startTime, "", "AWS SDK Go unmarshalling xml failed", err).Fatal() + return + } + + if errResp.Code != "XAmzContentSHA256Mismatch" { + failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go presigned PUT expected to fail with XAmzContentSHA256Mismatch but got %v", errResp.Code), errors.New("AWS S3 error code mismatch")).Fatal() + return + } + successLogger(function, args, startTime).Info() +} + +func main() { + endpoint := os.Getenv("SERVER_ENDPOINT") + accessKey := os.Getenv("ACCESS_KEY") + secretKey := os.Getenv("SECRET_KEY") + secure := os.Getenv("ENABLE_HTTPS") + sdkEndpoint := "http://" + endpoint + if secure == "1" { + sdkEndpoint = "https://" + } + + creds := credentials.NewStaticCredentials(accessKey, secretKey, "") + newSession := session.New() + s3Config := &aws.Config{ + Credentials: creds, + Endpoint: aws.String(sdkEndpoint), + Region: aws.String("us-east-1"), + S3ForcePathStyle: aws.Bool(true), + } + + s3Config = s3Config.WithLogLevel(aws.LogDebug) + + // Create an S3 service object in the default region. + s3Client := s3.New(newSession, s3Config) + + // Output to stdout instead of the default stderr + log.SetOutput(os.Stdout) + // create custom formatter + mintFormatter := mintJSONFormatter{} + // set custom formatter + log.SetFormatter(&mintFormatter) + // log Info or above -- success cases are Info level, failures are Fatal level + log.SetLevel(log.InfoLevel) + // execute tests + testPresignedPut(s3Client) +} diff --git a/run/core/aws-sdk-go/run.sh b/run/core/aws-sdk-go/run.sh new file mode 100755 index 00000000..1811bdd5 --- /dev/null +++ b/run/core/aws-sdk-go/run.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Mint (C) 2017 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# handle command line arguments +if [ $# -ne 2 ]; then + echo "usage: run.sh " + exit -1 +fi + +output_log_file="$1" +error_log_file="$2" + +# run tests +/mint/run/core/aws-sdk-go/aws-sdk-go 1>>"$output_log_file" 2>"$error_log_file" diff --git a/run/core/aws-sdk-php/quick-tests.php b/run/core/aws-sdk-php/quick-tests.php index 627a3d0c..4fffad7d 100644 --- a/run/core/aws-sdk-php/quick-tests.php +++ b/run/core/aws-sdk-php/quick-tests.php @@ -22,6 +22,7 @@ use Aws\Credentials; use Aws\Exception\AwsException; use GuzzleHttp\Psr7; +use GuzzleHttp\Psr7\Request; use GuzzleHttp\Client; // Constants @@ -872,30 +873,6 @@ function testPresignedPut($s3Client, $params) { } // ------ 2 ------ - // Test to presigned PUT with invalid sha256. - // ------ 3 ------ - $bucket = $params['Bucket']; - $object = $params['Object']; - $sha256 = "invalid-sha256"; - - $cmd = $s3Client->getCommand('PutObject', [ - 'Bucket' => $bucket, - 'Key' => $object, - 'ContentSHA256' => $sha256, - ]); - - $request = $s3Client->createPresignedRequest($cmd, '+20 minutes'); - $presignedUrl = (string) $request->getUri(); - - $client = new Client(); - $body = "Hello, World"; - $res = $client->request('PUT', $presignedUrl, ['body' => $body]); - - if ($res->getStatusCode() != HTTP_BADREQUEST) { - throw new Exception('presignedPutObject API failed for ' . - $bucket, $object); - } - // ------ 3 ------ } /** diff --git a/run/core/minio-go/README.md b/run/core/minio-go/README.md index 277d3ce4..6f1723b7 100644 --- a/run/core/minio-go/README.md +++ b/run/core/minio-go/README.md @@ -2,7 +2,7 @@ This directory serves as the location for Mint tests using `minio-go`. Top level `mint.sh` calls `run.sh` to execute tests. ## Adding new tests -New tests is added in functional tests of minio-go. Please check https://github.com/minio/minio-go +New tests are added in functional tests of minio-go. Please check https://github.com/minio/minio-go ## Running tests manually - Set environment variables `MINT_DATA_DIR`, `MINT_MODE`, `SERVER_ENDPOINT`, `ACCESS_KEY`, `SECRET_KEY`, `SERVER_REGION` and `ENABLE_HTTPS`