-
Notifications
You must be signed in to change notification settings - Fork 49
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Add aws-sdk-go to mint - Add presigned PUT test in aws-sdk-go - Remove presigned PUT test in aws-sdk-php since the SDK doesn't support 'X-Amz-Content-Sha256' header with presigned requests
- Loading branch information
Krishnan Parthasarathi
committed
Nov 16, 2017
1 parent
88b0040
commit 5394795
Showing
6 changed files
with
276 additions
and
25 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| #!/bin/bash -e | ||
| # | ||
| # Mint (C) 2017 Minio, Inc. | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| # | ||
|
|
||
| test_run_dir="$MINT_RUN_CORE_DIR/aws-sdk-go" | ||
| go get -u github.com/aws/aws-sdk-go/aws/... | ||
| go get -u github.com/aws/aws-sdk-go/aws/credentials/... | ||
| go get -u github.com/aws/aws-sdk-go/aws/session/... | ||
| go get -u github.com/aws/aws-sdk-go/service/s3/... | ||
| go get -u github.com/sirupsen/logrus/... | ||
| go build -o "$test_run_dir/aws-sdk-go" "$test_run_dir/quick-tests.go" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,219 @@ | ||
| /* | ||
| * | ||
| * Mint, (C) 2017 Minio, Inc. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| * | ||
| */ | ||
|
|
||
| package main | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "encoding/json" | ||
| "encoding/xml" | ||
| "errors" | ||
| "fmt" | ||
| "io/ioutil" | ||
| "math/rand" | ||
| "net/http" | ||
| "os" | ||
| "time" | ||
|
|
||
| "github.com/aws/aws-sdk-go/aws" | ||
| "github.com/aws/aws-sdk-go/aws/credentials" | ||
| "github.com/aws/aws-sdk-go/aws/session" | ||
| "github.com/aws/aws-sdk-go/service/s3" | ||
| log "github.com/sirupsen/logrus" | ||
| ) | ||
|
|
||
| const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" | ||
| const ( | ||
| letterIdxBits = 6 // 6 bits to represent a letter index | ||
| letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits | ||
| letterIdxMax = 63 / letterIdxBits // # of letter indices fitting in 63 bits | ||
| ) | ||
|
|
||
| type ErrorResponse struct { | ||
| XMLName xml.Name `xml:"Error" json:"-"` | ||
| Code string | ||
| Message string | ||
| BucketName string | ||
| Key string | ||
| RequestID string `xml:"RequestId"` | ||
| HostID string `xml:"HostId"` | ||
|
|
||
| // Region where the bucket is located. This header is returned | ||
| // only in HEAD bucket and ListObjects response. | ||
| Region string | ||
|
|
||
| // Headers of the returned S3 XML error | ||
| Headers http.Header `xml:"-" json:"-"` | ||
| } | ||
|
|
||
| type mintJSONFormatter struct { | ||
| } | ||
|
|
||
| func (f *mintJSONFormatter) Format(entry *log.Entry) ([]byte, error) { | ||
| data := make(log.Fields, len(entry.Data)) | ||
| for k, v := range entry.Data { | ||
| switch v := v.(type) { | ||
| case error: | ||
| // Otherwise errors are ignored by `encoding/json` | ||
| // https://github.com/sirupsen/logrus/issues/137 | ||
| data[k] = v.Error() | ||
| default: | ||
| data[k] = v | ||
| } | ||
| } | ||
|
|
||
| serialized, err := json.Marshal(data) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err) | ||
| } | ||
| return append(serialized, '\n'), nil | ||
| } | ||
|
|
||
| // log successful test runs | ||
| func successLogger(function string, args map[string]interface{}, startTime time.Time) *log.Entry { | ||
| // calculate the test case duration | ||
| duration := time.Since(startTime) | ||
| // log with the fields as per mint | ||
| fields := log.Fields{"name": "minio-go", "function": function, "args": args, "duration": duration.Nanoseconds() / 1000000, "status": "pass"} | ||
| return log.WithFields(fields) | ||
| } | ||
|
|
||
| // log failed test runs | ||
| func failureLog(function string, args map[string]interface{}, startTime time.Time, alert string, message string, err error) *log.Entry { | ||
| // calculate the test case duration | ||
| duration := time.Since(startTime) | ||
| var fields log.Fields | ||
| // log with the fields as per mint | ||
| if err != nil { | ||
| fields = log.Fields{"name": "minio-go", "function": function, "args": args, | ||
| "duration": duration.Nanoseconds() / 1000000, "status": "fail", "alert": alert, "message": message, "error": err} | ||
| } else { | ||
| fields = log.Fields{"name": "minio-go", "function": function, "args": args, | ||
| "duration": duration.Nanoseconds() / 1000000, "status": "fail", "alert": alert, "message": message} | ||
| } | ||
| return log.WithFields(fields) | ||
| } | ||
|
|
||
| func randString(n int, src rand.Source, prefix string) string { | ||
| b := make([]byte, n) | ||
| // A rand.Int63() generates 63 random bits, enough for letterIdxMax letters! | ||
| for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; { | ||
| if remain == 0 { | ||
| cache, remain = src.Int63(), letterIdxMax | ||
| } | ||
| if idx := int(cache & letterIdxMask); idx < len(letterBytes) { | ||
| b[i] = letterBytes[idx] | ||
| i-- | ||
| } | ||
| cache >>= letterIdxBits | ||
| remain-- | ||
| } | ||
| return prefix + string(b[0:30-len(prefix)]) | ||
| } | ||
|
|
||
| func testPresignedPut(s3Client *s3.S3) { | ||
| startTime := time.Now() | ||
| function := "PresignedPut" | ||
| bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test") | ||
| object := "presignedTest" | ||
| expiry := 1 * time.Minute | ||
| args := map[string]interface{}{ | ||
| "bucketName": bucket, | ||
| "objectName": object, | ||
| "expiry": expiry, | ||
| } | ||
|
|
||
| req, _ := s3Client.PutObjectRequest(&s3.PutObjectInput{ | ||
| Bucket: aws.String(bucket), | ||
| Key: aws.String(object), | ||
| ContentType: aws.String("application/octet-stream"), | ||
| }) | ||
|
|
||
| req.HTTPRequest.Header.Set("X-Amz-Content-Sha256", "invalid-sha256") | ||
| url, err := req.Presign(expiry) | ||
| if err != nil { | ||
| failureLog(function, args, startTime, "", "AWS SDK Go presigned Put request creation failed", err).Fatal() | ||
| return | ||
| } | ||
|
|
||
| rreq, err := http.NewRequest("PUT", url, bytes.NewReader([]byte(""))) | ||
| rreq.Header.Add("X-Amz-Content-Sha256", "invalid-sha256") | ||
| rreq.Header.Add("Content-Type", "application/octet-stream") | ||
|
|
||
| resp, err := http.DefaultClient.Do(rreq) | ||
| if err != nil { | ||
| failureLog(function, args, startTime, "", "AWS SDK Go presigned put request failed", err).Fatal() | ||
| return | ||
| } | ||
| body, err := ioutil.ReadAll(resp.Body) | ||
| resp.Body.Close() | ||
| if err != nil { | ||
| failureLog(function, args, startTime, "", "AWS SDK Go reading response body failed", err).Fatal() | ||
| return | ||
| } | ||
|
|
||
| errResp := ErrorResponse{} | ||
| err = xml.Unmarshal(body, &errResp) | ||
| if err != nil { | ||
| failureLog(function, args, startTime, "", "AWS SDK Go unmarshalling xml failed", err).Fatal() | ||
| return | ||
| } | ||
|
|
||
| if errResp.Code != "XAmzContentSHA256Mismatch" { | ||
| failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go presigned PUT expected to fail with XAmzContentSHA256Mismatch but got %v", errResp.Code), errors.New("AWS S3 error code mismatch")).Fatal() | ||
| return | ||
| } | ||
| successLogger(function, args, startTime).Info() | ||
| } | ||
|
|
||
| func main() { | ||
| endpoint := os.Getenv("SERVER_ENDPOINT") | ||
| accessKey := os.Getenv("ACCESS_KEY") | ||
| secretKey := os.Getenv("SECRET_KEY") | ||
| secure := os.Getenv("ENABLE_HTTPS") | ||
| sdkEndpoint := "http://" + endpoint | ||
| if secure == "1" { | ||
| sdkEndpoint = "https://" | ||
| } | ||
|
|
||
| creds := credentials.NewStaticCredentials(accessKey, secretKey, "") | ||
| newSession := session.New() | ||
| s3Config := &aws.Config{ | ||
| Credentials: creds, | ||
| Endpoint: aws.String(sdkEndpoint), | ||
| Region: aws.String("us-east-1"), | ||
| S3ForcePathStyle: aws.Bool(true), | ||
| } | ||
|
|
||
| s3Config = s3Config.WithLogLevel(aws.LogDebug) | ||
|
|
||
| // Create an S3 service object in the default region. | ||
| s3Client := s3.New(newSession, s3Config) | ||
|
|
||
| // Output to stdout instead of the default stderr | ||
| log.SetOutput(os.Stdout) | ||
| // create custom formatter | ||
| mintFormatter := mintJSONFormatter{} | ||
| // set custom formatter | ||
| log.SetFormatter(&mintFormatter) | ||
| // log Info or above -- success cases are Info level, failures are Fatal level | ||
| log.SetLevel(log.InfoLevel) | ||
| // execute tests | ||
| testPresignedPut(s3Client) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| #!/bin/bash | ||
| # | ||
| # Mint (C) 2017 Minio, Inc. | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| # | ||
|
|
||
| # handle command line arguments | ||
| if [ $# -ne 2 ]; then | ||
| echo "usage: run.sh <OUTPUT-LOG-FILE> <ERROR-LOG-FILE>" | ||
| exit -1 | ||
| fi | ||
|
|
||
| output_log_file="$1" | ||
| error_log_file="$2" | ||
|
|
||
| # run tests | ||
| /mint/run/core/aws-sdk-go/aws-sdk-go 1>>"$output_log_file" 2>"$error_log_file" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters