From a9e1a1df5df27f85e17a159a4f3600f5a676210c Mon Sep 17 00:00:00 2001 From: vishal Date: Mon, 23 Mar 2020 14:12:46 -0400 Subject: [PATCH 1/3] Create loggroup and bucket in go instead of bash --- cli/cmd/cluster.go | 47 ++++++++++++++++++++++++++++++++++++++ manager/install.sh | 37 ------------------------------ pkg/lib/aws/cloudwatch.go | 48 +++++++++++++++++++++++++++++++++++++++ pkg/lib/aws/errors.go | 6 ++++- pkg/lib/aws/s3.go | 34 +++++++++++++++++++++++++++ 5 files changed, 134 insertions(+), 38 deletions(-) create mode 100644 pkg/lib/aws/cloudwatch.go diff --git a/cli/cmd/cluster.go b/cli/cmd/cluster.go index 8186dbe0d0..c73fec5f9d 100644 --- a/cli/cmd/cluster.go +++ b/cli/cmd/cluster.go @@ -23,6 +23,7 @@ import ( "os" "time" + "github.com/cortexlabs/cortex/pkg/lib/aws" cr "github.com/cortexlabs/cortex/pkg/lib/configreader" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/exit" @@ -110,6 +111,17 @@ var _upCmd = &cobra.Command{ if err != nil { exit.Error(err) } + + err = CreateBucketIfNotFound(awsClient, clusterConfig.Bucket) + if err != nil { + exit.Error(err) + } + + err = CreateLogGroupIfNotFound(awsClient, clusterConfig.LogGroup) + if err != nil { + exit.Error(err) + } + out, exitCode, err := runManagerUpdateCommand("/root/install.sh", clusterConfig, awsCreds) if err != nil { exit.Error(err) @@ -452,3 +464,38 @@ func assertClusterStatus(accessConfig *clusterconfig.AccessConfig, status cluste func getCloudFormationURLWithAccessConfig(accessConfig *clusterconfig.AccessConfig) string { return getCloudFormationURL(*accessConfig.ClusterName, *accessConfig.Region) } + +func CreateBucketIfNotFound(awsClient *aws.Client, bucket string) error { + bucketFound, err := awsClient.DoesBucketExist(bucket) + if err != nil { + return err + } + if !bucketFound { + fmt.Println("creating a new s3 bucket: ", bucket) + err = awsClient.CreateBucket(bucket) + if err != nil { + return err + } + } else { + fmt.Println("using existing s3 bucket: ", bucket) + } + return nil +} + +func CreateLogGroupIfNotFound(awsClient *aws.Client, logGroup string) error { + logGroupFound, err := awsClient.DoesLogGroupExist(logGroup) + if err != nil { + return err + } + if !logGroupFound { + fmt.Println("creating a new cloudwatch log group: ", logGroup) + err = awsClient.CreateLogGroup(logGroup) + if err != nil { + return err + } + } else { + fmt.Println("using existing cloudwatch log group: ", logGroup) + } + + return nil +} diff --git a/manager/install.sh b/manager/install.sh index 22824a2e02..b747c0f5ad 100755 --- a/manager/install.sh +++ b/manager/install.sh @@ -131,9 +131,6 @@ function ensure_eks() { function main() { mkdir -p $CORTEX_CLUSTER_WORKSPACE - setup_bucket - setup_cloudwatch_logs - ensure_eks eksctl utils write-kubeconfig --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION | grep -v "saved kubeconfig as" | grep -v "using region" | grep -v "eksctl version" || true @@ -211,40 +208,6 @@ function main() { echo -e "\ncortex is ready!" } -function setup_bucket() { - if ! aws s3api head-bucket --bucket $CORTEX_BUCKET --output json 2>/dev/null; then - if aws s3 ls "s3://$CORTEX_BUCKET" --output json 2>&1 | grep -q 'NoSuchBucket'; then - echo -n "￮ creating s3 bucket: $CORTEX_BUCKET " - if [ "$CORTEX_REGION" == "us-east-1" ]; then - aws s3api create-bucket --bucket $CORTEX_BUCKET \ - --region $CORTEX_REGION \ - >/dev/null - else - aws s3api create-bucket --bucket $CORTEX_BUCKET \ - --region $CORTEX_REGION \ - --create-bucket-configuration LocationConstraint=$CORTEX_REGION \ - >/dev/null - fi - echo "✓" - else - echo "error: a bucket named \"${CORTEX_BUCKET}\" already exists, but you do not have access to it" - exit 1 - fi - else - echo "￮ using existing s3 bucket: $CORTEX_BUCKET" - fi -} - -function setup_cloudwatch_logs() { - if ! aws logs list-tags-log-group --log-group-name $CORTEX_LOG_GROUP --region $CORTEX_REGION --output json 2>&1 | grep -q "\"tags\":"; then - echo -n "￮ creating cloudwatch log group: $CORTEX_LOG_GROUP " - aws logs create-log-group --log-group-name $CORTEX_LOG_GROUP --region $CORTEX_REGION - echo "✓" - else - echo "￮ using existing cloudwatch log group: $CORTEX_LOG_GROUP" - fi -} - function setup_configmap() { kubectl -n=default create configmap 'cluster-config' \ --from-file='cluster.yaml'=$CORTEX_CLUSTER_CONFIG_FILE \ diff --git a/pkg/lib/aws/cloudwatch.go b/pkg/lib/aws/cloudwatch.go new file mode 100644 index 0000000000..fa8902b506 --- /dev/null +++ b/pkg/lib/aws/cloudwatch.go @@ -0,0 +1,48 @@ +/* +Copyright 2020 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aws + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/cortexlabs/cortex/pkg/lib/errors" +) + +func (c *Client) DoesLogGroupExist(logGroup string) (bool, error) { + _, err := c.CloudWatchLogs().ListTagsLogGroup(&cloudwatchlogs.ListTagsLogGroupInput{ + LogGroupName: aws.String(logGroup), + }) + if err != nil { + if CheckErrCode(err, "ResourceNotFoundException") { + return false, nil + } + return false, errors.Wrap(err, "log group", logGroup) + } + + return true, nil +} + +func (c *Client) CreateLogGroup(logGroup string) error { + _, err := c.CloudWatchLogs().CreateLogGroup(&cloudwatchlogs.CreateLogGroupInput{ + LogGroupName: aws.String(logGroup), + }) + if err != nil { + return errors.Wrap(err, "creating log group", logGroup) + } + + return nil +} diff --git a/pkg/lib/aws/errors.go b/pkg/lib/aws/errors.go index 0ee92bedcf..4dff9c540f 100644 --- a/pkg/lib/aws/errors.go +++ b/pkg/lib/aws/errors.go @@ -48,6 +48,10 @@ func IsNoSuchBucketErr(err error) bool { return CheckErrCode(err, "NoSuchBucket") } +func IsForbiddenErr(err error) bool { + return CheckErrCode(err, "Forbidden") +} + func IsGenericNotFoundErr(err error) bool { return IsNotFoundErr(err) || IsNoSuchKeyErr(err) || IsNoSuchBucketErr(err) } @@ -94,7 +98,7 @@ func ErrorAuth() error { func ErrorBucketInaccessible(bucket string) error { return errors.WithStack(&errors.Error{ Kind: ErrBucketInaccessible, - Message: fmt.Sprintf("bucket \"%s\" not found or insufficient permissions", bucket), + Message: fmt.Sprintf("bucket \"%s\" is not accessible with the specified AWS credentials", bucket), }) } diff --git a/pkg/lib/aws/s3.go b/pkg/lib/aws/s3.go index ce95243b4e..2e89213fbc 100644 --- a/pkg/lib/aws/s3.go +++ b/pkg/lib/aws/s3.go @@ -22,6 +22,7 @@ import ( "strings" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/endpoints" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" @@ -429,6 +430,39 @@ func (c *Client) DeletePrefix(bucket string, prefix string, continueIfFailure bo return nil } +func (c *Client) CreateBucket(bucket string) error { + _, err := c.S3().CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucket), + CreateBucketConfiguration: &s3.CreateBucketConfiguration{ + LocationConstraint: aws.String(c.Region), + }, + }) + if err != nil { + return errors.Wrap(err, "creating bucket", bucket) + } + return nil +} + +// Checks bucket existence and accessibility with credentials +func (c *Client) DoesBucketExist(bucket string) (bool, error) { + _, err := c.S3().HeadBucket(&s3.HeadBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + if aerr, ok := err.(awserr.Error); ok { + switch aerr.Code() { + case "NotFound": + return false, nil + case "Forbidden": + return false, ErrorBucketInaccessible(bucket) + } + } + return false, errors.Wrap(err, "bucket", bucket) + } + + return true, nil +} + func GetBucketRegion(bucket string) (string, error) { sess := session.Must(session.NewSession()) // credentials are not necessary for this request, and will not be used region, err := s3manager.GetBucketRegion(aws.BackgroundContext(), sess, bucket, endpoints.UsWest2RegionID) From c2b26489b89c2121b69144516edf939de6597314 Mon Sep 17 00:00:00 2001 From: vishal Date: Mon, 23 Mar 2020 16:19:50 -0400 Subject: [PATCH 2/3] Update cluster.go --- cli/cmd/cluster.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cli/cmd/cluster.go b/cli/cmd/cluster.go index c73fec5f9d..8d14036736 100644 --- a/cli/cmd/cluster.go +++ b/cli/cmd/cluster.go @@ -471,13 +471,14 @@ func CreateBucketIfNotFound(awsClient *aws.Client, bucket string) error { return err } if !bucketFound { - fmt.Println("creating a new s3 bucket: ", bucket) + fmt.Print("￮ creating a new s3 bucket: ", bucket) err = awsClient.CreateBucket(bucket) if err != nil { return err } + fmt.Println(" ✓") } else { - fmt.Println("using existing s3 bucket: ", bucket) + fmt.Println("￮ using existing s3 bucket:", bucket, "✓") } return nil } @@ -488,13 +489,14 @@ func CreateLogGroupIfNotFound(awsClient *aws.Client, logGroup string) error { return err } if !logGroupFound { - fmt.Println("creating a new cloudwatch log group: ", logGroup) + fmt.Print("￮ creating a new cloudwatch log group: ", logGroup) err = awsClient.CreateLogGroup(logGroup) if err != nil { return err } + fmt.Println(" ✓") } else { - fmt.Println("using existing cloudwatch log group: ", logGroup) + fmt.Println("￮ using existing cloudwatch log group:", logGroup, "✓") } return nil From b5253c097f444d001f47150438afac8ef28c669c Mon Sep 17 00:00:00 2001 From: vishal Date: Mon, 23 Mar 2020 19:12:14 -0400 Subject: [PATCH 3/3] Respond to PR comments --- cli/cmd/lib_cluster_config.go | 5 +++++ pkg/lib/aws/cloudwatch.go | 4 ++-- pkg/lib/aws/s3.go | 16 ++++++++++------ 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/cli/cmd/lib_cluster_config.go b/cli/cmd/lib_cluster_config.go index 5d9c226082..2a271587fc 100644 --- a/cli/cmd/lib_cluster_config.go +++ b/cli/cmd/lib_cluster_config.go @@ -199,6 +199,11 @@ func getClusterUpdateConfig(cachedClusterConfig clusterconfig.Config, awsCreds A } userClusterConfig.Bucket = cachedClusterConfig.Bucket + if userClusterConfig.LogGroup != "" && userClusterConfig.LogGroup != cachedClusterConfig.LogGroup { + return nil, clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.LogGroupKey, cachedClusterConfig.LogGroup) + } + userClusterConfig.LogGroup = cachedClusterConfig.LogGroup + if userClusterConfig.InstanceType != nil && *userClusterConfig.InstanceType != *cachedClusterConfig.InstanceType { return nil, clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.InstanceTypeKey, *cachedClusterConfig.InstanceType) } diff --git a/pkg/lib/aws/cloudwatch.go b/pkg/lib/aws/cloudwatch.go index fa8902b506..60e3de3c96 100644 --- a/pkg/lib/aws/cloudwatch.go +++ b/pkg/lib/aws/cloudwatch.go @@ -30,7 +30,7 @@ func (c *Client) DoesLogGroupExist(logGroup string) (bool, error) { if CheckErrCode(err, "ResourceNotFoundException") { return false, nil } - return false, errors.Wrap(err, "log group", logGroup) + return false, errors.Wrap(err, "log group "+logGroup) } return true, nil @@ -41,7 +41,7 @@ func (c *Client) CreateLogGroup(logGroup string) error { LogGroupName: aws.String(logGroup), }) if err != nil { - return errors.Wrap(err, "creating log group", logGroup) + return errors.Wrap(err, "creating log group "+logGroup) } return nil diff --git a/pkg/lib/aws/s3.go b/pkg/lib/aws/s3.go index 2e89213fbc..4e22a661e5 100644 --- a/pkg/lib/aws/s3.go +++ b/pkg/lib/aws/s3.go @@ -431,14 +431,18 @@ func (c *Client) DeletePrefix(bucket string, prefix string, continueIfFailure bo } func (c *Client) CreateBucket(bucket string) error { - _, err := c.S3().CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(bucket), - CreateBucketConfiguration: &s3.CreateBucketConfiguration{ + var bucketConfiguration *s3.CreateBucketConfiguration + if c.Region != "us-east-1" { + bucketConfiguration = &s3.CreateBucketConfiguration{ LocationConstraint: aws.String(c.Region), - }, + } + } + _, err := c.S3().CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucket), + CreateBucketConfiguration: bucketConfiguration, }) if err != nil { - return errors.Wrap(err, "creating bucket", bucket) + return errors.Wrap(err, "creating bucket "+bucket) } return nil } @@ -457,7 +461,7 @@ func (c *Client) DoesBucketExist(bucket string) (bool, error) { return false, ErrorBucketInaccessible(bucket) } } - return false, errors.Wrap(err, "bucket", bucket) + return false, errors.Wrap(err, "bucket "+bucket) } return true, nil