Automate the setup of DS Integ and E2E tests #186
Changes from 6 commits
e9addb1
e26a175
c9548ec
0359c7b
4704e9c
eeb0f70
dc7b659
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,17 +1,32 @@ | ||
# Setup | ||
- Have [etcd](https://github.com/coreos/etcd) running locally. | ||
- AWS Credentials available using default providers (environment variables, instance-profile) | ||
- AWS Profile appropriately configured (e.g. AWS_PROFILE=default) | ||
- AWS Region appropriately configured (e.g. AWS_REGION=us-west-2) | ||
- Start blox-daemon-scheduler (blox-daemon-scheduler --bind localhost:2000 --etcd-endpoint localhost:2379) | ||
- Create ECS cluster (e.g. name=q-daemon-scheduler-test) | ||
- Create AutoScaling group (e.g. name=ecs-daemon-scheduler-test-asg) which can launch instances into the above ECS cluster. Follow the steps [here](http://docs.aws.amazon.com/AmazonECS/latest/developerguide/launch_container_instance.html) | ||
- Start blox cluster state service: | ||
``` | ||
AWS_PROFILE=default AWS_REGION=us-west-2 CSS_LOG_FILE=var/output/logs/css.log ./out/cluster-state-service --queue event_stream --etcd-endpoint localhost:2379 --bind localhost:3000 | ||
``` | ||
- Start blox daemon scheduler: | ||
``` | ||
AWS_PROFILE=default AWS_REGION=us-west-2 DS_LOG_FILE=var/output/logs/ds.log DS_LOG_LEVEL=debug ./out/daemon-scheduler --bind localhost:2000 --etcd-endpoint localhost:2379 --css-endpoint localhost:3000 | ||
``` | ||
|
||
# Usage | ||
- In the package root directory (parent of internal) | ||
``` | ||
ECS_CLUSTER=... ECS_CLUSTER_ASG=... AWS_REGION=... AWS_PROFILE=... gucumber -tags=@e2e | ||
AWS_REGION=... AWS_PROFILE=... gucumber -tags=@e2e | ||
``` | ||
|
||
``` | ||
ECS_CLUSTER=... ECS_CLUSTER_ASG=... AWS_REGION=... AWS_PROFILE=... gucumber -tags=@integ | ||
AWS_REGION=... AWS_PROFILE=... gucumber -tags=@integ | ||
``` | ||
|
||
# Customization | ||
Users can pass in their own custom parameters to the tests through environment variables: | ||
* Cluster (custom name has to be different from default name: "DSTestCluster") | ||
* Autoscaling Group (custom name has to be different from default name: "DSClusterASG") | ||
* EC2 key pair | ||
|
||
``` | ||
ECS_CLUSTER=<custom-cluster> ECS_CLUSTER_ASG=<custom-autoscaling-group> EC2_KEY_PAIR=<ec2-key-pair> | ||
``` |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -29,6 +29,13 @@ import ( | |
. "github.com/gucumber/gucumber" | ||
"github.com/pkg/errors" | ||
"github.com/stretchr/testify/assert" | ||
"github.com/aws/aws-sdk-go/aws/awserr" | ||
) | ||
|
||
var ( | ||
roleName = "DSTestRole" | ||
instanceProfileName = "DSTestInstance" | ||
policyARN = "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role" | ||
) | ||
|
||
const ( | ||
|
@@ -38,11 +45,18 @@ const ( | |
invalidCluster = "cluster/cluster" | ||
badRequestHTTPResponse = "400 Bad Request" | ||
listEnvironmentsBadRequest = "ListEnvironmentsBadRequest" | ||
|
||
errorCode = 1 | ||
launchConfigurationName = "DSASGLaunchConfiguration" | ||
defaultASGClusterName = "DSClusterASG" | ||
defaultECSClusterName = "DSTestCluster" | ||
) | ||
|
||
func init() { | ||
asgWrapper := wrappers.NewAutoScalingWrapper() | ||
ecsWrapper := wrappers.NewECSWrapper() | ||
ec2Wrapper := wrappers.NewEC2Wrapper() | ||
iamWrapper := wrappers.NewIAMWrapper() | ||
edsWrapper := wrappers.NewEDSWrapper() | ||
ctx := context.Background() | ||
|
||
|
@@ -52,6 +66,120 @@ func init() { | |
return | ||
} | ||
|
||
// TODO: Change these os.Exit calls to T.Errorf. Currently unable to do so because T is not initialized until the first test. | ||
// (https://github.com/gucumber/gucumber/issues/28) | ||
BeforeAll(func() { | ||
clusterName := wrappers.GetClusterName() | ||
|
||
_, err := ecsWrapper.CreateCluster(clusterName) | ||
if err != nil { | ||
fmt.Println(err.Error()) | ||
os.Exit(errorCode) | ||
} | ||
|
||
err = terminateAllContainerInstances(ec2Wrapper, ecsWrapper, clusterName) | ||
if err != nil { | ||
fmt.Println(err.Error()) | ||
os.Exit(errorCode) | ||
} | ||
|
||
azs, err := ec2Wrapper.DescribeAvailabilityZones() | ||
if err != nil { | ||
fmt.Println(err.Error()) | ||
os.Exit(errorCode) | ||
} | ||
|
||
asg := wrappers.GetASGName() | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can you change this to |
||
if asg == defaultASGClusterName { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. What is this for? What if the asg name is different? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If users wanna pass in their custom ASG, the test will pick up that ASG and not create a new ASG itself. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should we validate the user-given ASG? |
||
keyPair := wrappers.GetKeyPairName() | ||
|
||
err = createInstanceProfile(iamWrapper) | ||
if err != nil { | ||
fmt.Println(err.Error()) | ||
os.Exit(errorCode) | ||
} | ||
|
||
amiID, err := wrappers.GetLatestECSOptimizedAMIID() | ||
if err != nil { | ||
fmt.Println(err.Error()) | ||
os.Exit(errorCode) | ||
} | ||
|
||
err = asgWrapper.CreateLaunchConfiguration(launchConfigurationName, clusterName, instanceProfileName, keyPair, amiID) | ||
if err != nil { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. What if it's not an |
||
if awsErr, ok := errors.Cause(err).(awserr.Error); !ok || awsErr.Code() != "AlreadyExists" { | ||
fmt.Println(err.Error()) | ||
os.Exit(errorCode) | ||
} | ||
} | ||
|
||
err = asgWrapper.CreateAutoScalingGroup(asg, launchConfigurationName, azs) | ||
if err != nil { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Same comment as above. |
||
if awsErr, ok := errors.Cause(err).(awserr.Error); !ok || awsErr.Code() != "AlreadyExists" { | ||
fmt.Println(err.Error()) | ||
os.Exit(errorCode) | ||
} else { | ||
asgStatus, err := asgWrapper.GetAutoScalingGroupStatus(asg) | ||
if err != nil { | ||
fmt.Println(err.Error()) | ||
os.Exit(errorCode) | ||
} | ||
if asgStatus == "Delete in progress" { | ||
fmt.Println("The current Autoscaling Group is in progress of deleting. Wait for the deletion to complete and restart the test.") | ||
os.Exit(errorCode) | ||
} | ||
} | ||
} | ||
} | ||
}) | ||
|
||
AfterAll(func() { | ||
clusterName := wrappers.GetClusterName() | ||
|
||
err := stopAllTasks(ecsWrapper, clusterName) | ||
if err != nil { | ||
T.Errorf(err.Error()) | ||
return | ||
} | ||
|
||
if wrappers.GetASGName() == defaultASGClusterName { | ||
forceDelete := true | ||
// With ForceDelete set to true, this will delete all instances attached to the Autoscaling group | ||
asgWrapper.DeleteAutoScalingGroup(asg, forceDelete) | ||
if err != nil { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. What err are you checking here? |
||
T.Errorf(err.Error()) | ||
return | ||
} | ||
|
||
asgWrapper.DeleteLaunchConfiguration(launchConfigurationName) | ||
if err != nil { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. What err are you checking here? |
||
T.Errorf(err.Error()) | ||
return | ||
} | ||
} else { | ||
err = terminateAllContainerInstances(ec2Wrapper, ecsWrapper, clusterName) | ||
if err != nil { | ||
T.Errorf(err.Error()) | ||
return | ||
} | ||
} | ||
|
||
if wrappers.GetClusterName() == defaultECSClusterName { | ||
_, err = ecsWrapper.DeleteCluster(clusterName) | ||
if err != nil { | ||
T.Errorf(err.Error()) | ||
return | ||
} | ||
} | ||
|
||
err = deleteInstanceProfile(iamWrapper) | ||
if err != nil { | ||
T.Errorf(err.Error()) | ||
return | ||
} | ||
|
||
}) | ||
|
||
When(`^I make a Ping call$`, func() { | ||
err = edsWrapper.Ping() | ||
}) | ||
|
@@ -595,6 +723,133 @@ func init() { | |
}) | ||
} | ||
|
||
func terminateAllContainerInstances(ec2Wrapper wrappers.EC2Wrapper, ecsWrapper wrappers.ECSWrapper, clusterName string) error { | ||
instanceARNs, err := ecsWrapper.ListContainerInstances(clusterName) | ||
if err != nil { | ||
return errors.Wrapf(err, "Failed to list container instances from cluster '%v'.", clusterName) | ||
} | ||
|
||
if len(instanceARNs) == 0 { | ||
return nil | ||
} | ||
|
||
err = ecsWrapper.DeregisterContainerInstances(&clusterName, instanceARNs) | ||
if err != nil { | ||
return errors.Wrapf(err, "Failed to deregister container instances '%v'.", instanceARNs) | ||
} | ||
|
||
ec2InstanceIDs := make([]*string, 0, len(instanceARNs)) | ||
for _, v := range instanceARNs { | ||
containerInstance, err := ecsWrapper.DescribeContainerInstance(clusterName, *v) | ||
if err != nil { | ||
return errors.Wrapf(err, "Failed to describe container instance '%v'.", v) | ||
} | ||
ec2InstanceIDs = append(ec2InstanceIDs, containerInstance.Ec2InstanceId) | ||
} | ||
|
||
err = ec2Wrapper.TerminateInstances(ec2InstanceIDs) | ||
if err != nil { | ||
return errors.Wrapf(err, "Failed to terminate container instances '%v'.", ec2InstanceIDs) | ||
} | ||
|
||
return nil | ||
} | ||
|
||
func stopAllTasks(ecsWrapper wrappers.ECSWrapper, clusterName string) error { | ||
taskARNList, err := ecsWrapper.ListTasks(clusterName, nil) | ||
if err != nil { | ||
return err | ||
} | ||
for _, t := range taskARNList { | ||
err = ecsWrapper.StopTask(clusterName, *t) | ||
if err != nil { | ||
return err | ||
} | ||
} | ||
return nil | ||
} | ||
|
||
func createInstanceProfile(iamWrapper wrappers.IAMWrapper) error { | ||
assumeRolePolicy := `{ | ||
"Version": "2012-10-17", | ||
"Statement": [ | ||
{ | ||
"Effect": "Allow", | ||
"Principal": { | ||
"Service": "ec2.amazonaws.com" | ||
}, | ||
"Action": "sts:AssumeRole" | ||
} | ||
] | ||
}` | ||
|
||
err := iamWrapper.GetRole(&roleName) | ||
if err != nil { | ||
if awsErr, ok := errors.Cause(err).(awserr.Error); ok && awsErr.Code() == "NoSuchEntity" { | ||
err = iamWrapper.CreateRole(&roleName, &assumeRolePolicy) | ||
if err != nil { | ||
return err | ||
} | ||
} else { | ||
return err | ||
} | ||
} | ||
|
||
err = iamWrapper.GetInstanceProfile(&instanceProfileName) | ||
if err != nil { | ||
if awsErr, ok := errors.Cause(err).(awserr.Error); ok && awsErr.Code() == "NoSuchEntity" { | ||
err = iamWrapper.CreateInstanceProfile(&instanceProfileName) | ||
if err != nil { | ||
return err | ||
} | ||
} else { | ||
return err | ||
} | ||
} | ||
|
||
err = iamWrapper.AttachRolePolicy(&policyARN, &roleName) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
err = iamWrapper.AddRoleToInstanceProfile(&roleName, &instanceProfileName) | ||
if err != nil { | ||
if awsErr, ok := errors.Cause(err).(awserr.Error); ok { | ||
if awsErr.Code() != "EntityAlreadyExists" && awsErr.Code() != "LimitExceeded" { | ||
return err | ||
} | ||
} else { | ||
return err | ||
} | ||
} | ||
|
||
return nil | ||
} | ||
|
||
func deleteInstanceProfile(iamWrapper wrappers.IAMWrapper) error { | ||
err := iamWrapper.DetachRolePolicy(&policyARN, &roleName) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
err = iamWrapper.RemoveRoleFromInstanceProfile(&roleName, &instanceProfileName) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
err = iamWrapper.DeleteRole(&roleName) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
err = iamWrapper.DeleteInstanceProfile(&instanceProfileName) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
return nil | ||
} | ||
|
||
func deleteEnvironment(environment string, edsWrapper wrappers.EDSWrapper) { | ||
err := edsWrapper.DeleteEnvironment(&environment) | ||
if err != nil { | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Can you wrap these two commands into ``` blocks similar to the gucumber ones down below?