Skip to content

Commit

Permalink
emr demo and allocation strategies demo
Browse files Browse the repository at this point in the history
  • Loading branch information
@schmutze
schmutze committed Aug 28, 2019
1 parent 31515d8 commit e8cc485
Show file tree
Hide file tree
Showing 9 changed files with 495 additions and 0 deletions.
18 changes: 18 additions & 0 deletions ec2-spot-allocation-strategies/README.txt
View file
@@ -0,0 +1,18 @@
README.txt

aws ec2 describe-images --owners amazon --filters 'Name=name,Values=amzn2-ami-hvm-2.0.????????-x86_64-gp2' 'Name=state,Values=available' --output json | jq -r '.Images | sort_by(.CreationDate) | last(.[]).ImageId'

aws ec2 create-launch-template --cli-input-json file://ec2-launch-template.json

aws autoscaling create-auto-scaling-group --cli-input-json file://asg-lowest-price-n-1.json

aws autoscaling create-auto-scaling-group --cli-input-json file://asg-capacity-optimized.json



while read i a p m; do s=$(aws ec2 describe-spot-price-history --instance-types $i --availability-zone $a --product-descriptions "Linux/UNIX (Amazon VPC)" --start-time $(date +%FT%TZ) --end-time $(date +%FT%TZ) --output text); echo -e $s'\t'$m | awk -F'\t' '{print $2, $3, $7, $5, ($7-$5)/$7*100}'; done < <(for sir in $(aws ec2 describe-instances --filters "Name=tag:aws:autoscaling:groupName,Values=lowest-price-n-1" "Name=instance-state-name,Values=running" "Name=instance-lifecycle,Values=spot" --query 'Reservations[*].Instances[*].[SpotInstanceRequestId]' --output text); do aws ec2 describe-spot-instance-requests --spot-instance-request-ids $sir --query 'SpotInstanceRequests[*].[LaunchSpecification.InstanceType, LaunchedAvailabilityZone, ProductDescription, SpotPrice]' --output text; done)


while read i a p m; do s=$(aws ec2 describe-spot-price-history --instance-types $i --availability-zone $a --product-descriptions "Linux/UNIX (Amazon VPC)" --start-time $(date +%FT%TZ) --end-time $(date +%FT%TZ) --output text); echo -e $s'\t'$m | awk -F'\t' '{print $2, $3, $7, $5, ($7-$5)/$7*100}'; done < <(for sir in $(aws ec2 describe-instances --filters "Name=tag:aws:autoscaling:groupName,Values=capacity-optimized" "Name=instance-state-name,Values=running" "Name=instance-lifecycle,Values=spot" --query 'Reservations[*].Instances[*].[SpotInstanceRequestId]' --output text); do aws ec2 describe-spot-instance-requests --spot-instance-request-ids $sir --query 'SpotInstanceRequests[*].[LaunchSpecification.InstanceType, LaunchedAvailabilityZone, ProductDescription, SpotPrice]' --output text; done)


51 changes: 51 additions & 0 deletions ec2-spot-allocation-strategies/asg-capacity-optimized.json
View file
@@ -0,0 +1,51 @@
{
"AutoScalingGroupName": "capacity-optimized",
"MixedInstancesPolicy": {
"LaunchTemplate": {
"LaunchTemplateSpecification": {
"LaunchTemplateName": "amazon-linux-2",
"Version": "$Latest"
},
"Overrides": [
{
"InstanceType": "c3.large"
},
{
"InstanceType": "c4.large"
},
{
"InstanceType": "c5.large"
},
{
"InstanceType": "t3a.large"
},
{
"InstanceType": "t3.large"
},
{
"InstanceType": "m5a.large"
},
{
"InstanceType": "m5.large"
},
{
"InstanceType": "r5a.large"
},
{
"InstanceType": "r5.large"
}
]
},
"InstancesDistribution": {
"OnDemandAllocationStrategy": "prioritized",
"OnDemandBaseCapacity": 0,
"OnDemandPercentageAboveBaseCapacity": 0,
"SpotAllocationStrategy": "capacity-optimized"
}
},
"MinSize": 10,
"MaxSize": 40,
"DesiredCapacity": 20,
"HealthCheckType": "EC2",
"VPCZoneIdentifier": "subnet-0fc4d2543f3dcc255,subnet-09b9d0dbca4bae8a5,subnet-0437dfe009676c434"
}
52 changes: 52 additions & 0 deletions ec2-spot-allocation-strategies/asg-lowest-price-n-1.json
View file
@@ -0,0 +1,52 @@
{
"AutoScalingGroupName": "lowest-price-n-1",
"MixedInstancesPolicy": {
"LaunchTemplate": {
"LaunchTemplateSpecification": {
"LaunchTemplateName": "amazon-linux-2",
"Version": "$Latest"
},
"Overrides": [
{
"InstanceType": "c3.large"
},
{
"InstanceType": "c4.large"
},
{
"InstanceType": "c5.large"
},
{
"InstanceType": "t3a.large"
},
{
"InstanceType": "t3.large"
},
{
"InstanceType": "m5a.large"
},
{
"InstanceType": "m5.large"
},
{
"InstanceType": "r5a.large"
},
{
"InstanceType": "r5.large"
}
]
},
"InstancesDistribution": {
"OnDemandAllocationStrategy": "prioritized",
"OnDemandBaseCapacity": 0,
"OnDemandPercentageAboveBaseCapacity": 0,
"SpotAllocationStrategy": "lowest-price",
"SpotInstancePools": 1
}
},
"MinSize": 10,
"MaxSize": 40,
"DesiredCapacity": 20,
"HealthCheckType": "EC2",
"VPCZoneIdentifier": "subnet-0fc4d2543f3dcc255,subnet-09b9d0dbca4bae8a5,subnet-0437dfe009676c434"
}
130 changes: 130 additions & 0 deletions ec2-spot-allocation-strategies/ec2-fleet-capacity-optimized.json
View file
@@ -0,0 +1,130 @@
{
"SpotOptions": {
"AllocationStrategy": "capacity-optimized",
"MinTargetCapacity": 10,
"InstanceInterruptionBehavior": "terminate"
},
"LaunchTemplateConfigs": [
{
"LaunchTemplateSpecification": {
"LaunchTemplateName": "amazon-linux-2",
"Version": "1"
},
"Overrides": [
{
"InstanceType": "c3.large",
"SubnetId": "subnet-0fc4d2543f3dcc255"
},
{
"InstanceType": "c3.large",
"SubnetId": "subnet-09b9d0dbca4bae8a5"
},
{
"InstanceType": "c3.large",
"SubnetId": "subnet-0437dfe009676c434"
},
{
"InstanceType": "c4.large",
"SubnetId": "subnet-0fc4d2543f3dcc255"
},
{
"InstanceType": "c4.large",
"SubnetId": "subnet-09b9d0dbca4bae8a5"
},
{
"InstanceType": "c4.large",
"SubnetId": "subnet-0437dfe009676c434"
},
{
"InstanceType": "c5.large",
"SubnetId": "subnet-0fc4d2543f3dcc255"
},
{
"InstanceType": "c5.large",
"SubnetId": "subnet-09b9d0dbca4bae8a5"
},
{
"InstanceType": "c5.large",
"SubnetId": "subnet-0437dfe009676c434"
},
{
"InstanceType": "t3a.large",
"SubnetId": "subnet-0fc4d2543f3dcc255"
},
{
"InstanceType": "t3a.large",
"SubnetId": "subnet-09b9d0dbca4bae8a5"
},
{
"InstanceType": "t3a.large",
"SubnetId": "subnet-0437dfe009676c434"
},
{
"InstanceType": "t3.large",
"SubnetId": "subnet-0fc4d2543f3dcc255"
},
{
"InstanceType": "t3.large",
"SubnetId": "subnet-09b9d0dbca4bae8a5"
},
{
"InstanceType": "t3.large",
"SubnetId": "subnet-0437dfe009676c434"
},
{
"InstanceType": "m5a.large",
"SubnetId": "subnet-0fc4d2543f3dcc255"
},
{
"InstanceType": "m5a.large",
"SubnetId": "subnet-09b9d0dbca4bae8a5"
},
{
"InstanceType": "m5a.large",
"SubnetId": "subnet-0437dfe009676c434"
},
{
"InstanceType": "m5.large",
"SubnetId": "subnet-0fc4d2543f3dcc255"
},
{
"InstanceType": "m5.large",
"SubnetId": "subnet-09b9d0dbca4bae8a5"
},
{
"InstanceType": "m5.large",
"SubnetId": "subnet-0437dfe009676c434"
},
{
"InstanceType": "r5a.large",
"SubnetId": "subnet-0fc4d2543f3dcc255"
},
{
"InstanceType": "r5a.large",
"SubnetId": "subnet-09b9d0dbca4bae8a5"
},
{
"InstanceType": "r5a.large",
"SubnetId": "subnet-0437dfe009676c434"
},
{
"InstanceType": "r5.large",
"SubnetId": "subnet-0fc4d2543f3dcc255"
},
{
"InstanceType": "r5.large",
"SubnetId": "subnet-09b9d0dbca4bae8a5"
},
{
"InstanceType": "r5.large",
"SubnetId": "subnet-0437dfe009676c434"
}
]
}
],
"TargetCapacitySpecification": {
"TotalTargetCapacity": 1,
"DefaultTargetCapacityType": "spot"
},
"Type": "maintain"
}
131 changes: 131 additions & 0 deletions ec2-spot-allocation-strategies/ec2-fleet-lowest-price-n-1.json
View file
@@ -0,0 +1,131 @@
{
"SpotOptions": {
"AllocationStrategy": "lowestPrice",
"InstancePoolsToUseCount": 1,
"MinTargetCapacity": 10,
"InstanceInterruptionBehavior": "terminate"
},
"LaunchTemplateConfigs": [
{
"LaunchTemplateSpecification": {
"LaunchTemplateName": "amazon-linux-2",
"Version": "1"
},
"Overrides": [
{
"InstanceType": "c3.large",
"SubnetId": "subnet-0fc4d2543f3dcc255"
},
{
"InstanceType": "c3.large",
"SubnetId": "subnet-09b9d0dbca4bae8a5"
},
{
"InstanceType": "c3.large",
"SubnetId": "subnet-0437dfe009676c434"
},
{
"InstanceType": "c4.large",
"SubnetId": "subnet-0fc4d2543f3dcc255"
},
{
"InstanceType": "c4.large",
"SubnetId": "subnet-09b9d0dbca4bae8a5"
},
{
"InstanceType": "c4.large",
"SubnetId": "subnet-0437dfe009676c434"
},
{
"InstanceType": "c5.large",
"SubnetId": "subnet-0fc4d2543f3dcc255"
},
{
"InstanceType": "c5.large",
"SubnetId": "subnet-09b9d0dbca4bae8a5"
},
{
"InstanceType": "c5.large",
"SubnetId": "subnet-0437dfe009676c434"
},
{
"InstanceType": "t3a.large",
"SubnetId": "subnet-0fc4d2543f3dcc255"
},
{
"InstanceType": "t3a.large",
"SubnetId": "subnet-09b9d0dbca4bae8a5"
},
{
"InstanceType": "t3a.large",
"SubnetId": "subnet-0437dfe009676c434"
},
{
"InstanceType": "t3.large",
"SubnetId": "subnet-0fc4d2543f3dcc255"
},
{
"InstanceType": "t3.large",
"SubnetId": "subnet-09b9d0dbca4bae8a5"
},
{
"InstanceType": "t3.large",
"SubnetId": "subnet-0437dfe009676c434"
},
{
"InstanceType": "m5a.large",
"SubnetId": "subnet-0fc4d2543f3dcc255"
},
{
"InstanceType": "m5a.large",
"SubnetId": "subnet-09b9d0dbca4bae8a5"
},
{
"InstanceType": "m5a.large",
"SubnetId": "subnet-0437dfe009676c434"
},
{
"InstanceType": "m5.large",
"SubnetId": "subnet-0fc4d2543f3dcc255"
},
{
"InstanceType": "m5.large",
"SubnetId": "subnet-09b9d0dbca4bae8a5"
},
{
"InstanceType": "m5.large",
"SubnetId": "subnet-0437dfe009676c434"
},
{
"InstanceType": "r5a.large",
"SubnetId": "subnet-0fc4d2543f3dcc255"
},
{
"InstanceType": "r5a.large",
"SubnetId": "subnet-09b9d0dbca4bae8a5"
},
{
"InstanceType": "r5a.large",
"SubnetId": "subnet-0437dfe009676c434"
},
{
"InstanceType": "r5.large",
"SubnetId": "subnet-0fc4d2543f3dcc255"
},
{
"InstanceType": "r5.large",
"SubnetId": "subnet-09b9d0dbca4bae8a5"
},
{
"InstanceType": "r5.large",
"SubnetId": "subnet-0437dfe009676c434"
}
]
}
],
"TargetCapacitySpecification": {
"TotalTargetCapacity": 20,
"DefaultTargetCapacityType": "spot"
},
"Type": "maintain"
}

0 comments on commit e8cc485

Please sign in to comment.