-
-
Notifications
You must be signed in to change notification settings - Fork 0
/
marathon-lb-autoscaler.py
123 lines (95 loc) · 5 KB
/
marathon-lb-autoscaler.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
import boto3
import datetime
import math
import requests
from parse_it import ParseIt
parser = ParseIt()
marathon_url = parser.read_configuration_variable("marathon_url")
marathon_port = parser.read_configuration_variable("marathon_port")
elb_name = parser.read_configuration_variable("elb_name")
min_num_of_lb = parser.read_configuration_variable("min_num_of_lb")
lb_per_x_connections = parser.read_configuration_variable("lb_per_x_connections")
spotinst_auth_token = parser.read_configuration_variable("spotinst_auth_token")
spotinst_account_id = parser.read_configuration_variable("spotinst_account_id")
elastigroup_id = parser.read_configuration_variable("elastigroup_id")
app_name = parser.read_configuration_variable("app_name", default_value="marathon-lb")
def get_elb_requests(aws_elb_name):
client = boto3.client(
'cloudwatch'
)
cloudwatch_metric_data = client.get_metric_statistics(
Period=300,
StartTime=datetime.datetime.utcnow() - datetime.timedelta(minutes=6),
EndTime=datetime.datetime.utcnow() - datetime.timedelta(minutes=1),
MetricName='RequestCount',
Namespace='AWS/ELB',
Statistics=['Sum'],
Dimensions=[{'Name': "LoadBalancerName", 'Value': aws_elb_name}],
Unit="Count"
)
return int(cloudwatch_metric_data["Datapoints"][0]["Sum"] / 5)
def change_marathon_lb_size(marathon_host_url, marathon_host_port, new_size, marathon_app_name):
url = marathon_host_url + ":" + str(marathon_host_port) + "/v2/apps/" + marathon_app_name + "/"
querystring = {"force": "true"}
payload = "{\"instances\": " + str(new_size) + "}"
headers = {
'content-type': "application/json",
'cache-control': "no-cache",
'postman-token': "7b955bc7-d7ee-4b02-3821-95b57f151bca"
}
response = requests.request("PUT", url, data=payload, headers=headers, params=querystring)
return response.status_code
def get_spotinst_instances(auth_token, elastigroup):
url = "https://api.spotinst.io/aws/ec2/group/" + elastigroup + "/instanceHealthiness?accountId=" \
+ spotinst_account_id
headers = {
'authorization': "Bearer " + auth_token,
'cache-control': "no-cache"
}
response = requests.request("GET", url, headers=headers)
response_json = response.json()
return int(response_json["response"]["count"])
def get_marathon_lb_tasks(marathon_host_url, marathon_host_port, marathon_app_name):
url = marathon_host_url + ":" + str(marathon_host_port) + "/v2/apps/" + marathon_app_name + "/tasks"
headers = {
'cache-control': "no-cache"
}
response = requests.request("GET", url, headers=headers)
response_json = response.json()
return int(len(response_json["tasks"]))
def set_spotinst_elastigroup_size(auth_token, elastigroup, instance_size):
url = "https://api.spotinst.io/aws/ec2/group/" + elastigroup + "?accountId=" + spotinst_account_id
payload = "{\"group\": { \"capacity\": { \"target\": " + str(instance_size) + ", \"minimum\": " \
+ str(instance_size) + ", \"maximum\":" + str(instance_size) + "}}}"
headers = {
'authorization': "Bearer " + auth_token,
'content-type': "application/json",
'cache-control': "no-cache"
}
response = requests.request("PUT", url, data=payload, headers=headers)
return response.status_code
requests_last_minute = get_elb_requests(elb_name)
print("there was an average of " + str(requests_last_minute) + " requests per minute over the last 5 minutes")
marathon_lb_needed = int(math.ceil(int(requests_last_minute) / int(lb_per_x_connections)))
if int(marathon_lb_needed) < int(min_num_of_lb):
marathon_lb_needed = int(min_num_of_lb)
print("there are " + str(marathon_lb_needed) + " marathon-lb instances needed")
current_spotinst_instances = get_spotinst_instances(spotinst_auth_token, elastigroup_id)
print("there are currently " + str(current_spotinst_instances) + " marathon-lb spot instances")
current_marathon_lb_tasks = get_marathon_lb_tasks(marathon_url, marathon_port, app_name)
print("there are currently " + str(current_marathon_lb_tasks) + " marathon-lb tasks")
if current_marathon_lb_tasks == current_spotinst_instances == marathon_lb_needed:
print("no changes needed - exiting")
exit(0)
else:
print("scaling spot instances & marathon-lb tasks to " + str(marathon_lb_needed))
spotinst_rescale_status_code = set_spotinst_elastigroup_size(spotinst_auth_token, elastigroup_id,
marathon_lb_needed)
marathon_rescale_status_code = change_marathon_lb_size(marathon_url, marathon_port, str(marathon_lb_needed),
app_name)
if marathon_rescale_status_code != spotinst_rescale_status_code != 200:
if marathon_rescale_status_code != 200:
print("critical - failed to scale marathon-lb tasks")
if spotinst_rescale_status_code != 200:
print("critical - failed to scale spotinst elastigroup")
exit(2)