(12.1.0) INFRASYS-6779 - disable the cloud-watch_alarms while rolling deployment.

License2Deploy/AWSConn.py

@@ -3,6 +3,7 @@
 import boto.ec2 as ec2
 import boto.ec2.autoscale as a
 import boto.ec2.elb as elb
+import boto.ec2.cloudwatch as cloudwatch
 import logging
 import yaml
 
@@ -32,6 +33,14 @@ def aws_conn_elb(region, profile='default'):
     except Exception as e:
       logging.error("Unable to connect to region, please investigate: {0}".format(e))
 
+  @staticmethod
+  def aws_conn_cloudwatch(region, profile='default'):
+    try: 
+      conn = cloudwatch.connect_to_region(region, profile_name=profile)
+      return conn
+    except Exception as e:
+      logging.error("Unable to connect to region, please investigate: {0}".format(e))
+
   @staticmethod
   def load_config(config):
     with open(config, 'r') as stream:

License2Deploy/rolling_deploy.py

@@ -23,6 +23,7 @@ def __init__(self, env=None, project=None, buildNum=None, ami_id=None, profile_n
     self.conn_ec2 = AWSConn.aws_conn_ec2(self.region, self.profile_name)
     self.conn_elb = AWSConn.aws_conn_elb(self.region, self.profile_name)
     self.conn_auto = AWSConn.aws_conn_auto(self.region, self.profile_name)
+    self.conn_cloudwatch = AWSConn.aws_conn_cloudwatch(self.region, self.profile_name)
     self.exit_error_code = 2
     self.load_balancer = self.get_lb()
 
@@ -222,18 +223,55 @@ def healthcheck_new_instances(self, group_name): # pragma: no cover
     self.wait_for_new_instances(new_instance_ids) #Wait for new instances to be up and ready
     self.lb_healthcheck(new_instance_ids) #Once instances are ready, healthcheck. If successful, decrease desired count.
 
+  def retrieve_project_cloudwatch_alarms(self):
+    """ Retrieve all the Cloud-Watch alarms for the given project and environment """
+    try:
+      all_cloud_watch_alarms = self.conn_cloudwatch.describe_alarms()
+    except Exception as e:
+      logging.error("Error while retrieving the list of cloud-watch alarms. Error: {0}".format(e))
+      exit(self.exit_error_code)
+    project_cloud_watch_alarms = filter(lambda alarm: self.project in alarm.name and self.env in alarm.name, all_cloud_watch_alarms)
+    if len(project_cloud_watch_alarms) == 0:
+       logging.info("No cloud-watch alarm found")
+    return project_cloud_watch_alarms
+
+  def disable_project_cloudwatch_alarms(self):
+    ''' Disable all the cloud watch alarms '''
+    project_cloud_watch_alarms = self.retrieve_project_cloudwatch_alarms()
+    for alarm in project_cloud_watch_alarms:
+      try:
+        self.conn_cloudwatch.disable_alarm_actions(alarm.name)
+        logging.info("Disabled cloud-watch alarm. {0}".format(alarm.name))
+      except Exception as e:
+        logging.error("Unable to disable the cloud-watch alarm, please investigate: {0}".format(e))
+        exit(self.exit_error_code)
+
+  def enable_project_cloudwatch_alarms(self):
+    ''' Enable all the cloud watch alarms '''
+    project_cloud_watch_alarms = self.retrieve_project_cloudwatch_alarms()
+    for alarm in project_cloud_watch_alarms:
+      logging.info("Found an alarm. {0}".format(alarm.name))
+      try:
+        self.conn_cloudwatch.enable_alarm_actions(alarm.name)
+        logging.info("Enabled cloud-watch alarm. {0}".format(alarm.name))
+      except Exception as e:
+        logging.error("Unable to enable the cloud-watch alarm, please investigate: {0}".format(e))
+        exit(self.exit_error_code)
+
   def deploy(self): # pragma: no cover
     ''' Rollin Rollin Rollin, Rawhide! '''
     group_name = self.get_autoscale_group_name()
     self.wait_ami_availability(self.ami_id)
     logging.info("Build #: {0} ::: Autoscale Group: {1}".format(self.buildNum, group_name))
+    self.disable_project_cloudwatch_alarms()
     self.set_autoscale_instance_desired_count(self.calculate_autoscale_desired_instance_count(group_name, 'increase'), group_name)
     logging.info("Sleeping for 240 seconds to allow for instances to spin up")
     sleep(240) #Need to wait until the instances come up in the load balancer
     self.healthcheck_new_instances(group_name)
     self.set_autoscale_instance_desired_count(self.calculate_autoscale_desired_instance_count(group_name, 'decrease'), group_name)
     self.confirm_lb_has_only_new_instances()
     self.tag_ami(self.ami_id, self.env)
+    self.enable_project_cloudwatch_alarms()
     logging.info("Deployment Complete!")
 
   def revert_deployment(self): #pragma: no cover

tests/rolling_deploy_test.py

@@ -5,9 +5,12 @@
 import os
 from boto.ec2.autoscale.launchconfig import LaunchConfiguration
 from boto.ec2.autoscale.group import AutoScalingGroup
+from boto.ec2.cloudwatch.alarm import MetricAlarm
+from boto.ec2.cloudwatch.dimension import Dimension
 from moto import mock_autoscaling
 from moto import mock_ec2
 from moto import mock_elb
+from moto.cloudwatch import mock_cloudwatch
 from License2Deploy.rolling_deploy import RollingDeploy
 from License2Deploy.AWSConn import AWSConn
 import sys
@@ -21,6 +24,7 @@ class RollingDeployTest(unittest.TestCase):
   GMS_LAUNCH_CONFIGURATION_PRD = 'server-backend-prd-servergmsextenderLCprd-46TIE5ZFQTLB'
   GMS_AUTOSCALING_GROUP_STG = 'server-backend-stg-servergmsextenderASGstg-3ELOD1FOTESTING'
   GMS_AUTOSCALING_GROUP_PRD = 'server-backend-prd-servergmsextenderASGprd-3ELOD1FOTESTING'
+
   @mock_autoscaling
   @mock_elb
   @mock_ec2
@@ -35,14 +39,15 @@ def get_autoscaling_configurations(self, launch_configuration_name, autoscaling_
     }
 
   @mock_autoscaling
-  def setUpAutoScaleGroup(self, configurations):
+  def setUpAutoScaleGroup(self, configurations, env="stg"):
     conn = boto.connect_autoscale()
     for configuration in configurations:
       config = LaunchConfiguration(
         name=configuration[self.launch_configuration_name],
         image_id='ami-abcd1234',
         instance_type='m1.medium',
       )
+      load_balancer_name = 'servergmsextenderELB{0}'.format(env)
       group = AutoScalingGroup(
         name=configuration[self.autoscaling_group_name],
         availability_zones=['us-east-1a'],
@@ -53,7 +58,7 @@ def setUpAutoScaleGroup(self, configurations):
         max_size=10,
         min_size=2,
         launch_config=config,
-        load_balancers=['servergmsextenderELBstg'],
+        load_balancers=[load_balancer_name],
         vpc_zone_identifier='subnet-1234abcd',
         termination_policies=["Default"],
       )
@@ -89,6 +94,76 @@ def setUpEC2(self):
 
     return [conn, instance_id_list]
 
+  @mock_cloudwatch
+  def setUpCloudWatch(self, instance_ids, env="stg"):
+    alarm = MetricAlarm(
+      name = "servergmsextender_CloudWatchAlarm" + env,
+      namespace = "AWS/EC2",
+      metric = "CPUUtilization",
+      comparison = ">=",
+      threshold = "90",
+      evaluation_periods = 1,
+      statistic = "Average",
+      period = 300,
+      dimensions = {'InstanceId': instance_ids},
+      alarm_actions=['arn:alarm'],
+      ok_actions=['arn:ok']
+    )
+    watch_conn = boto.connect_cloudwatch()
+    watch_conn.put_metric_alarm(alarm)
+
+  @mock_cloudwatch
+  def setUpCloudWatchWithWrongConfig(self, instance_ids, env="stg"):
+    alarm = MetricAlarm(
+      name = "servergmsextender_CloudWatchAlarm" + env,
+      namespace = "AWS/EC2",
+      metric = "CPUUtilization",
+      comparison = "GreaterThanThreshold", # wrong configuration that would generate error.
+      threshold = "90",
+      evaluation_periods = 1,
+      statistic = "Average",
+      period = 300,
+      dimensions = {'InstanceId': instance_ids},
+      alarm_actions=['arn:alarm'],
+      ok_actions=['arn:ok']
+    )
+    watch_conn = boto.connect_cloudwatch()
+    watch_conn.put_metric_alarm(alarm)
+
+  @mock_cloudwatch
+  def test_retrieve_project_cloudwatch_alarms(self):
+    instance_ids = self.setUpEC2()
+    self.setUpCloudWatch(instance_ids)
+    cloud_watch_alarms = self.rolling_deploy.retrieve_project_cloudwatch_alarms()
+    print cloud_watch_alarms
+    self.assertEqual(1, len(cloud_watch_alarms))
+
+  @mock_cloudwatch
+  def test_retrieve_project_cloudwatch_alarms_with_no_valid_alarms(self):
+    instance_ids = self.setUpEC2()
+    self.setUpCloudWatch(instance_ids)
+    self.rolling_deploy.env = "wrong_env_prd" # set a wrong environment 
+    cloud_watch_alarms = self.rolling_deploy.retrieve_project_cloudwatch_alarms()
+    self.assertEqual(0, len(cloud_watch_alarms))
+
+  @mock_cloudwatch
+  def test_retrieve_project_cloudwatch_alarms_with_wrong_config(self):
+    instance_ids = self.setUpEC2()
+    self.setUpCloudWatchWithWrongConfig(instance_ids)
+    self.assertRaises(SystemExit, lambda: self.rolling_deploy.retrieve_project_cloudwatch_alarms())
+
+  @mock_cloudwatch
+  def test_enable_project_cloudwatch_alarms_Error(self):
+    instance_ids = self.setUpEC2()
+    self.setUpCloudWatch(instance_ids)
+    self.assertRaises(SystemExit, lambda: self.rolling_deploy.enable_project_cloudwatch_alarms())
+
+  @mock_cloudwatch
+  def test_disable_project_cloudwatch_alarms_Error(self):
+    instance_ids = self.setUpEC2()
+    self.setUpCloudWatch(instance_ids)
+    self.assertRaises(SystemExit, lambda: self.rolling_deploy.disable_project_cloudwatch_alarms())
+
   @mock_ec2
   def test_tag_ami(self):
     conn = self.setUpEC2()[0]
@@ -185,9 +260,8 @@ def test_get_autoscale_group_name_prd(self):
     self.setUpELB(env='prd')
     self.rolling_deploy = RollingDeploy('prd', 'server-gms-extender', '0', 'ami-test212', None, './regions.yml')
     autoscaling_configurations = list()
-    autoscaling_configurations.append(self.get_autoscaling_configurations(self.GMS_LAUNCH_CONFIGURATION_STG, self.GMS_AUTOSCALING_GROUP_STG))
     autoscaling_configurations.append(self.get_autoscaling_configurations(self.GMS_LAUNCH_CONFIGURATION_PRD, self.GMS_AUTOSCALING_GROUP_PRD))
-    self.setUpAutoScaleGroup(autoscaling_configurations)
+    self.setUpAutoScaleGroup(autoscaling_configurations, env='prd')
     group = self.rolling_deploy.get_autoscale_group_name()
     self.assertEqual(group, self.GMS_AUTOSCALING_GROUP_PRD)
     self.assertNotEqual(group, self.GMS_AUTOSCALING_GROUP_STG)
@@ -215,6 +289,7 @@ def test_get_instance_ip_addrs(self):
   @mock_autoscaling
   @mock_elb
   def test_get_all_instance_ids(self):
+    self.setUpELB()
     self.setUpAutoScaleGroup([self.get_autoscaling_configurations(self.GMS_LAUNCH_CONFIGURATION_STG, self.GMS_AUTOSCALING_GROUP_STG)])
     conn = boto.connect_ec2()
     reservation = conn.run_instances('ami-1234abcd', min_count=2, private_ip_address="10.10.10.10")
@@ -273,5 +348,4 @@ def main():
     unittest.main()
 
 if __name__ == "__main__":
-    main()
-
+    main()