Merge pull request #250 from cloudify-cosmo/1.5.1.2

1.5.1.2

CHANGELOG.txt

@@ -44,3 +44,9 @@
     - Add connect Security Groups to Instance via InstanceConnectedToSecurityGroup relationship
     - Support adding VPC Peering Connection + Use external routes
     - Fix bug that disassociate the wrong elastic ip from its instance
+1.5.1
+    - Support Windows instances with init_script agent installation method.
+1.5.1.1
+    - Fix bug when init_script is empty string.
+1.5.1.2
+    - Execute user-provided user data before agent install user data.

cloudify_aws/ec2/instance.py

@@ -14,6 +14,7 @@
 #    * limitations under the License.
 
 import os
+import json
 
 # Third-party Imports
 from boto import exception
@@ -30,6 +31,9 @@
 from cloudify_aws import utils, constants
 from cloudify.exceptions import NonRecoverableError
 
+PS_OPEN = '<powershell>'
+PS_CLOSE = '</powershell>'
+
 
 @operation
 def creation_validation(**_):
@@ -389,21 +393,95 @@ def _get_instance_attribute(self, attribute):
         attribute = getattr(instance_object, attribute)
         return attribute
 
+    def extract_powershell_content(self, string_with_powershell):
+        """We want to filter user data for powershell scripts.
+        However, AWS EC2 allows only one segment that is Powershell.
+        So we have to concat separate Powershell scripts into one.
+        First we separate all Powershell scripts without their tags.
+        Later we will add the tags back.
+        """
+
+        split_string = string_with_powershell.splitlines()
+
+        if not split_string:
+            return ''
+
+        if split_string[0] == '#ps1_sysnative' or \
+                split_string[0] == '#ps1_x86':
+            split_string.pop(0)
+
+        if PS_OPEN not in split_string:
+            script_start = -1  # Because we join at +1.
+        else:
+            script_start = split_string.index(PS_OPEN)
+
+        if PS_CLOSE not in split_string:
+            script_end = len(split_string)
+        else:
+            script_end = split_string.index(PS_CLOSE)
+
+        # Return everything between Powershell back as a string.
+        return '\n'.join(split_string[script_start+1:script_end])
+
     def _handle_userdata(self, parameters):
 
         existing_userdata = parameters.get('user_data')
+
+        if existing_userdata is None:
+            existing_userdata = ''
+        elif isinstance(existing_userdata, dict) or \
+                isinstance(existing_userdata, list):
+            existing_userdata = json.dumps(existing_userdata)
+        elif not isinstance(existing_userdata, basestring):
+            existing_userdata = str(existing_userdata)
+
         install_agent_userdata = ctx.agent.init_script()
+        os_family = ctx.node.properties['os_family']
 
         if not (existing_userdata or install_agent_userdata):
             return parameters
 
-        if not existing_userdata:
+        # AWS EC2 Windows instances require no more than one
+        # Powershell script, which must be surrounded by
+        # Powershell tags.
+        if install_agent_userdata and os_family == 'windows':
+
+            # Get the powershell content from install_agent_userdata
+            install_agent_userdata = \
+                self.extract_powershell_content(install_agent_userdata)
+
+            # Get the powershell content from existing_userdata
+            # (If it exists.)
+            existing_userdata_powershell = \
+                self.extract_powershell_content(existing_userdata)
+
+            # Combine the powershell content from two sources.
+            install_agent_userdata = \
+                '#ps1_sysnative\n{0}\n{1}\n{2}\n{3}\n'.format(
+                    PS_OPEN,
+                    existing_userdata_powershell,
+                    install_agent_userdata,
+                    PS_CLOSE)
+
+            # Additional work on the existing_userdata.
+            # Remove duplicate Powershell content.
+            # Get rid of unnecessary newlines.
+            existing_userdata = \
+                existing_userdata.replace(
+                    existing_userdata_powershell,
+                    '').replace(
+                        PS_OPEN,
+                        '').replace(
+                            PS_CLOSE,
+                            '').strip()
+
+        if not existing_userdata or existing_userdata.isspace():
             final_userdata = install_agent_userdata
         elif not install_agent_userdata:
             final_userdata = existing_userdata
         else:
             final_userdata = compute.create_multi_mimetype_userdata(
-                    [existing_userdata, install_agent_userdata])
+                [existing_userdata, install_agent_userdata])
 
         parameters['user_data'] = final_userdata
 
@@ -445,8 +523,8 @@ def _get_instance_parameters(self, args=None):
         })
 
         parameters.update(ctx.node.properties['parameters'])
-        parameters = self._handle_userdata(parameters)
         parameters = utils.update_args(parameters, args)
+        parameters = self._handle_userdata(parameters)
         parameters['block_device_map'] = \
             self._create_block_device_mapping(
                 parameters.get('block_device_map', {})

cloudify_aws/ec2/tests/test_ec2_elb.py

@@ -61,10 +61,10 @@ def _get_elbs(self):
     def _get_elb_instances(self):
         instance_list = boto.connect_elb().get_all_load_balancers(
                 load_balancer_names=['myelb'])[0].instances
-        l = []
+        my_list = []
         for i in instance_list:
-            l.append(i.id)
-        return l
+            my_list.append(i.id)
+        return my_list
 
     def _create_external_instance(self):
         return boto.connect_ec2().run_instances(

cloudify_aws/ec2/tests/test_ec2_instance.py

@@ -60,6 +60,7 @@ def mock_ctx(self, test_name, retry_number=0, operation_name='create'):
             'tags': {},
             'image_id': TEST_AMI_IMAGE_ID,
             'instance_type': TEST_INSTANCE_TYPE,
+            'os_family': 'linux',
             'cloudify_agent': {},
             'agent_config': {},
             'use_password': False,
@@ -259,7 +260,6 @@ def test_with_existing_userdata_clean(self):
         ctx.agent.init_script = lambda: 'EXISTING'
         current_ctx.set(ctx=ctx)
         test_instance = self.create_instance_for_checking()
-
         handle_userdata_output = \
             test_instance._handle_userdata(ctx.node.properties['parameters'])
         expected_userdata = 'EXISTING'
@@ -282,6 +282,32 @@ def test_with_both_userdata_clean(self):
         self.assertTrue(handle_userdata_output['user_data'].startswith(
                 'Content-Type: multi'))
 
+    @mock_ec2
+    def test_with_both_userdata_clean_windows(self):
+        """ this tests that handle user data returns the expected output when merging
+        """
+
+        ctx = self.mock_ctx('test_with_both_userdata_clean_windows')
+        ctx.agent.init_script = lambda: '#ps1_sysnative\nSCRIPT'
+        ctx.node.properties['os_family'] = 'windows'
+        ctx.node.properties['agent_config']['install_method'] = 'init_script'
+        ctx.node.properties['parameters']['user_data'] = \
+            '<powershell>\nfunction Existing{}\n'\
+            '</powershell>\nrem cmd\n'
+        current_ctx.set(ctx=ctx)
+        test_instance = self.create_instance_for_checking()
+        handle_userdata_output = \
+            test_instance._handle_userdata(ctx.node.properties['parameters'])
+        self.assertTrue(handle_userdata_output['user_data'].startswith(
+                'Content-Type: multi'))
+        self.assertIn(
+            '#ps1_sysnative\n<powershell>\nfunction Existing{}\n'
+            'SCRIPT\n</powershell>',
+            handle_userdata_output['user_data'])
+        self.assertIn(
+            'rem cmd',
+            handle_userdata_output['user_data'])
+
     @mock_ec2
     def test_without_userdata_clean(self):
         """ this tests that handle user data returns the expected output

cloudify_aws/vpc/tests/blueprint/plugin.yaml

@@ -5,9 +5,9 @@
 plugins:
   aws:
     executor: central_deployment_agent
-    source: https://github.com/cloudify-cosmo/cloudify-aws-plugin/archive/1.5.zip
+    source: https://github.com/cloudify-cosmo/cloudify-aws-plugin/archive/1.5.1.1.zip
     package_name: cloudify-aws-plugin
-    package_version: '1.5'
+    package_version: '1.5.1.1'
 
 data_types:
 

dev-requirements.txt

@@ -1,3 +1,3 @@
-https://github.com/cloudify-cosmo/cloudify-dsl-parser/archive/master.zip
-https://github.com/cloudify-cosmo/cloudify-rest-client/archive/master.zip
-https://github.com/cloudify-cosmo/cloudify-plugins-common/archive/master.zip
+https://github.com/cloudify-cosmo/cloudify-dsl-parser/archive/3.4.2.zip
+https://github.com/cloudify-cosmo/cloudify-rest-client/archive/3.4.2.zip
+https://github.com/cloudify-cosmo/cloudify-plugins-common/archive/3.4.2.zip

plugin.yaml

@@ -5,9 +5,9 @@
 plugins:
   aws:
     executor: central_deployment_agent
-    source: https://github.com/cloudify-cosmo/cloudify-aws-plugin/archive/1.5.zip
+    source: https://github.com/cloudify-cosmo/cloudify-aws-plugin/archive/1.5.1.2.zip
     package_name: cloudify-aws-plugin
-    package_version: '1.5'
+    package_version: '1.5.1.2'
 
 data_types:
 

setup.py

@@ -25,7 +25,7 @@
     author='Gigaspaces',
     author_email='cosmo-admin@gigaspaces.com',
 
-    version='1.5',
+    version='1.5.1.2',
     description='Cloudify plugin for AWS infrastructure.',
 
     # This must correspond to the actual packages in the plugin.
@@ -37,8 +37,8 @@
 
     license='LICENSE',
     install_requires=[
-        'cloudify-plugins-common>=3.3.1',
-        'boto==2.38.0',
+        'cloudify-plugins-common>=3.4.2',
+        'boto==2.48.0',
         'pycrypto==2.6.1',
         'ipaddress==1.0.18'
     ]

system_tests/manager/resources/relationships-blueprint.yaml

@@ -4,7 +4,7 @@ tosca_definitions_version: cloudify_dsl_1_3
 
 imports:
   - http://www.getcloudify.org/spec/cloudify/4.0.1/types.yaml
-  - https://raw.githubusercontent.com/cloudify-cosmo/cloudify-aws-plugin/1.5/plugin.yaml
+  - https://raw.githubusercontent.com/cloudify-cosmo/cloudify-aws-plugin/1.5.1.2/plugin.yaml
 
 inputs:
 

system_tests/manager/resources/sg-blueprint.yaml

@@ -4,7 +4,7 @@ tosca_definitions_version: cloudify_dsl_1_3
 
 imports:
   - http://www.getcloudify.org/spec/cloudify/4.0.1/types.yaml
-  - https://raw.githubusercontent.com/cloudify-cosmo/cloudify-aws-plugin/1.5/plugin.yaml
+  - https://raw.githubusercontent.com/cloudify-cosmo/cloudify-aws-plugin/1.5.1.2/plugin.yaml
 
 inputs:
 

system_tests/manager/resources/simple-blueprint.yaml

@@ -4,7 +4,7 @@ tosca_definitions_version: cloudify_dsl_1_3
 
 imports:
   - http://www.getcloudify.org/spec/cloudify/4.0.1/types.yaml
-  - https://raw.githubusercontent.com/cloudify-cosmo/cloudify-aws-plugin/1.5/plugin.yaml
+  - https://raw.githubusercontent.com/cloudify-cosmo/cloudify-aws-plugin/1.5.1.2/plugin.yaml
 
 inputs:
 

system_tests/manager/resources/user-data-agent-install-blueprint.yaml

@@ -4,7 +4,7 @@ tosca_definitions_version: cloudify_dsl_1_3
 
 imports:
   - http://www.getcloudify.org/spec/cloudify/4.0.1/types.yaml
-  - https://raw.githubusercontent.com/cloudify-cosmo/cloudify-aws-plugin/1.5/plugin.yaml
+  - https://raw.githubusercontent.com/cloudify-cosmo/cloudify-aws-plugin/1.5.1.2/plugin.yaml
 
 inputs:
 

system_tests/manager/resources/vpc_test_blueprint.yaml

@@ -4,7 +4,7 @@ tosca_definitions_version: cloudify_dsl_1_3
 
 imports:
   - http://www.getcloudify.org/spec/cloudify/4.0.1/types.yaml
-  - https://raw.githubusercontent.com/cloudify-cosmo/cloudify-aws-plugin/1.5/plugin.yaml
+  - https://raw.githubusercontent.com/cloudify-cosmo/cloudify-aws-plugin/1.5.1.2/plugin.yaml
 
 inputs: