diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 8fa36ec7..c3ae8dc4 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -44,3 +44,9 @@ - Add connect Security Groups to Instance via InstanceConnectedToSecurityGroup relationship - Support adding VPC Peering Connection + Use external routes - Fix bug that disassociate the wrong elastic ip from its instance +1.5.1 + - Support Windows instances with init_script agent installation method. +1.5.1.1 + - Fix bug when init_script is empty string. +1.5.1.2 + - Execute user-provided user data before agent install user data. diff --git a/cloudify_aws/ec2/instance.py b/cloudify_aws/ec2/instance.py index 2fb56ca5..9f1a2920 100644 --- a/cloudify_aws/ec2/instance.py +++ b/cloudify_aws/ec2/instance.py @@ -14,6 +14,7 @@ # * limitations under the License. import os +import json # Third-party Imports from boto import exception @@ -30,6 +31,9 @@ from cloudify_aws import utils, constants from cloudify.exceptions import NonRecoverableError +PS_OPEN = '' +PS_CLOSE = '' + @operation def creation_validation(**_): @@ -389,21 +393,95 @@ def _get_instance_attribute(self, attribute): attribute = getattr(instance_object, attribute) return attribute + def extract_powershell_content(self, string_with_powershell): + """We want to filter user data for powershell scripts. + However, AWS EC2 allows only one segment that is Powershell. + So we have to concat separate Powershell scripts into one. + First we separate all Powershell scripts without their tags. + Later we will add the tags back. + """ + + split_string = string_with_powershell.splitlines() + + if not split_string: + return '' + + if split_string[0] == '#ps1_sysnative' or \ + split_string[0] == '#ps1_x86': + split_string.pop(0) + + if PS_OPEN not in split_string: + script_start = -1 # Because we join at +1. + else: + script_start = split_string.index(PS_OPEN) + + if PS_CLOSE not in split_string: + script_end = len(split_string) + else: + script_end = split_string.index(PS_CLOSE) + + # Return everything between Powershell back as a string. + return '\n'.join(split_string[script_start+1:script_end]) + def _handle_userdata(self, parameters): existing_userdata = parameters.get('user_data') + + if existing_userdata is None: + existing_userdata = '' + elif isinstance(existing_userdata, dict) or \ + isinstance(existing_userdata, list): + existing_userdata = json.dumps(existing_userdata) + elif not isinstance(existing_userdata, basestring): + existing_userdata = str(existing_userdata) + install_agent_userdata = ctx.agent.init_script() + os_family = ctx.node.properties['os_family'] if not (existing_userdata or install_agent_userdata): return parameters - if not existing_userdata: + # AWS EC2 Windows instances require no more than one + # Powershell script, which must be surrounded by + # Powershell tags. + if install_agent_userdata and os_family == 'windows': + + # Get the powershell content from install_agent_userdata + install_agent_userdata = \ + self.extract_powershell_content(install_agent_userdata) + + # Get the powershell content from existing_userdata + # (If it exists.) + existing_userdata_powershell = \ + self.extract_powershell_content(existing_userdata) + + # Combine the powershell content from two sources. + install_agent_userdata = \ + '#ps1_sysnative\n{0}\n{1}\n{2}\n{3}\n'.format( + PS_OPEN, + existing_userdata_powershell, + install_agent_userdata, + PS_CLOSE) + + # Additional work on the existing_userdata. + # Remove duplicate Powershell content. + # Get rid of unnecessary newlines. + existing_userdata = \ + existing_userdata.replace( + existing_userdata_powershell, + '').replace( + PS_OPEN, + '').replace( + PS_CLOSE, + '').strip() + + if not existing_userdata or existing_userdata.isspace(): final_userdata = install_agent_userdata elif not install_agent_userdata: final_userdata = existing_userdata else: final_userdata = compute.create_multi_mimetype_userdata( - [existing_userdata, install_agent_userdata]) + [existing_userdata, install_agent_userdata]) parameters['user_data'] = final_userdata @@ -445,8 +523,8 @@ def _get_instance_parameters(self, args=None): }) parameters.update(ctx.node.properties['parameters']) - parameters = self._handle_userdata(parameters) parameters = utils.update_args(parameters, args) + parameters = self._handle_userdata(parameters) parameters['block_device_map'] = \ self._create_block_device_mapping( parameters.get('block_device_map', {}) diff --git a/cloudify_aws/ec2/tests/test_ec2_elb.py b/cloudify_aws/ec2/tests/test_ec2_elb.py index 8626da2c..98f6598c 100644 --- a/cloudify_aws/ec2/tests/test_ec2_elb.py +++ b/cloudify_aws/ec2/tests/test_ec2_elb.py @@ -61,10 +61,10 @@ def _get_elbs(self): def _get_elb_instances(self): instance_list = boto.connect_elb().get_all_load_balancers( load_balancer_names=['myelb'])[0].instances - l = [] + my_list = [] for i in instance_list: - l.append(i.id) - return l + my_list.append(i.id) + return my_list def _create_external_instance(self): return boto.connect_ec2().run_instances( diff --git a/cloudify_aws/ec2/tests/test_ec2_instance.py b/cloudify_aws/ec2/tests/test_ec2_instance.py index d3af5b26..f0a1774a 100644 --- a/cloudify_aws/ec2/tests/test_ec2_instance.py +++ b/cloudify_aws/ec2/tests/test_ec2_instance.py @@ -60,6 +60,7 @@ def mock_ctx(self, test_name, retry_number=0, operation_name='create'): 'tags': {}, 'image_id': TEST_AMI_IMAGE_ID, 'instance_type': TEST_INSTANCE_TYPE, + 'os_family': 'linux', 'cloudify_agent': {}, 'agent_config': {}, 'use_password': False, @@ -259,7 +260,6 @@ def test_with_existing_userdata_clean(self): ctx.agent.init_script = lambda: 'EXISTING' current_ctx.set(ctx=ctx) test_instance = self.create_instance_for_checking() - handle_userdata_output = \ test_instance._handle_userdata(ctx.node.properties['parameters']) expected_userdata = 'EXISTING' @@ -282,6 +282,32 @@ def test_with_both_userdata_clean(self): self.assertTrue(handle_userdata_output['user_data'].startswith( 'Content-Type: multi')) + @mock_ec2 + def test_with_both_userdata_clean_windows(self): + """ this tests that handle user data returns the expected output when merging + """ + + ctx = self.mock_ctx('test_with_both_userdata_clean_windows') + ctx.agent.init_script = lambda: '#ps1_sysnative\nSCRIPT' + ctx.node.properties['os_family'] = 'windows' + ctx.node.properties['agent_config']['install_method'] = 'init_script' + ctx.node.properties['parameters']['user_data'] = \ + '\nfunction Existing{}\n'\ + '\nrem cmd\n' + current_ctx.set(ctx=ctx) + test_instance = self.create_instance_for_checking() + handle_userdata_output = \ + test_instance._handle_userdata(ctx.node.properties['parameters']) + self.assertTrue(handle_userdata_output['user_data'].startswith( + 'Content-Type: multi')) + self.assertIn( + '#ps1_sysnative\n\nfunction Existing{}\n' + 'SCRIPT\n', + handle_userdata_output['user_data']) + self.assertIn( + 'rem cmd', + handle_userdata_output['user_data']) + @mock_ec2 def test_without_userdata_clean(self): """ this tests that handle user data returns the expected output diff --git a/cloudify_aws/vpc/tests/blueprint/plugin.yaml b/cloudify_aws/vpc/tests/blueprint/plugin.yaml index 6897d353..0292a5d5 100644 --- a/cloudify_aws/vpc/tests/blueprint/plugin.yaml +++ b/cloudify_aws/vpc/tests/blueprint/plugin.yaml @@ -5,9 +5,9 @@ plugins: aws: executor: central_deployment_agent - source: https://github.com/cloudify-cosmo/cloudify-aws-plugin/archive/1.5.zip + source: https://github.com/cloudify-cosmo/cloudify-aws-plugin/archive/1.5.1.1.zip package_name: cloudify-aws-plugin - package_version: '1.5' + package_version: '1.5.1.1' data_types: diff --git a/dev-requirements.txt b/dev-requirements.txt index 8671c6c4..4ce89853 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,3 @@ -https://github.com/cloudify-cosmo/cloudify-dsl-parser/archive/master.zip -https://github.com/cloudify-cosmo/cloudify-rest-client/archive/master.zip -https://github.com/cloudify-cosmo/cloudify-plugins-common/archive/master.zip +https://github.com/cloudify-cosmo/cloudify-dsl-parser/archive/3.4.2.zip +https://github.com/cloudify-cosmo/cloudify-rest-client/archive/3.4.2.zip +https://github.com/cloudify-cosmo/cloudify-plugins-common/archive/3.4.2.zip diff --git a/plugin.yaml b/plugin.yaml index 6897d353..ed6694f0 100644 --- a/plugin.yaml +++ b/plugin.yaml @@ -5,9 +5,9 @@ plugins: aws: executor: central_deployment_agent - source: https://github.com/cloudify-cosmo/cloudify-aws-plugin/archive/1.5.zip + source: https://github.com/cloudify-cosmo/cloudify-aws-plugin/archive/1.5.1.2.zip package_name: cloudify-aws-plugin - package_version: '1.5' + package_version: '1.5.1.2' data_types: diff --git a/setup.py b/setup.py index 923ef832..07f964e3 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ author='Gigaspaces', author_email='cosmo-admin@gigaspaces.com', - version='1.5', + version='1.5.1.2', description='Cloudify plugin for AWS infrastructure.', # This must correspond to the actual packages in the plugin. @@ -37,8 +37,8 @@ license='LICENSE', install_requires=[ - 'cloudify-plugins-common>=3.3.1', - 'boto==2.38.0', + 'cloudify-plugins-common>=3.4.2', + 'boto==2.48.0', 'pycrypto==2.6.1', 'ipaddress==1.0.18' ] diff --git a/system_tests/manager/resources/relationships-blueprint.yaml b/system_tests/manager/resources/relationships-blueprint.yaml index 47ce82a3..7f37045d 100644 --- a/system_tests/manager/resources/relationships-blueprint.yaml +++ b/system_tests/manager/resources/relationships-blueprint.yaml @@ -4,7 +4,7 @@ tosca_definitions_version: cloudify_dsl_1_3 imports: - http://www.getcloudify.org/spec/cloudify/4.0.1/types.yaml - - https://raw.githubusercontent.com/cloudify-cosmo/cloudify-aws-plugin/1.5/plugin.yaml + - https://raw.githubusercontent.com/cloudify-cosmo/cloudify-aws-plugin/1.5.1.2/plugin.yaml inputs: diff --git a/system_tests/manager/resources/sg-blueprint.yaml b/system_tests/manager/resources/sg-blueprint.yaml index a557ce12..1e24191c 100644 --- a/system_tests/manager/resources/sg-blueprint.yaml +++ b/system_tests/manager/resources/sg-blueprint.yaml @@ -4,7 +4,7 @@ tosca_definitions_version: cloudify_dsl_1_3 imports: - http://www.getcloudify.org/spec/cloudify/4.0.1/types.yaml - - https://raw.githubusercontent.com/cloudify-cosmo/cloudify-aws-plugin/1.5/plugin.yaml + - https://raw.githubusercontent.com/cloudify-cosmo/cloudify-aws-plugin/1.5.1.2/plugin.yaml inputs: diff --git a/system_tests/manager/resources/simple-blueprint.yaml b/system_tests/manager/resources/simple-blueprint.yaml index c9afdeae..3dbca475 100644 --- a/system_tests/manager/resources/simple-blueprint.yaml +++ b/system_tests/manager/resources/simple-blueprint.yaml @@ -4,7 +4,7 @@ tosca_definitions_version: cloudify_dsl_1_3 imports: - http://www.getcloudify.org/spec/cloudify/4.0.1/types.yaml - - https://raw.githubusercontent.com/cloudify-cosmo/cloudify-aws-plugin/1.5/plugin.yaml + - https://raw.githubusercontent.com/cloudify-cosmo/cloudify-aws-plugin/1.5.1.2/plugin.yaml inputs: diff --git a/system_tests/manager/resources/user-data-agent-install-blueprint.yaml b/system_tests/manager/resources/user-data-agent-install-blueprint.yaml index a22da559..f693ce9c 100644 --- a/system_tests/manager/resources/user-data-agent-install-blueprint.yaml +++ b/system_tests/manager/resources/user-data-agent-install-blueprint.yaml @@ -4,7 +4,7 @@ tosca_definitions_version: cloudify_dsl_1_3 imports: - http://www.getcloudify.org/spec/cloudify/4.0.1/types.yaml - - https://raw.githubusercontent.com/cloudify-cosmo/cloudify-aws-plugin/1.5/plugin.yaml + - https://raw.githubusercontent.com/cloudify-cosmo/cloudify-aws-plugin/1.5.1.2/plugin.yaml inputs: diff --git a/system_tests/manager/resources/vpc_test_blueprint.yaml b/system_tests/manager/resources/vpc_test_blueprint.yaml index db9a2871..cf37913b 100644 --- a/system_tests/manager/resources/vpc_test_blueprint.yaml +++ b/system_tests/manager/resources/vpc_test_blueprint.yaml @@ -4,7 +4,7 @@ tosca_definitions_version: cloudify_dsl_1_3 imports: - http://www.getcloudify.org/spec/cloudify/4.0.1/types.yaml - - https://raw.githubusercontent.com/cloudify-cosmo/cloudify-aws-plugin/1.5/plugin.yaml + - https://raw.githubusercontent.com/cloudify-cosmo/cloudify-aws-plugin/1.5.1.2/plugin.yaml inputs: