From b4d1ce0946f5ddf53a2d781fb70b438f10d1fd58 Mon Sep 17 00:00:00 2001 From: Andrew Glenn Date: Wed, 24 Jan 2018 15:18:13 -0600 Subject: [PATCH 1/2] Adding ability to fill ci/override.json with parameter overrides. --- taskcat/stacker.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/taskcat/stacker.py b/taskcat/stacker.py index 2657e111e..2819f2297 100755 --- a/taskcat/stacker.py +++ b/taskcat/stacker.py @@ -258,6 +258,35 @@ def set_parameter_path(self, parameter): def get_parameter_path(self): return self.parameter_path + def get_param_includes(self, s_params): + # Github/issue/57 + # Fetch overrides from S3. + param_path = self.parameter_path + _url_prefix=re.search(r'^http.*/.*/(?!\/.*?$)', param_path).group[:-1] + override_url = "{}/{}".format(_url_prefix, 'override.json') + r = requests.get(override_url) + if r.status_code != 200: + return None + + override = json.loads(r.text) + + # Setup a list index dictionary. + param_index = {} + for (idx, param_dict) in enumerate(s_params): + key = param_dict['ParameterKey'] + param_index[key] = idx + + # Merge the two lists, overriding the original values if necessary. + for override_pd in override: + key = override_pd['ParameterKey'] + if key in param_index.keys(): + idx = param_index[key] + s_params[idx] = override_pd + else: + s_params.append(override_pd) + + return s_params + def set_template_path(self, template): self.template_path = template @@ -943,6 +972,9 @@ def stackcreate(self, taskcat_cfg, test_list, sprefix): cfn = self._boto_client.get('cloudformation', region=region) s_parmsdata = requests.get(self.get_parameter_path()).text s_parms = json.loads(s_parmsdata) + s_include_params = self.get_param_includes(s_parms) + if s_include_params: + s_parms = s_include_params j_params = self.generate_input_param_values(s_parms, region) if self.verbose: print(D + "Creating Boto Connection region=%s" % region) From 4966cc2ae32700730054dcba7c5b12692a03f80e Mon Sep 17 00:00:00 2001 From: Andrew Glenn Date: Mon, 12 Feb 2018 14:13:51 -0600 Subject: [PATCH 2/2] Additional code changes as requested in pull/58 --- README.md | 45 +++++++++++++++++++++++---------- taskcat/stacker.py | 62 ++++++++++++++++++++++++++++++++-------------- 2 files changed, 76 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 5f5f12d01..a4937c897 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # TaskCat -> This program requires python3 +> This program requires python3 # Currently in (beta release) @@ -13,31 +13,31 @@ Class [taskcat.Sweeper] - -## What is TaskCat? + +## What is TaskCat? TaskCat is a tool that tests AWS CloudFormation templates. It deploys your AWS CloudFormation template in multiple AWS Regions and generates a report with a pass/fail grade for each region. You can specify the regions and number of Availability Zones you want to include in the test, and pass in parameter values from your AWS CloudFormation template. TaskCat is implemented as a Python class that you import, instantiate, and run. - -TestCat was developed by the AWS Quick Start team to test AWS CloudFormation templates that automatically deploy workloads on AWS. We’re pleased to make the tool available to all developers who want to validate their custom AWS CloudFormation + +TestCat was developed by the AWS Quick Start team to test AWS CloudFormation templates that automatically deploy workloads on AWS. We’re pleased to make the tool available to all developers who want to validate their custom AWS CloudFormation templates across AWS Regions ## Files you’ll need * **config.yml** - This file contains the test cases * **JSON input** - This file contains the inputs that you want to pass to AWS CloudFormation template that is being tested -* Step 1 Building your configuration file +* Step 1 Building your configuration file * Step 2 Building your JSON input file. #### Step 1 Creating a config.yml -Open the config.yml file with and editor and update the filenames to match your need. +Open the config.yml file with and editor and update the filenames to match your need. example here: [config.yml](https://raw.githubusercontent.com/aws-quickstart/taskcat/master/examples/sample-taskcat-project/ci/taskcat.yml) -#### Example of config.yml +#### Example of config.yml global: owner: owner@company.com qsname: sample-cloudformation-project <- Must match the root directory of project (usually the name of git repo) - #s3bucket: projectx-templates <- (Optional) Only needed if you want to use a specific bucket + #s3bucket: projectx-templates <- (Optional) Only needed if you want to use a specific bucket regions: - us-east-1 - us-east-2 @@ -61,7 +61,7 @@ example here: ├── LICENSE.txt ├── README.md ├── ci - │  ├── config.yml <- This the config file that will hold all the test definitions + │  ├── config.yml <- This the config file that will hold all the test definitions │  ├── sample-cloudformation-input-novpc.json <- This file contain input that will pass in during stack creation [vpc version] (See auto parms for more info) │  └── sample-cloudformation-input-withvpc.json <- This file contain input that will pass in during stack creation [no-vpc version](See auto parms for more info) ├── scripts @@ -71,8 +71,8 @@ example here: │   └── templates │   └── aws-vpc.template └── templates - ├── sample-cloudformation-project-novpc.template - └── sample-cloudformation-project-withvpc.template <- Second version on template that will create a vpc with the workload + ├── sample-cloudformation-project-novpc.template + └── sample-cloudformation-project-withvpc.template <- Second version on template that will create a vpc with the workload ### Step 2 Building a json input file The example below shows an input file for a stack that requires four parameters `KeyPair`,`InstanceType`, `AvailablityZones` and `Password` @@ -190,7 +190,7 @@ curl -s https://raw.githubusercontent.com/aws-quickstart/taskcat/master/installe ``` > Note: (If you do not have root privileges taskcat will install in the current directory) -### Installing via pip +### Installing via pip > Prerequisites: Python 3.5+ and pip3 ``` curl -s https://raw.githubusercontent.com/aws-quickstart/taskcat/master/installer/pip/pip3-install-master| python -E @@ -210,3 +210,22 @@ If you want to use a different account or profile ``` taskcat -c sample-taskcat-project/ci/config.yml -P boto-profile-name ``` + +### Local Parameter Overrides. +In certain situations it may be desirable to introduce local Parameter Override values. Taskcat supports this via two files. + +The first is located within the home-directory of the running user. +``` +~/.taskcat_global_override.json +``` + +The second applies per-project and is located the 'CI' directory. +``` +/ci/taskcat_project_override.json +``` + +Parameters defined in either file will supersede parameters within the normal parameter files. The override includes are read in the following order. +- Home Directory (~/.taskcat_global_override.json) +- Project Directory (ci/taskcat_project_override.json) + +Keys defined in the Project Override with supersede the same keys defined in the Global Override. diff --git a/taskcat/stacker.py b/taskcat/stacker.py index 2819f2297..9453e05ab 100755 --- a/taskcat/stacker.py +++ b/taskcat/stacker.py @@ -258,34 +258,60 @@ def set_parameter_path(self, parameter): def get_parameter_path(self): return self.parameter_path - def get_param_includes(self, s_params): + def get_param_includes(self, original_keys): + """ + This function searches for ~/.taskcat_global_override.json, then /ci/taskcat_project_override.json, in that order. + Keys defined in either of these files will override Keys defined in /ci/*.json. + + :param original_keys: json object derived from Parameter Input JSON in /ci/ + """ # Github/issue/57 - # Fetch overrides from S3. - param_path = self.parameter_path - _url_prefix=re.search(r'^http.*/.*/(?!\/.*?$)', param_path).group[:-1] - override_url = "{}/{}".format(_url_prefix, 'override.json') - r = requests.get(override_url) - if r.status_code != 200: - return None + # Look for ~/.taskcat_overrides.json - override = json.loads(r.text) + # Fetch overrides Homedir first. + dict_squash_list = [] + _homedir_override_file_path = "{}/{}".format(os.path.expanduser('~'), '.taskcat_global_override.json') + if os.path.isfile(_homedir_override_file_path): + with open(_homedir_override_file_path) as f: + _homedir_override_json = json.loads(f.read()) + print(D + "Values loaded from ~/.taskcat_global_override.json") + print(D + str(_homedir_override_json)) + dict_squash_list.append(_homedir_override_json) + + + # Now look for per-project override uploaded to S3. + override_file_key = "{}/ci/taskcat_project_override.json".format(self.project) + try: + # Intentional duplication of self.get_content() here, as I don't want to break that due to + # tweaks necessary here. + s3_client = self._boto_client.get('s3', region=self.get_default_region(), s3v4=True) + dict_object = s3_client.get_object(Bucket=self.s3bucket, Key=override_file_key) + content = dict_object['Body'].read().strip() + _obj = json.loads(content) + dict_squash_list.append(_obj) + print(D + "Values loaded from {}/ci/taskcat_project_override.json".format(self.project)) + print(D + str(_obj)) + except: + pass # Setup a list index dictionary. + # - Used to give an Parameter => Index mapping for replacement. param_index = {} - for (idx, param_dict) in enumerate(s_params): + for (idx, param_dict) in enumerate(original_keys): key = param_dict['ParameterKey'] param_index[key] = idx # Merge the two lists, overriding the original values if necessary. - for override_pd in override: - key = override_pd['ParameterKey'] - if key in param_index.keys(): - idx = param_index[key] - s_params[idx] = override_pd - else: - s_params.append(override_pd) + for override in dict_squash_list: + for override_pd in override: + key = override_pd['ParameterKey'] + if key in param_index.keys(): + idx = param_index[key] + original_keys[idx] = override_pd + else: + original_keys.append(override_pd) - return s_params + return original_keys def set_template_path(self, template): self.template_path = template