diff --git a/plugins/filter/filters.py b/plugins/filter/filters.py index 356ccd5b..597decc3 100644 --- a/plugins/filter/filters.py +++ b/plugins/filter/filters.py @@ -23,7 +23,10 @@ class FilterModule(object): def filters(self): return { 'flatten_dict_list': self.flatten_dict_list, + 'extract_custom_roles': self.extract_custom_roles, + 'extract_custom_role_groups': self.extract_custom_role_groups, 'extract_products_from_manifests': self.extract_products_from_manifests, + 'extract_role_and_group': self.extract_role_and_group, 'format_database_type': self.format_database_type, 'get_product_version': self.get_product_version, 'get_major_version': self.get_major_version, # Unused @@ -187,3 +190,29 @@ def find_clusters(self, clusters, name): cluster for cluster in clusters if cluster.get('name') == name] + + def extract_role_and_group(self, role_spec): + role = None + template_group = "BASE" + if '/' in role_spec: + role = role_spec[:role_spec.index('/')] + template_group = role_spec[role_spec.index('/')+1:] + else: + role = role_spec + return (role, template_group) + + def extract_custom_roles(self, host_templates, service): + custom_roles = [] + for role_mapping in host_templates.values(): + if service in role_mapping: + for custom_role in filter(lambda x: '/' in x, role_mapping[service]): + custom_roles.append(custom_role) + return custom_roles + + def extract_custom_role_groups(self, host_templates): + custom_role_groups = [] + for role_mapping in host_templates.values(): + for (service, roles) in role_mapping.items(): + for custom_role in filter(lambda x: '/' in x, roles): + custom_role_groups.append("-".join([service.lower()] + custom_role.split("/"))) + return custom_role_groups diff --git a/roles/cloudera_manager/config/templates/cm_api.j2 b/roles/cloudera_manager/config/templates/cm_api.j2 index 88ab1447..94cc2d0a 100644 --- a/roles/cloudera_manager/config/templates/cm_api.j2 +++ b/roles/cloudera_manager/config/templates/cm_api.j2 @@ -31,7 +31,7 @@ {%- if value_type == "ref" or value_type == "value" -%} {{ config.update({ value_type: value }) }} {%- else -%} - {%- set var_name = [service_type, role_type, key] | join("_") | replace('.','_') -%} + {%- set var_name = [service_type, role_type, key] | join("_") | replace('.','_') | replace('/','_') -%} {{ config.update({ "variable": var_name | upper }) }} {%- endif -%} {{ config | to_json }} diff --git a/roles/deployment/cluster/templates/cluster_template/base/hostTemplates.j2 b/roles/deployment/cluster/templates/cluster_template/base/hostTemplates.j2 index 7233b63f..242ba8c8 100644 --- a/roles/deployment/cluster/templates/cluster_template/base/hostTemplates.j2 +++ b/roles/deployment/cluster/templates/cluster_template/base/hostTemplates.j2 @@ -15,7 +15,8 @@ {{ roles.append("KT_RENEWER") }} {%- endif -%} {%- for role in roles -%} - {{ role_config_group_refs.append("{}-{}-BASE".format(service_ref, role)) }} + {%- set (role_type, config_group) = role | cloudera.cluster.extract_role_and_group -%} + {{ role_config_group_refs.append("{}-{}-{}".format(service_ref, role_type, config_group)) }} {%- endfor -%} {%- endif -%} {%- endfor -%} diff --git a/roles/deployment/cluster/templates/cluster_template/base/instantiator.j2 b/roles/deployment/cluster/templates/cluster_template/base/instantiator.j2 index 2ea6e97a..f52b2eea 100644 --- a/roles/deployment/cluster/templates/cluster_template/base/instantiator.j2 +++ b/roles/deployment/cluster/templates/cluster_template/base/instantiator.j2 @@ -34,4 +34,13 @@ {%- endif -%} } {%- endif -%} + , "roleConfigGroups": [ + {%- set role_group_sep = joiner(",") -%} + {%- for custom_rcg in cluster.host_templates | cloudera.cluster.extract_custom_role_groups -%} + {{ role_group_sep() }} + { + "rcgRefName": "{{ custom_rcg }}" + } + {%- endfor -%} + ] } diff --git a/roles/deployment/cluster/templates/cluster_template/base/services.j2 b/roles/deployment/cluster/templates/cluster_template/base/services.j2 index 644108e5..4dcab530 100644 --- a/roles/deployment/cluster/templates/cluster_template/base/services.j2 +++ b/roles/deployment/cluster/templates/cluster_template/base/services.j2 @@ -22,13 +22,24 @@ {%- if service_role_types is iterable -%} {%- for role_type in service_role_types -%} {{ role_group_sep() }} - { - "refName": "{{ service_ref }}-{{ role_type }}-BASE", - "roleType": "{{ role_type }}", - {% set config_sep = joiner(",") -%} - "configs": {{ cm_api.ApiClusterTemplateConfigList(merged_configs, service, role_type) }}, - "base": true - } + { + "refName": "{{ service_ref }}-{{ role_type }}-BASE", + "roleType": "{{ role_type }}", + {% set config_sep = joiner(",") -%} + "configs": {{ cm_api.ApiClusterTemplateConfigList(merged_configs, service, role_type) }}, + "base": true + } + {%- endfor -%} + {%- for custom_role in cluster.host_templates | cloudera.cluster.extract_custom_roles(service) -%} + {%- set (role_type, config_group) = custom_role | cloudera.cluster.extract_role_and_group -%} + {{ role_group_sep() }} + { + "refName": "{{ service_ref }}-{{ role_type }}-{{ config_group }}", + "roleType": "{{ role_type }}", + {% set config_sep = joiner(",") -%} + "configs": {{ cm_api.ApiClusterTemplateCustomConfigList(merged_configs, service, role_type, custom_role) }}, + "base": false + } {%- endfor -%} {%- endif -%} ] diff --git a/roles/deployment/cluster/templates/cluster_template/cm_api.j2 b/roles/deployment/cluster/templates/cluster_template/cm_api.j2 index 88ab1447..b6930a9d 100644 --- a/roles/deployment/cluster/templates/cluster_template/cm_api.j2 +++ b/roles/deployment/cluster/templates/cluster_template/cm_api.j2 @@ -31,7 +31,7 @@ {%- if value_type == "ref" or value_type == "value" -%} {{ config.update({ value_type: value }) }} {%- else -%} - {%- set var_name = [service_type, role_type, key] | join("_") | replace('.','_') -%} + {%- set var_name = [service_type, role_type, key] | join("_") | replace('.','_') | replace('/','_') -%} {{ config.update({ "variable": var_name | upper }) }} {%- endif -%} {{ config | to_json }} @@ -54,6 +54,28 @@ {{ config_list | to_json }} {%- endmacro %} +{% macro ApiClusterTemplateCustomConfigList(service_merged_configs, service_type, role_type, custom_role_type) -%} + {%- set config_list = [] -%} + {%- if service_type in service_merged_configs -%} + {%- if service_merged_configs[service_type][role_type] is mapping -%} + {%- set new_merged_configs = service_merged_configs[service_type][custom_role_type] | combine(service_merged_configs[service_type][role_type]) -%} + {%- for (k, v) in new_merged_configs.items() -%} + {%- if k.endswith("_service") and not "suppression" in k -%} + {%- set config = ApiClusterTemplateConfig(k, service_type, custom_role_type, v, "ref") -%} + {%- else -%} + {%- if k in service_merged_configs[service_type][custom_role_type] %} + {%- set config = ApiClusterTemplateConfig(k, service_type, custom_role_type, v, "variable") -%} + {%- else -%} + {%- set config = ApiClusterTemplateConfig(k, service_type, role_type, v, "variable") -%} + {%- endif -%} + {%- endif -%} + {{ config_list.append(config | from_json) }} + {%- endfor -%} + {%- endif -%} + {%- endif -%} + {{ config_list | to_json }} +{%- endmacro %} + {% macro ApiService(service) -%} {} {%- endmacro %} diff --git a/roles/deployment/cluster/templates/cluster_template/common/variables.j2 b/roles/deployment/cluster/templates/cluster_template/common/variables.j2 index 9f864857..0e46368c 100644 --- a/roles/deployment/cluster/templates/cluster_template/common/variables.j2 +++ b/roles/deployment/cluster/templates/cluster_template/common/variables.j2 @@ -7,7 +7,7 @@ {%- if configs is mapping -%} {%- for (key, val) in configs.items() -%} {{ config_joiner() }} - {%- set var_name = [service, role_name, key] | join("_") | replace('.','_') -%} + {%- set var_name = [service, role_name, key] | join("_") | replace('.','_') | replace('/','_') -%} {{ cm_api.ApiConfig(var_name, val, force_uppercase_keys=True) }} {%- endfor -%} {%- endif -%} diff --git a/roles/verify/parcels_and_roles/tasks/check_cluster_config_roles.yml b/roles/verify/parcels_and_roles/tasks/check_cluster_config_roles.yml index d8e3a4d4..3556456a 100644 --- a/roles/verify/parcels_and_roles/tasks/check_cluster_config_roles.yml +++ b/roles/verify/parcels_and_roles/tasks/check_cluster_config_roles.yml @@ -19,6 +19,7 @@ invalid_roles: >- {{ config.roles | flatten + | map('regex_replace','/.+','') | difference(['SERVICEWIDE']) | difference(role_mappings[config.service] | list) }} diff --git a/roles/verify/parcels_and_roles/tasks/check_template_roles.yml b/roles/verify/parcels_and_roles/tasks/check_template_roles.yml index 5b2e492d..6ab4be79 100644 --- a/roles/verify/parcels_and_roles/tasks/check_template_roles.yml +++ b/roles/verify/parcels_and_roles/tasks/check_template_roles.yml @@ -19,6 +19,7 @@ invalid_roles: >- {{ template.roles + | map('regex_replace','/.+','') | difference(role_mappings[template.service] | list) }} - name: Ensure the host template service roles are valid