Add the ability to specify custom role config groups for sets of hosts

plugins/filter/filters.py

@@ -23,7 +23,10 @@ class FilterModule(object):
     def filters(self):
         return {
             'flatten_dict_list': self.flatten_dict_list,
+            'extract_custom_roles': self.extract_custom_roles,
+            'extract_custom_role_groups': self.extract_custom_role_groups,
             'extract_products_from_manifests': self.extract_products_from_manifests,
+            'extract_role_and_group': self.extract_role_and_group,
             'format_database_type': self.format_database_type,
             'get_product_version': self.get_product_version,
             'get_major_version': self.get_major_version,  # Unused
@@ -187,3 +190,29 @@ def find_clusters(self, clusters, name):
             cluster
             for cluster in clusters
             if cluster.get('name') == name]
+
+    def extract_role_and_group(self, role_spec):
+        role = None
+        template_group = "BASE"
+        if '/' in role_spec:
+            role = role_spec[:role_spec.index('/')]
+            template_group = role_spec[role_spec.index('/')+1:]
+        else:
+            role = role_spec
+        return (role, template_group)
+
+    def extract_custom_roles(self, host_templates, service):
+        custom_roles = []
+        for role_mapping in host_templates.values():
+            if service in role_mapping:
+                for custom_role in filter(lambda x: '/' in x, role_mapping[service]):
+                    custom_roles.append(custom_role)
+        return custom_roles
+
+    def extract_custom_role_groups(self, host_templates):
+        custom_role_groups = []
+        for role_mapping in host_templates.values():
+            for (service, roles) in role_mapping.items():
+                for custom_role in filter(lambda x: '/' in x, roles):
+                    custom_role_groups.append("-".join([service.lower()] + custom_role.split("/")))
+        return custom_role_groups

roles/cloudera_manager/config/templates/cm_api.j2

@@ -31,7 +31,7 @@
   {%- if value_type == "ref" or value_type == "value" -%}
     {{ config.update({ value_type: value }) }}
   {%- else -%}
-    {%- set var_name = [service_type, role_type, key] | join("_") | replace('.','_') -%}
+    {%- set var_name = [service_type, role_type, key] | join("_") | replace('.','_') | replace('/','_') -%}
     {{ config.update({ "variable": var_name | upper }) }}
   {%- endif -%}
   {{ config | to_json }}

roles/deployment/cluster/templates/cluster_template/base/hostTemplates.j2

@@ -15,7 +15,8 @@
         {{ roles.append("KT_RENEWER") }}
       {%- endif -%}
       {%- for role in roles -%}
-        {{ role_config_group_refs.append("{}-{}-BASE".format(service_ref, role)) }}
+        {%- set (role_type, config_group) = role | cloudera.cluster.extract_role_and_group -%}
+        {{ role_config_group_refs.append("{}-{}-{}".format(service_ref, role_type, config_group)) }}
       {%- endfor -%}
     {%- endif -%}
   {%- endfor -%}

roles/deployment/cluster/templates/cluster_template/base/instantiator.j2

@@ -34,4 +34,13 @@
     {%- endif -%}
   }
   {%- endif -%}
+  , "roleConfigGroups": [
+  {%- set role_group_sep = joiner(",") -%}
+  {%- for custom_rcg in cluster.host_templates | cloudera.cluster.extract_custom_role_groups -%}
+    {{ role_group_sep() }}
+    {
+      "rcgRefName": "{{ custom_rcg }}"
+    }
+  {%- endfor -%}
+  ]
 }

roles/deployment/cluster/templates/cluster_template/base/services.j2

@@ -22,13 +22,24 @@
     {%- if service_role_types is iterable -%}
       {%- for role_type in service_role_types -%}
       {{ role_group_sep() }}
-      {
-        "refName": "{{ service_ref }}-{{ role_type }}-BASE",
-        "roleType": "{{ role_type }}",
-        {% set config_sep = joiner(",") -%}
-        "configs": {{ cm_api.ApiClusterTemplateConfigList(merged_configs, service, role_type) }},
-        "base": true
-      }
+        {
+          "refName": "{{ service_ref }}-{{ role_type }}-BASE",
+          "roleType": "{{ role_type }}",
+          {% set config_sep = joiner(",") -%}
+          "configs": {{ cm_api.ApiClusterTemplateConfigList(merged_configs, service, role_type) }},
+          "base": true
+        }
+      {%- endfor -%}
+      {%- for custom_role in cluster.host_templates | cloudera.cluster.extract_custom_roles(service) -%}
+        {%- set (role_type, config_group) = custom_role | cloudera.cluster.extract_role_and_group -%}
+        {{ role_group_sep() }}
+        {
+          "refName": "{{ service_ref }}-{{ role_type }}-{{ config_group }}",
+          "roleType": "{{ role_type }}",
+          {% set config_sep = joiner(",") -%}
+          "configs": {{ cm_api.ApiClusterTemplateCustomConfigList(merged_configs, service, role_type, custom_role) }},
+          "base": false
+        }
       {%- endfor -%}
     {%- endif -%}
     ]

roles/deployment/cluster/templates/cluster_template/cm_api.j2

@@ -31,7 +31,7 @@
   {%- if value_type == "ref" or value_type == "value" -%}
     {{ config.update({ value_type: value }) }}
   {%- else -%}
-    {%- set var_name = [service_type, role_type, key] | join("_") | replace('.','_') -%}
+    {%- set var_name = [service_type, role_type, key] | join("_") | replace('.','_') | replace('/','_') -%}
     {{ config.update({ "variable": var_name | upper }) }}
   {%- endif -%}
   {{ config | to_json }}
@@ -54,6 +54,28 @@
   {{ config_list | to_json }}
 {%- endmacro %}
 
+{% macro ApiClusterTemplateCustomConfigList(service_merged_configs, service_type, role_type, custom_role_type) -%}
+  {%- set config_list = [] -%}
+  {%- if service_type in service_merged_configs -%}
+    {%- if service_merged_configs[service_type][role_type] is mapping -%}
+      {%- set new_merged_configs = service_merged_configs[service_type][custom_role_type] | combine(service_merged_configs[service_type][role_type]) -%}
+      {%- for (k, v) in new_merged_configs.items() -%}
+        {%- if k.endswith("_service") and not "suppression" in k -%}
+          {%- set config = ApiClusterTemplateConfig(k, service_type, custom_role_type, v, "ref") -%}
+        {%- else -%}
+          {%- if k in service_merged_configs[service_type][custom_role_type] %}
+            {%- set config = ApiClusterTemplateConfig(k, service_type, custom_role_type, v, "variable") -%}
+          {%- else -%}
+            {%- set config = ApiClusterTemplateConfig(k, service_type, role_type, v, "variable") -%}
+          {%- endif -%}
+        {%- endif -%}
+        {{ config_list.append(config | from_json) }}
+      {%- endfor -%}
+    {%- endif -%}
+  {%- endif -%}
+  {{ config_list | to_json }}
+{%- endmacro %}
+
 {% macro ApiService(service) -%}
 {}
 {%- endmacro %}

roles/deployment/cluster/templates/cluster_template/common/variables.j2

@@ -7,7 +7,7 @@
       {%- if configs is mapping -%}
         {%- for (key, val) in configs.items() -%}
         {{ config_joiner() }}
-        {%- set var_name = [service, role_name, key] | join("_") | replace('.','_') -%}
+        {%- set var_name = [service, role_name, key] | join("_") | replace('.','_') | replace('/','_') -%}
         {{ cm_api.ApiConfig(var_name, val, force_uppercase_keys=True) }}
         {%- endfor -%}
       {%- endif -%}

roles/verify/parcels_and_roles/tasks/check_cluster_config_roles.yml

@@ -19,6 +19,7 @@
         invalid_roles: >-
           {{ config.roles
           | flatten
+          | map('regex_replace','/.+','')
           | difference(['SERVICEWIDE'])
           | difference(role_mappings[config.service] | list)
           }}

roles/verify/parcels_and_roles/tasks/check_template_roles.yml

@@ -19,6 +19,7 @@
         invalid_roles: >-
           {{
           template.roles
+          | map('regex_replace','/.+','')
           | difference(role_mappings[template.service] | list)
           }}
     - name: Ensure the host template service roles are valid