Fixes for deploying CM5/CDH5 clusters

galaxy.yml

@@ -55,7 +55,13 @@ tags: []
 # collection label 'namespace.name'. The value is a version range
 # L(specifiers,https://python-semanticversion.readthedocs.io/en/latest/#requirement-specification). Multiple version
 # range specifiers can be set and are separated by ','
-dependencies: {}
+dependencies:
+  'ansible.posix': '>=1.2.0'
+  'community.crypto': '>=1.6.0'
+  'community.general': '>=2.4.0'
+  'community.mysql': '>=1.3.0'
+  'community.postgresql': '>=1.3.0'
+  'freeipa.ansible_freeipa': '>=0.3.5'
 
 # The URL of the originating SCM repository
 repository: http://github.com/cloudera-labs

roles/cloudera_manager/database/defaults/main.yml

@@ -13,5 +13,5 @@
 # limitations under the License.
 
 ---
-cloudera_manager_database_prepare_script: /opt/cloudera/cm/schema/scm_prepare_database.sh
+cloudera_manager_database_prepare_script: "{{ '/opt/cloudera/cm/schema/scm_prepare_database.sh' if cloudera_manager_version is version('6.0.0','>=') else '/usr/share/cmf/schema/scm_prepare_database.sh' }}"
 cloudera_manager_database_ranger_script: /opt/cloudera/cm/bin/gen_embedded_ranger_db.sh

roles/cloudera_manager/external_auth/tasks/main.yml

@@ -17,15 +17,15 @@
 - name: Select external auth provider details
   set_fact:
     auth_provider: "{{ auth_providers[cloudera_manager_external_auth.provider] }}"
-  when: cloudera_manager_external_auth.provider is defined
+  when: cloudera_manager_external_auth.provider is defined and cloudera_manager_version is version('6.0.0','>=')
 
 - name: Set Cloudera Manager external auth configs
   include_role:
-    name: cloudera_manager/config
+    name: cloudera.cluster.cloudera_manager.config
   vars:
     api_config_keys_uppercase: True
     api_configs: "{{ lookup('template', 'external_auth_configs.j2') | from_yaml }}"
-  when: auth_provider is defined
+  when: auth_provider is defined and cloudera_manager_version is version('6.0.0','>=')
 
 - block:
 
@@ -60,4 +60,4 @@
     notify:
       - wait cloudera-scm-server
 
-  when: cloudera_manager_external_auth.role_mappings is defined
+  when: cloudera_manager_external_auth.role_mappings is defined and cloudera_manager_version is version('6.0.0','>=')

roles/cloudera_manager/external_auth/templates/external_auth_configs.j2

@@ -20,8 +20,10 @@ LDAP_URL: {{ auth_provider.ldap_url | default(None) }}
 LDAP_USER_SEARCH_BASE: {{ auth_provider.ldap_search_base.user | default(None) }}
 LDAP_USER_SEARCH_FILTER: "({{ auth_provider.ldap_attribute.user | default('sAMAccountName') }}={0})"
 NT_DOMAIN: {{ auth_provider.domain | default(None) }}
+{% if cloudera_manager_version is version('7.1.0','>=') %}
 FRONTEND_URL: {{ frontend_url | default(None) }}
 PROXYUSER_KNOX_GROUPS: "{{ proxyuser_knox_groups | default('*') }}"
 PROXYUSER_KNOX_USERS: "{{ proxyuser_knox_users | default('*') }}"
 PROXYUSER_KNOX_HOSTS: "{{ proxyuser_knox_hosts | default('*') }}"
 PROXYUSER_KNOX_PRINCIPAL: "{{ proxyuser_knox_principal | default('knox') }}"
+{% endif %}

roles/config/cluster/base/templates/configs/databases-7.1.0.j2

@@ -15,6 +15,14 @@ RANGER:
     ranger_database_name: {{ databases.RANGER.name }}
     ranger_database_user: {{ databases.RANGER.user }}
     ranger_database_password: {{ databases.RANGER.password }}
+RANGER_RMS:
+  SERVICEWIDE:
+    ranger_rms_database_host: {{ databases.RANGER.host }}
+    ranger_rms_database_port: {{ databases.RANGER.port }}
+    ranger_rms_database_type: {{ databases.RANGER.type | cloudera.cluster.format_database_type }}
+    ranger_rms_database_name: {{ databases.RANGER.name }}
+    ranger_rms_database_user: {{ databases.RANGER.user }}
+    ranger_rms_database_password: {{ databases.RANGER.password }}
 SCHEMAREGISTRY:
   SERVICEWIDE:
     database_host: {{ databases.SCHEMAREGISTRY.host }}

roles/config/cluster/base/templates/configs/kerberos-7.x.j2

@@ -0,0 +1,4 @@
+---
+HBASE:
+  SERVICEWIDE:
+    hadoop_secure_web_ui: true

roles/config/cluster/base/templates/configs/logdirs-7.1.0.j2

@@ -5,6 +5,8 @@ ATLAS:
     log_dir: "{{ log_base }}/atlas"
     ranger_atlas_plugin_hdfs_audit_spool_directory: "{{ log_base }}/atlas/audit/hdfs/spool"
     ranger_atlas_plugin_solr_audit_spool_directory: "{{ log_base }}/atlas/audit/solr/spool"
+  GATEWAY:
+    log_dir: "{{ log_base }}/atlas"
 CORE_SETTINGS:
   STORAGEOPERATIONS:
     storageoperations_log_dir: "{{ log_base }}/"

roles/config/cluster/base/templates/configs/ranger.j2

@@ -2,6 +2,9 @@
 HDFS:
   SERVICEWIDE:
     enable_ranger_authorization: true
+{% if 'RANGER_RMS' in cluster.services %}
+    ranger_security_safety_valve: "<property><name>ranger.plugin.hdfs.chained.services</name><value>cm_hive</value><description>Ranger RMS related config</description></property><property><name>ranger.plugin.hdfs.chained.services.cm_hive.impl</name><value>org.apache.ranger.chainedplugin.hdfs.hive.RangerHdfsHiveChainedPlugin</value><description>Ranger RMS related config</description></property>"
+{% endif %}
 RANGER:
   SERVICEWIDE:
     keyadmin_user_password: {{ ranger_keyadmin_user_password | default('password123') }}

roles/config/cluster/base/templates/configs/tls-7.1.0.j2

@@ -96,6 +96,10 @@ KNOX:
     ssl_enabled: true
     ssl_server_keystore_location: {{ tls_keystore_path_generic }}
     ssl_server_keystore_password: {{ tls_keystore_password }}
+KUDU:
+  MASTER:
+    ssl_client_truststore_location: {{ tls_truststore_path }}
+    ssl_client_truststore_password: {{ tls_truststore_password }}
 LIVY:
   GATEWAY:
     ssl_client_truststore_location: {{ tls_truststore_path }}

roles/config/cluster/base/templates/configs/tls-7.1.4.j2

@@ -5,3 +5,10 @@ OOZIE:
     oozie_zookeeper_https_keystore_password: {{ tls_keystore_password }}
     oozie_zookeeper_https_truststore_file: {{ tls_truststore_path }}
     oozie_zookeeper_https_truststore_password: {{ tls_truststore_password }}
+RANGER_RMS:
+  RANGER_RMS_SERVER:
+    ssl_client_truststore_location: {{ tls_truststore_path }}
+    ssl_client_truststore_password: {{ tls_truststore_password }}
+    ssl_enabled: true
+    ssl_server_keystore_location: {{ tls_keystore_path_generic }}
+    ssl_server_keystore_password: {{ tls_keystore_password }}

roles/config/cluster/base/templates/configs/tls-7.3.1.j2

@@ -0,0 +1,4 @@
+---
+OZONE:
+  OZONE_PROMETHEUS:
+    ozone.prometheus.ca.file: {{ tls_chain_path }}

roles/config/cluster/base/templates/configs/tls.j2

@@ -123,8 +123,6 @@ KS_INDEXER:
     keystore_indexer_truststore_password: {{ tls_truststore_password }}
 KUDU:
   KUDU_MASTER:
-    ssl_client_truststore_location: {{ tls_truststore_path }}
-    ssl_client_truststore_password: {{ tls_truststore_password }}
     ssl_enabled: True
     ssl_server_ca_certificate_location: {{ tls_chain_path }}
     ssl_server_certificate_location: {{ tls_cert_path_generic }}

roles/config/cluster/base/templates/configs/varlib-7.1.0.j2

@@ -63,7 +63,7 @@ OZONE:
     ozone.metadata.dirs: "{{ varlib_base }}/hadoop-ozone/om/ozone-metadata"
     ozone.om.db.dirs: "{{ varlib_base }}/hadoop-ozone/om/data"
     ozone.om.ratis.storage.dir: "{{ varlib_base }}/hadoop-ozone/om/ratis"
-  PROMETHEUS:
+  OZONE_PROMETHEUS:
     ozone.prometheus.db.dir: "{{ varlib_base }}/hadoop-ozone/prometheus/data"
   OZONE_RECON:
     ozone.metadata.dirs: "{{ varlib_base }}/hadoop-ozone/recon/ozone-metadata"

roles/config/cluster/base/vars/main.yml

@@ -42,6 +42,8 @@ custom_config_templates:
     condition: "{{ cluster.security.kerberos | default(False) and (cloudera_manager_version is version('6.0.0','<') or cluster.type | default('base') == 'compute') }}"
   - template: configs/kerberos-6.x-7.x.j2
     condition: "{{ cluster.security.kerberos | default(False) and cloudera_manager_version is version('6.0.0','>=') }}"
+  - template: configs/kerberos-7.x.j2
+    condition: "{{ cluster.security.kerberos | default(False) and cloudera_manager_version is version('7.1.0','>=') }}"
   - template: configs/trusted-realms.j2
     condition: "{{ cluster.security.kerberos | default(False) and auth_providers | default({}) | dict2items | json_query('[?value.type == `KERBEROS`]') | length > 0 }}"
 # Custom configurations for TLS
@@ -53,6 +55,8 @@ custom_config_templates:
     condition: "{{ cluster.security.tls | default(False) and cloudera_runtime_version is version('7.1.0','>=') }}"
   - template: configs/tls-7.1.4.j2
     condition: "{{ cluster.security.tls | default(False) and cloudera_runtime_version is version('7.1.4','>=') }}"
+  - template: configs/tls-7.3.1.j2
+    condition: "{{ cluster.security.tls | default(False) and cloudera_manager_version is version('7.3.1', '>=') }}"
 # Custom configurations for Cloudera Streams Processing components on CDH 6.x
   - template: configs/schemaregistry.j2
     condition: >-

roles/config/cluster/kts/tasks/main.yml

@@ -16,7 +16,7 @@
 
 - name: Retrieve repository metadata
   include_role:
-    name: deployment/repometa
+    name: cloudera.cluster.deployment.repometa
   vars:
     repositories: "{{ cluster.repositories | default({}) }}"
 

roles/deployment/cluster/tasks/create_base.yml

@@ -16,11 +16,11 @@
 
 - name: Generate complete base cluster configs
   include_role:
-    name: config/cluster/base
+    name: cloudera.cluster.config.cluster.base
 
 - name: Create databases and users
   include_role:
-    name: deployment/databases
+    name: cloudera.cluster.deployment.databases
   vars:
     services: "{{ cluster.services | default({}) }}"
 

roles/deployment/cluster/tasks/create_kts.yml

@@ -16,7 +16,7 @@
 
 - name: Generate complete kts cluster configs
   include_role:
-    name: config/cluster/kts
+    name: cloudera.cluster.config.cluster.kts
 
 - name: Generate cluster template file
   template:

roles/deployment/cluster/tasks/main.yml

@@ -20,7 +20,7 @@
 
 - name: Apply "all hosts" configs
   include_role:
-    name: cloudera_manager/config
+    name: cloudera.cluster.cloudera_manager.config
   vars:
     api_config_keys_uppercase: False
     api_config_endpoint: cm/allHosts/config

roles/deployment/repometa/templates/role_mappings/cdh7.j2

@@ -1,6 +1,7 @@
 ADLS_CONNECTOR:
 ATLAS:
   - ATLAS_SERVER
+  - GATEWAY
 AWS_S3:
 CORE_SETTINGS:
   - GATEWAY
@@ -29,7 +30,6 @@ HDFS:
 HIVE:
   - GATEWAY
   - HIVEMETASTORE
-  - HIVESERVER2
 HIVE_ON_TEZ:
   - GATEWAY
   - HIVESERVER2
@@ -92,6 +92,8 @@ RANGER_KMS_KTS:
   - RANGER_KMS_SERVER_KTS
 RANGER_RAZ:
   - RANGER_RAZ_SERVER
+RANGER_RMS:
+  - RANGER_RMS_SERVER
 SCHEMAREGISTRY:
   - GATEWAY
   - SCHEMA_REGISTRY_SERVER

roles/deployment/services/kms/tasks/create_kms.yml

@@ -75,7 +75,7 @@
 
 - name: Generate KMS configs
   include_role:
-    name: config/services/kms
+    name: cloudera.cluster.config.services.kms
 
 - name: Create KMS service
   cloudera.cluster.cm_api:

roles/prereqs/os/tasks/main.yml

@@ -34,7 +34,7 @@
   loop: "{{ kernel_flags }}"
   loop_control:
     loop_var: flag
-  when: not(ansible_virtualization_type == "docker" and ansible_virtualization_role == "guest")
+  when: not((ansible_virtualization_type == "docker" or ansible_virtualization_type == "container") and ansible_virtualization_role == "guest")
 
 - name: Populate service facts
   service_facts:
@@ -103,4 +103,4 @@
 - name: Apply OS-specific configurations
   include_tasks:
     file: "main-{{ ansible_os_family }}.yml"
-  when: not(ansible_virtualization_type == "docker" and ansible_virtualization_role == "guest")
+  when: not((ansible_virtualization_type == "docker" or ansible_virtualization_type == "container") and ansible_virtualization_role == "guest")

roles/prereqs/user_accounts/tasks/main.yml

@@ -49,7 +49,7 @@
         path: "{{ account.home }}"
         owner: "{{ account.user }}"
         group: "{{ account.user }}"
-        mode: "{{ account.mode | default(0755) }}"
+        mode: "{{ account.mode | default('0755') }}"
       loop: "{{ local_accounts }}"
       loop_control:
         loop_var: account

roles/teardown/tasks/main.yml

@@ -77,7 +77,7 @@
 
 - name: Remove Clusters from Cloudera Manager (compute)
   include_role:
-    name: operations/delete_cluster
+    name: cloudera.cluster.operations.delete_cluster
   vars:
     stop_cluster_before_delete: true
     cluster: "{{ default_cluster_compute | combine(_cluster) }}"
@@ -95,7 +95,7 @@
 
 - name: Remove Clusters from Cloudera Manager (base)
   include_role:
-    name: operations/delete_cluster
+    name: cloudera.cluster.operations.delete_cluster
   vars:
     stop_cluster_before_delete: true
     cluster: "{{ default_cluster_base | combine(_cluster) }}"
@@ -113,7 +113,7 @@
 
 - name: Remove Clusters from Cloudera Manager (kts)
   include_role:
-    name: operations/delete_cluster
+    name: cloudera.cluster.operations.delete_cluster
   vars:
     stop_cluster_before_delete: true
     cluster: "{{ default_cluster_kts | combine(_cluster) }}"
@@ -132,7 +132,7 @@
 # delete the cms from cm if we are not tearing cm down
 - name: Remove CMS from Cloudera Manager
   import_role:
-    name: operations/delete_cms
+    name: cloudera.cluster.operations.delete_cms
   vars:
     stop_cms_before_delete: true
   run_once: true

roles/teardown/tasks/teardown_cdsw.yml

@@ -16,7 +16,7 @@
 
 - name: Generate merged configs (base)
   include_role:
-    name: config/cluster/base
+    name: cloudera.cluster.config.cluster.base
 
 - name: Stop the CDSW node
   shell: /opt/cloudera/parcels/CDSW/scripts/cdsw-stop-node.sh

roles/teardown/tasks/teardown_cluster.yml

@@ -16,12 +16,12 @@
 
 - name: Generate merged configs (base, compute)
   include_role:
-    name: config/cluster/base
+    name: cloudera.cluster.config.cluster.base
   when: cluster.type | default('base') in ['base', 'compute']
 
 - name: Generate merged configs (kts)
   include_role:
-    name: config/cluster/kts
+    name: cloudera.cluster.config.cluster.kts
   when: cluster.type | default('base') == 'kts'
 
 - name: Remove cluster service directories (base, compute)

roles/teardown/tasks/teardown_cms.yml

@@ -16,7 +16,7 @@
 
 - name: Generate merged configs
   include_role:
-    name: config/services/mgmt
+    name: cloudera.cluster.config.services.mgmt
 
 - name: Delete service database
   include_tasks: teardown_database.yml

roles/verify/definition/tasks/main.yml

@@ -249,6 +249,15 @@
         - "'zookeeper_tls_keystore' not in zookeeper_servicewide_configs"
         - "'zookeeper_tls_keystore' not in zookeeper_servicewide_configs"
 
+## Passwords
+- block:
+    - name: Ensure that the admin password is not part of the hostname(s)
+      assert:
+        that: groups.cluster is not search(cloudera_manager_admin_password)
+        success_msg: "The CM admin password is not part of the hostname"
+        fail_msg: "The CM admin password must not be part of the hostname"
+      when: cloudera_manager_admin_password is defined
+
 # Version specific
 
 # Add version specific issues here (e.g. Database versions)

roles/verify/parcels_and_roles/tasks/check_cluster.yml

@@ -16,7 +16,7 @@
 
 - name: Retrieve repository metadata
   include_role:
-    name: deployment/repometa
+    name: cloudera.cluster.deployment.repometa
   vars:
     repositories: "{{ cluster.repositories | default({}) }}"