diff --git a/galaxy.yml b/galaxy.yml index 05ee0601..bff7f817 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -55,7 +55,13 @@ tags: [] # collection label 'namespace.name'. The value is a version range # L(specifiers,https://python-semanticversion.readthedocs.io/en/latest/#requirement-specification). Multiple version # range specifiers can be set and are separated by ',' -dependencies: {} +dependencies: + 'ansible.posix': '>=1.2.0' + 'community.crypto': '>=1.6.0' + 'community.general': '>=2.4.0' + 'community.mysql': '>=1.3.0' + 'community.postgresql': '>=1.3.0' + 'freeipa.ansible_freeipa': '>=0.3.5' # The URL of the originating SCM repository repository: http://github.com/cloudera-labs diff --git a/roles/cloudera_manager/database/defaults/main.yml b/roles/cloudera_manager/database/defaults/main.yml index 9bf17031..8cdd7cb8 100644 --- a/roles/cloudera_manager/database/defaults/main.yml +++ b/roles/cloudera_manager/database/defaults/main.yml @@ -13,5 +13,5 @@ # limitations under the License. --- -cloudera_manager_database_prepare_script: /opt/cloudera/cm/schema/scm_prepare_database.sh +cloudera_manager_database_prepare_script: "{{ '/opt/cloudera/cm/schema/scm_prepare_database.sh' if cloudera_manager_version is version('6.0.0','>=') else '/usr/share/cmf/schema/scm_prepare_database.sh' }}" cloudera_manager_database_ranger_script: /opt/cloudera/cm/bin/gen_embedded_ranger_db.sh diff --git a/roles/cloudera_manager/external_auth/tasks/main.yml b/roles/cloudera_manager/external_auth/tasks/main.yml index 2a3ecf6d..2f48076d 100644 --- a/roles/cloudera_manager/external_auth/tasks/main.yml +++ b/roles/cloudera_manager/external_auth/tasks/main.yml @@ -17,15 +17,15 @@ - name: Select external auth provider details set_fact: auth_provider: "{{ auth_providers[cloudera_manager_external_auth.provider] }}" - when: cloudera_manager_external_auth.provider is defined + when: cloudera_manager_external_auth.provider is defined and cloudera_manager_version is version('6.0.0','>=') - name: Set Cloudera Manager external auth configs include_role: - name: cloudera_manager/config + name: cloudera.cluster.cloudera_manager.config vars: api_config_keys_uppercase: True api_configs: "{{ lookup('template', 'external_auth_configs.j2') | from_yaml }}" - when: auth_provider is defined + when: auth_provider is defined and cloudera_manager_version is version('6.0.0','>=') - block: @@ -60,4 +60,4 @@ notify: - wait cloudera-scm-server - when: cloudera_manager_external_auth.role_mappings is defined + when: cloudera_manager_external_auth.role_mappings is defined and cloudera_manager_version is version('6.0.0','>=') diff --git a/roles/cloudera_manager/external_auth/templates/external_auth_configs.j2 b/roles/cloudera_manager/external_auth/templates/external_auth_configs.j2 index bd23c832..18cf21d5 100644 --- a/roles/cloudera_manager/external_auth/templates/external_auth_configs.j2 +++ b/roles/cloudera_manager/external_auth/templates/external_auth_configs.j2 @@ -20,8 +20,10 @@ LDAP_URL: {{ auth_provider.ldap_url | default(None) }} LDAP_USER_SEARCH_BASE: {{ auth_provider.ldap_search_base.user | default(None) }} LDAP_USER_SEARCH_FILTER: "({{ auth_provider.ldap_attribute.user | default('sAMAccountName') }}={0})" NT_DOMAIN: {{ auth_provider.domain | default(None) }} +{% if cloudera_manager_version is version('7.1.0','>=') %} FRONTEND_URL: {{ frontend_url | default(None) }} PROXYUSER_KNOX_GROUPS: "{{ proxyuser_knox_groups | default('*') }}" PROXYUSER_KNOX_USERS: "{{ proxyuser_knox_users | default('*') }}" PROXYUSER_KNOX_HOSTS: "{{ proxyuser_knox_hosts | default('*') }}" PROXYUSER_KNOX_PRINCIPAL: "{{ proxyuser_knox_principal | default('knox') }}" +{% endif %} \ No newline at end of file diff --git a/roles/config/cluster/base/templates/configs/databases-7.1.0.j2 b/roles/config/cluster/base/templates/configs/databases-7.1.0.j2 index ce62c5f2..81668d8e 100644 --- a/roles/config/cluster/base/templates/configs/databases-7.1.0.j2 +++ b/roles/config/cluster/base/templates/configs/databases-7.1.0.j2 @@ -15,6 +15,14 @@ RANGER: ranger_database_name: {{ databases.RANGER.name }} ranger_database_user: {{ databases.RANGER.user }} ranger_database_password: {{ databases.RANGER.password }} +RANGER_RMS: + SERVICEWIDE: + ranger_rms_database_host: {{ databases.RANGER.host }} + ranger_rms_database_port: {{ databases.RANGER.port }} + ranger_rms_database_type: {{ databases.RANGER.type | cloudera.cluster.format_database_type }} + ranger_rms_database_name: {{ databases.RANGER.name }} + ranger_rms_database_user: {{ databases.RANGER.user }} + ranger_rms_database_password: {{ databases.RANGER.password }} SCHEMAREGISTRY: SERVICEWIDE: database_host: {{ databases.SCHEMAREGISTRY.host }} diff --git a/roles/config/cluster/base/templates/configs/kerberos-7.x.j2 b/roles/config/cluster/base/templates/configs/kerberos-7.x.j2 new file mode 100644 index 00000000..744efc08 --- /dev/null +++ b/roles/config/cluster/base/templates/configs/kerberos-7.x.j2 @@ -0,0 +1,4 @@ +--- +HBASE: + SERVICEWIDE: + hadoop_secure_web_ui: true \ No newline at end of file diff --git a/roles/config/cluster/base/templates/configs/logdirs-7.1.0.j2 b/roles/config/cluster/base/templates/configs/logdirs-7.1.0.j2 index ce841ede..077fe350 100644 --- a/roles/config/cluster/base/templates/configs/logdirs-7.1.0.j2 +++ b/roles/config/cluster/base/templates/configs/logdirs-7.1.0.j2 @@ -5,6 +5,8 @@ ATLAS: log_dir: "{{ log_base }}/atlas" ranger_atlas_plugin_hdfs_audit_spool_directory: "{{ log_base }}/atlas/audit/hdfs/spool" ranger_atlas_plugin_solr_audit_spool_directory: "{{ log_base }}/atlas/audit/solr/spool" + GATEWAY: + log_dir: "{{ log_base }}/atlas" CORE_SETTINGS: STORAGEOPERATIONS: storageoperations_log_dir: "{{ log_base }}/" diff --git a/roles/config/cluster/base/templates/configs/ranger.j2 b/roles/config/cluster/base/templates/configs/ranger.j2 index 9d73dfb5..aecdf0d3 100644 --- a/roles/config/cluster/base/templates/configs/ranger.j2 +++ b/roles/config/cluster/base/templates/configs/ranger.j2 @@ -2,6 +2,9 @@ HDFS: SERVICEWIDE: enable_ranger_authorization: true +{% if 'RANGER_RMS' in cluster.services %} + ranger_security_safety_valve: "ranger.plugin.hdfs.chained.servicescm_hiveRanger RMS related configranger.plugin.hdfs.chained.services.cm_hive.implorg.apache.ranger.chainedplugin.hdfs.hive.RangerHdfsHiveChainedPluginRanger RMS related config" +{% endif %} RANGER: SERVICEWIDE: keyadmin_user_password: {{ ranger_keyadmin_user_password | default('password123') }} diff --git a/roles/config/cluster/base/templates/configs/tls-7.1.0.j2 b/roles/config/cluster/base/templates/configs/tls-7.1.0.j2 index b127b191..2d1f7d2c 100644 --- a/roles/config/cluster/base/templates/configs/tls-7.1.0.j2 +++ b/roles/config/cluster/base/templates/configs/tls-7.1.0.j2 @@ -96,6 +96,10 @@ KNOX: ssl_enabled: true ssl_server_keystore_location: {{ tls_keystore_path_generic }} ssl_server_keystore_password: {{ tls_keystore_password }} +KUDU: + MASTER: + ssl_client_truststore_location: {{ tls_truststore_path }} + ssl_client_truststore_password: {{ tls_truststore_password }} LIVY: GATEWAY: ssl_client_truststore_location: {{ tls_truststore_path }} diff --git a/roles/config/cluster/base/templates/configs/tls-7.1.4.j2 b/roles/config/cluster/base/templates/configs/tls-7.1.4.j2 index d250b90b..62e7f266 100644 --- a/roles/config/cluster/base/templates/configs/tls-7.1.4.j2 +++ b/roles/config/cluster/base/templates/configs/tls-7.1.4.j2 @@ -5,3 +5,10 @@ OOZIE: oozie_zookeeper_https_keystore_password: {{ tls_keystore_password }} oozie_zookeeper_https_truststore_file: {{ tls_truststore_path }} oozie_zookeeper_https_truststore_password: {{ tls_truststore_password }} +RANGER_RMS: + RANGER_RMS_SERVER: + ssl_client_truststore_location: {{ tls_truststore_path }} + ssl_client_truststore_password: {{ tls_truststore_password }} + ssl_enabled: true + ssl_server_keystore_location: {{ tls_keystore_path_generic }} + ssl_server_keystore_password: {{ tls_keystore_password }} \ No newline at end of file diff --git a/roles/config/cluster/base/templates/configs/tls-7.3.1.j2 b/roles/config/cluster/base/templates/configs/tls-7.3.1.j2 new file mode 100644 index 00000000..bf0545d5 --- /dev/null +++ b/roles/config/cluster/base/templates/configs/tls-7.3.1.j2 @@ -0,0 +1,4 @@ +--- +OZONE: + OZONE_PROMETHEUS: + ozone.prometheus.ca.file: {{ tls_chain_path }} \ No newline at end of file diff --git a/roles/config/cluster/base/templates/configs/tls.j2 b/roles/config/cluster/base/templates/configs/tls.j2 index 14b6498d..178e5278 100644 --- a/roles/config/cluster/base/templates/configs/tls.j2 +++ b/roles/config/cluster/base/templates/configs/tls.j2 @@ -123,8 +123,6 @@ KS_INDEXER: keystore_indexer_truststore_password: {{ tls_truststore_password }} KUDU: KUDU_MASTER: - ssl_client_truststore_location: {{ tls_truststore_path }} - ssl_client_truststore_password: {{ tls_truststore_password }} ssl_enabled: True ssl_server_ca_certificate_location: {{ tls_chain_path }} ssl_server_certificate_location: {{ tls_cert_path_generic }} diff --git a/roles/config/cluster/base/templates/configs/varlib-7.1.0.j2 b/roles/config/cluster/base/templates/configs/varlib-7.1.0.j2 index 43e5123a..5ef85239 100644 --- a/roles/config/cluster/base/templates/configs/varlib-7.1.0.j2 +++ b/roles/config/cluster/base/templates/configs/varlib-7.1.0.j2 @@ -63,7 +63,7 @@ OZONE: ozone.metadata.dirs: "{{ varlib_base }}/hadoop-ozone/om/ozone-metadata" ozone.om.db.dirs: "{{ varlib_base }}/hadoop-ozone/om/data" ozone.om.ratis.storage.dir: "{{ varlib_base }}/hadoop-ozone/om/ratis" - PROMETHEUS: + OZONE_PROMETHEUS: ozone.prometheus.db.dir: "{{ varlib_base }}/hadoop-ozone/prometheus/data" OZONE_RECON: ozone.metadata.dirs: "{{ varlib_base }}/hadoop-ozone/recon/ozone-metadata" diff --git a/roles/config/cluster/base/vars/main.yml b/roles/config/cluster/base/vars/main.yml index 37bda732..7e5af57c 100644 --- a/roles/config/cluster/base/vars/main.yml +++ b/roles/config/cluster/base/vars/main.yml @@ -42,6 +42,8 @@ custom_config_templates: condition: "{{ cluster.security.kerberos | default(False) and (cloudera_manager_version is version('6.0.0','<') or cluster.type | default('base') == 'compute') }}" - template: configs/kerberos-6.x-7.x.j2 condition: "{{ cluster.security.kerberos | default(False) and cloudera_manager_version is version('6.0.0','>=') }}" + - template: configs/kerberos-7.x.j2 + condition: "{{ cluster.security.kerberos | default(False) and cloudera_manager_version is version('7.1.0','>=') }}" - template: configs/trusted-realms.j2 condition: "{{ cluster.security.kerberos | default(False) and auth_providers | default({}) | dict2items | json_query('[?value.type == `KERBEROS`]') | length > 0 }}" # Custom configurations for TLS @@ -53,6 +55,8 @@ custom_config_templates: condition: "{{ cluster.security.tls | default(False) and cloudera_runtime_version is version('7.1.0','>=') }}" - template: configs/tls-7.1.4.j2 condition: "{{ cluster.security.tls | default(False) and cloudera_runtime_version is version('7.1.4','>=') }}" + - template: configs/tls-7.3.1.j2 + condition: "{{ cluster.security.tls | default(False) and cloudera_manager_version is version('7.3.1', '>=') }}" # Custom configurations for Cloudera Streams Processing components on CDH 6.x - template: configs/schemaregistry.j2 condition: >- diff --git a/roles/config/cluster/kts/tasks/main.yml b/roles/config/cluster/kts/tasks/main.yml index 32462dd1..b7ff1bdb 100644 --- a/roles/config/cluster/kts/tasks/main.yml +++ b/roles/config/cluster/kts/tasks/main.yml @@ -16,7 +16,7 @@ - name: Retrieve repository metadata include_role: - name: deployment/repometa + name: cloudera.cluster.deployment.repometa vars: repositories: "{{ cluster.repositories | default({}) }}" diff --git a/roles/deployment/cluster/tasks/create_base.yml b/roles/deployment/cluster/tasks/create_base.yml index b476c026..ac080232 100644 --- a/roles/deployment/cluster/tasks/create_base.yml +++ b/roles/deployment/cluster/tasks/create_base.yml @@ -16,11 +16,11 @@ - name: Generate complete base cluster configs include_role: - name: config/cluster/base + name: cloudera.cluster.config.cluster.base - name: Create databases and users include_role: - name: deployment/databases + name: cloudera.cluster.deployment.databases vars: services: "{{ cluster.services | default({}) }}" diff --git a/roles/deployment/cluster/tasks/create_kts.yml b/roles/deployment/cluster/tasks/create_kts.yml index dea39fc4..e2e0303e 100644 --- a/roles/deployment/cluster/tasks/create_kts.yml +++ b/roles/deployment/cluster/tasks/create_kts.yml @@ -16,7 +16,7 @@ - name: Generate complete kts cluster configs include_role: - name: config/cluster/kts + name: cloudera.cluster.config.cluster.kts - name: Generate cluster template file template: diff --git a/roles/deployment/cluster/tasks/main.yml b/roles/deployment/cluster/tasks/main.yml index 87d4d41e..a5c45f06 100644 --- a/roles/deployment/cluster/tasks/main.yml +++ b/roles/deployment/cluster/tasks/main.yml @@ -20,7 +20,7 @@ - name: Apply "all hosts" configs include_role: - name: cloudera_manager/config + name: cloudera.cluster.cloudera_manager.config vars: api_config_keys_uppercase: False api_config_endpoint: cm/allHosts/config diff --git a/roles/deployment/repometa/templates/role_mappings/cdh7.j2 b/roles/deployment/repometa/templates/role_mappings/cdh7.j2 index 0b82adb4..d5dc2a63 100644 --- a/roles/deployment/repometa/templates/role_mappings/cdh7.j2 +++ b/roles/deployment/repometa/templates/role_mappings/cdh7.j2 @@ -1,6 +1,7 @@ ADLS_CONNECTOR: ATLAS: - ATLAS_SERVER + - GATEWAY AWS_S3: CORE_SETTINGS: - GATEWAY @@ -29,7 +30,6 @@ HDFS: HIVE: - GATEWAY - HIVEMETASTORE - - HIVESERVER2 HIVE_ON_TEZ: - GATEWAY - HIVESERVER2 @@ -92,6 +92,8 @@ RANGER_KMS_KTS: - RANGER_KMS_SERVER_KTS RANGER_RAZ: - RANGER_RAZ_SERVER +RANGER_RMS: + - RANGER_RMS_SERVER SCHEMAREGISTRY: - GATEWAY - SCHEMA_REGISTRY_SERVER diff --git a/roles/deployment/services/kms/tasks/create_kms.yml b/roles/deployment/services/kms/tasks/create_kms.yml index e6b07f80..372a7f0c 100644 --- a/roles/deployment/services/kms/tasks/create_kms.yml +++ b/roles/deployment/services/kms/tasks/create_kms.yml @@ -75,7 +75,7 @@ - name: Generate KMS configs include_role: - name: config/services/kms + name: cloudera.cluster.config.services.kms - name: Create KMS service cloudera.cluster.cm_api: diff --git a/roles/prereqs/os/tasks/main.yml b/roles/prereqs/os/tasks/main.yml index e6b3489a..443e70c1 100644 --- a/roles/prereqs/os/tasks/main.yml +++ b/roles/prereqs/os/tasks/main.yml @@ -34,7 +34,7 @@ loop: "{{ kernel_flags }}" loop_control: loop_var: flag - when: not(ansible_virtualization_type == "docker" and ansible_virtualization_role == "guest") + when: not((ansible_virtualization_type == "docker" or ansible_virtualization_type == "container") and ansible_virtualization_role == "guest") - name: Populate service facts service_facts: @@ -103,4 +103,4 @@ - name: Apply OS-specific configurations include_tasks: file: "main-{{ ansible_os_family }}.yml" - when: not(ansible_virtualization_type == "docker" and ansible_virtualization_role == "guest") + when: not((ansible_virtualization_type == "docker" or ansible_virtualization_type == "container") and ansible_virtualization_role == "guest") diff --git a/roles/prereqs/user_accounts/tasks/main.yml b/roles/prereqs/user_accounts/tasks/main.yml index 16eb6880..7f126be8 100644 --- a/roles/prereqs/user_accounts/tasks/main.yml +++ b/roles/prereqs/user_accounts/tasks/main.yml @@ -49,7 +49,7 @@ path: "{{ account.home }}" owner: "{{ account.user }}" group: "{{ account.user }}" - mode: "{{ account.mode | default(0755) }}" + mode: "{{ account.mode | default('0755') }}" loop: "{{ local_accounts }}" loop_control: loop_var: account diff --git a/roles/teardown/tasks/main.yml b/roles/teardown/tasks/main.yml index c636ce11..695fe6cf 100644 --- a/roles/teardown/tasks/main.yml +++ b/roles/teardown/tasks/main.yml @@ -77,7 +77,7 @@ - name: Remove Clusters from Cloudera Manager (compute) include_role: - name: operations/delete_cluster + name: cloudera.cluster.operations.delete_cluster vars: stop_cluster_before_delete: true cluster: "{{ default_cluster_compute | combine(_cluster) }}" @@ -95,7 +95,7 @@ - name: Remove Clusters from Cloudera Manager (base) include_role: - name: operations/delete_cluster + name: cloudera.cluster.operations.delete_cluster vars: stop_cluster_before_delete: true cluster: "{{ default_cluster_base | combine(_cluster) }}" @@ -113,7 +113,7 @@ - name: Remove Clusters from Cloudera Manager (kts) include_role: - name: operations/delete_cluster + name: cloudera.cluster.operations.delete_cluster vars: stop_cluster_before_delete: true cluster: "{{ default_cluster_kts | combine(_cluster) }}" @@ -132,7 +132,7 @@ # delete the cms from cm if we are not tearing cm down - name: Remove CMS from Cloudera Manager import_role: - name: operations/delete_cms + name: cloudera.cluster.operations.delete_cms vars: stop_cms_before_delete: true run_once: true diff --git a/roles/teardown/tasks/teardown_cdsw.yml b/roles/teardown/tasks/teardown_cdsw.yml index f5044b0a..e5eb7e52 100644 --- a/roles/teardown/tasks/teardown_cdsw.yml +++ b/roles/teardown/tasks/teardown_cdsw.yml @@ -16,7 +16,7 @@ - name: Generate merged configs (base) include_role: - name: config/cluster/base + name: cloudera.cluster.config.cluster.base - name: Stop the CDSW node shell: /opt/cloudera/parcels/CDSW/scripts/cdsw-stop-node.sh diff --git a/roles/teardown/tasks/teardown_cluster.yml b/roles/teardown/tasks/teardown_cluster.yml index 6b1c2047..6ab5b189 100644 --- a/roles/teardown/tasks/teardown_cluster.yml +++ b/roles/teardown/tasks/teardown_cluster.yml @@ -16,12 +16,12 @@ - name: Generate merged configs (base, compute) include_role: - name: config/cluster/base + name: cloudera.cluster.config.cluster.base when: cluster.type | default('base') in ['base', 'compute'] - name: Generate merged configs (kts) include_role: - name: config/cluster/kts + name: cloudera.cluster.config.cluster.kts when: cluster.type | default('base') == 'kts' - name: Remove cluster service directories (base, compute) diff --git a/roles/teardown/tasks/teardown_cms.yml b/roles/teardown/tasks/teardown_cms.yml index cf0a4591..f2a80044 100644 --- a/roles/teardown/tasks/teardown_cms.yml +++ b/roles/teardown/tasks/teardown_cms.yml @@ -16,7 +16,7 @@ - name: Generate merged configs include_role: - name: config/services/mgmt + name: cloudera.cluster.config.services.mgmt - name: Delete service database include_tasks: teardown_database.yml diff --git a/roles/verify/definition/tasks/main.yml b/roles/verify/definition/tasks/main.yml index 98b2cef0..5a3b92b7 100644 --- a/roles/verify/definition/tasks/main.yml +++ b/roles/verify/definition/tasks/main.yml @@ -249,6 +249,15 @@ - "'zookeeper_tls_keystore' not in zookeeper_servicewide_configs" - "'zookeeper_tls_keystore' not in zookeeper_servicewide_configs" +## Passwords +- block: + - name: Ensure that the admin password is not part of the hostname(s) + assert: + that: groups.cluster is not search(cloudera_manager_admin_password) + success_msg: "The CM admin password is not part of the hostname" + fail_msg: "The CM admin password must not be part of the hostname" + when: cloudera_manager_admin_password is defined + # Version specific # Add version specific issues here (e.g. Database versions) diff --git a/roles/verify/parcels_and_roles/tasks/check_cluster.yml b/roles/verify/parcels_and_roles/tasks/check_cluster.yml index c397054f..6bff0b24 100644 --- a/roles/verify/parcels_and_roles/tasks/check_cluster.yml +++ b/roles/verify/parcels_and_roles/tasks/check_cluster.yml @@ -16,7 +16,7 @@ - name: Retrieve repository metadata include_role: - name: deployment/repometa + name: cloudera.cluster.deployment.repometa vars: repositories: "{{ cluster.repositories | default({}) }}"