Fix #117 : CDH to CDP Upgrade : YARN Queues are not migrated

docs/how-to/oracle.md

@@ -46,7 +46,7 @@ e.g. For Hive:
 ```
 HIVE:
   host: "{{ database_host }}"
-  port: "{{ database_type | default_database_port }}"
+  port: "{{ database_port | default_database_port }}"
   type: "{{ database_type }}"
   name: "{{ database_name }}"
   user: hive
@@ -65,14 +65,14 @@ clusters:
     databases:
       HIVE:
         host: "{{ database_host }}"
-        port: "{{ database_type | default_database_port }}"
+        port: "{{ database_port | default_database_port }}"
         type: "{{ database_type }}"
         name: "{{ database_name }}"
         user: hive
         password: "{{ database_default_password }}"
       OOZIE:
         host: "{{ database_host }}"
-        port: "{{ database_type | default_database_port }}"
+        port: "{{ database_port | default_database_port }}"
         type: "{{ database_type }}"
         name: "{{ database_name }}"
         user: oozie
@@ -104,7 +104,7 @@ mgmt:
   databases:
     REPORTSMANAGER:
       host: "{{ database_host }}"
-      port: "{{ database_type | default_database_port }}"
+      port: "{{ database_port | default_database_port }}"
       type: "{{ database_type }}"
       name: "{{ database_name }}"
       user: rman

galaxy.yml

@@ -23,7 +23,7 @@ namespace: cloudera
 name: cluster
 
 # The version of the collection. Must be compatible with semantic versioning
-version: 3.3.0
+version: 3.3.1
 
 # The path to the Markdown (.md) readme file. This path is relative to the root of the collection
 readme: README.md

roles/cloudera_manager/common/defaults/main.yml

@@ -27,6 +27,6 @@ cloudera_manager_database_type: "{{ database_type }}"
 cloudera_manager_database_name: scm
 cloudera_manager_database_user: scm
 cloudera_manager_database_password: changeme
-cloudera_manager_database_port: "{{ database_type | cloudera.cluster.default_database_port }}"
+cloudera_manager_database_port: "{{ database_port | cloudera.cluster.default_database_port }}"
 cloudera_manager_agent_lib_directory: /var/lib/cloudera-scm-agent
 cloudera_manager_cmf_java_opts_default: "-Xmx4G -XX:MaxPermSize=256m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp"

roles/config/cluster/base/templates/configs/inter-service-dependencies.j2

@@ -28,7 +28,6 @@ HIVE:
 {% if 'ATLAS' in cluster.services and not (cdh_cdp_upgrade|default(false)|bool) %}
     atlas_service: atlas
 {% endif %}
-    spark_on_yarn_service: spark_on_yarn
     zookeeper_service: zookeeper
 
 HIVE_ON_TEZ:
@@ -170,8 +169,12 @@ SPARK3_ON_YARN:
 {% if 'HBASE' in cluster.services %}
     hbase_service: hbase
 {% endif %}
-{% if 'HIVE' in cluster.services %}
-    hive_service: hive
+
+{% if 'LIVY_FOR_SPARK3' in cluster.services %}
+LIVY_FOR_SPARK3:
+  SERVICEWIDE:
+    yarn_service: yarn
+    spark3_on_yarn_service: spark3_on_yarn
 {% endif %}
 
 TEZ:

roles/config/cluster/base/vars/main.yml

@@ -46,7 +46,7 @@ custom_config_templates:
   - template: configs/kerberos-6.x-7.x.j2
     condition: "{{ cluster.security.kerberos | default(False) and cloudera_manager_version is version('6.0.0','>=') }}"
   - template: configs/kerberos-7.x.j2
-    condition: "{{ cluster.security.kerberos | default(False) and cloudera_manager_version is version('7.1.0','>=') }}"
+    condition: "{{ cluster.security.kerberos | default(False) and cloudera_runtime_version is version('7.1.0','>=') and (cloudera_runtime_pre_upgrade is undefined or cloudera_runtime_pre_upgrade is version('7.1.0','>=')) }}"
   - template: configs/trusted-realms.j2
     condition: "{{ cluster.security.kerberos | default(False) and auth_providers | default({}) | dict2items | json_query('[?value.type == `KERBEROS`]') | length > 0 }}"
 # Custom configurations for TLS

roles/config/cluster/common/defaults/main.yml

@@ -80,6 +80,7 @@ cluster_services_ordered:
   - SPARK3_ON_YARN
   - SPARK_ON_YARN
   - LIVY
+  - LIVY_FOR_SPARK3
   - FLUME
   - KS_INDEXER
   - LUNA_KMS

roles/deployment/cluster/templates/upgrade_scripts/fs2cs.j2

@@ -48,6 +48,6 @@ mkdir fs2cs
 cp ${FAIR_SCHEDULER} fs2cs/fair-scheduler.xml
 cp ${CORE_SITE} fs2cs/core-site.xml
 
-/opt/cloudera/parcels/CDH-{{ new_parcel_version }}/bin/yarn fs2cs ${RESOURCE_OPTS}  --no-terminal-rule-check --yarnsiteconfig ${YARN_SITE}  --fsconfig fs2cs/fair-scheduler.xml --output-directory {{ fs2cs_temp_dir }}
+/opt/cloudera/parcels/CDH-{{ new_parcel_version }}/bin/yarn fs2cs ${RESOURCE_OPTS}  --no-terminal-rule-check --yarnsiteconfig ${YARN_SITE} --fsconfig $(realpath fs2cs/fair-scheduler.xml) --output-directory {{ fs2cs_temp_dir }}
 
 exit $?

roles/deployment/definition/defaults/main.yml

@@ -31,84 +31,84 @@ local_temp_dir: '/tmp'
 database_defaults:
   ACTIVITYMONITOR:
     host: "{{ database_host }}"
-    port: "{{ database_type | cloudera.cluster.default_database_port }}"
+    port: "{{ database_port | cloudera.cluster.default_database_port }}"
     type: "{{ database_type }}"
     name: amon
     user: amon
     password: "{{ database_default_password }}"
   DAS:
     host: "{{ database_host }}"
-    port: "{{ database_type | cloudera.cluster.default_database_port }}"
+    port: "{{ database_port | cloudera.cluster.default_database_port }}"
     type: "{{ database_type }}"
     name: das
     user: das
     password: "{{ database_default_password }}"
   HIVE:
     host: "{{ database_host }}"
-    port: "{{ database_type | cloudera.cluster.default_database_port }}"
+    port: "{{ database_port | cloudera.cluster.default_database_port }}"
     type: "{{ database_type }}"
     name: metastore
     user: hive
     password: "{{ database_default_password }}"
   HUE:
     host: "{{ database_host }}"
-    port: "{{ database_type | cloudera.cluster.default_database_port }}"
+    port: "{{ database_port | cloudera.cluster.default_database_port }}"
     type: "{{ database_type }}"
     name: hue
     user: hue
     password: "{{ database_default_password }}"
   NAVIGATOR:
     host: "{{ database_host }}"
-    port: "{{ database_type | cloudera.cluster.default_database_port }}"
+    port: "{{ database_port | cloudera.cluster.default_database_port }}"
     type: "{{ database_type }}"
     name: nav
     user: nav
     password: "{{ database_default_password }}"
   NAVIGATORMETASERVER:
     host: "{{ database_host }}"
-    port: "{{ database_type | cloudera.cluster.default_database_port }}"
+    port: "{{ database_port | cloudera.cluster.default_database_port }}"
     type: "{{ database_type }}"
     name: navms
     user: navms
     password: "{{ database_default_password }}"
   OOZIE:
     host: "{{ database_host }}"
-    port: "{{ database_type | cloudera.cluster.default_database_port }}"
+    port: "{{ database_port | cloudera.cluster.default_database_port }}"
     type: "{{ database_type }}"
     name: oozie
     user: oozie
     password: "{{ database_default_password }}"
   RANGER:
     host: "{{ database_host }}"
-    port: "{{ database_type | cloudera.cluster.default_database_port }}"
+    port: "{{ database_port | cloudera.cluster.default_database_port }}"
     type: "{{ database_type }}"
     name: ranger
     user: rangeradmin
     password: "{{ database_default_password }}"
   REPORTSMANAGER:
     host: "{{ database_host }}"
-    port: "{{ database_type | cloudera.cluster.default_database_port }}"
+    port: "{{ database_port | cloudera.cluster.default_database_port }}"
     type: "{{ database_type }}"
     name: rman
     user: rman
     password: "{{ database_default_password }}"
   SCHEMAREGISTRY:
     host: "{{ database_host }}"
-    port: "{{ database_type | cloudera.cluster.default_database_port }}"
+    port: "{{ database_port | cloudera.cluster.default_database_port }}"
     type: "{{ database_type }}"
     name: schemaregistry
     user: schemaregistry
     password: "{{ database_default_password }}"
   STREAMS_MESSAGING_MANAGER:
     host: "{{ database_host }}"
-    port: "{{ database_type | cloudera.cluster.default_database_port }}"
+    port: "{{ database_port | cloudera.cluster.default_database_port }}"
     type: "{{ database_type }}"
     name: streamsmsgmgr
     user: streamsmsgmgr
     password: "{{ database_default_password }}"
   SENTRY:
     host: "{{ database_host }}"
-    port: "{{ database_type | cloudera.cluster.default_database_port }}"
+    port: "{{ database_port | cloudera.cluster.default_database_port }}"
     type: "{{ database_type }}"
     name: sentry
     user: sentry

roles/deployment/repometa/templates/role_mappings/spark3.j2

@@ -1,3 +1,6 @@
 SPARK3_ON_YARN:
   - GATEWAY
   - SPARK3_YARN_HISTORY_SERVER
+LIVY_FOR_SPARK3:
+  - GATEWAY
+  - LIVY_SERVER_FOR_SPARK3

roles/infrastructure/ca_server/vars/RedHat.yml

@@ -13,5 +13,4 @@
 # limitations under the License.
 
 ---
-
-ca_server_root_key_cipher: aes256
+ca_server_root_key_cipher: auto

roles/security/tls_generate_csr/tasks/main.yml

@@ -43,7 +43,7 @@
       -keystore {{ tls_keystore_path }}
       -keysize 2048
       -dname "{{ keytool_subject }}"
-      -ext san=dns:{{ inventory_hostname }}
+      -ext "san=dns:{{ inventory_hostname }}{% if subject_alternative_names | default([]) %},dns:{{ subject_alternative_names | join(',dns:') }}{% endif %}"
       -storetype JKS
       -storepass {{ tls_keystore_password }}
       -keypass {{ tls_keystore_password }}

roles/security/tls_generate_csr/templates/csr.cnf.j2

@@ -19,5 +19,5 @@ subjectAltName = @alt_names
 [alt_names]
 DNS.1 = {{ inventory_hostname }}
 {% for san in subject_alternative_names | default([]) %}
-DNS.{{ loop.index + 1 }} =  {{ san }}
+DNS.{{ loop.index + 1 }} = {{ san }}
 {% endfor %}