Add selectable distribution support for cloudera.cluster

README.md

@@ -23,7 +23,9 @@ This Ansible Collection is designed to work hand-in-hand with [Cloudera Deploy](
 ### Operating Systems
 
 - Red Hat / CentOS 7.x
-- Ubuntu 18.04.04 LTS (Bionic Beaver)
+- Red Hat / CentOS 8.x
+- Ubuntu 18.04 LTS (Bionic Beaver)
+- Ubuntu 20.04 LTS (Focal Fossa)
 
 Active development is focused on **CDP Private Cloud Base** (formerly CDP-DC) deployments and their respective platform compatibility matrices.
 

plugins/filter/filters.py

@@ -11,8 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from __future__ import (absolute_import, division, print_function)
+from __future__ import absolute_import, division, print_function
+from typing import Optional
 import re
 
 __metaclass__ = type
@@ -74,15 +74,24 @@ def _flatten_dict_list(i, l, parents):
 
         return state
 
-    def extract_products_from_manifests(self, manifests):
+    def extract_products_from_manifests(self, manifests, os_distribution: Optional[str] = None):
         products = dict()
         for manifest in manifests:
-            for parcel in manifest['parcels']:
+            for parcel in manifest["parcels"]:
+                # fetch the full parcel name from the manifest
+                full_parcel_name = str(parcel["parcelName"])
+                # the parcel OS distribution is between the last "-" and the ".parcel" extension
+                parcel_os_distribution = full_parcel_name[
+                    full_parcel_name.rindex("-")
+                    + 1: full_parcel_name.rindex(".parcel")
+                ]
                 # take first parcel, strip off OS name and file extension
-                parcel_name = re.sub(r"-[a-z0-9]+\.parcel$", "", str(parcel['parcelName']))
+                parcel_name = re.sub(r"-[a-z0-9]+\.parcel$", "", full_parcel_name)
                 # the product name is before the first dash
-                product = parcel_name[:parcel_name.index("-")]
-                if product not in products:
+                product = parcel_name[: parcel_name.index("-")]
+                if product not in products and (
+                    os_distribution == parcel_os_distribution or os_distribution is None
+                ):
                     # the version string is everything after the first dash
                     version = parcel_name[parcel_name.index("-") + 1:]
                     products[product] = version

roles/cloudera_manager/agent/tasks/main.yml

@@ -37,7 +37,7 @@
 # Note on the CM server this will perform an upgrade of the server if it hasn't already been upgraded
 - name: Install or Upgrade Cloudera Manager agent packages
   ansible.builtin.package:
-    lock_timeout: 60
+    lock_timeout: "{{ (ansible_os_family == 'RedHat') | ternary(60, omit) }}"
     name: cloudera-manager-agent
     update_cache: yes
     state: latest

roles/cloudera_manager/daemons/tasks/main.yml

@@ -16,7 +16,7 @@
 
 - name: Install Cloudera Manager daemons package
   ansible.builtin.package:
-    lock_timeout: 180
+    lock_timeout: "{{ (ansible_os_family == 'RedHat') | ternary(180, omit) }}"
     name: cloudera-manager-daemons
     update_cache: yes
     state: present

roles/cloudera_manager/database/tasks/embedded.yml

@@ -16,7 +16,7 @@
 
 - name: Install Cloudera Manager embedded database
   ansible.builtin.package:
-    lock_timeout: 60
+    lock_timeout: "{{ (ansible_os_family == 'RedHat') | ternary(60, omit) }}"
     name:
     - cloudera-manager-server-db-2
     state: present

roles/cloudera_manager/database/tasks/external.yml

@@ -14,7 +14,7 @@
 
 ---
 
-- name: Create Cloudera Manager database user
+- name: Create Cloudera Manager database user for Postgres
   postgresql_user:
     name: "{{ cloudera_manager_database_user }}"
     password: "{{ cloudera_manager_database_password }}"
@@ -39,7 +39,7 @@
   - cloudera_manager_database_type == 'postgresql'
   - cloudera_manager_database_host in groups.db_server | default([])
 
-- name: Create Cloudera Manager database user
+- name: Create Cloudera Manager database user for MySQL
   mysql_user:
     name: "{{ cloudera_manager_database_user }}"
     password: "{{ cloudera_manager_database_password }}"
@@ -66,7 +66,7 @@
 
 - name: Prepare Cloudera Manager Server external database
   command: |
-    {{ cloudera_manager_database_prepare_script }}  -f --host {{ cloudera_manager_database_host }} --port {{ cloudera_manager_database_port }}
+    {{ cloudera_manager_database_prepare_script }} -f --host {{ cloudera_manager_database_host }} --port {{ cloudera_manager_database_port }}
     {{ cloudera_manager_database_type | regex_replace('mariadb', 'mysql') }}
     {{ cloudera_manager_database_name }}
     {{ cloudera_manager_database_user }}

roles/cloudera_manager/repo/defaults/main.yml

@@ -14,6 +14,6 @@
 
 ---
 cloudera_archive_base_url: https://archive.cloudera.com
-cloudera_manager_version: 7.1.4
+cloudera_manager_version: 7.4.4
 
 install_repo_on_host: yes

roles/cloudera_manager/repo/tasks/main-Debian.yml

@@ -18,18 +18,18 @@
   set_fact:
     cloudera_manager_repo_url_with_creds: "{{ cloudera_manager_repo_url | regex_replace('^(?P<proto>http[s]?://)','\\g<proto>' + cloudera_manager_repo_username|string + ':' + cloudera_manager_repo_password|string + '@')}}"
     cloudera_manager_repo_key_with_creds: "{{ cloudera_manager_repo_key | regex_replace('^(?P<proto>http[s]?://)','\\g<proto>' + cloudera_manager_repo_username|string + ':' + cloudera_manager_repo_password|string + '@')}}"
-  no_log: cloudera_manager_repo_username is defined
+  no_log: yes
   when: cloudera_manager_repo_username is defined
 
 - name: Add Cloudera Manager apt repository key
   apt_key:
     url: "{{ cloudera_manager_repo_key_with_creds | default(cloudera_manager_repo_key) }}"
     state: present
-  no_log: cloudera_manager_repo_username is defined
+  no_log: yes
 
 - name: Add Cloudera Manager apt repository
   apt_repository:
     repo: "deb [arch=amd64] {{ cloudera_manager_repo_url_with_creds | default(cloudera_manager_repo_url) }} {{ cloudera_manager_repo_apt_codename }} {{ cloudera_manager_repo_apt_component }}"
     filename: cloudera-manager
-  no_log: cloudera_manager_repo_username is defined
+  no_log: yes
 

roles/cloudera_manager/repo/tasks/main.yml

@@ -14,10 +14,9 @@
 
 ---
 
-- name: Include variables for not-Debian in cm6/7
+- name: Include variables
   include_vars:
     file: "{{ ansible_os_family }}.yml"
-  when: ansible_os_family != "Debian"
 
 - name: Correct repo URL for Redhat with cm5
   ansible.builtin.set_fact:
@@ -26,11 +25,6 @@
     - ansible_os_family != "Debian"
     - cloudera_manager_version.split('.')[0] == "5"
 
-- name: Include variables
-  include_vars:
-    file: "{{ ansible_distribution }}{{ ansible_distribution_major_version }}.yml"
-  when: ansible_os_family == "Debian"
-
 - name: Define Cloudera Manager trial repository details
   set_fact:
     cloudera_manager_repo_url: "{{ __cloudera_manager_repo_url_trial }}"

roles/cloudera_manager/repo/vars/Ubuntu18.yml → roles/cloudera_manager/repo/vars/Debian.yml

@@ -14,8 +14,8 @@
 
 ---
 __cloudera_manager_major_version: "{{ cloudera_manager_version.split('.')[0] }}"
-__cloudera_manager_repo_url_trial: "{{ cloudera_archive_base_url }}/cm{{ __cloudera_manager_major_version }}/{{ cloudera_manager_version }}/ubuntu1804/apt"
-__cloudera_manager_repo_url_paywall: "{{ cloudera_archive_base_url }}/p/cm{{ __cloudera_manager_major_version }}/{{ cloudera_manager_version }}/ubuntu1804/apt"
+__cloudera_manager_repo_url_trial: "{{ cloudera_archive_base_url }}/cm{{ __cloudera_manager_major_version }}/{{ cloudera_manager_version }}/{{ ansible_distribution | lower }}{{ ansible_distribution_version | replace('.','') }}/apt"
+__cloudera_manager_repo_url_paywall: "{{ cloudera_archive_base_url }}/p/cm{{ __cloudera_manager_major_version }}/{{ cloudera_manager_version }}/{{ ansible_distribution | lower }}{{ ansible_distribution_version | replace('.','') }}/apt"
 __cloudera_manager_repo_key_filename: archive.key
 __cloudera_manager_repo_key_trial: "{{ __cloudera_manager_repo_url_trial }}/{{ __cloudera_manager_repo_key_filename }}"
 __cloudera_manager_repo_key_paywall: "{{ __cloudera_manager_repo_url_paywall }}/{{ __cloudera_manager_repo_key_filename }}"

roles/cloudera_manager/repo/vars/RedHat.yml

@@ -13,9 +13,14 @@
 # limitations under the License.
 
 ---
+__cloudera_manager_distro_name: "{{ ansible_os_family | lower }}{{ ansible_distribution_major_version }}"
 __cloudera_manager_major_version: "{{ cloudera_manager_version.split('.')[0] }}"
-__cloudera_manager_repo_url_trial: "{{ cloudera_archive_base_url | regex_replace('/?$','') }}/cm{{ __cloudera_manager_major_version }}/{{ cloudera_manager_version }}/redhat7/yum"
-__cloudera_manager_repo_url_paywall: "{{ cloudera_archive_base_url | regex_replace('/?$','') }}/p/cm{{ __cloudera_manager_major_version }}/{{ cloudera_manager_version }}/redhat7/yum"
+__cloudera_manager_cm5_path: "{{ ansible_os_family | lower }}/{{ ansible_distribution_major_version }}/x86_64/cm/{{ cloudera_manager_version }}"
+__cloudera_manager_cm6_path: "{{ cloudera_manager_version }}/{{ __cloudera_manager_distro_name }}/yum"
+
+__cloudera_manager_repo_url_trial: "{{ cloudera_archive_base_url | regex_replace('/?$','') }}/cm{{ __cloudera_manager_major_version }}/{{ cloudera_manager_version }}/{{ __cloudera_manager_distro_name }}/yum"
+__cloudera_manager_repo_url_paywall: "{{ cloudera_archive_base_url | regex_replace('/?$','') }}/p/cm{{ __cloudera_manager_major_version }}/{{ (__cloudera_manager_major_version == '5' ) | ternary(__cloudera_manager_cm5_path, __cloudera_manager_cm6_path) }}"
+
 __cloudera_manager_repo_key_filename: "RPM-GPG-KEY-cloudera"
 __cloudera_manager_repo_key_trial: "{{ __cloudera_manager_repo_url_trial }}/{{ __cloudera_manager_repo_key_filename }}"
 __cloudera_manager_repo_key_paywall: "{{ __cloudera_manager_repo_url_paywall }}/{{ __cloudera_manager_repo_key_filename }}"

roles/cloudera_manager/server/tasks/main.yml

@@ -37,7 +37,7 @@
 # This will update the daemons and the agent on this host as well.
 - name: Install Cloudera Manager Server
   ansible.builtin.package:
-    lock_timeout: 60
+    lock_timeout: "{{ (ansible_os_family == 'RedHat') | ternary(60, omit) }}"
     name: cloudera-manager-server
     state: latest
 

roles/deployment/repometa/tasks/main.yml

@@ -13,6 +13,9 @@
 # limitations under the License.
 
 ---
+- name: Determine OS-specific values
+  include_tasks:
+    file: "prepare-{{ ansible_os_family }}.yml"
 
 - name: Set products and parcel repositories
   include_tasks: parcels.yml

roles/deployment/repometa/tasks/parcels.yml

@@ -30,13 +30,17 @@
     loop_var: repository
   check_mode: no
 
+- debug:
+    msg: "This Play Host OS Distro is {{ cluster_os_distribution }}"
+    verbosity: 3
+
 - name: Extract product details from parcel manifests
-  set_fact:
+  ansible.builtin.set_fact:
     products: >
       {{ manifests.results
       | map(attribute='json')
       | list
-      | cloudera.cluster.extract_products_from_manifests
+      | cloudera.cluster.extract_products_from_manifests(os_distribution=cluster_os_distribution)
       | dict2items(key_name='product', value_name='version')
       }}
   run_once: true

roles/deployment/repometa/tasks/prepare-Debian.yml

@@ -0,0 +1,20 @@
+# Copyright 2021 Cloudera, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+- name: Set OS Distribution for parcel filtering
+  when: cluster_os_distribution is undefined
+  ansible.builtin.set_fact:
+    cluster_os_distribution: "{{ ansible_distribution_release }}"

roles/deployment/repometa/tasks/prepare-RedHat.yml

@@ -0,0 +1,20 @@
+# Copyright 2021 Cloudera, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+- name: Set OS Distribution for parcel filtering
+  when: cluster_os_distribution is undefined
+  ansible.builtin.set_fact:
+    cluster_os_distribution: "el{{ ansible_distribution_major_version }}"

roles/infrastructure/ca_server/tasks/main.yml

@@ -20,19 +20,29 @@
 
 - name: Install openssl
   ansible.builtin.package:
-    lock_timeout: 60
+    lock_timeout: "{{ (ansible_os_family == 'RedHat') | ternary(60, omit) }}"
     name: openssl
     state: present
 
-- name: Install PyOpenSSL
+- name: Install PyOpenSSL on Rhel7
   ansible.builtin.package:
-    lock_timeout: 60
+    lock_timeout: "{{ (ansible_os_family == 'RedHat') | ternary(60, omit) }}"
     name: pyOpenSSL
     state: present
-  when: ansible_os_family == "RedHat"
+  when:
+    - ansible_os_family == "RedHat"
+    - ansible_distribution_major_version | int < 8
+
+- name: Install PyOpenSSL on Rhel8
+  ansible.builtin.pip:
+    name: pyOpenSSL
+    state: present
+  when:
+    - ansible_os_family == "RedHat"
+    - ansible_distribution_major_version | int >= 8
 
 - name: Install cryptography Python package
-  pip:
+  ansible.builtin.pip:
     name: cryptography
     state: present
   when: ansible_os_family == "Debian"

roles/infrastructure/custom_repo/tasks/main.yml

@@ -20,7 +20,8 @@
 
 - name: Install {{ httpd_package }}
   ansible.builtin.package:
-    lock_timeout: 60
+    lock_timeout: "{{ (ansible_os_family == 'RedHat') | ternary(60, omit) }}"
+    update_cache: yes
     name: "{{ httpd_package }}"
     state: present
 

roles/infrastructure/haproxy/tasks/main.yml

@@ -16,7 +16,7 @@
 
 - name: Install HAProxy
   ansible.builtin.package:
-    lock_timeout: 60
+    lock_timeout: "{{ (ansible_os_family == 'RedHat') | ternary(60, omit) }}"
     name: haproxy
     state: latest
 

roles/infrastructure/krb5_client/tasks/mit.yml

@@ -19,7 +19,7 @@
 
 - name: Install KRB5 client libraries
   ansible.builtin.package:
-    lock_timeout: 60
+    lock_timeout: "{{ (ansible_os_family == 'RedHat') | ternary(60, omit) }}"
     name: "{{ krb5_packages }}"
     state: present
 

roles/infrastructure/krb5_server/tasks/mit.yml

@@ -19,7 +19,7 @@
 
 - name: Install KRB5 server
   ansible.builtin.package:
-    lock_timeout: 60
+    lock_timeout: "{{ (ansible_os_family == 'RedHat') | ternary(60, omit) }}"
     name: "{{ krb5_packages }}"
     state: present
 

roles/infrastructure/rdbms/tasks/postgresql-Debian.yml

@@ -22,7 +22,7 @@
 
 - name: Install PostgreSQL apt repository
   apt_repository:
-    repo: deb https://apt.postgresql.org/pub/repos/apt bionic-pgdg main
+    repo: "deb https://apt.postgresql.org/pub/repos/apt {{ ansible_distribution_release }}-pgdg main"
     state: present
   when: not (skip_rdbms_repo_setup | default(False))
 
@@ -31,10 +31,9 @@
     name: geerlingguy.postgresql
 
 - name: Install python-psycopg2
-  ansible.builtin.package:
-    lock_timeout: 60
-    name: python-psycopg2
-    state: present
+  ansible.builtin.pip:
+    name: psycopg2-binary
+    state: latest
 
 - name: Fix UTF-8 template
   include_tasks:

roles/infrastructure/rdbms/tasks/postgresql-RedHat.yml

@@ -30,6 +30,15 @@
     gpgkey: https://download.postgresql.org/pub/repos/yum/RPM-GPG-KEY-PGDG
   when: not (skip_rdbms_repo_setup | default(False))
 
+- name: disable default Postgres module in Rhel 8
+  command: dnf module disable -y postgresql
+  register: __postgres_module_result
+  changed_when:
+    - '"Disabling modules" in __postgres_module_result.stdout'
+  when:
+    - ansible_os_family == 'RedHat'
+    - ansible_distribution_major_version | int >= 8
+
 - name: Install PostgreSQL
   include_role:
     name: geerlingguy.postgresql