Changes required to deploy SQL Stream Builder

roles/config/cluster/base/templates/configs/databases-7.1.0.j2

@@ -31,6 +31,18 @@ SCHEMAREGISTRY:
     database_name: {{ databases.SCHEMAREGISTRY.name }}
     database_user: {{ databases.SCHEMAREGISTRY.user }}
     database_password: {{ databases.SCHEMAREGISTRY.password }}
+SQL_STREAM_BUILDER:
+  SERVICEWIDE:
+    database_host: {{ databases.SQL_STREAM_BUILDER.host }}
+    database_port: {{ databases.SQL_STREAM_BUILDER.port }}
+    database_type: {{ databases.SQL_STREAM_BUILDER.type | cloudera.cluster.format_database_type }}
+    database_schema: {{ databases.SQL_STREAM_BUILDER.name }}
+    database_user: {{ databases.SQL_STREAM_BUILDER.user }}
+    database_password: {{ databases.SQL_STREAM_BUILDER.password }}
+  MATERIALIZED_VIEW_ENGINE:
+    ssb.mve.datasource.url: jdbc:{{ databases.SQL_STREAM_BUILDER_MVE.type | cloudera.cluster.format_database_type }}://{{ databases.SQL_STREAM_BUILDER_MVE.host }}:{{ databases.SQL_STREAM_BUILDER_MVE.port }}/{{ databases.SQL_STREAM_BUILDER_MVE.name }}
+    ssb.mve.datasource.username: {{ databases.SQL_STREAM_BUILDER_MVE.user }}
+    ssb.mve.datasource.password: {{ databases.SQL_STREAM_BUILDER_MVE.password }}
 STREAMS_MESSAGING_MANAGER:
   SERVICEWIDE:
     smm_database_host: {{ databases.STREAMS_MESSAGING_MANAGER.host }}

roles/config/cluster/base/templates/configs/inter-service-dependencies.j2

@@ -71,9 +71,11 @@ HUE:
 FLINK:
   SERVICEWIDE:
     hdfs_service: hdfs
-    hive_service: hive
     yarn_service: yarn
-    zookeeper_service: zoookeeper
+    zookeeper_service: zookeeper
+{% if 'HIVE' in cluster.services %}
+    hive_service: hive
+{% endif %}
 {% if 'ATLAS' in cluster.services and not (cdh_cdp_upgrade|default(false)|bool) %}
     atlas_service: atlas
 {% endif %}
@@ -174,6 +176,17 @@ SPARK3_ON_YARN:
     hive_service: hive
 {% endif %}
 
+SQL_STREAM_BUILDER:
+  SERVICEWIDE:
+    flink_service: flink
+    kafka_service: kafka
+{% if 'HIVE' in cluster.services %}
+    hive_service: hive
+{% endif %}
+{% if 'KNOX' in cluster.services %}
+    knox_service: knox
+{% endif %}
+
 TEZ:
   SERVICEWIDE:
     yarn_service: yarn

roles/config/cluster/base/templates/configs/kerberos-6.x-7.x.j2

@@ -2,6 +2,9 @@
 CORE_SETTINGS:
   SERVICEWIDE:
     hadoop_secure_web_ui: true
+FLINK:
+  SERVICEWIDE:
+    kerberos.auth.enabled: true
 HBASE:
   SERVICEWIDE:
     hbase_restserver_security_authentication: kerberos

roles/config/cluster/base/templates/configs/tls-7.3.1.j2

@@ -1,4 +1,26 @@
 ---
 OZONE:
   OZONE_PROMETHEUS:
-    ozone.prometheus.ca.file: {{ tls_chain_path }}
+    ozone.prometheus.ca.file: {{ tls_chain_path }}
+SQL_STREAM_BUILDER:
+  STREAMING_SQL_ENGINE:
+    ssl_client_truststore_location: {{ tls_truststore_path }}
+    ssl_client_truststore_password: {{ tls_truststore_password }}
+    ssl_enabled: true
+    ssl_server_keystore_keypassword: {{ tls_keystore_password }}
+    ssl_server_keystore_location: {{ tls_keystore_path_generic }}
+    ssl_server_keystore_password: {{ tls_keystore_password }}
+  MATERIALIZED_VIEW_ENGINE:
+    ssl_client_truststore_location: {{ tls_truststore_path }}
+    ssl_client_truststore_password: {{ tls_truststore_password }}
+    ssl_enabled: true
+    ssl_server_keystore_keypassword: {{ tls_keystore_password }}
+    ssl_server_keystore_location: {{ tls_keystore_path_generic }}
+    ssl_server_keystore_password: {{ tls_keystore_password }}
+  STREAMING_SQL_CONSOLE:
+    ssl_client_truststore_location: {{ tls_chain_path }}
+    ssl_enabled: true
+    ssl_server_ca_certificate_location: {{ tls_chain_path }}
+    ssl_server_certificate_location: {{ tls_cert_path_generic }}
+    ssl_server_privatekey_location: {{ tls_key_path_generic }}
+    ssl_server_privatekey_password: {{ tls_key_password }}

roles/config/cluster/common/defaults/main.yml

@@ -75,6 +75,7 @@ cluster_services_ordered:
   - CRUISE_CONTROL
   - DAS
   - FLINK
+  - SQL_STREAM_BUILDER
   - SPARK
   - SPARK2_ON_YARN
   - SPARK3_ON_YARN

roles/deployment/definition/defaults/main.yml

@@ -99,6 +99,20 @@ database_defaults:
     name: schemaregistry
     user: schemaregistry
     password: "{{ database_default_password }}"
+  SQL_STREAM_BUILDER:
+    host: "{{ database_host }}"
+    port: "{{ database_type | cloudera.cluster.default_database_port }}"
+    type: "{{ database_type }}"
+    name: ssb_admin
+    user: ssb_admin
+    password: "{{ database_default_password }}"
+  SQL_STREAM_BUILDER_MVE:
+    host: "{{ database_host }}"
+    port: "{{ database_type | cloudera.cluster.default_database_port }}"
+    type: "{{ database_type }}"
+    name: ssb_mve
+    user: ssb_mve
+    password: "{{ database_default_password }}"
   STREAMS_MESSAGING_MANAGER:
     host: "{{ database_host }}"
     port: "{{ database_type | cloudera.cluster.default_database_port }}"

roles/deployment/groupby/tasks/main.yml

@@ -19,10 +19,15 @@
     key: "{{ 'host_template_' ~ host_template if host_template is defined else 'no_template' }}"
 
 - name: Find the correct host template
-  set_fact:
-    host_template_content: "{{ _pre_template_cluster | json_query(query) | first }}"
+  block:
+  - fail:
+      msg: "Unable to host template {{ host_template }} in the cluster definition"
+    when: content | length == 0
+  - set_fact:
+      host_template_content: "{{ content | first }}"
   vars:
     query: "clusters[].host_templates[].\"{{ host_template }}\""
+    content: "{{ _pre_template_cluster | json_query(query) }}"
   when: host_template is defined
 
 - name: Group by service

roles/deployment/repometa/templates/all_services.j2

@@ -1,57 +1,58 @@
-- KEYTRUSTEE_SERVER
-- ZOOKEEPER
-- INFRA_SOLR
-- RANGER
-- RANGER_KMS
-- RANGER_KMS_KTS
-- RANGER_RAZ
-- KMS
-- HDFS
-- SENTRY
-- LIVY
-- HBASE
-- SOLR
-- SQOOP
-- SQOOP_CLIENT
 - ACCUMULO16
 - ACCUMULO_C6
-- HIVE
-- TEZ
-- HIVE_ON_TEZ
-- KUDU
-- IMPALA
-- YARN
 - ADLS_CONNECTOR
+- ATLAS
 - AWS_S3
 - CDSW
 - CORE_SETTINGS
 - CRUISE_CONTROL
 - DAS
 - FLINK
 - FLUME
-- SPARK
-- SPARK2_ON_YARN
-- SPARK3_ON_YARN
-- SPARK_ON_YARN
+- HBASE
+- HDFS
+- HIVE
+- HIVE_ON_TEZ
+- HUE
+- IMPALA
+- INFRA_SOLR
 - ISILON
 - KAFKA
 - KEYTRUSTEE
+- KEYTRUSTEE_SERVER
+- KMS
 - KNOX
 - KS_INDEXER
+- KUDU
+- LIVY
 - LUNA_KMS
 - MAPREDUCE
 - NIFI
 - NIFIREGISTRY
 - NIFITOOLKITCA
+- OOZIE
 - OZONE
 - PHOENIX
 - QUEUEMANAGER
+- RANGER
+- RANGER_KMS
+- RANGER_KMS_KTS
+- RANGER_RAZ
 - SCHEMAREGISTRY
+- SENTRY
+- SOLR
+- SPARK
+- SPARK2_ON_YARN
+- SPARK3_ON_YARN
+- SPARK_ON_YARN
+- SQL_STREAM_BUILDER
+- SQOOP
+- SQOOP_CLIENT
 - STREAMS_MESSAGING_MANAGER
 - STREAMS_REPLICATION_MANAGER
+- TEZ
 - THALES_KMS
 - WXM
+- YARN
 - ZEPPELIN
-- ATLAS
-- OOZIE
-- HUE
+- ZOOKEEPER

roles/deployment/repometa/templates/role_mappings/cdh7.j2

@@ -11,6 +11,8 @@ CRUISE_CONTROL:
 DAS:
   - DAS_EVENT_PROCESSOR
   - DAS_WEBAPP
+FLINK:
+  - FLINK_HISTORY_SERVER
 HBASE:
   - GATEWAY
   - HBASERESTSERVER
@@ -103,6 +105,10 @@ SOLR:
 SPARK_ON_YARN:
   - GATEWAY
   - SPARK_YARN_HISTORY_SERVER
+SQL_STREAM_BUILDER:
+  - STREAMING_SQL_ENGINE
+  - MATERIALIZED_VIEW_ENGINE
+  - STREAMING_SQL_CONSOLE
 SQOOP_CLIENT:
   - GATEWAY
 STREAMS_MESSAGING_MANAGER:

roles/prereqs/local_accounts_common/defaults/main.yml

@@ -48,6 +48,13 @@ local_accounts:
     comment: Flink
     keystore_acl: True
 
+  - user: ssb
+    home: /var/lib/ssb
+    comment: SQL Stream Builder
+    keystore_acl: True
+    key_acl: True
+    key_password_acl: True
+
   - user: flume
     home: /var/lib/flume-ng
     comment: Flume

roles/prereqs/postgresql_connector/defaults/main.yml

@@ -0,0 +1,18 @@
+# Copyright 2021 Cloudera, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+postgresql_connector_url: https://jdbc.postgresql.org/download/postgresql-42.3.3.jar
+postgresql_connector_checksum: md5:bef0b2e1c6edcd8647c24bed31e1a4ac

roles/prereqs/postgresql_connector/meta/main.yml

@@ -0,0 +1,17 @@
+# Copyright 2021 Cloudera, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+dependencies:
+  - role: cloudera.cluster.deployment.definition

roles/prereqs/postgresql_connector/tasks/main.yml

@@ -0,0 +1,37 @@
+# Copyright 2021 Cloudera, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+- name: Download PostgreSQL Connector
+  get_url:
+    url: "{{ postgresql_connector_url }}"
+    dest: "{{ local_temp_dir }}/postgresql-connector-java.jar"
+    checksum: "{{ postgresql_connector_checksum }}"
+    mode: 0644
+  become: no
+  run_once: true
+  delegate_to: localhost
+
+- name: Create /usr/share/java directory
+  file:
+    path: /usr/share/java
+    state: directory
+    mode: 0755
+
+- name: Copy PostgreSQL Connector jar file to correct location
+  copy:
+    src: "{{ local_temp_dir }}/postgresql-connector-java.jar"
+    dest: /usr/share/java/postgresql-connector-java.jar
+    mode: 0644

roles/verify/definition/tasks/main.yml

@@ -29,7 +29,9 @@
         success_msg: "TLS is configured on a set of nodes"
         fail_msg: >-
           TLS certificate distribution is not configured in the cluster inventory
-          file (set manual_tls_cert_distribution if this is intentional)
+          file.
+          If this is intential, set manual_tls_cert_distribution=true.
+          Otherwise, set tls=true for each host requiring a certificate.
       when: has_tls and not (manual_tls_cert_distribution | default(false))
     - name: Ensure that TLS distribution is not configured in the inventory
       assert:

roles/verify/parcels_and_roles/tasks/check_template_roles.yml

@@ -22,6 +22,7 @@
           | map('regex_replace','/.+','')
           | difference(role_mappings[template.service] | list)
           }}
+
     - name: Ensure the host template service roles are valid
       assert:
         that: "{{ invalid_roles | length == 0 }}"
@@ -31,3 +32,12 @@
         fail_msg: >-
           Unknown role(s) {{ invalid_roles }} for service '{{ template.service }}'
           defined in host template '{{ host_template.name }}'.
+
+    - name: Ensure the Tez gateway has been deployed
+      assert:
+        that: "{{ 'GATEWAY' in (host_template.mappings['TEZ'] | default({})) }}"
+        success_msg: The Tez gateway has been included as required
+        fail_msg: The Tez gateway should be colocated with Hive On Tez roles
+      when:
+        - template.service == 'HIVE_ON_TEZ'
+        - "'HIVESERVER2' in template.roles or 'GATEWAY' in template.roles"