Merge pull request #8 from JumaX/6.1.x

Merge latest 6.1.x changes into Replicator RBAC dev branch

VARIABLES.md

@@ -4,6 +4,14 @@ Below are the supported variables for the role confluent.variables
 
 ***
 
+### confluent_package_version
+
+Version of Confluent Platform to install
+
+Default:  6.1.2
+
+***
+
 ### jolokia_url_remote
 
 To copy from Ansible control host or download
@@ -292,6 +300,14 @@ Default:  "{{ false if ssl_provided_keystore_and_truststore|bool or ssl_custom_c
 
 ***
 
+### ssl_file_dir
+
+Directory on hosts to store all ssl files.
+
+Default:  /var/ssl/private/
+
+***
+
 ### regenerate_ca
 
 Boolean to have reruns of all.yml regenerate the certificate authority used for self signed certs.
@@ -476,6 +492,14 @@ Default:  "{{zookeeper.log_path}}"
 
 ***
 
+### zookeeper_chroot
+
+Chroot path in Zookeeper used by Kafka. Defaults to no chroot. Must begin with a /
+
+Default:  ""
+
+***
+
 ### zookeeper_jolokia_enabled
 
 Boolean to enable Jolokia Agent installation and configuration on zookeeper
@@ -1788,6 +1812,14 @@ Default:  "{{mds_http_protocol}}://{{ groups['kafka_broker'] | default(['localho
 
 ***
 
+### regenerate_token_pem
+
+To regenerate MDS Token Pem files on subsequent runs of the playbook, set this to true.
+
+Default:  false
+
+***
+
 ### rbac_component_additional_system_admins
 
 List of users to be granted system admin Role Bindings across all components
@@ -1836,19 +1868,19 @@ Default:  "{{rbac_component_additional_system_admins}}"
 
 ***
 
-### secrets_protection_enabled
+### mask_secrets
 
-Boolean to enable secrets protection on all components except Zookeeper
+Boolean to mask secrets in playbook output
 
-Default:  false
+Default:  true
 
 ***
 
-### mask_secrets
+### secrets_protection_enabled
 
-Boolean to mask secrets in playbook output
+Boolean to enable secrets protection on all components except Zookeeper
 
-Default:  true
+Default:  false
 
 ***
 
@@ -1876,6 +1908,14 @@ Default:  generated_ssl_files/security.properties
 
 ***
 
+### secrets_protection_encrypt_passwords
+
+Boolean to encrypt sensitive properties, such as those containing 'password', 'basic.auth.user.info', or 'sasl.jaas.config'.
+
+Default:  "{{secrets_protection_enabled}}"
+
+***
+
 ### kafka_broker_secrets_protection_enabled
 
 Boolean to enable secrets protection in Kafka broker.
@@ -1886,9 +1926,9 @@ Default:  "{{secrets_protection_enabled}}"
 
 ### kafka_broker_secrets_protection_encrypt_passwords
 
-Boolean to encrypt all properties containing 'password' for Kafka.
+Boolean to encrypt sensitive properties, such as those containing 'password', 'basic.auth.user.info', or 'sasl.jaas.config' for Kafka.
 
-Default:  "{{kafka_broker_secrets_protection_enabled}}"
+Default:  "{{secrets_protection_encrypt_passwords}}"
 
 ***
 
@@ -1910,9 +1950,9 @@ Default:  "{{secrets_protection_enabled}}"
 
 ### schema_registry_secrets_protection_encrypt_passwords
 
-Boolean to encrypt all properties containing 'password' for Schema Registry.
+Boolean to encrypt sensitive properties, such as those containing 'password', 'basic.auth.user.info', or 'sasl.jaas.config' for Schema Registry.
 
-Default:  "{{schema_registry_secrets_protection_enabled}}"
+Default:  "{{secrets_protection_encrypt_passwords}}"
 
 ***
 
@@ -1934,9 +1974,9 @@ Default:  "{{secrets_protection_enabled}}"
 
 ### kafka_connect_secrets_protection_encrypt_passwords
 
-Boolean to encrypt all properties containing 'password' for Connect.
+Boolean to encrypt sensitive properties, such as those containing 'password', 'basic.auth.user.info', or 'sasl.jaas.config' for Connect.
 
-Default:  "{{kafka_connect_secrets_protection_enabled}}"
+Default:  "{{secrets_protection_encrypt_passwords}}"
 
 ***
 
@@ -1958,9 +1998,9 @@ Default:  "{{secrets_protection_enabled}}"
 
 ### kafka_rest_secrets_protection_encrypt_passwords
 
-Boolean to encrypt all properties containing 'password' for Rest Proxy.
+Boolean to encrypt sensitive properties, such as those containing 'password', 'basic.auth.user.info', or 'sasl.jaas.config' for Rest Proxy.
 
-Default:  "{{kafka_rest_secrets_protection_enabled}}"
+Default:  "{{secrets_protection_encrypt_passwords}}"
 
 ***
 
@@ -1982,9 +2022,9 @@ Default:  "{{secrets_protection_enabled}}"
 
 ### ksql_secrets_protection_encrypt_passwords
 
-Boolean to encrypt all properties containing 'password' for KSQL.
+Boolean to encrypt sensitive properties, such as those containing 'password', 'basic.auth.user.info', or 'sasl.jaas.config' for KSQL.
 
-Default:  "{{ksql_secrets_protection_enabled}}"
+Default:  "{{secrets_protection_encrypt_passwords}}"
 
 ***
 
@@ -2006,9 +2046,9 @@ Default:  "{{secrets_protection_enabled}}"
 
 ### control_center_secrets_protection_encrypt_passwords
 
-Boolean to encrypt all properties containing 'password' for Control Center.
+Boolean to encrypt sensitive properties, such as those containing 'password', 'basic.auth.user.info', or 'sasl.jaas.config' for Control Center.
 
-Default:  "{{control_center_secrets_protection_enabled}}"
+Default:  "{{secrets_protection_encrypt_passwords}}"
 
 ***
 

ansible.cfg

@@ -1,6 +1,7 @@
 [defaults]
 hash_behaviour=merge
 callback_whitelist=profile_tasks
+remote_tmp=~/.ansible/tmp
 
 [ssh_connection]
 pipelining = True

control_center.yml

@@ -34,6 +34,7 @@
 - name: Control Center Serial Provisioning
   hosts: control_center_serial
   serial: 1
+  any_errors_fatal: true
   gather_facts: false
   tags: control_center
   environment: "{{ proxy_env }}"

hosts_example.yml

@@ -271,11 +271,12 @@ kafka_broker:
   #   ## To configure Kafka to run as a custom user, uncomment below
   #   kafka_broker_user: custom-user
   #   kafka_broker_group: custom-group
-  #   # To update the log.dirs property within the kafka server.properties, uncomment below
+  #
+  #   # To update data log location use custom properties:
   #   # By default the log directory is /var/lib/kafka/data
-  #   kafka_broker:
-  #     datadir:
-  #       - /var/lib/kafka/my-data
+  #
+  #    kafka_broker_custom_properties:
+  #      log.dirs: dir1,dir2
   #
   #   ## To enabled Self Balancing Kafka Brokers, uncomment the below lines
   #   kafka_broker_custom_properties:

kafka_broker.yml

@@ -44,6 +44,7 @@
 - name: Kafka Broker Non Controllers Provisioning
   hosts: kafka_broker_non_controller
   serial: 1
+  any_errors_fatal: true
   gather_facts: false
   tags: kafka_broker
   environment: "{{ proxy_env }}"
@@ -58,6 +59,7 @@
 
 - name: Kafka Broker Controller Provisioning
   hosts: kafka_broker_controller
+  any_errors_fatal: true
   gather_facts: false
   tags: kafka_broker
   environment: "{{ proxy_env }}"

kafka_connect.yml

@@ -34,6 +34,7 @@
 - name: Kafka Connect Serial Provisioning
   hosts: kafka_connect_serial
   serial: 1
+  any_errors_fatal: true
   gather_facts: false
   tags: kafka_connect
   environment: "{{ proxy_env }}"

kafka_connect_replicator.yml

@@ -34,6 +34,7 @@
 - name: Kafka Connect Replicator Serial Provisioning
   hosts: kafka_connect_replicator_serial
   serial: 1
+  any_errors_fatal: true
   gather_facts: false
   tags: kafka_connect_replicator
   environment: "{{ proxy_env }}"

kafka_rest.yml

@@ -34,6 +34,7 @@
 - name: Kafka Rest Serial Provisioning
   hosts: kafka_rest_serial
   serial: 1
+  any_errors_fatal: true
   gather_facts: false
   tags: kafka_rest
   environment: "{{ proxy_env }}"

roles/confluent.common/defaults/main.yml

@@ -25,7 +25,7 @@ confluent_common_repository_redhat_main_gpgcheck: 1
 confluent_common_repository_redhat_main_gpgkey: "{{confluent_common_repository_redhat_baseurl}}/{{confluent_repo_version}}/archive.key"
 confluent_common_repository_redhat_main_enabled: 1
 
-confluent_common_repository_redhat_dist_baseurl: "{{confluent_common_repository_redhat_baseurl}}/{{confluent_repo_version}}/7"
+confluent_common_repository_redhat_dist_baseurl: "{{confluent_common_repository_redhat_baseurl}}/{{confluent_repo_version}}/$releasever"
 confluent_common_repository_redhat_dist_gpgcheck: 1
 confluent_common_repository_redhat_dist_gpgkey: "{{confluent_common_repository_redhat_baseurl}}/{{confluent_repo_version}}/archive.key"
 confluent_common_repository_redhat_dist_enabled: 1

roles/confluent.common/tasks/rbac_setup.yml

@@ -36,7 +36,7 @@
 
 - name: Create SSL Certificate Directory
   file:
-    path: /var/ssl/private
+    path: "{{ ssl_file_dir_final }}"
     state: directory
     mode: 0755
   when: copy_certs | default(True)

roles/confluent.common/tasks/remove_packages.yml

@@ -30,7 +30,7 @@
 ### - ansible_facts.services[service_name + '.service'] is defined
 
 - name: Get Service Facts
-  ansible.builtin.service_facts:
+  service_facts:
 
 - name: Stop Service before Removing Confluent Packages
   systemd:

roles/confluent.common/tasks/secrets_protection.yml

@@ -10,21 +10,24 @@
     dest: "{{ config_path }}-backup"
     remote_src: true
     mode: 0640
-    owner: "{{secrets_file_owner}}"
-    group: "{{secrets_file_group}}"
+    owner: "{{ secrets_file_owner }}"
+    group: "{{ secrets_file_group }}"
   when: config_stat.stat.exists
 
 - name: Create Unmasked Config
   template:
-    src: "{{ config_template }}"
+    src: properties.j2
     dest: "{{ config_path }}"
     mode: 0640
-    owner: "{{secrets_file_owner}}"
-    group: "{{secrets_file_group}}"
+    owner: "{{ secrets_file_owner }}"
+    group: "{{ secrets_file_group }}"
+  vars:
+    # Secrets Protection CLI needs all backslashes escaped by another backslash. This filter turns \ into \\.
+    properties: '{{ final_properties | regex_replace("\\", "\\\\") }}' # noqa var-naming
 
 - name: Create Secrets Protection Directory
   file:
-    path: /var/ssl/private/
+    path: "{{ ssl_file_dir_final }}"
     state: directory
     mode: 0755
 
@@ -51,25 +54,33 @@
     secrets_protection_masterkey: "{{ slurped_masterkey.content|b64decode}}"
   when: not secrets_protection_masterkey
 
-- name: Encrypt All Properties that Contain 'password'
-  shell: |
-    {{ confluent_cli_path }} secret file encrypt --config-file {{ config_path }} \
-      --local-secrets-file {{ secrets_file }} \
-      --remote-secrets-file {{ secrets_file }}
-  environment:
-    CONFLUENT_SECURITY_MASTER_KEY: "{{ secrets_protection_masterkey }}"
-  when: encrypt_passwords|bool
+- name: Create Encrypt Properties List
+  set_fact:
+    final_encrypt_properties: "{{ (final_properties | dict2items | map(attribute='key') | select('match', properties_patterns|join('|'))
+      + encrypt_properties) | unique if encrypt_passwords|bool else encrypt_properties }}"
+  vars:
+    properties_patterns:
+      - '.*password.*'
+      - '.*basic.auth.user.info.*'
+      - '^ldap.java.naming.security.credentials$'
+      - '^confluent.license$'
+      - '.*sasl.jaas.config'
+
+- name: Debug Properties to Mask
+  debug:
+    var: final_encrypt_properties
+  run_once: true
 
-- name: Encrypt Selected Properties
+- name: Encrypt Properties
   shell: |
     {{ confluent_cli_path }} secret file encrypt --config-file {{ config_path }} \
       --local-secrets-file {{ secrets_file }} \
       --remote-secrets-file {{ secrets_file }} \
-      --config {{ item }}
-  loop: "{{ properties }}"
+      --config "{{ final_encrypt_properties | join (',') }}"
   environment:
     CONFLUENT_SECURITY_MASTER_KEY: "{{ secrets_protection_masterkey }}"
-  when: properties|length>0
+  changed_when: true
+  when: final_encrypt_properties|length > 0
 
 # If config is different than the backup, need to restart
 - name: Test for Config File Changes from Backup - Trigger Handler

roles/confluent.common/templates/properties.j2

@@ -0,0 +1,4 @@
+# Maintained by Ansible
+{% for key, value in properties|dictsort%}
+{{key}}={{value}}
+{% endfor %}

roles/confluent.control_center/defaults/main.yml

@@ -48,4 +48,4 @@ control_center_service_unit_overrides:
 ### Time in seconds to wait before starting Control Center Health Checks.
 control_center_health_check_delay: 30
 
-control_center_secrets_protection_file: /var/ssl/private/control-center-security.properties
+control_center_secrets_protection_file: "{{ ssl_file_dir_final }}/control-center-security.properties"

roles/confluent.control_center/tasks/main.yml

@@ -150,10 +150,10 @@
     name: confluent.common
     tasks_from: secrets_protection.yml
   vars:
+    final_properties: "{{ control_center_final_properties }}"
     encrypt_passwords: "{{ control_center_secrets_protection_encrypt_passwords }}"
-    properties: "{{ control_center_secrets_protection_encrypt_properties }}"
+    encrypt_properties: "{{ control_center_secrets_protection_encrypt_properties }}"
     config_path: "{{ control_center.config_file }}"
-    config_template: control-center.properties.j2
     secrets_file: "{{ control_center_secrets_protection_file }}"
     secrets_file_owner: "{{ control_center_user }}"
     secrets_file_group: "{{ control_center_group }}"

roles/confluent.kafka_broker/defaults/main.yml

@@ -58,4 +58,4 @@ kafka_broker_sysctl_file: /etc/sysctl.conf
 ### Time in seconds to wait before starting Kafka Health Checks.
 kafka_broker_health_check_delay: 20
 
-kafka_broker_secrets_protection_file: /var/ssl/private/kafka-broker-security.properties
+kafka_broker_secrets_protection_file: "{{ ssl_file_dir_final }}/kafka-broker-security.properties"

roles/confluent.kafka_broker/tasks/dynamic_groups.yml

@@ -39,7 +39,7 @@
 - name: Get Controller Broker ID
   shell: >
     set -o pipefail &&
-      {{ binary_base_path }}/bin/zookeeper-shell {{ groups['zookeeper'][0] }}:{{ zookeeper_client_port }}
+      {{ binary_base_path }}/bin/zookeeper-shell {{ groups['zookeeper'][0] }}:{{ zookeeper_client_port }}{{zookeeper_chroot}}
           {%- if 'zookeeper.ssl.client.enable = true' in slurped_properties.content|b64decode
                 or 'zookeeper.ssl.client.enable=true' in slurped_properties.content|b64decode %}
             -zk-tls-config-file {{ kafka_broker.zookeeper_tls_client_config_file if kafka_broker_secrets_protection_enabled else kafka_broker.config_file }}