Merge pull request #8181 from vrutkovs/3.6-etcd-backport

[3.6] Backport etcd certificate updates from 3.7

filter_plugins/oo_filters.py

@@ -607,6 +607,56 @@ def oo_parse_named_certificates(certificates, named_certs_dir, internal_hostname
     return certificates
 
 
+def oo_parse_certificate_san(certificate):
+    """ Parses SubjectAlternativeNames from a PEM certificate.
+        Ex: certificate = '''-----BEGIN CERTIFICATE-----
+                MIIEcjCCAlqgAwIBAgIBAzANBgkqhkiG9w0BAQsFADAhMR8wHQYDVQQDDBZldGNk
+                LXNpZ25lckAxNTE2ODIwNTg1MB4XDTE4MDEyNDE5MDMzM1oXDTIzMDEyMzE5MDMz
+                M1owHzEdMBsGA1UEAwwUbWFzdGVyMS5hYnV0Y2hlci5jb20wggEiMA0GCSqGSIb3
+                DQEBAQUAA4IBDwAwggEKAoIBAQD4wBdWXNI3TF1M0b0bEIGyJPvdqKeGwF5XlxWg
+                NoA1Ain/Xz0N1SW5pXW2CDo9HX+ay8DyhzR532yrBa+RO3ivNCmfnexTQinfSLWG
+                mBEdiu7HO3puR/GNm74JNyXoEKlMAIRiTGq9HPoTo7tNV5MLodgYirpHrkSutOww
+                DfFSrNjH/ehqxwQtrIOnTAHigdTOrKVdoYxqXblDEMONTPLI5LMvm4/BqnAVaOyb
+                9RUzND6lxU/ei3FbUS5IoeASOHx0l1ifxae3OeSNAimm/RIRo9rieFNUFh45TzID
+                elsdGrLB75LH/gnRVV1xxVbwPN6xW1mEwOceRMuhIArJQ2G5AgMBAAGjgbYwgbMw
+                UQYDVR0jBEowSIAUXTqN88vCI6E7wONls3QJ4/63unOhJaQjMCExHzAdBgNVBAMM
+                FmV0Y2Qtc2lnbmVyQDE1MTY4MjA1ODWCCQDMaopfom6OljAMBgNVHRMBAf8EAjAA
+                MBMGA1UdJQQMMAoGCCsGAQUFBwMBMAsGA1UdDwQEAwIFoDAdBgNVHQ4EFgQU7l05
+                OYeY3HppL6/0VJSirudj8t0wDwYDVR0RBAgwBocEwKh6ujANBgkqhkiG9w0BAQsF
+                AAOCAgEAFU8sicE5EeQsUPnFEqDvoJd1cVE+8aCBqkW0++4GsVw2A/JOJ3OBJL6r
+                BV3b1u8/e8xBNi8hPi42Q+LWBITZZ/COFyhwEAK94hcr7eZLCV2xfUdMJziP4Qkh
+                /WRN7vXHTtJ6NP/d6A22SPbtnMSt9Y6G8y9qa5HBrqIqmkYbLzDw/SdZbDbuGhRk
+                xUwg2ahXNblVoE5P6rxPONgXliA94telZ1/61iyrVaiGQb1/GUP/DRfvvR4dOCrA
+                lMosW6fm37Wdi/8iYW+aDPWGS+yVK/sjSnHNjxqvrzkfGk+COa5riT9hJ7wZY0Hb
+                YiJS74SZgZt/nnr5PI2zFRUiZLECqCkZnC/sz29i+irLabnq7Cif9Mv+TUcXWvry
+                TdJuaaYdTSMRSUkDd/c9Ife8tOr1i1xhFzDNKNkZjTVRk1MBquSXndVCDKucdfGi
+                YoWm+NDFrayw8yxK/KTHo3Db3lu1eIXTHxriodFx898b//hysHr4hs4/tsEFUTZi
+                705L2ScIFLfnyaPby5GK/3sBIXtuhOFM3QV3JoYKlJB5T6wJioVoUmSLc+UxZMeE
+                t9gGVQbVxtLvNHUdW7uKQ5pd76nIJqApQf8wg2Pja8oo56fRZX2XLt8nm9cswcC4
+                Y1mDMvtfxglQATwMTuoKGdREuu1mbdb8QqdyQmZuMa72q+ax2kQ=
+                -----END CERTIFICATE-----'''
+            returns ['192.168.122.186']
+    """
+
+    if not HAS_OPENSSL:
+        raise errors.AnsibleFilterError("|missing OpenSSL python bindings")
+
+    names = []
+
+    try:
+        lcert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, certificate)
+        for i in range(lcert.get_extension_count()):
+            if lcert.get_extension(i).get_short_name() == 'subjectAltName':
+                sanstr = str(lcert.get_extension(i))
+                sanstr = sanstr.replace('DNS:', '')
+                sanstr = sanstr.replace('IP Address:', '')
+                names = sanstr.split(', ')
+    except Exception:
+        raise errors.AnsibleFilterError("|failed to parse certificate")
+
+    return names
+
+
 def oo_pretty_print_cluster(data, prefix='tag_'):
     """ Read a subset of hostvars and build a summary of the cluster
         in the following layout:
@@ -1050,6 +1100,7 @@ def filters(self):
             "oo_parse_named_certificates": oo_parse_named_certificates,
             "oo_haproxy_backend_masters": oo_haproxy_backend_masters,
             "oo_pretty_print_cluster": oo_pretty_print_cluster,
+            "oo_parse_certificate_san": oo_parse_certificate_san,
             "oo_generate_secret": oo_generate_secret,
             "oo_nodes_with_label": oo_nodes_with_label,
             "oo_openshift_env": oo_openshift_env,

playbooks/common/openshift-cluster/redeploy-certificates/etcd-ca.yml

@@ -60,47 +60,9 @@
   roles:
   - role: etcd_common
     r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
-  tasks:
-  - name: Create a tarball of the etcd ca certs
-    command: >
-      tar -czvf {{ etcd_conf_dir }}/{{ etcd_ca_name }}.tgz
-        -C {{ etcd_ca_dir }} .
-    args:
-      creates: "{{ etcd_conf_dir }}/{{ etcd_ca_name }}.tgz"
-      warn: no
-    delegate_to: "{{ etcd_ca_host }}"
-    run_once: true
-  - name: Retrieve etcd ca cert tarball
-    fetch:
-      src: "{{ etcd_conf_dir }}/{{ etcd_ca_name }}.tgz"
-      dest: "{{ hostvars['localhost'].g_etcd_mktemp.stdout }}/"
-      flat: yes
-      fail_on_missing: yes
-      validate_checksum: yes
-    delegate_to: "{{ etcd_ca_host }}"
-    run_once: true
-  - name: Ensure ca directory exists
-    file:
-      path: "{{ etcd_ca_dir }}"
-      state: directory
-  - name: Unarchive etcd ca cert tarballs
-    unarchive:
-      src: "{{ hostvars['localhost'].g_etcd_mktemp.stdout }}/{{ etcd_ca_name }}.tgz"
-      dest: "{{ etcd_ca_dir }}"
-  - name: Read current etcd CA
-    slurp:
-      src: "{{ etcd_conf_dir }}/ca.crt"
-    register: g_current_etcd_ca_output
-  - name: Read new etcd CA
-    slurp:
-      src: "{{ etcd_ca_dir }}/ca.crt"
-    register: g_new_etcd_ca_output
-  - copy:
-      content: "{{ (g_new_etcd_ca_output.content|b64decode) + (g_current_etcd_ca_output.content|b64decode) }}"
-      dest: "{{ item }}/ca.crt"
-    with_items:
-    - "{{ etcd_conf_dir }}"
-    - "{{ etcd_ca_dir }}"
+  - role: etcd_server_certificates
+    r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+    etcd_sync_cert_dir: "{{ hostvars['localhost'].g_etcd_mktemp.stdout }}"
 
 - include: ../../openshift-etcd/restart.yml
   # Do not restart etcd when etcd certificates were previously expired.

playbooks/common/openshift-cluster/redeploy-certificates/etcd.yml

@@ -1,6 +1,6 @@
 ---
 - name: Backup and remove generated etcd certificates
-  hosts: oo_first_etcd
+  hosts: oo_etcd_to_config
   any_errors_fatal: true
   roles:
     - role: etcd_common

playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml

@@ -33,6 +33,31 @@
         embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}"
         debug_level: "{{ openshift_master_debug_level | default(openshift.common.debug_level | default(2)) }}"
 
+# Prior to 3.6, openshift-ansible created etcd serving certificates
+# without a SubjectAlternativeName entry for the system hostname. The
+# SAN list in Go 1.8 is now (correctly) authoritative and since
+# openshift-ansible configures masters to talk to etcd hostnames
+# rather than IP addresses, we must correct etcd certificates.
+#
+# This play examines the etcd serving certificate SANs on each etcd
+# host and records whether or not the system hostname is missing.
+- name: Examine etcd serving certificate SAN
+  hosts: oo_etcd_to_config
+  tasks:
+  - slurp:
+      src: /etc/etcd/server.crt
+    register: etcd_serving_cert
+  - set_fact:
+      __etcd_cert_lacks_hostname: "{{ (openshift.common.hostname not in (etcd_serving_cert.content | b64decode | oo_parse_certificate_san)) | bool }}"
+
+# Redeploy etcd certificates when hostnames were missing from etcd
+# serving certificate SANs.
+- include: ../../openshift-etcd/certificates.yml
+  when:
+  - true in hostvars | oo_select_keys(groups['oo_etcd_to_config']) | oo_collect('__etcd_cert_lacks_hostname') | default([false])
+  vars:
+    etcd_certificates_redeploy: True
+
 - name: Upgrade and backup etcd
   include: ./etcd/main.yml
 

playbooks/common/openshift-etcd/ca.yml

@@ -0,0 +1,15 @@
+---
+- name: Generate new etcd CA
+  hosts: oo_first_etcd
+  roles:
+  - role: openshift_etcd_facts
+  tasks:
+  - include_role:
+      name: etcd
+      tasks_from: ca
+    vars:
+      etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}"
+      etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}"
+    when:
+    - etcd_ca_setup | default(True) | bool
+    static: true

playbooks/common/openshift-etcd/certificates.yml

@@ -0,0 +1,4 @@
+---
+- include: server_certificates.yml
+
+- include: master_etcd_certificates.yml

playbooks/common/openshift-etcd/config.yml

@@ -5,7 +5,6 @@
   roles:
   - role: openshift_etcd
     etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}"
-    etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
     etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}"
     r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
   - role: nickhammond.logrotate

playbooks/common/openshift-etcd/master_etcd_certificates.yml

@@ -0,0 +1,12 @@
+---
+- name: Create etcd client certificates for master hosts
+  hosts: oo_masters_to_config
+  any_errors_fatal: true
+  roles:
+    - role: openshift_etcd_facts
+    - role: openshift_etcd_client_certificates
+      etcd_cert_subdir: "openshift-master-{{ openshift.common.hostname }}"
+      etcd_cert_config_dir: "{{ openshift.common.config_base }}/master"
+      etcd_cert_prefix: "master.etcd-"
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+      when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config

playbooks/common/openshift-etcd/migrate.yml

@@ -5,13 +5,16 @@
 
 - name: Run pre-checks
   hosts: oo_etcd_to_migrate
-  tags:
-  - always
-  roles:
-  - role: etcd_migrate
-    r_etcd_migrate_action: check
-    r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}"
-    etcd_peer: "{{ ansible_default_ipv4.address }}"
+  tasks:
+  - set_fact:
+      etcd_migration_in_progress: true
+  - include_role:
+      name: etcd_migrate
+      tasks_from: check
+    vars:
+      r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}"
+      etcd_peer: "{{ ansible_default_ipv4.address }}"
+    static: true
 
 - include: ../openshift-cluster/initialize_facts.yml
   tags:
@@ -255,3 +258,7 @@
       msg: "Migration failed. The following hosts were not properly migrated: {{ etcd_migration_failed | join(',') }}"
     when:
     - etcd_migration_failed | length > 0
+  - set_fact:
+      etcd_migration_in_progress: false
+    when:
+    - etcd_migration_failed | length > 0

playbooks/common/openshift-etcd/scaleup.yml

@@ -12,8 +12,6 @@
   hosts: oo_new_etcd_to_config
   serial: 1
   any_errors_fatal: true
-  vars:
-    etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
   pre_tasks:
   - name: Add new etcd members to cluster
     command: >
@@ -34,7 +32,6 @@
   - role: openshift_etcd
     when: etcd_add_check.rc == 0
     etcd_peers: "{{ groups.oo_etcd_to_config | union(groups.oo_new_etcd_to_config)| default([], true) }}"
-    etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
     etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}"
     etcd_initial_cluster_state: "existing"
     initial_etcd_cluster: "{{ etcd_add_check.stdout_lines[3] | regex_replace('ETCD_INITIAL_CLUSTER=','') | regex_replace('\"','') }}"

playbooks/common/openshift-etcd/server_certificates.yml

@@ -0,0 +1,14 @@
+---
+- name: Create etcd server certificates for etcd hosts
+  hosts: oo_etcd_to_config
+  any_errors_fatal: true
+  roles:
+    - role: openshift_etcd_facts
+  post_tasks:
+    - include_role:
+        name: etcd_server_certificates
+      vars:
+        etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}"
+        etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}"
+        r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+      static: true

playbooks/common/openshift-node/etcd_client_config.yml

@@ -0,0 +1,10 @@
+---
+- name: etcd_client node config
+  hosts: "{{ openshift_node_scale_up_group | default('this_group_does_not_exist') }}"
+  roles:
+  - role: openshift_facts
+  - role: openshift_etcd_facts
+  - role: openshift_etcd_client_certificates
+    etcd_cert_prefix: flannel.etcd-
+    etcd_cert_subdir: "openshift-node-{{ openshift.common.hostname }}"
+    etcd_cert_config_dir: "{{ openshift.common.config_base }}/node"

roles/etcd_client_certificates/tasks/main.yml

@@ -86,6 +86,13 @@
   when: etcd_client_certs_missing | bool
   become: no
 
+- name: Delete existing certificate tarball if certs need to be regenerated
+  file:
+    path: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz"
+    state: absent
+  when: etcd_client_certs_missing | bool
+  delegate_to: "{{ etcd_ca_host }}"
+
 - name: Create a tarball of the etcd certs
   command: >
     tar -czvf {{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz

roles/etcd_server_certificates/tasks/main.yml

@@ -111,6 +111,13 @@
   changed_when: False
   when: etcd_server_certs_missing | bool
 
+- name: Delete existing certificate tarball if certs need to be regenerated
+  file:
+    path: "{{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz"
+    state: absent
+  when: etcd_server_certs_missing | bool
+  delegate_to: "{{ etcd_ca_host }}"
+
 - name: Create a tarball of the etcd certs
   command: >
     tar -czvf {{ etcd_generated_certs_dir }}/{{ etcd_cert_subdir }}.tgz

roles/openshift_etcd_facts/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+etcd_ca_host_group: "oo_etcd_to_config"

roles/openshift_etcd_facts/tasks/main.yml

@@ -3,3 +3,5 @@
     role: etcd
     local_facts:
       etcd_image: "{{ osm_etcd_image | default(None) }}"
+- include: set_etcd_ca_host.yml
+  static: true

roles/openshift_etcd_facts/tasks/set_etcd_ca_host.yml

@@ -0,0 +1,44 @@
+---
+- name: Check for CA indicator files
+  stat:
+    path: "{{ item.0 }}"
+  delegate_to: "{{ item.1 }}"
+  with_nested:
+  - - /etc/etcd/ca
+    - /etc/etcd/generated_certs
+  - "{{ groups[etcd_ca_host_group] }}"
+  register: __etcd_ca_host_stat
+  run_once: true
+
+# Collect ansible_host (inventory hostname) of hosts with /etc/etcd/ca
+# and /etc/etcd/generated_certs directories.
+- set_fact:
+    __etcd_ca_dir_hosts: "{{ __etcd_ca_host_stat.results
+                             | oo_collect('_ansible_delegated_vars.ansible_host',
+                                                    filters={'stat.path':'/etc/etcd/ca','stat.exists':True}) }}"
+    __etcd_generated_certs_dir_hosts: "{{ __etcd_ca_host_stat.results
+                                          | oo_collect('_ansible_delegated_vars.ansible_host',
+                                                                 filters={'stat.path':'/etc/etcd/generated_certs','stat.exists':True}) }}"
+  run_once: true
+
+# __etcd_ca_hosts is the intersection of hosts which have /etc/etcd/ca
+# and /etc/etcd/generated_certs directories.
+- set_fact:
+    __etcd_ca_hosts: "{{ __etcd_ca_dir_hosts | intersect(__etcd_generated_certs_dir_hosts) }}"
+  run_once: true
+
+# __etcd_ca_hosts should only contain one host. If more than one host
+# is able to be an etcd CA host then we will use the first.
+- set_fact:
+    etcd_ca_host: "{{ __etcd_ca_hosts[0] }}"
+  when:
+  - __etcd_ca_hosts | length > 0
+  - etcd_ca_host is not defined
+
+# No etcd_ca_host was found in __etcd_ca_hosts. This is probably a
+# fresh installation so we will default to the first member of the
+# etcd host group.
+- set_fact:
+    etcd_ca_host: "{{ groups[etcd_ca_host_group].0 }}"
+  when:
+  - etcd_ca_host is not defined

roles/openshift_version/tasks/main.yml

@@ -183,6 +183,7 @@
     when:
     - not is_containerized | bool
     - openshift_release is defined
+    - etcd_migration_completed is not defined and etcd_migration_in_progress is not defined
     assert:
       that:
       - openshift_version.startswith(openshift_release) | bool