From e9b04777369fe6f7c92880d806ba8a4420979a8e Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Fri, 17 Nov 2023 15:30:22 +0000 Subject: [PATCH 1/5] Add rekey-hosts.yml playbook --- etc/kayobe/ansible/rekey-hosts.yml | 72 +++++++++++++++++++ .../add-rekey-playbook-0065c5057b1639f8.yaml | 5 ++ 2 files changed, 77 insertions(+) create mode 100644 etc/kayobe/ansible/rekey-hosts.yml create mode 100644 releasenotes/notes/add-rekey-playbook-0065c5057b1639f8.yaml diff --git a/etc/kayobe/ansible/rekey-hosts.yml b/etc/kayobe/ansible/rekey-hosts.yml new file mode 100644 index 000000000..445e7224f --- /dev/null +++ b/etc/kayobe/ansible/rekey-hosts.yml @@ -0,0 +1,72 @@ +--- +- name: Rekey hosts + hosts: overcloud,seed,seed-hypervisor,infra-vms + gather_facts: false + vars: + ansible_user: stack + ansible_python_interpreter: /usr/bin/python3 + tasks: + - name: Generate a fresh SSH key + community.crypto.openssh_keypair: + path: ~/.ssh/id_rsa_new + delegate_to: localhost + + # - name: Copy new key to hosts + # ansible.builtin.copy: + # src: /tmp/id_rsa_new.pub + # dest: /tmp/id_rsa_new.pub + # mode: '0600' + # become: true + + - name: Copy old key to hosts + ansible.builtin.copy: + src: ~/.ssh/id_rsa.pub + dest: /tmp/id_rsa_old.pub + mode: '0777' + become: true + + - name: Set new stack authorized keys + ansible.posix.authorized_key: + user: "{{ item }}" + state: present + key: "{{ lookup('file', '~/.ssh/id_rsa_new.pub') }}" + loop: + - "stack" + - "kolla" + become: true + + - name: Set new stack authorized keys + ansible.posix.authorized_key: + user: "{{ item }}" + state: present + key: "{{ lookup('file', '~/.ssh/id_rsa_new.pub') }}" + loop: + - "stack" + - "kolla" + become: true + + - name: Locally deprecate old key (private) + command: "mv ~/.ssh/id_rsa ~/.ssh/id_rsa_old" + delegate_to: localhost + + - name: Locally deprecate old key (public) + command: "mv ~/.ssh/id_rsa.pub ~/.ssh/id_rsa_old.pub" + delegate_to: localhost + + - name: Locally promote new key (private) + command: "mv ~/.ssh/id_rsa_new ~/.ssh/id_rsa" + delegate_to: localhost + + - name: Locally promote new key (public) + command: " mv ~/.ssh/id_rsa_new.pub ~/.ssh/id_rsa.pub" + delegate_to: localhost + + - name: Remove old key from hosts + ansible.posix.authorized_key: + user: "{{ item }}" + state: absent + key: "{{ lookup('file', '/tmp/id_rsa_old.pub') }}" + loop: + - "stack" + - "kolla" + become: true diff --git a/releasenotes/notes/add-rekey-playbook-0065c5057b1639f8.yaml b/releasenotes/notes/add-rekey-playbook-0065c5057b1639f8.yaml new file mode 100644 index 000000000..a9d78cc12 --- /dev/null +++ b/releasenotes/notes/add-rekey-playbook-0065c5057b1639f8.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Added the ``rekey-hosts.yml`` playbook to automatically rotate the SSH + keys on all hosts in the cloud for the stack and kolla users. From c527579ef38381c196d1c1cb67a5146124a6a6da Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Fri, 17 Nov 2023 16:13:16 +0000 Subject: [PATCH 2/5] Rekey playbook misc improvements --- etc/kayobe/ansible/rekey-hosts.yml | 87 ++++++++++--------- .../add-rekey-playbook-0065c5057b1639f8.yaml | 2 +- 2 files changed, 46 insertions(+), 43 deletions(-) diff --git a/etc/kayobe/ansible/rekey-hosts.yml b/etc/kayobe/ansible/rekey-hosts.yml index 445e7224f..8641d5791 100644 --- a/etc/kayobe/ansible/rekey-hosts.yml +++ b/etc/kayobe/ansible/rekey-hosts.yml @@ -3,70 +3,73 @@ hosts: overcloud,seed,seed-hypervisor,infra-vms gather_facts: false vars: - ansible_user: stack - ansible_python_interpreter: /usr/bin/python3 + new_key_type: ed25519 + ansible_ssh_common_args: "-o StrictHostKeyChecking=no" + rekey_users: + - stack + - kolla + existing_key_path: "~/.ssh/id_rsa" + rekey_remove_existing_key: true tasks: - - name: Generate a fresh SSH key - community.crypto.openssh_keypair: - path: ~/.ssh/id_rsa_new + - name: Stat existing key file + ansible.builtin.stat: + path: "{{ existing_key_path }}" + register: stat_result delegate_to: localhost + run_once: true - # - name: Copy new key to hosts - # ansible.builtin.copy: - # src: /tmp/id_rsa_new.pub - # dest: /tmp/id_rsa_new.pub - # mode: '0600' - # become: true - - - name: Copy old key to hosts - ansible.builtin.copy: - src: ~/.ssh/id_rsa.pub - dest: /tmp/id_rsa_old.pub - mode: '0777' - become: true + - name: Fail when existing key does not exist + ansible.builtin.fail: + msg: "No existing key file found. Check existing_key_path is set correctly." + when: + - not stat_result.stat.exists + delegate_to: localhost + run_once: true - - name: Set new stack authorized keys - ansible.posix.authorized_key: - user: "{{ item }}" - state: present - key: "{{ lookup('file', '~/.ssh/id_rsa_new.pub') }}" - loop: - - "stack" - - "kolla" - become: true + - name: Generate a new SSH key + community.crypto.openssh_keypair: + path: "~/.ssh/id_{{ new_key_type }}_new" + type: "{{ new_key_type }}" + delegate_to: localhost + run_once: true - - name: Set new stack authorized keys + - name: Set new authorized keys + vars: + lookup_path: "~/.ssh/id_{{ new_key_type }}_new.pub" ansible.posix.authorized_key: user: "{{ item }}" state: present - key: "{{ lookup('file', '~/.ssh/id_rsa_new.pub') }}" - loop: - - "stack" - - "kolla" + key: "{{ lookup('file', lookup_path) }}" + loop: "{{ rekey_users }}" become: true - - name: Locally deprecate old key (private) - command: "mv ~/.ssh/id_rsa ~/.ssh/id_rsa_old" + - name: Locally deprecate existing key (private) + command: "mv {{ existing_key_path }} {{ existing_key_path }}_old" delegate_to: localhost + run_once: true - - name: Locally deprecate old key (public) - command: "mv ~/.ssh/id_rsa.pub ~/.ssh/id_rsa_old.pub" + - name: Locally deprecate existing key (public) + command: "mv {{ existing_key_path }}.pub {{ existing_key_path }}_old.pub" delegate_to: localhost + run_once: true - name: Locally promote new key (private) - command: "mv ~/.ssh/id_rsa_new ~/.ssh/id_rsa" + command: "mv ~/.ssh/id_{{ new_key_type }}_new ~/.ssh/id_{{ new_key_type }}" delegate_to: localhost + run_once: true - name: Locally promote new key (public) - command: " mv ~/.ssh/id_rsa_new.pub ~/.ssh/id_rsa.pub" + command: " mv ~/.ssh/id_{{ new_key_type }}_new.pub ~/.ssh/id_{{ new_key_type }}.pub" delegate_to: localhost + run_once: true - name: Remove old key from hosts + vars: + lookup_path: "{{ existing_key_path }}_old.pub" ansible.posix.authorized_key: user: "{{ item }}" state: absent - key: "{{ lookup('file', '/tmp/id_rsa_old.pub') }}" - loop: - - "stack" - - "kolla" + key: "{{ lookup('file', lookup_path) }}" + loop: "{{ rekey_users }}" become: true + when: rekey_remove_existing_key diff --git a/releasenotes/notes/add-rekey-playbook-0065c5057b1639f8.yaml b/releasenotes/notes/add-rekey-playbook-0065c5057b1639f8.yaml index a9d78cc12..5e75a51ad 100644 --- a/releasenotes/notes/add-rekey-playbook-0065c5057b1639f8.yaml +++ b/releasenotes/notes/add-rekey-playbook-0065c5057b1639f8.yaml @@ -2,4 +2,4 @@ features: - | Added the ``rekey-hosts.yml`` playbook to automatically rotate the SSH - keys on all hosts in the cloud for the stack and kolla users. + keys on all hosts. From 0123c1f541b5553b3de788be169bb4ddaf6a9e10 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Mon, 20 Nov 2023 16:54:46 +0000 Subject: [PATCH 3/5] Change rekey playbook to use existing ssh vars --- etc/kayobe/ansible/rekey-hosts.yml | 43 +++++++++++++++++++----------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/etc/kayobe/ansible/rekey-hosts.yml b/etc/kayobe/ansible/rekey-hosts.yml index 8641d5791..bd903d010 100644 --- a/etc/kayobe/ansible/rekey-hosts.yml +++ b/etc/kayobe/ansible/rekey-hosts.yml @@ -3,24 +3,37 @@ hosts: overcloud,seed,seed-hypervisor,infra-vms gather_facts: false vars: - new_key_type: ed25519 ansible_ssh_common_args: "-o StrictHostKeyChecking=no" rekey_users: - stack - kolla - existing_key_path: "~/.ssh/id_rsa" - rekey_remove_existing_key: true + rekey_remove_existing_key: false tasks: - - name: Stat existing key file + - name: Stat existing private key file ansible.builtin.stat: - path: "{{ existing_key_path }}" + path: "{{ ssh_private_key_path }}" register: stat_result delegate_to: localhost run_once: true - - name: Fail when existing key does not exist + - name: Fail when existing private key does not exist ansible.builtin.fail: - msg: "No existing key file found. Check existing_key_path is set correctly." + msg: "No existing private key file found. Check ssh_private_key_path and is set correctly." + when: + - not stat_result.stat.exists + delegate_to: localhost + run_once: true + + - name: Stat existing public key file + ansible.builtin.stat: + path: "{{ ssh_public_key_path }}" + register: stat_result + delegate_to: localhost + run_once: true + + - name: Fail when existing public key does not exist + ansible.builtin.fail: + msg: "No existing public key file found. Check ssh_public_key_path and is set correctly." when: - not stat_result.stat.exists delegate_to: localhost @@ -28,14 +41,14 @@ - name: Generate a new SSH key community.crypto.openssh_keypair: - path: "~/.ssh/id_{{ new_key_type }}_new" - type: "{{ new_key_type }}" + path: "{{ ssh_private_key_path }}_new" + type: "{{ ssh_key_type }}" delegate_to: localhost run_once: true - name: Set new authorized keys vars: - lookup_path: "~/.ssh/id_{{ new_key_type }}_new.pub" + lookup_path: "{{ ssh_private_key_path }}_new.pub" ansible.posix.authorized_key: user: "{{ item }}" state: present @@ -44,28 +57,28 @@ become: true - name: Locally deprecate existing key (private) - command: "mv {{ existing_key_path }} {{ existing_key_path }}_old" + command: "mv {{ ssh_private_key_path }} {{ ssh_private_key_path }}_old" delegate_to: localhost run_once: true - name: Locally deprecate existing key (public) - command: "mv {{ existing_key_path }}.pub {{ existing_key_path }}_old.pub" + command: "mv {{ ssh_public_key_path }} {{ ssh_public_key_path }}_old" delegate_to: localhost run_once: true - name: Locally promote new key (private) - command: "mv ~/.ssh/id_{{ new_key_type }}_new ~/.ssh/id_{{ new_key_type }}" + command: "mv {{ ssh_private_key_path }}_new {{ ssh_private_key_path }}" delegate_to: localhost run_once: true - name: Locally promote new key (public) - command: " mv ~/.ssh/id_{{ new_key_type }}_new.pub ~/.ssh/id_{{ new_key_type }}.pub" + command: "mv {{ ssh_private_key_path }}_new.pub {{ ssh_public_key_path }}" delegate_to: localhost run_once: true - name: Remove old key from hosts vars: - lookup_path: "{{ existing_key_path }}_old.pub" + lookup_path: "{{ ssh_public_key_path }}_old" ansible.posix.authorized_key: user: "{{ item }}" state: absent From 6931e1cb6e80523475c0a4af97cc56944f0db7bf Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Wed, 29 Nov 2023 17:19:30 +0000 Subject: [PATCH 4/5] Rework rekey-hosts.yml playbook --- etc/kayobe/ansible/rekey-hosts.yml | 35 +++++++++++++++++++----------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/etc/kayobe/ansible/rekey-hosts.yml b/etc/kayobe/ansible/rekey-hosts.yml index bd903d010..6d3b358a6 100644 --- a/etc/kayobe/ansible/rekey-hosts.yml +++ b/etc/kayobe/ansible/rekey-hosts.yml @@ -1,9 +1,18 @@ --- +# Playbook to rotate SSH keys across the cloud. By default it will rotate the +# standard keys used by kayobe/kolla-ansible, but it can be configured for any +# keys. + - name: Rekey hosts hosts: overcloud,seed,seed-hypervisor,infra-vms gather_facts: false vars: ansible_ssh_common_args: "-o StrictHostKeyChecking=no" + existing_private_key_path: "{{ ssh_private_key_path }}" + existing_public_key_path: "{{ ssh_public_key_path }}" + new_private_key_path: "{{ ssh_private_key_path }}" + new_public_key_path: "{{ ssh_public_key_path }}" + new_key_type: "{{ ssh_key_type }}" rekey_users: - stack - kolla @@ -11,14 +20,14 @@ tasks: - name: Stat existing private key file ansible.builtin.stat: - path: "{{ ssh_private_key_path }}" + path: "{{ existing_private_key_path }}" register: stat_result delegate_to: localhost run_once: true - name: Fail when existing private key does not exist ansible.builtin.fail: - msg: "No existing private key file found. Check ssh_private_key_path and is set correctly." + msg: "No existing private key file found. Check existing_private_key_path is set correctly." when: - not stat_result.stat.exists delegate_to: localhost @@ -26,14 +35,14 @@ - name: Stat existing public key file ansible.builtin.stat: - path: "{{ ssh_public_key_path }}" + path: "{{ existing_public_key_path }}" register: stat_result delegate_to: localhost run_once: true - name: Fail when existing public key does not exist ansible.builtin.fail: - msg: "No existing public key file found. Check ssh_public_key_path and is set correctly." + msg: "No existing public key file found. Check existing_public_key_path is set correctly." when: - not stat_result.stat.exists delegate_to: localhost @@ -41,14 +50,14 @@ - name: Generate a new SSH key community.crypto.openssh_keypair: - path: "{{ ssh_private_key_path }}_new" - type: "{{ ssh_key_type }}" + path: "{{ existing_private_key_path }}_new" + type: "{{ new_key_type }}" delegate_to: localhost run_once: true - name: Set new authorized keys vars: - lookup_path: "{{ ssh_private_key_path }}_new.pub" + lookup_path: "{{ existing_private_key_path }}_new.pub" ansible.posix.authorized_key: user: "{{ item }}" state: present @@ -57,32 +66,32 @@ become: true - name: Locally deprecate existing key (private) - command: "mv {{ ssh_private_key_path }} {{ ssh_private_key_path }}_old" + command: "mv {{ existing_private_key_path }} {{ existing_public_key_path }}_old" delegate_to: localhost run_once: true - name: Locally deprecate existing key (public) - command: "mv {{ ssh_public_key_path }} {{ ssh_public_key_path }}_old" + command: "mv {{ existing_public_key_path }} {{ existing_public_key_path }}_old" delegate_to: localhost run_once: true - name: Locally promote new key (private) - command: "mv {{ ssh_private_key_path }}_new {{ ssh_private_key_path }}" + command: "mv {{ existing_private_key_path }}_new {{ new_private_key_path }}" delegate_to: localhost run_once: true - name: Locally promote new key (public) - command: "mv {{ ssh_private_key_path }}_new.pub {{ ssh_public_key_path }}" + command: "mv {{ existing_private_key_path }}_new.pub {{ new_public_key_path }}" delegate_to: localhost run_once: true - name: Remove old key from hosts vars: - lookup_path: "{{ ssh_public_key_path }}_old" + lookup_path: "{{ existing_public_key_path }}_old" ansible.posix.authorized_key: user: "{{ item }}" state: absent key: "{{ lookup('file', lookup_path) }}" loop: "{{ rekey_users }}" become: true - when: rekey_remove_existing_key + when: rekey_remove_existing_key | bool From 14caeb48e796a15e0922e0950cb3f0c87d581dc9 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Thu, 30 Nov 2023 13:24:19 +0000 Subject: [PATCH 5/5] rekey-host.yml remove-key tag --- etc/kayobe/ansible/rekey-hosts.yml | 44 ++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/etc/kayobe/ansible/rekey-hosts.yml b/etc/kayobe/ansible/rekey-hosts.yml index 6d3b358a6..a72da3ac7 100644 --- a/etc/kayobe/ansible/rekey-hosts.yml +++ b/etc/kayobe/ansible/rekey-hosts.yml @@ -7,12 +7,15 @@ hosts: overcloud,seed,seed-hypervisor,infra-vms gather_facts: false vars: - ansible_ssh_common_args: "-o StrictHostKeyChecking=no" + # The existing key is the key that is currently used to access overcloud hosts existing_private_key_path: "{{ ssh_private_key_path }}" existing_public_key_path: "{{ ssh_public_key_path }}" + # The new key is the key that will be generated by this playbook new_private_key_path: "{{ ssh_private_key_path }}" new_public_key_path: "{{ ssh_public_key_path }}" new_key_type: "{{ ssh_key_type }}" + # The existing key will locally be moved to deprecated_key_path once it is replaced + deprecated_key_path: ~/old_ssh_key rekey_users: - stack - kolla @@ -66,12 +69,12 @@ become: true - name: Locally deprecate existing key (private) - command: "mv {{ existing_private_key_path }} {{ existing_public_key_path }}_old" + command: "mv {{ existing_private_key_path }} {{ deprecated_key_path }}" delegate_to: localhost run_once: true - name: Locally deprecate existing key (public) - command: "mv {{ existing_public_key_path }} {{ existing_public_key_path }}_old" + command: "mv {{ existing_public_key_path }} {{ deprecated_key_path }}.pub" delegate_to: localhost run_once: true @@ -85,13 +88,30 @@ delegate_to: localhost run_once: true - - name: Remove old key from hosts - vars: - lookup_path: "{{ existing_public_key_path }}_old" - ansible.posix.authorized_key: - user: "{{ item }}" - state: absent - key: "{{ lookup('file', lookup_path) }}" - loop: "{{ rekey_users }}" - become: true + - block: + - name: Stat old key file + ansible.builtin.stat: + path: "{{ deprecated_key_path }}.pub" + register: stat_result + delegate_to: localhost + run_once: true + + - name: Fail when deprecated public key does not exist + ansible.builtin.fail: + msg: "No deprecated public key file found. Check deprecated_key_path is set correctly." + when: + - not stat_result.stat.exists + delegate_to: localhost + run_once: true + + - name: Remove old key from hosts + vars: + lookup_path: "{{ deprecated_key_path }}.pub" + ansible.posix.authorized_key: + user: "{{ item }}" + state: absent + key: "{{ lookup('file', lookup_path) }}" + loop: "{{ rekey_users }}" + become: true + tags: remove-key when: rekey_remove_existing_key | bool