diff --git a/ansible/deploy-openstack-config.yml b/ansible/deploy-openstack-config.yml index 8a83b8d..c7b0ae3 100644 --- a/ansible/deploy-openstack-config.yml +++ b/ansible/deploy-openstack-config.yml @@ -118,25 +118,6 @@ vars: kayobe_environment_path: "{{ src_directory }}/kayobe-config/etc/kayobe/environments/{{ kayobe_config_environment }}" - # WORKAROUND: - # There is a race condition in Ansible that can result in this failure: - # msg: |- - # MODULE FAILURE - # See stdout/stderr for the exact error - # rc: -13 - # See https://github.com/ansible/ansible/issues/78344 and - # https://github.com/ansible/ansible/issues/81777. - # In https://github.com/stackhpc/stackhpc-kayobe-config/pull/1108 we applied - # a workaround to increase the ControlPersist timeout to 1 hour, but this - # does not always work. - # Disabling SSH pipelining prevents the issue at the cost of Ansible - # execution duration. - - name: Disable SSH pipelining - ansible.builtin.replace: - path: "{{ src_directory }}/{{ kayobe_config_name }}/etc/kayobe/ansible.cfg" - regexp: "^pipelining.*=.*" - replace: "pipelining = False" - - name: Ensure hooks directory are present ansible.builtin.file: dest: "{{ src_directory }}/{{ kayobe_config_name }}/etc/kayobe/hooks/{{ item }}" diff --git a/ansible/files/multinode.sh b/ansible/files/multinode.sh index c23a3d9..78e82be 100755 --- a/ansible/files/multinode.sh +++ b/ansible/files/multinode.sh @@ -82,13 +82,40 @@ function decrypt_file() { ansible-vault decrypt --vault-password-file ~/vault.password $file } +function workaround_ansible_rc13_bug() { + # Call this function in between long-running Ansible executions to attempt to + # work around an Ansible race condition. + + # There is a race condition in Ansible that can result in this failure: + # msg: |- + # MODULE FAILURE + # See stdout/stderr for the exact error + # rc: -13 + # See https://github.com/ansible/ansible/issues/78344 and + # https://github.com/ansible/ansible/issues/81777. + # In https://github.com/stackhpc/stackhpc-kayobe-config/pull/1108 we applied + # a workaround to increase the ControlPersist timeout to 1 hour, but this + # does not always work. + # Here we use a different workaround of disabling SSH pipelining. This has + # performance implications for Ansible, but is a reasonable trade-off for + # reliability. + # We set the config option as an environment variable rather than in + # ansible.cfg in Kayobe configuration, to avoid a merge conflict on upgrade. + export ANSIBLE_PIPELINING=False +} + +function run_kayobe() { + workaround_ansible_rc13_bug + kayobe $* +} + function deploy_seed() { - kayobe seed host configure + run_kayobe seed host configure } function deploy_seed_vault() { # Deploy hashicorp vault to the seed - kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-deploy-seed.yml + run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-deploy-seed.yml encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/vault/OS-TLS-INT.pem encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/vault/seed-vault-keys.json encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/vault/*.key @@ -97,8 +124,8 @@ function deploy_seed_vault() { function get_seed_ssh() { # NOTE: Bash clears the -e option in subshells when not in Posix mode. set -e - ssh_user=$(kayobe configuration dump --host seed[0] --var-name ansible_user | tr -d '"') - seed_addr=$(kayobe configuration dump --host seed[0] --var-name ansible_host | tr -d '"') + ssh_user=$(run_kayobe configuration dump --host seed[0] --var-name ansible_user | tr -d '"') + seed_addr=$(run_kayobe configuration dump --host seed[0] --var-name ansible_host | tr -d '"') echo "${ssh_user}@${seed_addr}" } @@ -117,37 +144,37 @@ function copy_ca_to_seed() { } function deploy_ceph() { - kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-deploy.yml + run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-deploy.yml sleep 30 - kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm.yml - kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-gather-keys.yml + run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm.yml + run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-gather-keys.yml } function deploy_overcloud_vault() { # NOTE: Previously it was necessary to first deploy HAProxy with TLS disabled. if [[ -f $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/globals-tls-config.yml ]]; then # Skip os_capacity deployment since it requires admin-openrc.sh which doesn't exist yet. - kayobe overcloud service deploy --skip-tags os_capacity -kt haproxy + run_kayobe overcloud service deploy --skip-tags os_capacity -kt haproxy fi # Deploy hashicorp vault to the controllers - kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-deploy-overcloud.yml + run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-deploy-overcloud.yml encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/vault/overcloud-vault-keys.json } function generate_overcloud_certs() { # Generate external tls certificates if [[ -f $KAYOBE_CONFIG_PATH/ansible/vault-generate-test-external-tls.yml ]]; then - kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-test-external-tls.yml + run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-test-external-tls.yml encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/certificates/haproxy.pem fi # Generate internal tls certificates - kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-internal-tls.yml + run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-internal-tls.yml encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/certificates/haproxy-internal.pem # Generate backend tls certificates - kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-backend-tls.yml + run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-backend-tls.yml for cert in $(ls -1 $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/certificates/*-key.pem); do encrypt_file $cert done @@ -164,7 +191,7 @@ function generate_barbican_secrets() { decrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/secrets.yml sed -i "s/secret_id:.*/secret_id: $(uuidgen)/g" $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/secrets.yml encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/secrets.yml - kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-deploy-barbican.yml + run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-deploy-barbican.yml decrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/secrets.yml sed -i "s/role_id:.*/role_id: $(cat /tmp/barbican-role-id)/g" $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/secrets.yml encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/secrets.yml @@ -172,7 +199,7 @@ function generate_barbican_secrets() { } function deploy_overcloud() { - kayobe overcloud host configure + run_kayobe overcloud host configure deploy_ceph @@ -185,19 +212,19 @@ function deploy_overcloud() { generate_barbican_secrets # Deploy all services - kayobe overcloud service deploy + run_kayobe overcloud service deploy copy_ca_to_seed } function deploy_wazuh() { - kayobe infra vm host configure + run_kayobe infra vm host configure # Deploy Wazuh - kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-secrets.yml + run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-secrets.yml encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/wazuh-secrets.yml - kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-manager.yml - kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-agent.yml + run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-manager.yml + run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/wazuh-agent.yml } function create_resources() { @@ -317,7 +344,7 @@ function deploy_full() { deploy_seed deploy_overcloud - if kayobe configuration dump --host wazuh-manager --var-name group_names | grep wazuh-manager &>/dev/null; then + if run_kayobe configuration dump --host wazuh-manager --var-name group_names | grep wazuh-manager &>/dev/null; then deploy_wazuh fi create_resources @@ -327,13 +354,13 @@ function deploy_full() { function upgrade_overcloud() { # Generate external tls certificates if it was previously disabled. if [[ -f $KAYOBE_CONFIG_PATH/ansible/vault-generate-test-external-tls.yml ]] && [[ ! -f $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/certificates/haproxy.pem ]]; then - kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-test-external-tls.yml + run_kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-test-external-tls.yml encrypt_file $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/certificates/haproxy.pem fi - kayobe overcloud host upgrade - kayobe overcloud host configure - kayobe overcloud service upgrade + run_kayobe overcloud host upgrade + run_kayobe overcloud host configure + run_kayobe overcloud service upgrade } function usage() {