From a0944139b725f7ba73f3865399c3e03ea33cb58c Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Sat, 4 Nov 2023 20:53:14 +0000 Subject: [PATCH 01/52] feat: add runner and workflow playbooks Add two playbooks responsible for deploying GitHub runners and generating workflows for Kayobe automation. This commit also includes updated `requirements.yml` required by these playbooks. --- etc/kayobe/ansible/deploy-github-runner.yml | 32 +++++++++++++++++++++ etc/kayobe/ansible/requirements.yml | 8 ++++++ etc/kayobe/ansible/write-workflows.yml | 5 ++++ 3 files changed, 45 insertions(+) create mode 100644 etc/kayobe/ansible/deploy-github-runner.yml create mode 100644 etc/kayobe/ansible/write-workflows.yml diff --git a/etc/kayobe/ansible/deploy-github-runner.yml b/etc/kayobe/ansible/deploy-github-runner.yml new file mode 100644 index 000000000..211406f29 --- /dev/null +++ b/etc/kayobe/ansible/deploy-github-runner.yml @@ -0,0 +1,32 @@ +--- +- name: Deploy GitHub Runner + hosts: github-runners + become: yes + roles: + - role: geerlingguy.pip + - role: geerlingguy.docker + tasks: + - name: Deploy runners + ansible.builtin.include_role: + role: monolithprojects.github_actions_runner + vars: + runner_name: "{{ ansible_facts.nodename }}-{{ runner.key }}" + runner_dir: "{{ base_runner_dir }}/{{ runner.key }}" + runner_labels: "{{ runner.value.labels | default(default_runner_labels) }}" + runner_state: "{{ runner.value.state | default('started') }}" + with_dict: + "{{ github_runners }}" + loop_control: + loop_var: runner + + - name: Ensure runner service is running + ansible.builtin.service: + name: actions.runner.{{ github_account }}-{{ github_repo }}.{{ ansible_facts.nodename }}-{{ runner.key }}.service + state: started + enabled: true + become: true + when: runner_state | default('started') == 'started' + with_dict: + "{{ github_runners }}" + loop_control: + loop_var: runner diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index e54069d90..f534eda5c 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -6,6 +6,8 @@ collections: version: 0.4.1 - name: stackhpc.hashicorp version: 2.4.0 + - name: stackhpc.kayobe_workflows + version: 1.0.1 roles: - src: stackhpc.vxlan - name: ansible-lockdown.rhel8_cis @@ -14,3 +16,9 @@ roles: - name: wazuh-ansible src: https://github.com/stackhpc/wazuh-ansible version: stackhpc + - name: geerlingguy.pip + version: 2.2.0 + - name: monolithprojects.github_actions_runner + version: 1.18.5 + - src: https://github.com/stackhpc/ansible-role-docker.git + name: geerlingguy.docker \ No newline at end of file diff --git a/etc/kayobe/ansible/write-workflows.yml b/etc/kayobe/ansible/write-workflows.yml new file mode 100644 index 000000000..708bc22eb --- /dev/null +++ b/etc/kayobe/ansible/write-workflows.yml @@ -0,0 +1,5 @@ +--- +- name: Write Kayobe Automation Workflows for GitHub + hosts: github-writer + roles: + - stackhpc.kayobe_workflows.github From 45af36147d09b0f6e631222210ebd275aa8248b0 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Sat, 4 Nov 2023 20:51:02 +0000 Subject: [PATCH 02/52] feat: define groups and variables for runners and workflows --- etc/kayobe/inventory/group_vars/github-writer/writer.yml | 2 ++ etc/kayobe/inventory/groups | 6 ++++++ 2 files changed, 8 insertions(+) create mode 100644 etc/kayobe/inventory/group_vars/github-writer/writer.yml diff --git a/etc/kayobe/inventory/group_vars/github-writer/writer.yml b/etc/kayobe/inventory/group_vars/github-writer/writer.yml new file mode 100644 index 000000000..be3d3a195 --- /dev/null +++ b/etc/kayobe/inventory/group_vars/github-writer/writer.yml @@ -0,0 +1,2 @@ +--- +github_output_directory: "{{ kayobe_config_path }}/../../.github/workflows" diff --git a/etc/kayobe/inventory/groups b/etc/kayobe/inventory/groups index fa0a55a1c..d368b1bb2 100644 --- a/etc/kayobe/inventory/groups +++ b/etc/kayobe/inventory/groups @@ -31,6 +31,12 @@ seed-hypervisor seed overcloud +[github-runners] +# Empty group to provide declaration of github-runner group. + +[github-writer] +localhost + ############################################################################### # Overcloud groups. From dc02a18b5c8c46872c8bb0d2e1026bcbed44cae5 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 7 Nov 2023 09:16:06 +0000 Subject: [PATCH 03/52] feat: add documentation for `kayobe-automation` --- doc/source/configuration/index.rst | 1 + .../configuration/kayobe-automation.rst | 125 ++++++++++++++++++ 2 files changed, 126 insertions(+) create mode 100644 doc/source/configuration/kayobe-automation.rst diff --git a/doc/source/configuration/index.rst b/doc/source/configuration/index.rst index f19775700..0931a1282 100644 --- a/doc/source/configuration/index.rst +++ b/doc/source/configuration/index.rst @@ -18,3 +18,4 @@ the various features provided. wazuh vault magnum-capi + kayobe-automation diff --git a/doc/source/configuration/kayobe-automation.rst b/doc/source/configuration/kayobe-automation.rst new file mode 100644 index 000000000..5694d088f --- /dev/null +++ b/doc/source/configuration/kayobe-automation.rst @@ -0,0 +1,125 @@ +================= +Kayobe Automation +================= + +What is Kayobe Automation +========================= + +`Kayobe automation `__ is a collection of scripts and tools that enable the automation of kayobe related operations through the use of CI/CD platforms such as those provided by GitHub and GitLab. +Kayobe automation provides users an easy process of performing tasks such as; overcloud service deploy, config-diff, tempest testing and many more. +With it being integrated into platforms such as GitHub or GitLab it builds a close relationship between the contents of the deployments kayobe configuration and what is currently deployed. +This is because operations such as opening a pull request will trigger a config diff to be generated providing insight on what impact it might have on services or a tempest test that could be scheduled to run daily providing knowledge of faults earlier than before. + +Kayobe automation has been designed to be independent of any CI/CD platform with all tasks running inside of a purpose build Kayobe container. +However, platform specific workflows need to be deployed to bridge the gap between the contents of Kayobe Automation and these CI/CD platforms. +To achieve this work has been carried to template workflows as deployment specific choices have to be made when writing workflows, such as; multiple environment support, container registry location and secret sharing. +The templating of workflows is offered through the `stackhpc.kayobe_workflows `__ collection which currently supports GitHub workflows and should enable the easy and error free deployment of workflows. + +GitHub Deployment +================= + +To enable Kayobe Automation where GitHub Actions is used please follow the steps described below starting with the deployment the runners. + +Runner Deployment +----------------- + +1. Identify a suitable host for hosting the runners. + GitHub runners need to be deployed on a host which has not had Docker deployed using kolla. + This is because GitHub runners cannot provide `network options when running in a container `__. + + Ideally an Infra VM could be used here or failing that the control host. + Wherever it is deployed the host will need access to the :code:`admin_network`, :code:`public_network` and the :code:`pulp registry` on the seed. + +2. Edit the environments :code:`inventory/groups` to add the predefined :code:`github-runners` group to :code:`infra-vms` + +.. code-block:: ini + + [infra-vms:children] + github-runners + +3. Edit the environments :code:`inventory/hosts` to define the host(s) that will host the runners. + +.. code-block:: ini + + [github-runner] + runner-01 + +4. Provide all the relevant Kayobe :code:`group_vars` for :code:`github-runners` under :code:`inventory/hosts/github-runners` + * `infra-vms` ensuring all required `infra_vm_extra_network_interface` are defined + * `network-interfaces` + * `python-interpreter.yml` ensuring that `ansible_python_interpreter: /usr/bin/python3` has been set + +5. Create `runner.yml` file which will contain the variables required to deploy a series of runners + +.. code-block:: yaml + + --- + runner_user: VM_USER_NAME_HERE + github_account: ORG_NAME_HERE + github_repo: KAYOBE_CONFIG_REPO_NAME_HERE + access_token: "{{ secrets_github_access_token }}" + + base_runner_dir: /opt/actions-runner + + default_runner_labels: + - kayobe + - openstack + + github_runners: + runner_01: {} + runner_02: {} + runner_03: {} + + docker_users: + - "{{ runner_user }}" + + pip_install_packages: + - name: docker + +If using multiple environments add an extra label to :code:`default_runner_labels` to distinguish these runners from runners belonging to other environments. +Also feel free to change the number of runners and their names. + +6. Obtain a personal access token that would enable the registration of GitHub runners against the `github_account` and `github_repo` defined above. + This token ideally should be `fine-grain personal access token `__ which may require the organisation to enable such tokens beforehand steps can be found `here `__ + The repository permissions for a fine-grain personal access token should be; :code:`Actions: R/W, Administration: R/W, Metadata: R` + Once obtained the key add to :code:`secrets.yml` under :code:`secrets_github_access_token` + +7. Run :code:`kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/deploy-github-runner.yml` + +8. Check runners have registered properly by visiting the repositories :code:`Action` tab -> :code:`Runners` -> :code:`Self-hosted runners` + +9. Repeat the above steps for each environment you intend to deploy runners within. + You can share the fine-grain access token between environments. + +Workflow Deployment +------------------- + +1. Edit `inventory/group_vars/github-writer/writer.yml` in the base configuration making the appropriate changes to your deployments specific needs see documentation for `stackhpc.kayobe_workflows.github `__ + +2. Run :code:`kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/write-workflows.yml` + +3. Add all required secrets to repository either via the GitHub UI or GitHub CLI (may require repository owner) + * KAYOBE_AUTOMATION_SSH_PRIVATE_KEY + * KAYOBE_VAULT_PASSWORD + * REGISTRY_PASSWORD + * TEMPEST_OPENRC + +Note if using multiple environments and are not sharing secrets between environments then each of must have the environment name prefix for each enviromment for example + * PRODUCTION_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY + * PRODUCTION_KAYOBE_VAULT_PASSWORD + * PRODUCTION_REGISTRY_PASSWORD + * PRODUCTION_TEMPEST_OPENRC + * STAGING_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY + * STAGING_KAYOBE_VAULT_PASSWORD + * STAGING_REGISTRY_PASSWORD + * STAGING_TEMPEST_OPENRC + +4. Commit and push all newly generated workflows found under :code:`.github/workflows` + +Final Steps +----------- + +Some final steps include the following; running config-diff will require that :code:`.automation.conf/config.sh` contains a list :code:`KAYOBE_CONFIG_VAULTED_FILES_PATHS_EXTRA` of all vaulted files contained within the config. +All such files can be found with :code:`grep -r "$ANSIBLE_VAULT;1.1;AES256" .` though make sure NOT to include `kolla/passwords.yml` and `secrets.yml` +Also make sure tempest has been configured appropriately in :code:`.automation.conf/config.sh` to meet the limitations of a given deployment such as not using a too high of :code:`TEMPEST_CONCURRENCY` value and that overrides and load/skips lists are correct. +Finally, once all the workflows and configuration has been pushed and reviewed you can build a kayobe image using `Build Kayobe Docker Image` once succesfully built and pushed to a container registry other workflows can be used. From 6ee524300e3c942fd3679ce80047d81bbb922d32 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 7 Nov 2023 09:23:32 +0000 Subject: [PATCH 04/52] feat: add release note --- ...ithub-kayobe-automation-support-33dc85bbc8746d82.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 releasenotes/notes/add-github-kayobe-automation-support-33dc85bbc8746d82.yaml diff --git a/releasenotes/notes/add-github-kayobe-automation-support-33dc85bbc8746d82.yaml b/releasenotes/notes/add-github-kayobe-automation-support-33dc85bbc8746d82.yaml new file mode 100644 index 000000000..e18490003 --- /dev/null +++ b/releasenotes/notes/add-github-kayobe-automation-support-33dc85bbc8746d82.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + Adds support for deploying GitHub runners and creating GitHub workflows + for use within Kayobe Automation. Two playbooks and their requirements + have been added to `ansible/` in addition to the relevant groups defined + with some useful default variables where appropriate. Finally, + documentation has been added to cover how to deploy these runners and + workflows. From fca6ed059ae25f30ed053f44ef51dead0fea4b63 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 7 Nov 2023 21:02:54 +0000 Subject: [PATCH 05/52] fix: `new-line-at-end-of-file` --- etc/kayobe/ansible/requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index f534eda5c..678812b92 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -21,4 +21,4 @@ roles: - name: monolithprojects.github_actions_runner version: 1.18.5 - src: https://github.com/stackhpc/ansible-role-docker.git - name: geerlingguy.docker \ No newline at end of file + name: geerlingguy.docker From b8582d0416389a8ddf94ebf2edc5cd787dc04825 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 8 Nov 2023 08:49:39 +0000 Subject: [PATCH 06/52] fix: `D002 Trailing whitespace` --- doc/source/configuration/kayobe-automation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/configuration/kayobe-automation.rst b/doc/source/configuration/kayobe-automation.rst index 5694d088f..d3a0b0957 100644 --- a/doc/source/configuration/kayobe-automation.rst +++ b/doc/source/configuration/kayobe-automation.rst @@ -88,7 +88,7 @@ Also feel free to change the number of runners and their names. 8. Check runners have registered properly by visiting the repositories :code:`Action` tab -> :code:`Runners` -> :code:`Self-hosted runners` -9. Repeat the above steps for each environment you intend to deploy runners within. +9. Repeat the above steps for each environment you intend to deploy runners within. You can share the fine-grain access token between environments. Workflow Deployment From 4491f5b9eef88270fd3226d9c426a590465ef603 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 9 Nov 2023 10:05:57 +0000 Subject: [PATCH 07/52] feat: add amendments Co-authored-by: Alex-Welsh <112560678+Alex-Welsh@users.noreply.github.com> --- .../configuration/kayobe-automation.rst | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/doc/source/configuration/kayobe-automation.rst b/doc/source/configuration/kayobe-automation.rst index d3a0b0957..03fd836ef 100644 --- a/doc/source/configuration/kayobe-automation.rst +++ b/doc/source/configuration/kayobe-automation.rst @@ -5,15 +5,16 @@ Kayobe Automation What is Kayobe Automation ========================= -`Kayobe automation `__ is a collection of scripts and tools that enable the automation of kayobe related operations through the use of CI/CD platforms such as those provided by GitHub and GitLab. -Kayobe automation provides users an easy process of performing tasks such as; overcloud service deploy, config-diff, tempest testing and many more. +`Kayobe automation `__ is a collection of scripts and tools that automate kayobe operations. +It is deployed and controlled by CI/CD platforms such as GitHub actions and GitLab pipelines. +Kayobe automation provides users with an easy process to perform tasks such as; overcloud service deploy, config-diff, tempest testing and many more. With it being integrated into platforms such as GitHub or GitLab it builds a close relationship between the contents of the deployments kayobe configuration and what is currently deployed. This is because operations such as opening a pull request will trigger a config diff to be generated providing insight on what impact it might have on services or a tempest test that could be scheduled to run daily providing knowledge of faults earlier than before. -Kayobe automation has been designed to be independent of any CI/CD platform with all tasks running inside of a purpose build Kayobe container. +Kayobe automation has been designed to be independent of any CI/CD platform with all tasks running inside of a purpose built Kayobe container. However, platform specific workflows need to be deployed to bridge the gap between the contents of Kayobe Automation and these CI/CD platforms. To achieve this work has been carried to template workflows as deployment specific choices have to be made when writing workflows, such as; multiple environment support, container registry location and secret sharing. -The templating of workflows is offered through the `stackhpc.kayobe_workflows `__ collection which currently supports GitHub workflows and should enable the easy and error free deployment of workflows. +The templating of workflows is offered through the `stackhpc.kayobe_workflows `__ collection which currently supports GitHub workflows. GitHub Deployment ================= @@ -30,14 +31,14 @@ Runner Deployment Ideally an Infra VM could be used here or failing that the control host. Wherever it is deployed the host will need access to the :code:`admin_network`, :code:`public_network` and the :code:`pulp registry` on the seed. -2. Edit the environments :code:`inventory/groups` to add the predefined :code:`github-runners` group to :code:`infra-vms` +2. Edit the environment's :code:`inventory/groups` to add the predefined :code:`github-runners` group to :code:`infra-vms` .. code-block:: ini [infra-vms:children] github-runners -3. Edit the environments :code:`inventory/hosts` to define the host(s) that will host the runners. +3. Edit the environment's :code:`inventory/hosts` to define the host(s) that will host the runners. .. code-block:: ini @@ -82,11 +83,11 @@ Also feel free to change the number of runners and their names. 6. Obtain a personal access token that would enable the registration of GitHub runners against the `github_account` and `github_repo` defined above. This token ideally should be `fine-grain personal access token `__ which may require the organisation to enable such tokens beforehand steps can be found `here `__ The repository permissions for a fine-grain personal access token should be; :code:`Actions: R/W, Administration: R/W, Metadata: R` - Once obtained the key add to :code:`secrets.yml` under :code:`secrets_github_access_token` + Once the key has been obtained, add it to :code:`secrets.yml` under :code:`secrets_github_access_token` 7. Run :code:`kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/deploy-github-runner.yml` -8. Check runners have registered properly by visiting the repositories :code:`Action` tab -> :code:`Runners` -> :code:`Self-hosted runners` +8. Check runners have registered properly by visiting the repository's :code:`Action` tab -> :code:`Runners` -> :code:`Self-hosted runners` 9. Repeat the above steps for each environment you intend to deploy runners within. You can share the fine-grain access token between environments. @@ -104,7 +105,7 @@ Workflow Deployment * REGISTRY_PASSWORD * TEMPEST_OPENRC -Note if using multiple environments and are not sharing secrets between environments then each of must have the environment name prefix for each enviromment for example +Note if you are using multiple environments and not sharing secrets between environments then each of must have the environment name prefix for each enviromment for example * PRODUCTION_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY * PRODUCTION_KAYOBE_VAULT_PASSWORD * PRODUCTION_REGISTRY_PASSWORD @@ -119,7 +120,7 @@ Note if using multiple environments and are not sharing secrets between environm Final Steps ----------- -Some final steps include the following; running config-diff will require that :code:`.automation.conf/config.sh` contains a list :code:`KAYOBE_CONFIG_VAULTED_FILES_PATHS_EXTRA` of all vaulted files contained within the config. +Some final steps include the following: running config-diff will require that :code:`.automation.conf/config.sh` contains a list :code:`KAYOBE_CONFIG_VAULTED_FILES_PATHS_EXTRA` of all vaulted files contained within the config. All such files can be found with :code:`grep -r "$ANSIBLE_VAULT;1.1;AES256" .` though make sure NOT to include `kolla/passwords.yml` and `secrets.yml` Also make sure tempest has been configured appropriately in :code:`.automation.conf/config.sh` to meet the limitations of a given deployment such as not using a too high of :code:`TEMPEST_CONCURRENCY` value and that overrides and load/skips lists are correct. -Finally, once all the workflows and configuration has been pushed and reviewed you can build a kayobe image using `Build Kayobe Docker Image` once succesfully built and pushed to a container registry other workflows can be used. +Finally, once all the workflows and configuration has been pushed and reviewed you can build a kayobe image using the `Build Kayobe Docker Image` workflow. Once it is successfully built and pushed to a container registry, other workflows can be used. From 9988ec7e1fdc2fcd925f10f018a2a23ad590aa9e Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 9 Nov 2023 10:21:26 +0000 Subject: [PATCH 08/52] feat: add description of secrets --- doc/source/configuration/kayobe-automation.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/source/configuration/kayobe-automation.rst b/doc/source/configuration/kayobe-automation.rst index 03fd836ef..282b82bb1 100644 --- a/doc/source/configuration/kayobe-automation.rst +++ b/doc/source/configuration/kayobe-automation.rst @@ -100,10 +100,10 @@ Workflow Deployment 2. Run :code:`kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/write-workflows.yml` 3. Add all required secrets to repository either via the GitHub UI or GitHub CLI (may require repository owner) - * KAYOBE_AUTOMATION_SSH_PRIVATE_KEY - * KAYOBE_VAULT_PASSWORD - * REGISTRY_PASSWORD - * TEMPEST_OPENRC + * KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: private key used by Ansible to authenticate with machines. + * KAYOBE_VAULT_PASSWORD: password used by the config to encrypt Ansible Vault secrets. + * REGISTRY_PASSWORD: password used to login to the docker registry such as Pulp. + * TEMPEST_OPENRC: contents of :code:`kolla/public-openrc.sh` Note if you are using multiple environments and not sharing secrets between environments then each of must have the environment name prefix for each enviromment for example * PRODUCTION_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY From b6bde8765330bb4bc1aa369cbdfc5888b71387b8 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 9 Nov 2023 15:46:36 +0000 Subject: [PATCH 09/52] Update doc/source/configuration/kayobe-automation.rst Co-authored-by: Alex-Welsh <112560678+Alex-Welsh@users.noreply.github.com> --- doc/source/configuration/kayobe-automation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/configuration/kayobe-automation.rst b/doc/source/configuration/kayobe-automation.rst index 282b82bb1..b28f897f4 100644 --- a/doc/source/configuration/kayobe-automation.rst +++ b/doc/source/configuration/kayobe-automation.rst @@ -7,7 +7,7 @@ What is Kayobe Automation `Kayobe automation `__ is a collection of scripts and tools that automate kayobe operations. It is deployed and controlled by CI/CD platforms such as GitHub actions and GitLab pipelines. -Kayobe automation provides users with an easy process to perform tasks such as; overcloud service deploy, config-diff, tempest testing and many more. +Kayobe automation provides users with an easy process to perform tasks such as: overcloud service deploy, config-diff, tempest testing, and many more. With it being integrated into platforms such as GitHub or GitLab it builds a close relationship between the contents of the deployments kayobe configuration and what is currently deployed. This is because operations such as opening a pull request will trigger a config diff to be generated providing insight on what impact it might have on services or a tempest test that could be scheduled to run daily providing knowledge of faults earlier than before. From 118c713714f6841d734dbefa7ac9c2b2e153ede0 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Fri, 10 Nov 2023 12:40:05 +0000 Subject: [PATCH 10/52] feat: add suggested amendments Co-authored-by: Mark Goddard --- .../configuration/kayobe-automation.rst | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/doc/source/configuration/kayobe-automation.rst b/doc/source/configuration/kayobe-automation.rst index b28f897f4..cb6ba8f90 100644 --- a/doc/source/configuration/kayobe-automation.rst +++ b/doc/source/configuration/kayobe-automation.rst @@ -13,13 +13,13 @@ This is because operations such as opening a pull request will trigger a config Kayobe automation has been designed to be independent of any CI/CD platform with all tasks running inside of a purpose built Kayobe container. However, platform specific workflows need to be deployed to bridge the gap between the contents of Kayobe Automation and these CI/CD platforms. -To achieve this work has been carried to template workflows as deployment specific choices have to be made when writing workflows, such as; multiple environment support, container registry location and secret sharing. +Workflows are templated for each Kayobe configuration repository, ensuring appropriate workflow input parameters are defined, and any necessary customisations can be applied. The templating of workflows is offered through the `stackhpc.kayobe_workflows `__ collection which currently supports GitHub workflows. -GitHub Deployment +GitHub Actions ================= -To enable Kayobe Automation where GitHub Actions is used please follow the steps described below starting with the deployment the runners. +To enable CI/CD where GitHub Actions is used please follow the steps described below starting with the deployment of the runners. Runner Deployment ----------------- @@ -42,11 +42,11 @@ Runner Deployment .. code-block:: ini - [github-runner] + [github-runners] runner-01 -4. Provide all the relevant Kayobe :code:`group_vars` for :code:`github-runners` under :code:`inventory/hosts/github-runners` - * `infra-vms` ensuring all required `infra_vm_extra_network_interface` are defined +4. Provide all the relevant Kayobe :code:`group_vars` for :code:`github-runners` under :code:`inventory/group_vars/github-runners` + * `infra-vms` ensuring all required `infra_vm_extra_network_interfaces` are defined * `network-interfaces` * `python-interpreter.yml` ensuring that `ansible_python_interpreter: /usr/bin/python3` has been set @@ -81,8 +81,9 @@ If using multiple environments add an extra label to :code:`default_runner_label Also feel free to change the number of runners and their names. 6. Obtain a personal access token that would enable the registration of GitHub runners against the `github_account` and `github_repo` defined above. - This token ideally should be `fine-grain personal access token `__ which may require the organisation to enable such tokens beforehand steps can be found `here `__ - The repository permissions for a fine-grain personal access token should be; :code:`Actions: R/W, Administration: R/W, Metadata: R` + This token ideally should be `fine-grained personal access token `__ which may require the organisation to enable such tokens beforehand. + Steps can be found `here `__. + The repository permissions for a fine-grained personal access token should be; :code:`Actions: R/W, Administration: R/W, Metadata: R` Once the key has been obtained, add it to :code:`secrets.yml` under :code:`secrets_github_access_token` 7. Run :code:`kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/deploy-github-runner.yml` @@ -90,12 +91,12 @@ Also feel free to change the number of runners and their names. 8. Check runners have registered properly by visiting the repository's :code:`Action` tab -> :code:`Runners` -> :code:`Self-hosted runners` 9. Repeat the above steps for each environment you intend to deploy runners within. - You can share the fine-grain access token between environments. + You can share the fine-grained access token between environments. Workflow Deployment ------------------- -1. Edit `inventory/group_vars/github-writer/writer.yml` in the base configuration making the appropriate changes to your deployments specific needs see documentation for `stackhpc.kayobe_workflows.github `__ +1. Edit `inventory/group_vars/github-writer/writer.yml` in the base configuration making the appropriate changes to your deployments specific needs. See documentation for `stackhpc.kayobe_workflows.github `__. 2. Run :code:`kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/write-workflows.yml` @@ -105,7 +106,7 @@ Workflow Deployment * REGISTRY_PASSWORD: password used to login to the docker registry such as Pulp. * TEMPEST_OPENRC: contents of :code:`kolla/public-openrc.sh` -Note if you are using multiple environments and not sharing secrets between environments then each of must have the environment name prefix for each enviromment for example +Note if you are using multiple environments and not sharing secrets between environments then each of these must have the environment name prefix for each environment, for example: * PRODUCTION_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY * PRODUCTION_KAYOBE_VAULT_PASSWORD * PRODUCTION_REGISTRY_PASSWORD From fed02da4429fcf016a191393a1fc9a64c5012349 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Fri, 10 Nov 2023 13:05:11 +0000 Subject: [PATCH 11/52] feat: rename `kayobe-automation` -> `ci-cd` --- .../configuration/{kayobe-automation.rst => ci-cd.rst} | 6 +++--- doc/source/configuration/index.rst | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) rename doc/source/configuration/{kayobe-automation.rst => ci-cd.rst} (99%) diff --git a/doc/source/configuration/kayobe-automation.rst b/doc/source/configuration/ci-cd.rst similarity index 99% rename from doc/source/configuration/kayobe-automation.rst rename to doc/source/configuration/ci-cd.rst index cb6ba8f90..114c7a2c4 100644 --- a/doc/source/configuration/kayobe-automation.rst +++ b/doc/source/configuration/ci-cd.rst @@ -1,6 +1,6 @@ -================= -Kayobe Automation -================= +===== +CI/CD +===== What is Kayobe Automation ========================= diff --git a/doc/source/configuration/index.rst b/doc/source/configuration/index.rst index 0931a1282..09c91dba8 100644 --- a/doc/source/configuration/index.rst +++ b/doc/source/configuration/index.rst @@ -18,4 +18,4 @@ the various features provided. wazuh vault magnum-capi - kayobe-automation + ci-cd From 0d0af26d40da9a304446fed33d092ef7c7a153da Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Fri, 10 Nov 2023 13:09:25 +0000 Subject: [PATCH 12/52] feat: add link to upstream `Infra VMs` docs --- doc/source/configuration/ci-cd.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/source/configuration/ci-cd.rst b/doc/source/configuration/ci-cd.rst index 114c7a2c4..7a555d674 100644 --- a/doc/source/configuration/ci-cd.rst +++ b/doc/source/configuration/ci-cd.rst @@ -86,11 +86,13 @@ Also feel free to change the number of runners and their names. The repository permissions for a fine-grained personal access token should be; :code:`Actions: R/W, Administration: R/W, Metadata: R` Once the key has been obtained, add it to :code:`secrets.yml` under :code:`secrets_github_access_token` -7. Run :code:`kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/deploy-github-runner.yml` +7. If the host is an actual Infra VM then please refer to upstream `Infrastructure VMs `__ documentation for additional configuration and steps. -8. Check runners have registered properly by visiting the repository's :code:`Action` tab -> :code:`Runners` -> :code:`Self-hosted runners` +8. Run :code:`kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/deploy-github-runner.yml` -9. Repeat the above steps for each environment you intend to deploy runners within. +9. Check runners have registered properly by visiting the repository's :code:`Action` tab -> :code:`Runners` -> :code:`Self-hosted runners` + +10. Repeat the above steps for each environment you intend to deploy runners within. You can share the fine-grained access token between environments. Workflow Deployment From f0ab2f663cdaaeb7ae2771d16b738117e6b49c39 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Fri, 10 Nov 2023 13:41:52 +0000 Subject: [PATCH 13/52] feat: provide reasons for building new image --- doc/source/configuration/ci-cd.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/source/configuration/ci-cd.rst b/doc/source/configuration/ci-cd.rst index 7a555d674..bcb6efbd0 100644 --- a/doc/source/configuration/ci-cd.rst +++ b/doc/source/configuration/ci-cd.rst @@ -127,3 +127,12 @@ Some final steps include the following: running config-diff will require that :c All such files can be found with :code:`grep -r "$ANSIBLE_VAULT;1.1;AES256" .` though make sure NOT to include `kolla/passwords.yml` and `secrets.yml` Also make sure tempest has been configured appropriately in :code:`.automation.conf/config.sh` to meet the limitations of a given deployment such as not using a too high of :code:`TEMPEST_CONCURRENCY` value and that overrides and load/skips lists are correct. Finally, once all the workflows and configuration has been pushed and reviewed you can build a kayobe image using the `Build Kayobe Docker Image` workflow. Once it is successfully built and pushed to a container registry, other workflows can be used. + +Sometimes the kayobe docker image must be rebuilt the reasons for this include but are not limited to the following; + + * Change $KAYOBE_CONFIG_PATH/ansible/requirements.yml + * Change to requirements.txt + * Update Kayobe + * Update kolla-ansible + * UID/GID collision when deploying workflows to a new environment + * Prior to deployment of new a OpenStack release From d72f0ef07776f4ae7bbfa82d7a952a545401ea16 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Fri, 10 Nov 2023 14:19:17 +0000 Subject: [PATCH 14/52] fix: use explicit paths --- doc/source/configuration/ci-cd.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/source/configuration/ci-cd.rst b/doc/source/configuration/ci-cd.rst index bcb6efbd0..2948e1828 100644 --- a/doc/source/configuration/ci-cd.rst +++ b/doc/source/configuration/ci-cd.rst @@ -31,26 +31,26 @@ Runner Deployment Ideally an Infra VM could be used here or failing that the control host. Wherever it is deployed the host will need access to the :code:`admin_network`, :code:`public_network` and the :code:`pulp registry` on the seed. -2. Edit the environment's :code:`inventory/groups` to add the predefined :code:`github-runners` group to :code:`infra-vms` +2. Edit the environment's :code:`${KAYOBE_CONFIG_PATH}/environments/${KAYOBE_ENVIRONMENT}/inventory/groups` to add the predefined :code:`github-runners` group to :code:`infra-vms` .. code-block:: ini [infra-vms:children] github-runners -3. Edit the environment's :code:`inventory/hosts` to define the host(s) that will host the runners. +3. Edit the environment's :code:`${KAYOBE_CONFIG_PATH}/environments/${KAYOBE_ENVIRONMENT}/inventory/hosts` to define the host(s) that will host the runners. .. code-block:: ini [github-runners] runner-01 -4. Provide all the relevant Kayobe :code:`group_vars` for :code:`github-runners` under :code:`inventory/group_vars/github-runners` +4. Provide all the relevant Kayobe :code:`group_vars` for :code:`github-runners` under :code:`${KAYOBE_CONFIG_PATH}/environments/${KAYOBE_ENVIRONMENT}/inventory/group_vars/github-runners` * `infra-vms` ensuring all required `infra_vm_extra_network_interfaces` are defined * `network-interfaces` * `python-interpreter.yml` ensuring that `ansible_python_interpreter: /usr/bin/python3` has been set -5. Create `runner.yml` file which will contain the variables required to deploy a series of runners +5. Create `${KAYOBE_CONFIG_PATH}/environments/${KAYOBE_ENVIRONMENT}/inventory/group_vars/github-runners/runner.yml` file which will contain the variables required to deploy a series of runners .. code-block:: yaml @@ -98,7 +98,7 @@ Also feel free to change the number of runners and their names. Workflow Deployment ------------------- -1. Edit `inventory/group_vars/github-writer/writer.yml` in the base configuration making the appropriate changes to your deployments specific needs. See documentation for `stackhpc.kayobe_workflows.github `__. +1. Edit `${KAYOBE_CONFIG_PATH}/inventory/group_vars/github-writer/writer.yml` in the base configuration making the appropriate changes to your deployments specific needs. See documentation for `stackhpc.kayobe_workflows.github `__. 2. Run :code:`kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/write-workflows.yml` @@ -130,7 +130,7 @@ Finally, once all the workflows and configuration has been pushed and reviewed y Sometimes the kayobe docker image must be rebuilt the reasons for this include but are not limited to the following; - * Change $KAYOBE_CONFIG_PATH/ansible/requirements.yml + * Change :code:`$KAYOBE_CONFIG_PATH/ansible/requirements.yml` * Change to requirements.txt * Update Kayobe * Update kolla-ansible From 839844e18505ec1f40fd9b7385da61d14b989016 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Fri, 10 Nov 2023 14:57:59 +0000 Subject: [PATCH 15/52] feat: add comment to explain service start --- etc/kayobe/ansible/deploy-github-runner.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etc/kayobe/ansible/deploy-github-runner.yml b/etc/kayobe/ansible/deploy-github-runner.yml index 211406f29..47c8211a1 100644 --- a/etc/kayobe/ansible/deploy-github-runner.yml +++ b/etc/kayobe/ansible/deploy-github-runner.yml @@ -19,6 +19,8 @@ loop_control: loop_var: runner + # FIXME: Sometimes the runner service is not running at the end of the role. + # Start the service manually. - name: Ensure runner service is running ansible.builtin.service: name: actions.runner.{{ github_account }}-{{ github_repo }}.{{ ansible_facts.nodename }}-{{ runner.key }}.service From c3298e22da3f706c40ad00a44c01a7926cfd1f95 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Fri, 10 Nov 2023 15:21:42 +0000 Subject: [PATCH 16/52] feat: high level overview of `CI/CD` --- doc/source/configuration/ci-cd.rst | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/doc/source/configuration/ci-cd.rst b/doc/source/configuration/ci-cd.rst index 2948e1828..bff0834f0 100644 --- a/doc/source/configuration/ci-cd.rst +++ b/doc/source/configuration/ci-cd.rst @@ -2,8 +2,23 @@ CI/CD ===== -What is Kayobe Automation -========================= +Concepts +======== + +The CI/CD system developed for managing Kayobe based OpenStack clouds is composed of three main components; workflows, runners and kayobe automation. +Firstly, the workflows are files which describe a series of tasks to be performed in relation to the deployed cloud. +These workflows are executed on request, on schedule or in response to an event such as a pull request being opened. +The workflows are designed to carry out various day-to-day activites such as; running Tempest tests, configuring running services or displaying the change to configuration files if a pull request is merged. +Secondly, in order for the workflows to run against a cloud we would need private runners present within the cloud positioned in such a way they can reach the internal network and public API. +Deployment of private runners is supported by all major providers with the use of community developed Ansible roles. +Finally, due to the requirement that we support various different platforms tooling in the form of `Kayobe automation `__ was developed. +This tooling is not tied to any single CI/CD platform as all tasks are a series of shell script and Ansible playbooks which are designed to run in a purpose build kayobe container. +This is complemented by the use of an Ansible collection known as `stackhpc.kayobe_workflows `__ which aims to provide users with a quick and easy way of customising all workflows to fit within a customer's cloud. + +Currently we support the creation and deployment of workflows for GitHub with Gitlab support being actively worked upon. + +Kayobe Automation +----------------- `Kayobe automation `__ is a collection of scripts and tools that automate kayobe operations. It is deployed and controlled by CI/CD platforms such as GitHub actions and GitLab pipelines. From 4bb0e066b9caadc9bdc65f1fe29b4a0d852e53d2 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Fri, 10 Nov 2023 15:33:44 +0000 Subject: [PATCH 17/52] feat: provide comment of `github_runners` format --- doc/source/configuration/ci-cd.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/doc/source/configuration/ci-cd.rst b/doc/source/configuration/ci-cd.rst index bff0834f0..4c51f160a 100644 --- a/doc/source/configuration/ci-cd.rst +++ b/doc/source/configuration/ci-cd.rst @@ -31,6 +31,13 @@ However, platform specific workflows need to be deployed to bridge the gap betwe Workflows are templated for each Kayobe configuration repository, ensuring appropriate workflow input parameters are defined, and any necessary customisations can be applied. The templating of workflows is offered through the `stackhpc.kayobe_workflows `__ collection which currently supports GitHub workflows. +Runners +------- + +Runners are purpose built services tied to a particular service vendor such as GitHub Actions or GitLab CI. +These services will listen for jobs which have been tagged appropriately and dispatched to these specific runners. +The runners will need to be deployed using existing roles and playbooks whereby the binary/package is downloaded and registered using a special token. + GitHub Actions ================= @@ -81,6 +88,21 @@ Runner Deployment - kayobe - openstack + # Dictionary of runners to be deployed using the role. + # Each dict item can be provided with optional attributes + # * labels - provide a list of labels for a specific runner + # overriding the contents of `default_runner_labels` + # * state - either `started`` or `absent`. By default it will + # be started if however the runner needs to be removed + # then setting it to `absent` will unregister the runner with + # GitHub and remove it from the system. + # Example + # github_runners: + # runner_01: {} + # runner_02: + # labels: ['foo', 'bar', 'baz'] + # runner_03: + # state: absent github_runners: runner_01: {} runner_02: {} From c1153eccd31fbcd96e906a5f3cbb935b5faff410 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Fri, 10 Nov 2023 15:54:04 +0000 Subject: [PATCH 18/52] feat: use `default(omit)` in `default_runner_labels` --- doc/source/configuration/ci-cd.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/configuration/ci-cd.rst b/doc/source/configuration/ci-cd.rst index 4c51f160a..4eac4a017 100644 --- a/doc/source/configuration/ci-cd.rst +++ b/doc/source/configuration/ci-cd.rst @@ -87,6 +87,7 @@ Runner Deployment default_runner_labels: - kayobe - openstack + - "{{ kayobe_environment | default(omit) }}" # Dictionary of runners to be deployed using the role. # Each dict item can be provided with optional attributes From 34249ab8e43c78d91eb49e2c05d510f302b44541 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 14 Nov 2023 16:34:56 +0000 Subject: [PATCH 19/52] feat: placeholder `runners.yml` to group_vars --- doc/source/configuration/ci-cd.rst | 2 +- .../group_vars/github-runners/runners.yml | 38 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 etc/kayobe/inventory/group_vars/github-runners/runners.yml diff --git a/doc/source/configuration/ci-cd.rst b/doc/source/configuration/ci-cd.rst index 4eac4a017..73e2b47d0 100644 --- a/doc/source/configuration/ci-cd.rst +++ b/doc/source/configuration/ci-cd.rst @@ -72,7 +72,7 @@ Runner Deployment * `network-interfaces` * `python-interpreter.yml` ensuring that `ansible_python_interpreter: /usr/bin/python3` has been set -5. Create `${KAYOBE_CONFIG_PATH}/environments/${KAYOBE_ENVIRONMENT}/inventory/group_vars/github-runners/runner.yml` file which will contain the variables required to deploy a series of runners +5. Create or edit `${KAYOBE_CONFIG_PATH}/inventory/group_vars/github-runners/runners.yml` file which will contain the variables required to deploy a series of runners .. code-block:: yaml diff --git a/etc/kayobe/inventory/group_vars/github-runners/runners.yml b/etc/kayobe/inventory/group_vars/github-runners/runners.yml new file mode 100644 index 000000000..26c4e7fa0 --- /dev/null +++ b/etc/kayobe/inventory/group_vars/github-runners/runners.yml @@ -0,0 +1,38 @@ +--- +runner_user: VM_USER_NAME_HERE +github_account: ORG_NAME_HERE +github_repo: KAYOBE_CONFIG_REPO_NAME_HERE +access_token: "{{ secrets_github_access_token }}" + +base_runner_dir: /opt/actions-runner + +default_runner_labels: + - kayobe + - openstack + - "{{ kayobe_environment | default(omit) }}" + +# Dictionary of runners to be deployed using the role. +# Each dict item can be provided with optional attributes +# * labels - provide a list of labels for a specific runner +# overriding the contents of `default_runner_labels` +# * state - either `started`` or `absent`. By default it will +# be started if however the runner needs to be removed +# then setting it to `absent` will unregister the runner with +# GitHub and remove it from the system. +# Example +# github_runners: +# runner_01: {} +# runner_02: +# labels: ['foo', 'bar', 'baz'] +# runner_03: +# state: absent +github_runners: + runner_01: {} + runner_02: {} + runner_03: {} + +docker_users: + - "{{ runner_user }}" + +pip_install_packages: + - name: docker From ae8d8f28d975d0f2c51ac96eb6758542c68c630f Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 14 Nov 2023 16:36:42 +0000 Subject: [PATCH 20/52] feat: update `runners.yml` defaults --- etc/kayobe/inventory/group_vars/github-runners/runners.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/etc/kayobe/inventory/group_vars/github-runners/runners.yml b/etc/kayobe/inventory/group_vars/github-runners/runners.yml index 26c4e7fa0..5b6e26274 100644 --- a/etc/kayobe/inventory/group_vars/github-runners/runners.yml +++ b/etc/kayobe/inventory/group_vars/github-runners/runners.yml @@ -1,7 +1,7 @@ --- -runner_user: VM_USER_NAME_HERE -github_account: ORG_NAME_HERE -github_repo: KAYOBE_CONFIG_REPO_NAME_HERE +runner_user: stack +github_account: stackhpc +github_repo: stackhpc-kayobe-config access_token: "{{ secrets_github_access_token }}" base_runner_dir: /opt/actions-runner From 91a0ef6ac9e1bd67cb7c9c3dbd7bfb8d3e267246 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 14 Nov 2023 16:41:07 +0000 Subject: [PATCH 21/52] feat: add workflows section --- doc/source/configuration/ci-cd.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/source/configuration/ci-cd.rst b/doc/source/configuration/ci-cd.rst index 73e2b47d0..96d39c24a 100644 --- a/doc/source/configuration/ci-cd.rst +++ b/doc/source/configuration/ci-cd.rst @@ -20,6 +20,9 @@ Currently we support the creation and deployment of workflows for GitHub with Gi Kayobe Automation ----------------- +Workflows +--------- + `Kayobe automation `__ is a collection of scripts and tools that automate kayobe operations. It is deployed and controlled by CI/CD platforms such as GitHub actions and GitLab pipelines. Kayobe automation provides users with an easy process to perform tasks such as: overcloud service deploy, config-diff, tempest testing, and many more. From 8ff6442e46b194ec09b6c7ff6558bb67047c90df Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 14 Nov 2023 16:43:50 +0000 Subject: [PATCH 22/52] feat: `github_output_directory` play variable --- etc/kayobe/ansible/write-workflows.yml | 4 +++- etc/kayobe/inventory/group_vars/github-writer/writer.yml | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 etc/kayobe/inventory/group_vars/github-writer/writer.yml diff --git a/etc/kayobe/ansible/write-workflows.yml b/etc/kayobe/ansible/write-workflows.yml index 708bc22eb..fbedafbdf 100644 --- a/etc/kayobe/ansible/write-workflows.yml +++ b/etc/kayobe/ansible/write-workflows.yml @@ -1,5 +1,7 @@ --- - name: Write Kayobe Automation Workflows for GitHub hosts: github-writer + vars: + github_output_directory: "{{ kayobe_config_path }}/../../.github/workflows" roles: - - stackhpc.kayobe_workflows.github + - stackhpc.kayobe_workflows.github \ No newline at end of file diff --git a/etc/kayobe/inventory/group_vars/github-writer/writer.yml b/etc/kayobe/inventory/group_vars/github-writer/writer.yml deleted file mode 100644 index be3d3a195..000000000 --- a/etc/kayobe/inventory/group_vars/github-writer/writer.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -github_output_directory: "{{ kayobe_config_path }}/../../.github/workflows" From 4293d64b7011c389e790f82fad8a5bf3377e2560 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 15 Nov 2023 10:32:39 +0000 Subject: [PATCH 23/52] feat: add suggestions Co-authored-by: Mark Goddard --- doc/source/configuration/ci-cd.rst | 2 +- etc/kayobe/inventory/group_vars/github-runners/runners.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/source/configuration/ci-cd.rst b/doc/source/configuration/ci-cd.rst index 96d39c24a..8c4f4ae1d 100644 --- a/doc/source/configuration/ci-cd.rst +++ b/doc/source/configuration/ci-cd.rst @@ -75,7 +75,7 @@ Runner Deployment * `network-interfaces` * `python-interpreter.yml` ensuring that `ansible_python_interpreter: /usr/bin/python3` has been set -5. Create or edit `${KAYOBE_CONFIG_PATH}/inventory/group_vars/github-runners/runners.yml` file which will contain the variables required to deploy a series of runners +5. Edit the ``${KAYOBE_CONFIG_PATH}/inventory/group_vars/github-runners/runners.yml`` file which will contain the variables required to deploy a series of runners .. code-block:: yaml diff --git a/etc/kayobe/inventory/group_vars/github-runners/runners.yml b/etc/kayobe/inventory/group_vars/github-runners/runners.yml index 5b6e26274..24b04b431 100644 --- a/etc/kayobe/inventory/group_vars/github-runners/runners.yml +++ b/etc/kayobe/inventory/group_vars/github-runners/runners.yml @@ -1,5 +1,5 @@ --- -runner_user: stack +runner_user: "{{ kayobe_ansible_user }}" github_account: stackhpc github_repo: stackhpc-kayobe-config access_token: "{{ secrets_github_access_token }}" @@ -9,9 +9,9 @@ base_runner_dir: /opt/actions-runner default_runner_labels: - kayobe - openstack - - "{{ kayobe_environment | default(omit) }}" + - "{{ kayobe_environment }}" -# Dictionary of runners to be deployed using the role. +# Dictionary of GitHub runners to be deployed to each runner host. # Each dict item can be provided with optional attributes # * labels - provide a list of labels for a specific runner # overriding the contents of `default_runner_labels` From 771315366e62dedabf7ced1a2f9a6004878e62dc Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 15 Nov 2023 10:37:58 +0000 Subject: [PATCH 24/52] feat: add `undef` hints for runner variables --- etc/kayobe/inventory/group_vars/github-runners/runners.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/inventory/group_vars/github-runners/runners.yml b/etc/kayobe/inventory/group_vars/github-runners/runners.yml index 24b04b431..1d2d5235e 100644 --- a/etc/kayobe/inventory/group_vars/github-runners/runners.yml +++ b/etc/kayobe/inventory/group_vars/github-runners/runners.yml @@ -1,7 +1,7 @@ --- runner_user: "{{ kayobe_ansible_user }}" -github_account: stackhpc -github_repo: stackhpc-kayobe-config +github_account: "{{ undef(hint='Name of customer's GitHub account') }}" +github_repo: "{{ undef(hint='Name of customer's kayobe config repository') }}" access_token: "{{ secrets_github_access_token }}" base_runner_dir: /opt/actions-runner From 65994be0faa1b4349eda400beb1beda5dc24aad9 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 15 Nov 2023 10:47:09 +0000 Subject: [PATCH 25/52] fix: make playbook `github` specific --- doc/source/configuration/ci-cd.rst | 2 +- .../ansible/{write-workflows.yml => write-github-workflows.yml} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename etc/kayobe/ansible/{write-workflows.yml => write-github-workflows.yml} (100%) diff --git a/doc/source/configuration/ci-cd.rst b/doc/source/configuration/ci-cd.rst index 8c4f4ae1d..638ee1e38 100644 --- a/doc/source/configuration/ci-cd.rst +++ b/doc/source/configuration/ci-cd.rst @@ -141,7 +141,7 @@ Workflow Deployment 1. Edit `${KAYOBE_CONFIG_PATH}/inventory/group_vars/github-writer/writer.yml` in the base configuration making the appropriate changes to your deployments specific needs. See documentation for `stackhpc.kayobe_workflows.github `__. -2. Run :code:`kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/write-workflows.yml` +2. Run :code:`kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/write-github-workflows.yml` 3. Add all required secrets to repository either via the GitHub UI or GitHub CLI (may require repository owner) * KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: private key used by Ansible to authenticate with machines. diff --git a/etc/kayobe/ansible/write-workflows.yml b/etc/kayobe/ansible/write-github-workflows.yml similarity index 100% rename from etc/kayobe/ansible/write-workflows.yml rename to etc/kayobe/ansible/write-github-workflows.yml From a3d40a26c6a651f1baa756732aac0cb9a514862d Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 15 Nov 2023 10:52:49 +0000 Subject: [PATCH 26/52] feat: provide `writer.yml` as stub --- etc/kayobe/inventory/group_vars/github-writer/writer.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 etc/kayobe/inventory/group_vars/github-writer/writer.yml diff --git a/etc/kayobe/inventory/group_vars/github-writer/writer.yml b/etc/kayobe/inventory/group_vars/github-writer/writer.yml new file mode 100644 index 000000000..4c536087f --- /dev/null +++ b/etc/kayobe/inventory/group_vars/github-writer/writer.yml @@ -0,0 +1,8 @@ +--- +# Configuration of GitHub workflows generated with stackhpc.kayobe_workflows.github should go here. +# See documentation for more information +# https://github.com/stackhpc/ansible-collection-kayobe-workflows/blob/main/roles/github/README.md + +############################################################################### +# Dummy variable to allow Ansible to accept this file. +workaround_ansible_issue_8743: yes From f9a5ebbf38979f51c724a89208515704643276db Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 15 Nov 2023 11:13:33 +0000 Subject: [PATCH 27/52] feat: amend step five and runner count --- doc/source/configuration/ci-cd.rst | 32 +++++------------------------- 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/doc/source/configuration/ci-cd.rst b/doc/source/configuration/ci-cd.rst index 638ee1e38..c5b0972e0 100644 --- a/doc/source/configuration/ci-cd.rst +++ b/doc/source/configuration/ci-cd.rst @@ -75,7 +75,11 @@ Runner Deployment * `network-interfaces` * `python-interpreter.yml` ensuring that `ansible_python_interpreter: /usr/bin/python3` has been set -5. Edit the ``${KAYOBE_CONFIG_PATH}/inventory/group_vars/github-runners/runners.yml`` file which will contain the variables required to deploy a series of runners +5. Edit the ``${KAYOBE_CONFIG_PATH}/inventory/group_vars/github-runners/runners.yml`` file which will contain the variables required to deploy a series of runners. + Below is a core set of variables that will require consideration and modification for successful deployment of the runners. + The number of runners deployed can be configured by removing and extending the dict :code:`github-runners`. + As for how many runners present three is suitable number as this would prevent situations where long running jobs could halt progress other tasks whilst waiting for a free runner. + You might want to increase the number of runners if usage demands it or new workflows make use of multiple parallel jobs. .. code-block:: yaml @@ -85,42 +89,16 @@ Runner Deployment github_repo: KAYOBE_CONFIG_REPO_NAME_HERE access_token: "{{ secrets_github_access_token }}" - base_runner_dir: /opt/actions-runner - default_runner_labels: - kayobe - openstack - "{{ kayobe_environment | default(omit) }}" - # Dictionary of runners to be deployed using the role. - # Each dict item can be provided with optional attributes - # * labels - provide a list of labels for a specific runner - # overriding the contents of `default_runner_labels` - # * state - either `started`` or `absent`. By default it will - # be started if however the runner needs to be removed - # then setting it to `absent` will unregister the runner with - # GitHub and remove it from the system. - # Example - # github_runners: - # runner_01: {} - # runner_02: - # labels: ['foo', 'bar', 'baz'] - # runner_03: - # state: absent github_runners: runner_01: {} runner_02: {} runner_03: {} - docker_users: - - "{{ runner_user }}" - - pip_install_packages: - - name: docker - -If using multiple environments add an extra label to :code:`default_runner_labels` to distinguish these runners from runners belonging to other environments. -Also feel free to change the number of runners and their names. - 6. Obtain a personal access token that would enable the registration of GitHub runners against the `github_account` and `github_repo` defined above. This token ideally should be `fine-grained personal access token `__ which may require the organisation to enable such tokens beforehand. Steps can be found `here `__. From 477e7001be06e997be805656706e0d8d3372fa16 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 15 Nov 2023 11:16:24 +0000 Subject: [PATCH 28/52] fix: distinguish runner hostname from runner name --- doc/source/configuration/ci-cd.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/configuration/ci-cd.rst b/doc/source/configuration/ci-cd.rst index c5b0972e0..e55b23926 100644 --- a/doc/source/configuration/ci-cd.rst +++ b/doc/source/configuration/ci-cd.rst @@ -68,7 +68,7 @@ Runner Deployment .. code-block:: ini [github-runners] - runner-01 + prod-runner-01 4. Provide all the relevant Kayobe :code:`group_vars` for :code:`github-runners` under :code:`${KAYOBE_CONFIG_PATH}/environments/${KAYOBE_ENVIRONMENT}/inventory/group_vars/github-runners` * `infra-vms` ensuring all required `infra_vm_extra_network_interfaces` are defined From 3a3f4e3f0a8848f189fdc239ec621e55063210f3 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 15 Nov 2023 11:17:06 +0000 Subject: [PATCH 29/52] fix: move `workflows` heading --- doc/source/configuration/ci-cd.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/configuration/ci-cd.rst b/doc/source/configuration/ci-cd.rst index e55b23926..63662c31c 100644 --- a/doc/source/configuration/ci-cd.rst +++ b/doc/source/configuration/ci-cd.rst @@ -20,15 +20,15 @@ Currently we support the creation and deployment of workflows for GitHub with Gi Kayobe Automation ----------------- -Workflows ---------- - `Kayobe automation `__ is a collection of scripts and tools that automate kayobe operations. It is deployed and controlled by CI/CD platforms such as GitHub actions and GitLab pipelines. Kayobe automation provides users with an easy process to perform tasks such as: overcloud service deploy, config-diff, tempest testing, and many more. With it being integrated into platforms such as GitHub or GitLab it builds a close relationship between the contents of the deployments kayobe configuration and what is currently deployed. This is because operations such as opening a pull request will trigger a config diff to be generated providing insight on what impact it might have on services or a tempest test that could be scheduled to run daily providing knowledge of faults earlier than before. +Workflows +--------- + Kayobe automation has been designed to be independent of any CI/CD platform with all tasks running inside of a purpose built Kayobe container. However, platform specific workflows need to be deployed to bridge the gap between the contents of Kayobe Automation and these CI/CD platforms. Workflows are templated for each Kayobe configuration repository, ensuring appropriate workflow input parameters are defined, and any necessary customisations can be applied. From e85da1803e12f7aa4f3db46a0607fa53c7049d9c Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 15 Nov 2023 11:34:18 +0000 Subject: [PATCH 30/52] feat: state runners are needed in multiple environments --- doc/source/configuration/ci-cd.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/configuration/ci-cd.rst b/doc/source/configuration/ci-cd.rst index 63662c31c..6cad8566e 100644 --- a/doc/source/configuration/ci-cd.rst +++ b/doc/source/configuration/ci-cd.rst @@ -40,6 +40,7 @@ Runners Runners are purpose built services tied to a particular service vendor such as GitHub Actions or GitLab CI. These services will listen for jobs which have been tagged appropriately and dispatched to these specific runners. The runners will need to be deployed using existing roles and playbooks whereby the binary/package is downloaded and registered using a special token. +In some deployments runner hosts can be shared between environments however this is not always true and dedicated hosts will need to be used for each environment you intend to deploy kayobe automation within. GitHub Actions ================= From 2564a2f4fabdc98b7de924e7e8932b59a81cb53e Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 15 Nov 2023 11:49:23 +0000 Subject: [PATCH 31/52] fix: `new-line-at-end-of-file` --- etc/kayobe/ansible/write-github-workflows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/write-github-workflows.yml b/etc/kayobe/ansible/write-github-workflows.yml index fbedafbdf..f45319833 100644 --- a/etc/kayobe/ansible/write-github-workflows.yml +++ b/etc/kayobe/ansible/write-github-workflows.yml @@ -4,4 +4,4 @@ vars: github_output_directory: "{{ kayobe_config_path }}/../../.github/workflows" roles: - - stackhpc.kayobe_workflows.github \ No newline at end of file + - stackhpc.kayobe_workflows.github From 1f5a0413383b2024d73865a2e5c66a92e9631cbb Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Wed, 15 Nov 2023 13:27:51 +0000 Subject: [PATCH 32/52] fix: `D002 Trailing whitespace` --- doc/source/configuration/ci-cd.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/configuration/ci-cd.rst b/doc/source/configuration/ci-cd.rst index 6cad8566e..25621fd62 100644 --- a/doc/source/configuration/ci-cd.rst +++ b/doc/source/configuration/ci-cd.rst @@ -15,7 +15,7 @@ Finally, due to the requirement that we support various different platforms tool This tooling is not tied to any single CI/CD platform as all tasks are a series of shell script and Ansible playbooks which are designed to run in a purpose build kayobe container. This is complemented by the use of an Ansible collection known as `stackhpc.kayobe_workflows `__ which aims to provide users with a quick and easy way of customising all workflows to fit within a customer's cloud. -Currently we support the creation and deployment of workflows for GitHub with Gitlab support being actively worked upon. +Currently we support the creation and deployment of workflows for GitHub with Gitlab support being actively worked upon. Kayobe Automation ----------------- From d73f6e86782fca743ed5720b3bee420ce3ca43dd Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Fri, 17 Nov 2023 14:55:32 +0000 Subject: [PATCH 33/52] feat: add additional information about the `registry` --- doc/source/configuration/ci-cd.rst | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/doc/source/configuration/ci-cd.rst b/doc/source/configuration/ci-cd.rst index 25621fd62..a99e8e17e 100644 --- a/doc/source/configuration/ci-cd.rst +++ b/doc/source/configuration/ci-cd.rst @@ -95,6 +95,11 @@ Runner Deployment - openstack - "{{ kayobe_environment | default(omit) }}" + github_registry: + url: pulp.example.com + username: admin + password: ${{ secrets.REGISTRY_PASSWORD }} + github_runners: runner_01: {} runner_02: {} @@ -122,21 +127,32 @@ Workflow Deployment 2. Run :code:`kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/write-github-workflows.yml` -3. Add all required secrets to repository either via the GitHub UI or GitHub CLI (may require repository owner) +3. Add all required secrets and variables to repository either via the GitHub UI or GitHub CLI (may require repository owner) * KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: private key used by Ansible to authenticate with machines. * KAYOBE_VAULT_PASSWORD: password used by the config to encrypt Ansible Vault secrets. * REGISTRY_PASSWORD: password used to login to the docker registry such as Pulp. * TEMPEST_OPENRC: contents of :code:`kolla/public-openrc.sh` + * REGISTRY_PASSWORD: the password to access the docker registry for pushing and pulling containers. Recommend to use Pulp on the seed node. Note if you are using multiple environments and not sharing secrets between environments then each of these must have the environment name prefix for each environment, for example: * PRODUCTION_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY * PRODUCTION_KAYOBE_VAULT_PASSWORD * PRODUCTION_REGISTRY_PASSWORD * PRODUCTION_TEMPEST_OPENRC + * PRODUCTION_REGISTRY_URL* + * PRODUCTION_REGISTRY_USERNAME* + * PRODUCTION_REGISTRY_PASSWORD* * STAGING_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY * STAGING_KAYOBE_VAULT_PASSWORD * STAGING_REGISTRY_PASSWORD * STAGING_TEMPEST_OPENRC + * STAGING_REGISTRY_URL* + * STAGING_REGISTRY_USERNAME* + * STAGING_REGISTRY_PASSWORD* + +Note regarding the :code:`REGISTRY_` secrets and variables if you are using a single environment then :code:`REGISTRY_URL` and :code:`REGISTRY_PASSWORD` can be added directly to the workflows. +This is also true in the event of using multiple environments with a single shared Pulp registry. +You only need to add the secrets and variables in the event of using multiple environments each with their own registry. 4. Commit and push all newly generated workflows found under :code:`.github/workflows` From a5a2dd6d725827a7c9d73c2404a47e4aea3e06d8 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Tue, 21 Nov 2023 06:48:09 +0000 Subject: [PATCH 34/52] fix: reworded section on `github_registry` --- doc/source/configuration/ci-cd.rst | 66 ++++++++++++++++++------------ 1 file changed, 40 insertions(+), 26 deletions(-) diff --git a/doc/source/configuration/ci-cd.rst b/doc/source/configuration/ci-cd.rst index a99e8e17e..fdbb0d009 100644 --- a/doc/source/configuration/ci-cd.rst +++ b/doc/source/configuration/ci-cd.rst @@ -82,6 +82,12 @@ Runner Deployment As for how many runners present three is suitable number as this would prevent situations where long running jobs could halt progress other tasks whilst waiting for a free runner. You might want to increase the number of runners if usage demands it or new workflows make use of multiple parallel jobs. + Note :code:`github_registry` and the elements of the dict control the registry settings for pulling and pushing container images used by the workflows. + In the example below the registry settings have been adapted to demonstrate what a shared registry between environments might look like. + This values maybe suitable for your deployment providing all environments can reach the same registry. + If the all of the environments use their own registry and nothing is shared between them then :code:`github_registry` can omitted from the file and the template will expect environment specific secrets and variables to be added to the repository settings. + This is discussed further in the next section. + .. code-block:: yaml --- @@ -99,6 +105,7 @@ Runner Deployment url: pulp.example.com username: admin password: ${{ secrets.REGISTRY_PASSWORD }} + share: true github_runners: runner_01: {} @@ -123,36 +130,43 @@ Runner Deployment Workflow Deployment ------------------- -1. Edit `${KAYOBE_CONFIG_PATH}/inventory/group_vars/github-writer/writer.yml` in the base configuration making the appropriate changes to your deployments specific needs. See documentation for `stackhpc.kayobe_workflows.github `__. +1. Edit :code:`${KAYOBE_CONFIG_PATH}/inventory/group_vars/github-writer/writer.yml` in the base configuration making the appropriate changes to your deployments specific needs. See documentation for `stackhpc.kayobe_workflows.github `__. 2. Run :code:`kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/write-github-workflows.yml` 3. Add all required secrets and variables to repository either via the GitHub UI or GitHub CLI (may require repository owner) - * KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: private key used by Ansible to authenticate with machines. - * KAYOBE_VAULT_PASSWORD: password used by the config to encrypt Ansible Vault secrets. - * REGISTRY_PASSWORD: password used to login to the docker registry such as Pulp. - * TEMPEST_OPENRC: contents of :code:`kolla/public-openrc.sh` - * REGISTRY_PASSWORD: the password to access the docker registry for pushing and pulling containers. Recommend to use Pulp on the seed node. - -Note if you are using multiple environments and not sharing secrets between environments then each of these must have the environment name prefix for each environment, for example: - * PRODUCTION_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY - * PRODUCTION_KAYOBE_VAULT_PASSWORD - * PRODUCTION_REGISTRY_PASSWORD - * PRODUCTION_TEMPEST_OPENRC - * PRODUCTION_REGISTRY_URL* - * PRODUCTION_REGISTRY_USERNAME* - * PRODUCTION_REGISTRY_PASSWORD* - * STAGING_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY - * STAGING_KAYOBE_VAULT_PASSWORD - * STAGING_REGISTRY_PASSWORD - * STAGING_TEMPEST_OPENRC - * STAGING_REGISTRY_URL* - * STAGING_REGISTRY_USERNAME* - * STAGING_REGISTRY_PASSWORD* - -Note regarding the :code:`REGISTRY_` secrets and variables if you are using a single environment then :code:`REGISTRY_URL` and :code:`REGISTRY_PASSWORD` can be added directly to the workflows. -This is also true in the event of using multiple environments with a single shared Pulp registry. -You only need to add the secrets and variables in the event of using multiple environments each with their own registry. + +.. raw:: html + +
+ + + + + + + + + + + + + + + + + + + + + + + +
SecretsVariables
KAYOBE_AUTOMATION_SSH_PRIVATE_KEYREGISTRY_USERNAME
KAYOBE_VAULT_PASSWORDREGISTRY_URL
REGISTRY_PASSWORD
TEMPEST_OPENRC
+ +Note the above table shows the secrets and variable one may need to add to GitHub for a successful deployment. +However, these secrets and variables might not all be required for example if :code:`github_registry` has been configured with a single shared registry then :code:`REGISTRY_USERNAME` and :code:`REGISTRY_URL` can be ignored. +Also it is important that if you are using multiple environments and secrets and variables are not being shared then each will require an environment prefix added in the form `ENVIRONMENT_NAME_SECRET_OR_VARIABLE_NAME` for example if there was two environments each name :code:`production` and :code:`staging` then :code:`KAYOBE_AUTOMATION_SSH_PRIVATE_KEY` would be replaced by :code:`PRODUCTION_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY` and :code:`STAGING_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY`. 4. Commit and push all newly generated workflows found under :code:`.github/workflows` From 2222fc776f797202058f4fba5ed6d643bb706d1c Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 29 Nov 2023 11:41:41 +0000 Subject: [PATCH 35/52] Docs: Mention broken pxe boot_interface --- doc/source/operations/rocky-linux-9.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/doc/source/operations/rocky-linux-9.rst b/doc/source/operations/rocky-linux-9.rst index bf65181f8..c1835c6b7 100644 --- a/doc/source/operations/rocky-linux-9.rst +++ b/doc/source/operations/rocky-linux-9.rst @@ -110,6 +110,29 @@ The error from NetworkManager was: [1697192659.9611] keyfile: ipv4.routing-rules: invalid value for "routing-rule1": invalid value for "table" +Switching to iPXE +----------------- + +The ``pxe`` boot_interface is currently broken. When provisioning, you will see an error similar to: + + .. code-block:: shell + + Failed to prepare to deploy: Could not link image http://192.168.1.1:8080/ipa.vmlinuz from /httpboot/master_images/99d5b4b4-0420-578a-a327-acd88c1f1ff6.converted to /tftpboot/d6673eaa-17a4-4cd4-a4e7-8e8cbd4fca31/deploy_kernel, error: [Errno 18] Invalid cross-device link: '/httpboot/master_images/99d5b4b4-0420-578a-a327-acd88c1f1ff6.converted' -> '/tftpboot/d6673eaa-17a4-4cd4-a4e7-8e8cbd4fca31/deploy_kernel' + +It is suggested that you migrate to iPXE. + +.. code-block:: yaml + :caption: ``$KAYOBE_CONFIG_PATH/kolla/config/bifrost/bifrost.yml`` + + enabled_boot_interfaces: "ipxe,pxe" + ipxe_efi_binary: "ipxe-snponly-x86_64.efi" + +After deprovisioning a node, switch the boot interface to iPXE: + + .. code-block:: shell + + openstack baremetal node set --boot-interface ipxe + Prerequisites ============= From 0fb938d75b2dab8dac3379b40e0e1954551b6e33 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 6 Dec 2023 10:01:43 +0000 Subject: [PATCH 36/52] Remove piece about setting boot_interfaces --- doc/source/operations/rocky-linux-9.rst | 8 -------- 1 file changed, 8 deletions(-) diff --git a/doc/source/operations/rocky-linux-9.rst b/doc/source/operations/rocky-linux-9.rst index c1835c6b7..42a3e0120 100644 --- a/doc/source/operations/rocky-linux-9.rst +++ b/doc/source/operations/rocky-linux-9.rst @@ -119,14 +119,6 @@ The ``pxe`` boot_interface is currently broken. When provisioning, you will see Failed to prepare to deploy: Could not link image http://192.168.1.1:8080/ipa.vmlinuz from /httpboot/master_images/99d5b4b4-0420-578a-a327-acd88c1f1ff6.converted to /tftpboot/d6673eaa-17a4-4cd4-a4e7-8e8cbd4fca31/deploy_kernel, error: [Errno 18] Invalid cross-device link: '/httpboot/master_images/99d5b4b4-0420-578a-a327-acd88c1f1ff6.converted' -> '/tftpboot/d6673eaa-17a4-4cd4-a4e7-8e8cbd4fca31/deploy_kernel' -It is suggested that you migrate to iPXE. - -.. code-block:: yaml - :caption: ``$KAYOBE_CONFIG_PATH/kolla/config/bifrost/bifrost.yml`` - - enabled_boot_interfaces: "ipxe,pxe" - ipxe_efi_binary: "ipxe-snponly-x86_64.efi" - After deprovisioning a node, switch the boot interface to iPXE: .. code-block:: shell From fe4c973f87119a19eac3b6b6c339f45bf83e3a22 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Wed, 29 Nov 2023 16:11:42 +0000 Subject: [PATCH 37/52] Add stop-openstack-services.yml playbook --- .../ansible/stop-openstack-services.yml | 29 +++++++++++++++++++ ...op-services-playbook-b85b53d1a7571009.yaml | 5 ++++ 2 files changed, 34 insertions(+) create mode 100644 etc/kayobe/ansible/stop-openstack-services.yml create mode 100644 releasenotes/notes/stop-services-playbook-b85b53d1a7571009.yaml diff --git a/etc/kayobe/ansible/stop-openstack-services.yml b/etc/kayobe/ansible/stop-openstack-services.yml new file mode 100644 index 000000000..08f4e964a --- /dev/null +++ b/etc/kayobe/ansible/stop-openstack-services.yml @@ -0,0 +1,29 @@ +--- +# Stops containers running OpenStack services + +- name: Stop OpenStack services + hosts: overcloud + become: true + gather_facts: false + vars: + - stop_service_list: + - "blazar" + - "barbican" + - "cinder" + - "cloudkitty" + - "designate" + - "glance" + - "heat" + - "horizon" + - "ironic" + - "keystone" + - "magnum" + - "manila" + - "neutron" + - "nova" + - "octavia" + - "placement" + tasks: + - name: Stop OpenStack services + shell: >- + docker ps -a | egrep '({{ stop_service_list | join('|') }})' | awk '{ print $NF }' | xargs docker stop diff --git a/releasenotes/notes/stop-services-playbook-b85b53d1a7571009.yaml b/releasenotes/notes/stop-services-playbook-b85b53d1a7571009.yaml new file mode 100644 index 000000000..6516f8ec4 --- /dev/null +++ b/releasenotes/notes/stop-services-playbook-b85b53d1a7571009.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Added the ``stop-openstack-services.yml`` playbook, which can be used to + stop OpenStack services across the overcloud. From f12340fcd3f9a54ec6cf72d8ac02106515b30d7f Mon Sep 17 00:00:00 2001 From: Seunghun Lee Date: Thu, 7 Dec 2023 16:50:47 +0000 Subject: [PATCH 38/52] Fix Wazuh manager playbook with no custom policies --- etc/kayobe/ansible/wazuh-manager.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/ansible/wazuh-manager.yml b/etc/kayobe/ansible/wazuh-manager.yml index fedd1ad93..6865d3b6f 100644 --- a/etc/kayobe/ansible/wazuh-manager.yml +++ b/etc/kayobe/ansible/wazuh-manager.yml @@ -41,7 +41,9 @@ dest: "/var/ossec/etc/shared/default/" owner: wazuh group: wazuh - when: custom_sca_policies.files | length > 0 + when: + - custom_sca_policies_folder.stat.exists + - custom_sca_policies.files | length > 0 - name: Add custom policy definition(s) to the shared Agent config blockinfile: @@ -61,7 +63,9 @@ {% endfilter %} - when: custom_sca_policies.files | length > 0 + when: + - custom_sca_policies_folder.stat.exists + - custom_sca_policies.files | length > 0 notify: - Restart wazuh From fb9a53e9c993ae081e8c11f442c62f17d2a2c224 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Mon, 11 Dec 2023 12:24:34 +0000 Subject: [PATCH 39/52] Add password rotation docs page --- doc/source/operations/password-rotation.rst | 521 ++++++++++++++++++++ 1 file changed, 521 insertions(+) create mode 100644 doc/source/operations/password-rotation.rst diff --git a/doc/source/operations/password-rotation.rst b/doc/source/operations/password-rotation.rst new file mode 100644 index 000000000..ec5bdc89f --- /dev/null +++ b/doc/source/operations/password-rotation.rst @@ -0,0 +1,521 @@ +=============== +Secret Rotation +=============== + +General notes +============= + +This guide covers secret rotation in Kayobe and Kolla-Ansible for most services +in a standard deployment. It does not cover every secret. A full list of +passwords that have been successfully rotated is available at the bottom of +this page TODO: Link. + +Many of the secrets can simply be deleted from your passwords.yml and will be +automatically regenerated with a ``kayobe overcloud service deploy``. + +Some secrets require manual input from the operator to change. + +Following this process, there should be no interruption to service for running +VMs, but there will be API downtime for all services. The main reason for the +outage is that RabbitMQ must be completely stopped to change the secrets it +uses. The services must all be reconfigured to use the new RabbitMQ cluster. +Each service will come back once it has been reconfigured. The outage time for +each service is therefore approximately equal to the time between starting a +``kayobe overcloud service deploy``, and that service being reconfigured. + +Some secrets currently have to be regenerated by hand. Make sure you use a +reliable tool and match the formatting (length, character set etc) of the +existing secret. ``pwgen`` is recommended and used as an example throughout +this guide. Installation: + +.. code:: bash + + sudo apt/dnf install pwgen + + +As of writing, there are three upstream patches in the works to make this +process easier. + +1. A change to Kolla, to automate Step 2 of the full method (TODO: link). + + The upstream patch can be found `here + `__. + + This was previously mitigated with a change to the StackHPC fork of + Kolla-Ansible, which has since been reverted due to an unforeseen issue. + +2. A change to Nova, to automate Step 11 of the full method (TODO: link). + + The upstream patch can be found `here + `__. + +3. A change to Kolla-Ansible, to automate Step 13 of the full method (TODO: link). + + The upstream patch can be found `here + `__. + + +Full method +=========== + +.. warning:: + + You **must** back up your passwords.yml before making changes. You will need + to refer back to it later. + +1. Run a tempest refstack & check Kibana/OpenSearch Dashboards to check + the state of the cloud before any changes are made. + +2. Edit your kolla-ansible checkout to include this line within the + kolla_docker dict in + ``ansible/roles/nova/tasks/bootstrap_service.yml`` See + `here `__ + for an example. (If you are using the latest ``stackhpc/yoga`` branch + of kolla-ansible this should already be set) + + .. code:: bash + + command: bash -c 'sudo -E kolla_set_configs && nova-manage api_db sync && nova-manage db sync --local_cell' + + This change will break new deployments and should be reverted once the process is complete. + +3. Re-install kolla-ansible from source in your kolla-ansible python + environment + +4. Navigate to the directory containing your passwords.yml file + (``kayobe-config/etc/kolla/passwords.yml`` OR + ``kayobe-config/etc/kayobe/environments/envname/kolla/passwords.yml``) + +5. Create a file called deletelist.txt and populate it with this content + (including all whitespace): + +:: + + _keystone_password + _database_password + ^keystone_admin_password + ^memcache_secret_key + ^designate_rndc_key + ^docker_registry_password + ^keepalived_password + ^kibana_password + ^libvirt_sasl_password + ^metadata_secret + ^opensearch_dashboards_password + ^osprofiler_secret + ^prometheus_alertmanager_password + ^qdrouterd_password + ^redis_master_password + ^memcache_secret_key + _ssh_key + + private_key + public_key + ^$ + rabbitmq + ^haproxy_password + +NOTE: The above deletes bifrost secrets. Might need a seed s-d as well + +3. Decrypt your passwords.yml file with ansible vault + +4. Delete all the passwords in the deletion list + + .. code:: bash + + grep -vf deletelist.txt passwords.yml > new-passwords.yml + +5. Check the new file for basic formatting errors. If it looks correct, + replace the existing ``passwords.yml`` file with + ``new-passwords.yml`` + + .. code:: bash + + rm passwords.yml && mv new-passwords.yml passwords.yml + +6. Use the ``rekey-hosts.yml`` playbook to rotate your SSH keys for + hosts across the cloud. The playbook should exist under + ``kayobe-config/etc/kayobe/ansible/`` if not, merge the latest + ``stackhpc-kayobe-config``. + + .. code:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/rekey-hosts.yml + +7. Update the Pulp password + + 1. Generate a new Pulp password + + .. code:: bash + + pwgen -s 25 1 + + 2. Update secrets_pulp_password (usually found in secrets.yml) + + 3. Deploy changes + + .. code:: bash + + kayobe seed service deploy -t seed-deploy-containers -kt none + + (note you may need to skip docker registry login since the + password will now be ‘incorrect’ e.g. ``-e`` + ``deploy_containers_registry_attempt_login``) + +8. Rotate ``horizon_secret_key``. + + 1. Generate a new secret: + + .. code:: bash + + pwgen -s 40 1 + + 2. Add it to the passwords file, along with the old secret, in this + exact format (including quotes in the middle): + + .. code:: bash + + horizon_secret_key: newsecret' 'oldsecret + + This will allow both the old and new secrets to be used at the + same time, resulting in no interruption to service. The key is + mainly used for generating login and password reset tokens. The + old secret can be deleted & redeployed at a later date once all + users have closed & reopened their sessions. + +9. Update grafana_admin_password + + 1. Generate a new Grafana Admin password + + .. code:: bash + + pwgen -s 40 1 + + 2. Exec into the grafana container on a controller + + .. code:: bash + + sudo docker exec -it grafana bash + + 3. Run the password reset command, then enter the new password + + .. code:: bash + + grafana-cli admin reset-admin-password --password-from-stdin + + 4. Update the value of ``grafana_admin_password`` in passwords.yml + +10. Update the MariaDB database password + + 1. Generate a new secret: + + .. code:: bash + + pwgen -s 40 1 + + 2. Exec into the mariadb container on a controller + + .. code:: bash + + sudo docker exec -it mariadb bash + + 3. Log in to the database. You will be prompted for the password. + Use the existing value of ``database_password`` + + .. code:: bash + + mysql -uroot -p + + 4. Check the current state of the root user + + .. code:: bash + + SELECT Host,User,Password FROM mysql.user WHERE User='root'; + + 5. Update the password for the root user + + .. code:: bash + + SET PASSWORD FOR 'root'@'%' = PASSWORD('newpassword'); + + 6. Check that the password hash has changed in the user list + + .. code:: bash + + SELECT Host,User,Password FROM mysql.user WHERE User='root'; + + 7. If there are any remaining root users with the old password + e.g. ``root@localhost``, change the password for them too. + + 8. update ``database_password`` in ``passwords.yml`` with your new + password. + +:::warning From this point onward, service may be disrupted + +:: + + ::: + +11. Update the nova DB password + + 1. Create a new ``nova_database_password`` and store it in + ``passwords.yml`` + + .. code:: bash + + pwgen -s 40 1 + + 2. Exec into the nova_conductor container + + .. code:: bash + + sudo docker exec -it nova_conductor bash + + 3. List the cells + + .. code:: bash + + nova-manage cell_v2 list_cells --verbose + + 4. Find the entry for cell0, copy the Database Connection value, + replace the password in the string with the new value, and update + it with the following command: + + .. code:: bash + + nova-manage cell_v2 update_cell --cell_uuid 00000000-0000-0000-0000-000000000000 --database_connection "CONNECTION WITH NEW PASSWORD HERE" --transport-url "none:///" + + (If the ``cell_uuid`` for cell0 is not + ``00000000-0000-0000-0000-000000000000``, change the above + command accordingly) + +12. Re-encrypt your ``passwords.yml`` file + +13. Delete the service users in keystone. The exact users will depend on + the deployment. Multinode example: + +:::warning This will immediately cause an API outage + +:: + + Alt: Cherry pick this patch: + + ::: + + ```bash + openstack user delete glance cinder placement nova neutron heat magnum magnum_trustee_domain_admin barbican designate + ``` + +14. Stop services + + .. code:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/stop-openstack-services.yml + +15. Nuke RabbitMQ + + .. code:: bash + + kayobe overcloud host command run -l controllers --become --command "docker stop rabbitmq && docker rm rabbitmq && docker volume rm rabbitmq" + +16. Reconfigure overcloud services to push changes + +:::warning VMs should continue running, but connections to them will +likely be disrupted when neutron is redeployed + +:: + + ::: + + ```bash + kayobe overcloud service deploy + ``` + +17. Flush the memcached data on all controllers (any old data will now + be inaccessible) + + 1. Install telnet (on one of the controllers) + + .. code:: bash + + sudo apt -y install telnet + + 2. Check the config for the ip and port used by memcache (on every + controller) + + .. code:: bash + + sudo grep command /etc/kolla/memcached/config.json + + The IP and port will be printed after ``-l`` and ``-p`` + respectively. + + 3. For each controller start a telnet session, clear all data, then + exit + + .. code:: bash + + telnet + flush_all + quit + +18. Manually update heat_domain_admin_password + + 1. TODO: Instructions + +19. Re-run tempest to make sure everything has come back + +20. Inform other users of the steps they’ll need to take now that the + secrets have been rotated: + + 1. SSH keys have been rotated, so the new key will have to be + distributed if individual user accounts are used + 2. A PR with the new passwords will need to be merged in to the main + config branch (REMEMBER TO RE-ENCRYPT PASSWORDS.YML BEFORE + COMMITING) + 3. Any existing openrc files generated by Kolla Ansible will need to + be re-generated or edited to use the new Kolla admin password. + +21. At some point in the future (approx 1 week), remove the old horizon + secret key from ``passwords.yml`` and reconfigure horizon. + +Future improvements +------------------- + +- ☐ Regenerate passwords that we think we aren’t using, in case they + are actually being used +- ☐ Add the new database_password to passwords.yml before changing it, + in case it gets lost +- ☐ Stop services before deleting users? (Except for keystone) or allow + setting update_password in kolla-ansible +- ☒ At the end of the procedure write down what others need to do to + use the new passwords +- ☐ Can we get kolla-ansible to generate all/more of the passwords and + reduce the reliance on pwgen? +- ☐ Add a step to remove the old horizon_secret_key and redeploy + horizon at some point after the rotation + +Full password list +------------------- + +:: + + aodh_database_password + aodh_keystone_password + blazar_database_password + blazar_keystone_password + caso_keystone_password + ceilometer_database_password + ceilometer_keystone_password + cinder_database_password + cinder_keystone_password + barbican_database_password + barbican_keystone_password + cloudkitty_database_password + cloudkitty_keystone_password + congress_database_password + congress_keystone_password + cyborg_database_password + cyborg_keystone_password + designate_database_password + designate_keystone_password + freezer_database_password + freezer_keystone_password + glance_database_password + glance_keystone_password + gnocchi_database_password + gnocchi_keystone_password + heat_database_password + heat_keystone_password + horizon_database_password + ironic_database_password + ironic_inspector_database_password + ironic_inspector_keystone_password + ironic_keystone_password + karbor_database_password + karbor_keystone_password + keystone_database_password Check this one + magnum_database_password + manila_database_password + mariadb_backup_database_password + masakari_database_password + mistral_database_password + monasca_database_password + murano_database_password + neutron_database_password + nova_api_database_password + nova_database_password + octavia_database_password + panko_database_password + placement_database_password + prometheus_mysql_exporter_database_password + qinling_database_password + rally_database_password + sahara_database_password + senlin_database_password + solum_database_password + tacker_database_password + trove_database_password + vitrage_database_password + watcher_database_password + zun_database_password + keystone_admin_password + kuryr_keystone_password + magnum_keystone_password + manila_keystone_password + masakari_keystone_password + mistral_keystone_password + monasca_keystone_password + murano_keystone_password + neutron_keystone_password + nova_keystone_password + octavia_keystone_password + panko_keystone_password + rabbitmq_cluster_cookie + rabbitmq_monitoring_password + rabbitmq_password + database_password + heat_domain_admin_password + horizon_secret_key + placement_keystone_password + qinling_keystone_password + sahara_keystone_password + searchlight_keystone_password + senlin_keystone_password + solum_keystone_password + swift_keystone_password + tacker_keystone_password + trove_keystone_password + vitrage_keystone_password + watcher_keystone_password + zun_keystone_password + ceph_rgw_keystone_password + designate_rndc_key + keepalived_password + kibana_password + libvirt_sasl_password + metadata_secret + opensearch_dashboards_password + osprofiler_secret + prometheus_alertmanager_password + qdrouterd_password + grafana_admin_password + docker_registry_password + secrets_pulp_password + redis_master_password + keystone_ssh_key + private_key + public_key + neutron_ssh_key + private_key + public_key + nova_ssh_key + private_key + public_key + octavia_amp_ssh_key + private_key + public_key + bifrost_ssh_key + private_key + public_key + From 327e05282bda9e70e6b00117796612e1c60212fe Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Mon, 11 Dec 2023 16:13:49 +0000 Subject: [PATCH 40/52] Update secret rotation docs page --- doc/source/operations/index.rst | 1 + ...sword-rotation.rst => secret-rotation.rst} | 322 +++++++++--------- 2 files changed, 163 insertions(+), 160 deletions(-) rename doc/source/operations/{password-rotation.rst => secret-rotation.rst} (53%) diff --git a/doc/source/operations/index.rst b/doc/source/operations/index.rst index 1327b8c2e..284795d6a 100644 --- a/doc/source/operations/index.rst +++ b/doc/source/operations/index.rst @@ -11,3 +11,4 @@ This guide is for operators of the StackHPC Kayobe configuration project. octavia hotfix-playbook rocky-linux-9 + secret-rotation diff --git a/doc/source/operations/password-rotation.rst b/doc/source/operations/secret-rotation.rst similarity index 53% rename from doc/source/operations/password-rotation.rst rename to doc/source/operations/secret-rotation.rst index ec5bdc89f..63eef3f35 100644 --- a/doc/source/operations/password-rotation.rst +++ b/doc/source/operations/secret-rotation.rst @@ -8,20 +8,22 @@ General notes This guide covers secret rotation in Kayobe and Kolla-Ansible for most services in a standard deployment. It does not cover every secret. A full list of passwords that have been successfully rotated is available at the bottom of -this page TODO: Link. +this page (:ref:`link`). -Many of the secrets can simply be deleted from your passwords.yml and will be -automatically regenerated with a ``kayobe overcloud service deploy``. +Many of the secrets can simply be deleted from your ``passwords.yml`` and will +be automatically regenerated with a ``kayobe overcloud service deploy``. Some secrets require manual input from the operator to change. -Following this process, there should be no interruption to service for running -VMs, but there will be API downtime for all services. The main reason for the -outage is that RabbitMQ must be completely stopped to change the secrets it -uses. The services must all be reconfigured to use the new RabbitMQ cluster. -Each service will come back once it has been reconfigured. The outage time for -each service is therefore approximately equal to the time between starting a -``kayobe overcloud service deploy``, and that service being reconfigured. +Following this process, there may be a few seconds of network downtime for +running VMs when Neutron is reconfigured. + +There will be API downtime for all services. The main reason for the outage is +that RabbitMQ must be completely stopped to change the secrets it uses. The +services must all be reconfigured to use the new RabbitMQ cluster. Each service +will come back once it has been reconfigured. The outage time for each service +is therefore equal to the time between starting a ``kayobe overcloud service +deploy``, and that service being reconfigured. Some secrets currently have to be regenerated by hand. Make sure you use a reliable tool and match the formatting (length, character set etc) of the @@ -34,22 +36,25 @@ this guide. Installation: As of writing, there are three upstream patches in the works to make this -process easier. +process easier. -1. A change to Kolla, to automate Step 2 of the full method (TODO: link). +#. A change to Kolla, to automate :ref:`this` step to change the + extended start for the ``nova-api`` container. The upstream patch can be found `here `__. This was previously mitigated with a change to the StackHPC fork of - Kolla-Ansible, which has since been reverted due to an unforeseen issue. + Kolla-Ansible, which has since been reverted due to an unforeseen issue. -2. A change to Nova, to automate Step 11 of the full method (TODO: link). +#. A change to Nova, to automate :ref:`this` step to change the + nova cell0 database connection string. The upstream patch can be found `here `__. -3. A change to Kolla-Ansible, to automate Step 13 of the full method (TODO: link). +#. A change to Kolla-Ansible, to automate :ref:`this` step to + update service keystone user passwords. The upstream patch can be found `here `__. @@ -60,89 +65,90 @@ Full method .. warning:: - You **must** back up your passwords.yml before making changes. You will need - to refer back to it later. + You **must** back up your ``passwords.yml`` before making changes. You will + need to refer back to it later -1. Run a tempest refstack & check Kibana/OpenSearch Dashboards to check - the state of the cloud before any changes are made. +1. Run a Tempest ``refstack`` & check Kibana/OpenSearch Dashboards to check + the state of the cloud before any changes are made -2. Edit your kolla-ansible checkout to include this line within the - kolla_docker dict in - ``ansible/roles/nova/tasks/bootstrap_service.yml`` See - `here `__ - for an example. (If you are using the latest ``stackhpc/yoga`` branch - of kolla-ansible this should already be set) +.. _kolla-change: - .. code:: bash +2. Edit your Kolla-Ansible checkout to include this line within the + ``kolla_docker`` dict in ``ansible/roles/nova/tasks/bootstrap_service.yml`` See + `here + `__ + for an example. (If you are using the latest ``stackhpc/yoga`` branch of + Kolla-Ansible this should already be set) - command: bash -c 'sudo -E kolla_set_configs && nova-manage api_db sync && nova-manage db sync --local_cell' + .. code:: + + command: bash -c 'sudo -E kolla_set_configs && nova-manage api_db sync && nova-manage db sync --local_cell' - This change will break new deployments and should be reverted once the process is complete. + This change will break new deployments and should be reverted once this + process is complete -3. Re-install kolla-ansible from source in your kolla-ansible python +3. Re-install Kolla-Ansible from source in your Kolla-Ansible Python environment -4. Navigate to the directory containing your passwords.yml file +4. Navigate to the directory containing your ``passwords.yml`` file (``kayobe-config/etc/kolla/passwords.yml`` OR ``kayobe-config/etc/kayobe/environments/envname/kolla/passwords.yml``) -5. Create a file called deletelist.txt and populate it with this content +5. Create a file called ``deletelist.txt`` and populate it with this content (including all whitespace): -:: - - _keystone_password - _database_password - ^keystone_admin_password - ^memcache_secret_key - ^designate_rndc_key - ^docker_registry_password - ^keepalived_password - ^kibana_password - ^libvirt_sasl_password - ^metadata_secret - ^opensearch_dashboards_password - ^osprofiler_secret - ^prometheus_alertmanager_password - ^qdrouterd_password - ^redis_master_password - ^memcache_secret_key - _ssh_key - - private_key - public_key - ^$ - rabbitmq - ^haproxy_password - -NOTE: The above deletes bifrost secrets. Might need a seed s-d as well - -3. Decrypt your passwords.yml file with ansible vault - -4. Delete all the passwords in the deletion list + .. code:: + + _keystone_password + _database_password + ^keystone_admin_password + ^memcache_secret_key + ^designate_rndc_key + ^docker_registry_password + ^keepalived_password + ^kibana_password + ^libvirt_sasl_password + ^metadata_secret + ^opensearch_dashboards_password + ^osprofiler_secret + ^prometheus_alertmanager_password + ^qdrouterd_password + ^redis_master_password + ^memcache_secret_key + _ssh_key + + private_key + public_key + ^$ + rabbitmq + ^haproxy_password + + +6. Decrypt your ``passwords.yml`` file with ``ansible-vault`` + +7. Delete all the passwords in the deletion list .. code:: bash grep -vf deletelist.txt passwords.yml > new-passwords.yml -5. Check the new file for basic formatting errors. If it looks correct, - replace the existing ``passwords.yml`` file with - ``new-passwords.yml`` +8. Check the new file for basic formatting errors. If it looks correct, + replace the existing ``passwords.yml`` file with ``new-passwords.yml`` .. code:: bash rm passwords.yml && mv new-passwords.yml passwords.yml -6. Use the ``rekey-hosts.yml`` playbook to rotate your SSH keys for - hosts across the cloud. The playbook should exist under +9. Use the ``rekey-hosts.yml`` playbook to rotate your SSH keys for hosts + across the cloud. The playbook should exist under ``kayobe-config/etc/kayobe/ansible/`` if not, merge the latest - ``stackhpc-kayobe-config``. + ``stackhpc-kayobe-config`` .. code:: bash kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/rekey-hosts.yml -7. Update the Pulp password +10. Update the Pulp password 1. Generate a new Pulp password @@ -150,7 +156,7 @@ NOTE: The above deletes bifrost secrets. Might need a seed s-d as well pwgen -s 25 1 - 2. Update secrets_pulp_password (usually found in secrets.yml) + 2. Update ``secrets_pulp_password`` (usually found in ``secrets.yml``) 3. Deploy changes @@ -158,11 +164,11 @@ NOTE: The above deletes bifrost secrets. Might need a seed s-d as well kayobe seed service deploy -t seed-deploy-containers -kt none - (note you may need to skip docker registry login since the - password will now be ‘incorrect’ e.g. ``-e`` + (note you may need to skip docker registry login since the password will + now be ‘incorrect’ e.g. ``-e`` ``deploy_containers_registry_attempt_login``) -8. Rotate ``horizon_secret_key``. +11. Rotate ``horizon_secret_key`` 1. Generate a new secret: @@ -170,20 +176,20 @@ NOTE: The above deletes bifrost secrets. Might need a seed s-d as well pwgen -s 40 1 - 2. Add it to the passwords file, along with the old secret, in this + 2. Add it to the ``passwords.yml`` file, along with the old secret, in this exact format (including quotes in the middle): .. code:: bash horizon_secret_key: newsecret' 'oldsecret - This will allow both the old and new secrets to be used at the - same time, resulting in no interruption to service. The key is - mainly used for generating login and password reset tokens. The - old secret can be deleted & redeployed at a later date once all - users have closed & reopened their sessions. + This will allow both the old and new secrets to be used at the same + time, resulting in no interruption to service. The key is mainly used + for generating login and password reset tokens. The old secret can be + deleted & redeployed at a later date once all users have closed & + reopened their sessions. -9. Update grafana_admin_password +12. Update ``grafana_admin_password`` 1. Generate a new Grafana Admin password @@ -191,7 +197,7 @@ NOTE: The above deletes bifrost secrets. Might need a seed s-d as well pwgen -s 40 1 - 2. Exec into the grafana container on a controller + 2. Exec into the Grafana container on a controller .. code:: bash @@ -203,9 +209,9 @@ NOTE: The above deletes bifrost secrets. Might need a seed s-d as well grafana-cli admin reset-admin-password --password-from-stdin - 4. Update the value of ``grafana_admin_password`` in passwords.yml + 4. Update the value of ``grafana_admin_password`` in ``passwords.yml`` -10. Update the MariaDB database password +13. Update the MariaDB database password 1. Generate a new secret: @@ -213,26 +219,26 @@ NOTE: The above deletes bifrost secrets. Might need a seed s-d as well pwgen -s 40 1 - 2. Exec into the mariadb container on a controller + 2. Exec into the MariaDB container on a controller .. code:: bash sudo docker exec -it mariadb bash - 3. Log in to the database. You will be prompted for the password. - Use the existing value of ``database_password`` + 3. Log in to the database. You will be prompted for the password. Use the + existing value of ``database_password`` .. code:: bash mysql -uroot -p - 4. Check the current state of the root user + 4. Check the current state of the ``root`` user .. code:: bash SELECT Host,User,Password FROM mysql.user WHERE User='root'; - 5. Update the password for the root user + 5. Update the password for the ``root`` user .. code:: bash @@ -244,113 +250,115 @@ NOTE: The above deletes bifrost secrets. Might need a seed s-d as well SELECT Host,User,Password FROM mysql.user WHERE User='root'; - 7. If there are any remaining root users with the old password - e.g. ``root@localhost``, change the password for them too. + 7. If there are any remaining root users with the old password e.g. + ``root@localhost``, change the password for them too - 8. update ``database_password`` in ``passwords.yml`` with your new - password. + 8. Update ``database_password`` in ``passwords.yml`` with your new + password -:::warning From this point onward, service may be disrupted -:: +.. _nova-change: + +14. Update the Nova Database password + .. warning:: - ::: + From this point onward, service may be disrupted -11. Update the nova DB password - 1. Create a new ``nova_database_password`` and store it in + #. Create a new ``nova_database_password`` and store it in ``passwords.yml`` .. code:: bash pwgen -s 40 1 - 2. Exec into the nova_conductor container + #. Exec into the ``nova_conductor`` container .. code:: bash sudo docker exec -it nova_conductor bash - 3. List the cells + #. List the cells .. code:: bash nova-manage cell_v2 list_cells --verbose - 4. Find the entry for cell0, copy the Database Connection value, - replace the password in the string with the new value, and update - it with the following command: + #. Find the entry for ``cell0``, copy the Database Connection value, + replace the password in the string with the new value, and update it + with the following command: .. code:: bash nova-manage cell_v2 update_cell --cell_uuid 00000000-0000-0000-0000-000000000000 --database_connection "CONNECTION WITH NEW PASSWORD HERE" --transport-url "none:///" (If the ``cell_uuid`` for cell0 is not - ``00000000-0000-0000-0000-000000000000``, change the above - command accordingly) + ``00000000-0000-0000-0000-000000000000``, change the above command + accordingly) -12. Re-encrypt your ``passwords.yml`` file -13. Delete the service users in keystone. The exact users will depend on - the deployment. Multinode example: +15. Re-encrypt your ``passwords.yml`` file -:::warning This will immediately cause an API outage -:: +.. _k-a-change: - Alt: Cherry pick this patch: +16. Delete the service users in Keystone. The exact users will depend on the + deployment. Multinode example: - ::: + .. note:: - ```bash - openstack user delete glance cinder placement nova neutron heat magnum magnum_trustee_domain_admin barbican designate - ``` + Alternatively, cherry-pick + `this patch `__ -14. Stop services + + .. code:: bash + + openstack user delete glance cinder placement nova neutron heat magnum magnum_trustee_domain_admin barbican designate + +17. Stop services using RabbitMQ .. code:: bash kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/stop-openstack-services.yml -15. Nuke RabbitMQ +18. Nuke RabbitMQ .. code:: bash kayobe overcloud host command run -l controllers --become --command "docker stop rabbitmq && docker rm rabbitmq && docker volume rm rabbitmq" -16. Reconfigure overcloud services to push changes +19. Reconfigure Overcloud services to apply changes -:::warning VMs should continue running, but connections to them will -likely be disrupted when neutron is redeployed -:: + .. warning:: + + VMs should continue running, but connections to them will briefly be + disrupted when Neutron is redeployed - ::: + .. code:: bash - ```bash - kayobe overcloud service deploy - ``` + kayobe overcloud service deploy -17. Flush the memcached data on all controllers (any old data will now - be inaccessible) - 1. Install telnet (on one of the controllers) +20. Flush the Memcached data on all controllers (any old data will now be + inaccessible) + + #. Install Telnet (on one of the controllers) .. code:: bash sudo apt -y install telnet - 2. Check the config for the ip and port used by memcache (on every + #. Check the config for the IP and port used by Memcached (on every controller) .. code:: bash sudo grep command /etc/kolla/memcached/config.json - The IP and port will be printed after ``-l`` and ``-p`` - respectively. + The IP and port will be printed after ``-l`` and ``-p`` respectively - 3. For each controller start a telnet session, clear all data, then + #. For each controller start a Telnet session, clear all data, then exit .. code:: bash @@ -359,41 +367,35 @@ likely be disrupted when neutron is redeployed flush_all quit -18. Manually update heat_domain_admin_password +21. Manually update ``heat_domain_admin_password`` - 1. TODO: Instructions + #. TODO: Instructions + This has not been tested yet -19. Re-run tempest to make sure everything has come back +22. Re-run Tempest to make sure everything has come back -20. Inform other users of the steps they’ll need to take now that the - secrets have been rotated: +23. Inform other users of the steps they’ll need to take now that the secrets + have been rotated: - 1. SSH keys have been rotated, so the new key will have to be - distributed if individual user accounts are used - 2. A PR with the new passwords will need to be merged in to the main - config branch (REMEMBER TO RE-ENCRYPT PASSWORDS.YML BEFORE - COMMITING) - 3. Any existing openrc files generated by Kolla Ansible will need to - be re-generated or edited to use the new Kolla admin password. + 1. SSH keys have been rotated, so the new key will have to be distributed + if individual user accounts are used -21. At some point in the future (approx 1 week), remove the old horizon - secret key from ``passwords.yml`` and reconfigure horizon. + 2. Any existing ``openrc`` files generated by Kolla Ansible will need to be + re-generated or edited to use the new Kolla admin password + +24. Create a PR to merge the new secrets into your main Kayobe configuration + branch + + .. warning:: + + Unless you **really** enjoyed this process, RE-ENCRYPT + ``passwords.yml`` BEFORE COMMITTING + +25. Approximately 1 week after deploying, remove the old horizon secret key + from ``passwords.yml`` and reconfigure horizon -Future improvements -------------------- -- ☐ Regenerate passwords that we think we aren’t using, in case they - are actually being used -- ☐ Add the new database_password to passwords.yml before changing it, - in case it gets lost -- ☐ Stop services before deleting users? (Except for keystone) or allow - setting update_password in kolla-ansible -- ☒ At the end of the procedure write down what others need to do to - use the new passwords -- ☐ Can we get kolla-ansible to generate all/more of the passwords and - reduce the reliance on pwgen? -- ☐ Add a step to remove the old horizon_secret_key and redeploy - horizon at some point after the rotation +.. _full-password-list: Full password list ------------------- @@ -434,7 +436,7 @@ Full password list ironic_keystone_password karbor_database_password karbor_keystone_password - keystone_database_password Check this one + keystone_database_password magnum_database_password manila_database_password mariadb_backup_database_password From 43477f74b3c782f5f9988cef18433a8317a4ec01 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Mon, 11 Dec 2023 16:30:55 +0000 Subject: [PATCH 41/52] Fix secret rotation docs tox errors --- doc/source/operations/secret-rotation.rst | 20 ++++++++++---------- tox.ini | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/doc/source/operations/secret-rotation.rst b/doc/source/operations/secret-rotation.rst index 63eef3f35..3e80e7cab 100644 --- a/doc/source/operations/secret-rotation.rst +++ b/doc/source/operations/secret-rotation.rst @@ -36,28 +36,28 @@ this guide. Installation: As of writing, there are three upstream patches in the works to make this -process easier. +process easier. #. A change to Kolla, to automate :ref:`this` step to change the extended start for the ``nova-api`` container. The upstream patch can be found `here - `__. + `__. This was previously mitigated with a change to the StackHPC fork of Kolla-Ansible, which has since been reverted due to an unforeseen issue. #. A change to Nova, to automate :ref:`this` step to change the nova cell0 database connection string. - + The upstream patch can be found `here - `__. + `__. #. A change to Kolla-Ansible, to automate :ref:`this` step to update service keystone user passwords. - + The upstream patch can be found `here - `__. + `__. Full method @@ -83,7 +83,7 @@ Full method .. code:: command: bash -c 'sudo -E kolla_set_configs && nova-manage api_db sync && nova-manage db sync --local_cell' - + This change will break new deployments and should be reverted once this process is complete @@ -162,7 +162,7 @@ Full method .. code:: bash - kayobe seed service deploy -t seed-deploy-containers -kt none + kayobe seed service deploy -t seed-deploy-containers -kt none (note you may need to skip docker registry login since the password will now be ‘incorrect’ e.g. ``-e`` @@ -307,10 +307,10 @@ Full method .. note:: - Alternatively, cherry-pick + Alternatively, cherry-pick `this patch `__ - + .. code:: bash openstack user delete glance cinder placement nova neutron heat magnum magnum_trustee_domain_admin barbican designate diff --git a/tox.ini b/tox.ini index f79ac9701..c6b949efe 100644 --- a/tox.ini +++ b/tox.ini @@ -13,8 +13,8 @@ deps = commands = yamllint etc/kayobe reno lint - doc8 README.rst doc/source --ignore D001 - + # secret-rotation must be skipped because it includes purposeful whitespace + doc8 README.rst doc/source --ignore D001 --ignore-path doc/source/operations/secret-rotation.rst # StackHPC Kayobe configuration release notes: [testenv:releasenotes] allowlist_externals = rm From 0d969d7e0635cbb85c217ed44c42a4283043034e Mon Sep 17 00:00:00 2001 From: Bartosz Bezak Date: Tue, 12 Dec 2023 14:30:01 +0100 Subject: [PATCH 42/52] Pin R9 base image tag Co-Authored-By: Mark Goddard Co-Authored-By: Matt Anson --- etc/kayobe/kolla/kolla-build.conf | 4 ++++ .../notes/pin-rocky9-baseimage-3d224df5b6f56b10.yaml | 6 ++++++ 2 files changed, 10 insertions(+) create mode 100644 releasenotes/notes/pin-rocky9-baseimage-3d224df5b6f56b10.yaml diff --git a/etc/kayobe/kolla/kolla-build.conf b/etc/kayobe/kolla/kolla-build.conf index 83d8bb715..b879dba14 100644 --- a/etc/kayobe/kolla/kolla-build.conf +++ b/etc/kayobe/kolla/kolla-build.conf @@ -1,2 +1,6 @@ [DEFAULT] +{# Pinning to Rocky 9 minor version used in our repos #} +{% if kolla_base_distro == 'rocky' %} +base_tag = 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }} +{% endif %} build_args = {{ kolla_build_args.items() | map('join', ':') | join(',') }} diff --git a/releasenotes/notes/pin-rocky9-baseimage-3d224df5b6f56b10.yaml b/releasenotes/notes/pin-rocky9-baseimage-3d224df5b6f56b10.yaml new file mode 100644 index 000000000..5d99432b0 --- /dev/null +++ b/releasenotes/notes/pin-rocky9-baseimage-3d224df5b6f56b10.yaml @@ -0,0 +1,6 @@ +fixes: + - | + Pin the OCI image tag used for the base-image of Rocky 9 + Kolla image builds. This prevents packages in the image with + the latest tag getting in front of StackHPC release-train + package repositories. From adaafa4ab59624f09380bf636c3bb60cc2eaf012 Mon Sep 17 00:00:00 2001 From: Alex-Welsh Date: Tue, 12 Dec 2023 11:51:59 +0000 Subject: [PATCH 43/52] Secret rotation docs post-review changes --- doc/source/operations/secret-rotation.rst | 186 +++++++++++----------- tox.ini | 2 +- 2 files changed, 96 insertions(+), 92 deletions(-) diff --git a/doc/source/operations/secret-rotation.rst b/doc/source/operations/secret-rotation.rst index 3e80e7cab..7912530fb 100644 --- a/doc/source/operations/secret-rotation.rst +++ b/doc/source/operations/secret-rotation.rst @@ -16,7 +16,7 @@ be automatically regenerated with a ``kayobe overcloud service deploy``. Some secrets require manual input from the operator to change. Following this process, there may be a few seconds of network downtime for -running VMs when Neutron is reconfigured. +running VMs when Neutron is reconfigured when using ML2/OVS. There will be API downtime for all services. The main reason for the outage is that RabbitMQ must be completely stopped to change the secrets it uses. The @@ -45,7 +45,9 @@ process easier. `__. This was previously mitigated with a change to the StackHPC fork of - Kolla-Ansible, which has since been reverted due to an unforeseen issue. + Kolla-Ansible, which has since been reverted due to an unforeseen issue. See + `here ` for more + details. #. A change to Nova, to automate :ref:`this` step to change the nova cell0 database connection string. @@ -71,30 +73,41 @@ Full method 1. Run a Tempest ``refstack`` & check Kibana/OpenSearch Dashboards to check the state of the cloud before any changes are made +2. Edit your Kolla-Ansible checkout to include changes not yet included + upstream. + .. _kolla-change: -2. Edit your Kolla-Ansible checkout to include this line within the - ``kolla_docker`` dict in ``ansible/roles/nova/tasks/bootstrap_service.yml`` See - `here - `__ - for an example. (If you are using the latest ``stackhpc/yoga`` branch of - Kolla-Ansible this should already be set) + 1. Add this line within the ``kolla_docker`` dict in + ``ansible/roles/nova/tasks/bootstrap_service.yml`` See `here + `__ + for an example. - .. code:: + .. code:: + + command: bash -c 'sudo -E kolla_set_configs && nova-manage api_db sync && nova-manage db sync --local_cell' - command: bash -c 'sudo -E kolla_set_configs && nova-manage api_db sync && nova-manage db sync --local_cell' + This change will break new deployments and should be reverted once this + process is complete + +.. _k-a-change: - This change will break new deployments and should be reverted once this - process is complete + 2. Cherry-pick `this patch + `__ -3. Re-install Kolla-Ansible from source in your Kolla-Ansible Python - environment + .. code:: bash -4. Navigate to the directory containing your ``passwords.yml`` file + git fetch https://review.opendev.org/openstack/kolla-ansible refs/changes/78/903178/2 && git cherry-pick FETCH_HEAD + + 3. Re-install Kolla-Ansible from source in your Kolla-Ansible Python + environment + + +3. Navigate to the directory containing your ``passwords.yml`` file (``kayobe-config/etc/kolla/passwords.yml`` OR ``kayobe-config/etc/kayobe/environments/envname/kolla/passwords.yml``) -5. Create a file called ``deletelist.txt`` and populate it with this content +4. Create a file called ``deletelist.txt`` and populate it with this content (including all whitespace): .. code:: @@ -124,31 +137,42 @@ Full method ^haproxy_password -6. Decrypt your ``passwords.yml`` file with ``ansible-vault`` +5. Decrypt your ``passwords.yml`` file with ``ansible-vault`` -7. Delete all the passwords in the deletion list +6. Delete all the passwords in the deletion list .. code:: bash grep -vf deletelist.txt passwords.yml > new-passwords.yml -8. Check the new file for basic formatting errors. If it looks correct, +7. Check the new file for basic formatting errors. If it looks correct, replace the existing ``passwords.yml`` file with ``new-passwords.yml`` .. code:: bash rm passwords.yml && mv new-passwords.yml passwords.yml -9. Use the ``rekey-hosts.yml`` playbook to rotate your SSH keys for hosts +8. Use the ``rekey-hosts.yml`` playbook to rotate your SSH keys for hosts across the cloud. The playbook should exist under ``kayobe-config/etc/kayobe/ansible/`` if not, merge the latest ``stackhpc-kayobe-config`` - .. code:: bash + 1. Run the playbook to generate a new keypair and add it to the authorised + keys of your hosts. - kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/rekey-hosts.yml + .. code:: bash -10. Update the Pulp password + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/rekey-hosts.yml + + 2. Ensure you can SSH to other nodes using the new keypair + + 3. Re-run the playbook with arguments to remove the old keypair. + + .. code:: bash + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/rekey-hosts.yml -t remove-key -e rekey_remove_existing_key=true + +9. Update the Pulp password 1. Generate a new Pulp password @@ -164,11 +188,10 @@ Full method kayobe seed service deploy -t seed-deploy-containers -kt none - (note you may need to skip docker registry login since the password will - now be ‘incorrect’ e.g. ``-e`` - ``deploy_containers_registry_attempt_login``) + (note you will need to skip Docker registry login since the password will + now be ‘incorrect’ e.g. ``-e deploy_containers_registry_attempt_login=false``) -11. Rotate ``horizon_secret_key`` +10. Rotate ``horizon_secret_key`` 1. Generate a new secret: @@ -189,7 +212,7 @@ Full method deleted & redeployed at a later date once all users have closed & reopened their sessions. -12. Update ``grafana_admin_password`` +11. Update ``grafana_admin_password`` 1. Generate a new Grafana Admin password @@ -197,21 +220,21 @@ Full method pwgen -s 40 1 - 2. Exec into the Grafana container on a controller + 2. Update the value of ``grafana_admin_password`` in ``passwords.yml`` + + 3. Exec into the Grafana container on a controller .. code:: bash sudo docker exec -it grafana bash - 3. Run the password reset command, then enter the new password + 4. Run the password reset command, then enter the new password .. code:: bash grafana-cli admin reset-admin-password --password-from-stdin - 4. Update the value of ``grafana_admin_password`` in ``passwords.yml`` - -13. Update the MariaDB database password +12. Update the MariaDB database password 1. Generate a new secret: @@ -219,52 +242,51 @@ Full method pwgen -s 40 1 - 2. Exec into the MariaDB container on a controller + 2. Update ``database_password`` in ``passwords.yml`` with your new + password. Make a note of the old password. + + 3. Exec into the MariaDB container on a controller .. code:: bash sudo docker exec -it mariadb bash - 3. Log in to the database. You will be prompted for the password. Use the - existing value of ``database_password`` + 4. Log in to the database. You will be prompted for the password. Use the + old value of ``database_password`` .. code:: bash mysql -uroot -p - 4. Check the current state of the ``root`` user + 5. Check the current state of the ``root`` user .. code:: bash SELECT Host,User,Password FROM mysql.user WHERE User='root'; - 5. Update the password for the ``root`` user + 6. Update the password for the ``root`` user .. code:: bash SET PASSWORD FOR 'root'@'%' = PASSWORD('newpassword'); - 6. Check that the password hash has changed in the user list + 7. Check that the password hash has changed in the user list .. code:: bash SELECT Host,User,Password FROM mysql.user WHERE User='root'; - 7. If there are any remaining root users with the old password e.g. + 8. If there are any remaining root users with the old password e.g. ``root@localhost``, change the password for them too - 8. Update ``database_password`` in ``passwords.yml`` with your new - password - - .. _nova-change: -14. Update the Nova Database password +13. Update the Nova Database password + .. warning:: From this point onward, service may be disrupted - #. Create a new ``nova_database_password`` and store it in ``passwords.yml`` @@ -296,51 +318,15 @@ Full method ``00000000-0000-0000-0000-000000000000``, change the above command accordingly) +14. Re-encrypt your ``passwords.yml`` file -15. Re-encrypt your ``passwords.yml`` file - - -.. _k-a-change: - -16. Delete the service users in Keystone. The exact users will depend on the - deployment. Multinode example: - - .. note:: - - Alternatively, cherry-pick - `this patch `__ - - - .. code:: bash - - openstack user delete glance cinder placement nova neutron heat magnum magnum_trustee_domain_admin barbican designate - -17. Stop services using RabbitMQ +15. Stop all OpenStack services .. code:: bash kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/stop-openstack-services.yml -18. Nuke RabbitMQ - - .. code:: bash - - kayobe overcloud host command run -l controllers --become --command "docker stop rabbitmq && docker rm rabbitmq && docker volume rm rabbitmq" - -19. Reconfigure Overcloud services to apply changes - - - .. warning:: - - VMs should continue running, but connections to them will briefly be - disrupted when Neutron is redeployed - - .. code:: bash - - kayobe overcloud service deploy - - -20. Flush the Memcached data on all controllers (any old data will now be +16. Flush the Memcached data on all controllers (any old data will now be inaccessible) #. Install Telnet (on one of the controllers) @@ -367,23 +353,40 @@ Full method flush_all quit -21. Manually update ``heat_domain_admin_password`` +17. Nuke RabbitMQ + + .. code:: bash + + kayobe overcloud host command run -l controllers --become --command "docker stop rabbitmq && docker rm rabbitmq && docker volume rm rabbitmq" + +19. Reconfigure Overcloud services to apply changes + + .. warning:: + + VMs should continue running, but connections to them will briefly be + disrupted when Neutron is redeployed when using ML2/OVS + + .. code:: bash + + kayobe overcloud service deploy + +20. Manually update ``heat_domain_admin_password`` #. TODO: Instructions This has not been tested yet -22. Re-run Tempest to make sure everything has come back +21. Re-run Tempest to make sure everything has come back -23. Inform other users of the steps they’ll need to take now that the secrets +22. Inform other users of the steps they’ll need to take now that the secrets have been rotated: 1. SSH keys have been rotated, so the new key will have to be distributed if individual user accounts are used 2. Any existing ``openrc`` files generated by Kolla Ansible will need to be - re-generated or edited to use the new Kolla admin password + re-generated or edited to use the new Keystone admin password -24. Create a PR to merge the new secrets into your main Kayobe configuration +23. Create a PR to merge the new secrets into your main Kayobe configuration branch .. warning:: @@ -391,7 +394,7 @@ Full method Unless you **really** enjoyed this process, RE-ENCRYPT ``passwords.yml`` BEFORE COMMITTING -25. Approximately 1 week after deploying, remove the old horizon secret key +24. Approximately 1 week after deploying, remove the old horizon secret key from ``passwords.yml`` and reconfigure horizon @@ -505,6 +508,7 @@ Full password list docker_registry_password secrets_pulp_password redis_master_password + haproxy_password keystone_ssh_key private_key public_key diff --git a/tox.ini b/tox.ini index c6b949efe..e7f0d1d09 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,7 @@ commands = yamllint etc/kayobe reno lint # secret-rotation must be skipped because it includes purposeful whitespace - doc8 README.rst doc/source --ignore D001 --ignore-path doc/source/operations/secret-rotation.rst + doc8 README.rst doc/source --ignore D001 --ignore-path-errors doc/source/operations/secret-rotation.rst;D002 # StackHPC Kayobe configuration release notes: [testenv:releasenotes] allowlist_externals = rm From 8b9e3d7b18beaa540f927ed25ec7f0f43a64c2b9 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 15 Dec 2023 11:27:02 +0100 Subject: [PATCH 44/52] Disable os-capacity by default The HAProxy configuration for os-capacity is enabled by default and is causing alerts on every deployment until the exporter is deployed. Disable it while its configuration is being finalised [1]. [1] https://github.com/stackhpc/stackhpc-kayobe-config/pull/697 --- etc/kayobe/kolla/config/haproxy/services.d/os_exporter.cfg | 2 ++ .../{70-oscapacity.yml.j2 => 70-oscapacity.yml} | 2 ++ etc/kayobe/stackhpc-monitoring.yml | 6 ++++++ 3 files changed, 10 insertions(+) rename etc/kayobe/kolla/config/prometheus/prometheus.yml.d/{70-oscapacity.yml.j2 => 70-oscapacity.yml} (80%) diff --git a/etc/kayobe/kolla/config/haproxy/services.d/os_exporter.cfg b/etc/kayobe/kolla/config/haproxy/services.d/os_exporter.cfg index 1292f38bd..e40c27a38 100644 --- a/etc/kayobe/kolla/config/haproxy/services.d/os_exporter.cfg +++ b/etc/kayobe/kolla/config/haproxy/services.d/os_exporter.cfg @@ -1,3 +1,4 @@ +{% if stackhpc_enable_os_capacity | bool %} {% raw %} frontend os_capacity_frontend mode http @@ -17,3 +18,4 @@ backend os_capacity_backend server {{ host_name }} {{ host_ip }}:9000 check inter 2000 rise 2 fall 5 {% endfor %} {% endraw %} +{% endif %} diff --git a/etc/kayobe/kolla/config/prometheus/prometheus.yml.d/70-oscapacity.yml.j2 b/etc/kayobe/kolla/config/prometheus/prometheus.yml.d/70-oscapacity.yml similarity index 80% rename from etc/kayobe/kolla/config/prometheus/prometheus.yml.d/70-oscapacity.yml.j2 rename to etc/kayobe/kolla/config/prometheus/prometheus.yml.d/70-oscapacity.yml index 1c0c25c15..4a6d1a802 100644 --- a/etc/kayobe/kolla/config/prometheus/prometheus.yml.d/70-oscapacity.yml.j2 +++ b/etc/kayobe/kolla/config/prometheus/prometheus.yml.d/70-oscapacity.yml @@ -1,3 +1,4 @@ +{% if stackhpc_enable_os_capacity | bool %} {% raw %} scrape_configs: - job_name: os-capacity @@ -7,3 +8,4 @@ scrape_configs: scrape_interval: 15m scrape_timeout: 10m {% endraw %} +{% endif %} diff --git a/etc/kayobe/stackhpc-monitoring.yml b/etc/kayobe/stackhpc-monitoring.yml index 43f1f309f..b48646e79 100644 --- a/etc/kayobe/stackhpc-monitoring.yml +++ b/etc/kayobe/stackhpc-monitoring.yml @@ -9,3 +9,9 @@ alertmanager_low_memory_threshold_gib: 5 ############################################################################### +# Exporter configuration + +# Whether the OpenStack Capacity exporter is enabled. +# Enabling this flag will result in HAProxy configuration and Prometheus scrape +# targets being templated during deployment. +stackhpc_enable_os_capacity: false From e004af54a884241ad1b5b76447028ceb61e416c5 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 15 Dec 2023 11:33:04 +0100 Subject: [PATCH 45/52] Disable yamllint which fails on Jinja syntax --- .../kolla/config/prometheus/prometheus.yml.d/70-oscapacity.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etc/kayobe/kolla/config/prometheus/prometheus.yml.d/70-oscapacity.yml b/etc/kayobe/kolla/config/prometheus/prometheus.yml.d/70-oscapacity.yml index 4a6d1a802..659c26047 100644 --- a/etc/kayobe/kolla/config/prometheus/prometheus.yml.d/70-oscapacity.yml +++ b/etc/kayobe/kolla/config/prometheus/prometheus.yml.d/70-oscapacity.yml @@ -1,3 +1,5 @@ +# yamllint disable-file +--- {% if stackhpc_enable_os_capacity | bool %} {% raw %} scrape_configs: From 6ab159d96017574919e3d47b39bb1eb045ab52b3 Mon Sep 17 00:00:00 2001 From: Bartosz Bezak Date: Fri, 15 Dec 2023 12:49:08 +0100 Subject: [PATCH 46/52] use ark repos for R9 opensearch builds --- etc/kayobe/environments/ci-builder/stackhpc-ci.yml | 2 ++ etc/kayobe/kolla.yml | 6 ++++++ etc/kayobe/pulp-repo-versions.yml | 2 ++ etc/kayobe/pulp.yml | 12 ++++++++++++ etc/kayobe/stackhpc.yml | 8 ++++++++ 5 files changed, 30 insertions(+) diff --git a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml index 79ea4a7f8..efe4236e9 100644 --- a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml @@ -80,6 +80,8 @@ stackhpc_repo_rhel_9_treasuredata_4_version: "{{ stackhpc_pulp_repo_rhel_9_treas stackhpc_repo_rhel_9_mariadb_10_6_version: "{{ stackhpc_pulp_repo_rhel_9_mariadb_10_6_version }}" stackhpc_repo_rhel_9_influxdb_version: "{{ stackhpc_pulp_repo_rhel_9_influxdb_version }}" stackhpc_repo_epel_9_version: "{{ stackhpc_pulp_repo_epel_9_version }}" +stackhpc_repo_opensearch_2_x_version: "{{ stackhpc_pulp_repo_opensearch_2_x_version }}" +stackhpc_repo_opensearch_dashboards_2_x_version: "{{ stackhpc_pulp_repo_opensearch_dashboards_2_x_version }}" ## Use derived vars from etc/kayobe/pulp.yml to switch between ## minor Rocky versions using stackhpc_pulp_repo_rocky_x_minor_version stackhpc_repo_rocky_baseos_version: "{{ stackhpc_pulp_repo_rocky_8_baseos_version }}" diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 1f975c609..082e9243b 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -268,6 +268,12 @@ stackhpc_rocky_9_third_party_repos: - url: "{{ stackhpc_repo_rabbitmq_server_url }}" file: "rabbitmq_rabbitmq-server.repo" tag: "rabbitmq_rabbitmq-server" + - url: "{{ stackhpc_repo_opensearch_2_x_url }}" + file: "opensearch.repo" + tag: "opensearch-2.x" + - url: "{{ stackhpc_repo_opensearch_dashboards_2_x_url }}" + file: "opensearch.repo" + tag: "opensearch-dashboards-2.x" # List of EPEL 9 stackhpc_epel_9_repos: diff --git a/etc/kayobe/pulp-repo-versions.yml b/etc/kayobe/pulp-repo-versions.yml index 725629de5..f7ecb093b 100644 --- a/etc/kayobe/pulp-repo-versions.yml +++ b/etc/kayobe/pulp-repo-versions.yml @@ -26,6 +26,8 @@ stackhpc_pulp_repo_epel_version: 20231020T014922 stackhpc_pulp_repo_grafana_version: 20231020T014922 stackhpc_pulp_repo_mariadb_10_6_centos8_version: 20230815T010124 stackhpc_pulp_repo_mlnx_ofed_5_7_1_0_2_0_rhel8_6_version: 20220920T151419 +stackhpc_pulp_repo_opensearch_2_x_version: 20231202T013234 +stackhpc_pulp_repo_opensearch_dashboards_2_x_version: 20231202T013234 stackhpc_pulp_repo_rabbitmq_erlang_version: 20231015T004919 stackhpc_pulp_repo_rabbitmq_server_version: 20231018T041416 stackhpc_pulp_repo_rhel_9_influxdb_version: 20231019T010143 diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index 9e8af49ca..884bf679b 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -500,6 +500,18 @@ stackhpc_pulp_rpm_repos: base_path: "influxdb/rhel/9/x86_64/stable/" required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_9 | bool }}" + - name: OpenSearch 2.x + url: "{{ stackhpc_release_pulp_content_url }}/opensearch/2.x/yum/{{ stackhpc_pulp_repo_opensearch_2_x_version }}" + distribution_name: "opensearch-2-x-" + base_path: "opensearch/2.x/yum/" + required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_9 | bool }}" + + - name: OpenSearch Dashboards 2.x + url: "{{ stackhpc_release_pulp_content_url }}/opensearch-dashboards/2.x/yum/{{ stackhpc_pulp_repo_opensearch_dashboards_2_x_version }}" + distribution_name: "opensearch-dashboards-2-x-" + base_path: "opensearch-dashboards/2.x/yum/" + required: "{{ stackhpc_pulp_sync_for_local_container_build | bool and stackhpc_pulp_sync_el_9 | bool }}" + # RPM repositories stackhpc_pulp_repository_rpm_repos: >- {%- set rpm_repos = [] -%} diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml index 0ca77eb82..3c0e32430 100644 --- a/etc/kayobe/stackhpc.yml +++ b/etc/kayobe/stackhpc.yml @@ -165,6 +165,14 @@ stackhpc_repo_rhel_9_mariadb_10_6_version: "{{ stackhpc_repo_distribution }}" stackhpc_repo_rhel_9_influxdb_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/influxdb/rhel/9/x86_64/stable/{{ stackhpc_repo_rhel_9_influxdb_version }}" stackhpc_repo_rhel_9_influxdb_version: "{{ stackhpc_repo_distribution }}" +# OpenSearch for RHEL 9 +stackhpc_repo_opensearch_2_x_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/opensearch/2.x/yum/{{ stackhpc_repo_opensearch_2_x_version }}" +stackhpc_repo_opensearch_2_x_version: "{{ stackhpc_repo_distribution }}" + +# OpenSearch Dashboards for RHEL 9 +stackhpc_repo_opensearch_dashboards_2_x_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/opensearch-dashboards/2.x/yum/{{ stackhpc_repo_opensearch_dashboards_2_x_version }}" +stackhpc_repo_opensearch_dashboards_2_x_version: "{{ stackhpc_repo_distribution }}" + # Rocky 9 AppStream stackhpc_repo_rocky_9_appstream_url: "{{ stackhpc_repo_mirror_url }}/pulp/content/rocky/{{ stackhpc_rocky_9_url_version }}/AppStream/x86_64/os/{{ stackhpc_repo_rocky_9_appstream_version }}" stackhpc_repo_rocky_9_appstream_version: "{{ stackhpc_repo_distribution }}" From 74e7fff4d712596955514cfbba2a1762bae3e4d5 Mon Sep 17 00:00:00 2001 From: Bartosz Bezak Date: Tue, 19 Dec 2023 11:49:25 +0100 Subject: [PATCH 47/52] Pin Ubuntu Focal base image tag Co-Authored-By: Mark Goddard Co-Authored-By: Matt Anson --- etc/kayobe/kolla/kolla-build.conf | 9 +++++++-- .../notes/pin-focal-baseimage-baafa2583f952596.yaml | 8 ++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/pin-focal-baseimage-baafa2583f952596.yaml diff --git a/etc/kayobe/kolla/kolla-build.conf b/etc/kayobe/kolla/kolla-build.conf index b879dba14..2cd9c7a25 100644 --- a/etc/kayobe/kolla/kolla-build.conf +++ b/etc/kayobe/kolla/kolla-build.conf @@ -1,6 +1,11 @@ [DEFAULT] -{# Pinning to Rocky 9 minor version used in our repos #} -{% if kolla_base_distro == 'rocky' %} +{# Packages in the latest upstream Ubuntu base image can be ahead of our package repo #} +{# snapshots, so pin to a specific tag. #} +{# This tag should be updated when Ubuntu package repo snapshot versions are changed. #} +{% if kolla_base_distro == 'ubuntu' %} +base_tag = focal-20231003 +{# Similarly pinning to Rocky 9 minor version used in our repos #} +{% elif kolla_base_distro == 'rocky' %} base_tag = 9.{{ stackhpc_pulp_repo_rocky_9_minor_version }} {% endif %} build_args = {{ kolla_build_args.items() | map('join', ':') | join(',') }} diff --git a/releasenotes/notes/pin-focal-baseimage-baafa2583f952596.yaml b/releasenotes/notes/pin-focal-baseimage-baafa2583f952596.yaml new file mode 100644 index 000000000..b86e35d8f --- /dev/null +++ b/releasenotes/notes/pin-focal-baseimage-baafa2583f952596.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + Pin the OCI image tag used for the Ubuntu Focal base-image + of Kolla image builds. This prevents packages in the image + with the latest tag getting in front of StackHPC release-train + package repositories. Ubuntu tag should be bumped when new + packages are available in StackHPC release-train. From 0083e7228e9d406dd609317d4cc74f79bee07230 Mon Sep 17 00:00:00 2001 From: Bartosz Bezak Date: Wed, 20 Dec 2023 09:23:29 +0100 Subject: [PATCH 48/52] add opensearch tag 2.11.1 --- etc/kayobe/kolla/globals.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 337687d65..0abaeffaf 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -23,6 +23,7 @@ kayobe_image_tags: cloudkitty_tag: yoga-20231107T165648 nova_tag: "{% raw %}{{ kayobe_image_tags['nova'][kolla_base_distro] }}{% endraw %}" neutron_tag: yoga-20231114T125927 +opensearch_tag: yoga-20231219T221916 # These overrides are currently redundant, but are kept because it's not obvious that you need them if setting haproxy_tag glance_tls_proxy_tag: "{% raw %}{{ haproxy_tag | default(openstack_tag) }}{% endraw %}" From e111eb932a1cd7199f3eec85b8ce8c3f7e29b18a Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 21 Dec 2023 12:47:04 +0000 Subject: [PATCH 49/52] feat: bump `stackhpc.kayobe_workflows` --- etc/kayobe/ansible/requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index 0a30d7437..d72d41ae4 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -11,7 +11,7 @@ collections: - name: stackhpc.hashicorp version: 2.4.0 - name: stackhpc.kayobe_workflows - version: 1.0.1 + version: 1.0.2 roles: - src: stackhpc.vxlan - name: ansible-lockdown.rhel8_cis From 7bac603887125f69b63334461a2cba05e17d5c69 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 21 Dec 2023 14:15:02 +0000 Subject: [PATCH 50/52] feat: separate secrets and variables into separate tables --- doc/source/configuration/ci-cd.rst | 57 ++++++++++++++---------------- 1 file changed, 26 insertions(+), 31 deletions(-) diff --git a/doc/source/configuration/ci-cd.rst b/doc/source/configuration/ci-cd.rst index fdbb0d009..dc330370a 100644 --- a/doc/source/configuration/ci-cd.rst +++ b/doc/source/configuration/ci-cd.rst @@ -136,37 +136,32 @@ Workflow Deployment 3. Add all required secrets and variables to repository either via the GitHub UI or GitHub CLI (may require repository owner) -.. raw:: html - -
- - - - - - - - - - - - - - - - - - - - - - - -
SecretsVariables
KAYOBE_AUTOMATION_SSH_PRIVATE_KEYREGISTRY_USERNAME
KAYOBE_VAULT_PASSWORDREGISTRY_URL
REGISTRY_PASSWORD
TEMPEST_OPENRC
- -Note the above table shows the secrets and variable one may need to add to GitHub for a successful deployment. -However, these secrets and variables might not all be required for example if :code:`github_registry` has been configured with a single shared registry then :code:`REGISTRY_USERNAME` and :code:`REGISTRY_URL` can be ignored. -Also it is important that if you are using multiple environments and secrets and variables are not being shared then each will require an environment prefix added in the form `ENVIRONMENT_NAME_SECRET_OR_VARIABLE_NAME` for example if there was two environments each name :code:`production` and :code:`staging` then :code:`KAYOBE_AUTOMATION_SSH_PRIVATE_KEY` would be replaced by :code:`PRODUCTION_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY` and :code:`STAGING_KAYOBE_AUTOMATION_SSH_PRIVATE_KEY`. ++----------------------------------------------------------------------------------+ +| Secrets | ++===================================+==============================================+ +| Single Environment | Multiple Environments | ++-----------------------------------+----------------------------------------------+ +| KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | _KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | ++-----------------------------------+----------------------------------------------+ +| KAYOBE_VAULT_PASSWORD | _PRODUCTION_KAYOBE_VAULT_PASSWORD | ++-----------------------------------+----------------------------------------------+ +| REGISTRY_PASSWORD | _REGISTRY_PASSWORD | ++-----------------------------------+----------------------------------------------+ +| TEMPEST_OPENRC | _PRODUCTION_TEMPEST_OPENRC | ++-----------------------------------+----------------------------------------------+ + + +----------------------------------------------+ + | VARIABLES | + +====================+=========================+ + | Single Environment | Multiple Environments | + +--------------------+-------------------------+ + | REGISTRY_URL | _REGISTRY_URL | + +--------------------+-------------------------+ + | REGISTRY_USERNAME | _USERNAME | + +--------------------+-------------------------+ + +Note the above tables shows the secrets and variables one may need to add to GitHub for a successful deployment. +When adding secrets and variables make sure to adhere to the naming standards and ensure the :code:`` is replaced with all supported kayobe environments in uppercase. 4. Commit and push all newly generated workflows found under :code:`.github/workflows` From f48f5f7834b3f5c76fd44393f6e3c54135ba300f Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 21 Dec 2023 14:35:49 +0000 Subject: [PATCH 51/52] fix: secret and variable names in table Co-authored-by: Mark Goddard --- doc/source/configuration/ci-cd.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/configuration/ci-cd.rst b/doc/source/configuration/ci-cd.rst index dc330370a..51a18c073 100644 --- a/doc/source/configuration/ci-cd.rst +++ b/doc/source/configuration/ci-cd.rst @@ -143,11 +143,11 @@ Workflow Deployment +-----------------------------------+----------------------------------------------+ | KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | _KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | +-----------------------------------+----------------------------------------------+ -| KAYOBE_VAULT_PASSWORD | _PRODUCTION_KAYOBE_VAULT_PASSWORD | +| KAYOBE_VAULT_PASSWORD | _KAYOBE_VAULT_PASSWORD | +-----------------------------------+----------------------------------------------+ | REGISTRY_PASSWORD | _REGISTRY_PASSWORD | +-----------------------------------+----------------------------------------------+ -| TEMPEST_OPENRC | _PRODUCTION_TEMPEST_OPENRC | +| TEMPEST_OPENRC | _TEMPEST_OPENRC | +-----------------------------------+----------------------------------------------+ +----------------------------------------------+ @@ -157,7 +157,7 @@ Workflow Deployment +--------------------+-------------------------+ | REGISTRY_URL | _REGISTRY_URL | +--------------------+-------------------------+ - | REGISTRY_USERNAME | _USERNAME | + | REGISTRY_USERNAME | _REGISTRY_USERNAME | +--------------------+-------------------------+ Note the above tables shows the secrets and variables one may need to add to GitHub for a successful deployment. From 9e0032a0b387656fef1abf374428d0fde2b8f398 Mon Sep 17 00:00:00 2001 From: Jack Hodgkiss Date: Thu, 21 Dec 2023 14:38:01 +0000 Subject: [PATCH 52/52] fix: `ERROR: Malformed table` --- doc/source/configuration/ci-cd.rst | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/doc/source/configuration/ci-cd.rst b/doc/source/configuration/ci-cd.rst index 51a18c073..6e495c2e8 100644 --- a/doc/source/configuration/ci-cd.rst +++ b/doc/source/configuration/ci-cd.rst @@ -143,22 +143,22 @@ Workflow Deployment +-----------------------------------+----------------------------------------------+ | KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | _KAYOBE_AUTOMATION_SSH_PRIVATE_KEY | +-----------------------------------+----------------------------------------------+ -| KAYOBE_VAULT_PASSWORD | _KAYOBE_VAULT_PASSWORD | +| KAYOBE_VAULT_PASSWORD | _KAYOBE_VAULT_PASSWORD | +-----------------------------------+----------------------------------------------+ | REGISTRY_PASSWORD | _REGISTRY_PASSWORD | +-----------------------------------+----------------------------------------------+ -| TEMPEST_OPENRC | _TEMPEST_OPENRC | +| TEMPEST_OPENRC | _TEMPEST_OPENRC | +-----------------------------------+----------------------------------------------+ - +----------------------------------------------+ - | VARIABLES | - +====================+=========================+ - | Single Environment | Multiple Environments | - +--------------------+-------------------------+ - | REGISTRY_URL | _REGISTRY_URL | - +--------------------+-------------------------+ + +-------------------------------------------------------+ + | VARIABLES | + +====================+==================================+ + | Single Environment | Multiple Environments | + +--------------------+----------------------------------+ + | REGISTRY_URL | _REGISTRY_URL | + +--------------------+----------------------------------+ | REGISTRY_USERNAME | _REGISTRY_USERNAME | - +--------------------+-------------------------+ + +--------------------+----------------------------------+ Note the above tables shows the secrets and variables one may need to add to GitHub for a successful deployment. When adding secrets and variables make sure to adhere to the naming standards and ensure the :code:`` is replaced with all supported kayobe environments in uppercase.