From 0381e16bf2591dc431dd6b69967a9b1bf77564d3 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 28 Jul 2022 15:45:13 +0000 Subject: [PATCH 01/36] Add StackHPC overcloud DIB configuration --- etc/kayobe/overcloud-dib.yml | 10 +- etc/kayobe/stackhpc-overcloud-dib.yml | 212 ++++++++++++++++++ .../notes/overcloud-dib-938eac005662a2cb.yaml | 4 + 3 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 etc/kayobe/stackhpc-overcloud-dib.yml create mode 100644 releasenotes/notes/overcloud-dib-938eac005662a2cb.yaml diff --git a/etc/kayobe/overcloud-dib.yml b/etc/kayobe/overcloud-dib.yml index 3d69f8eb1..78fff9471 100644 --- a/etc/kayobe/overcloud-dib.yml +++ b/etc/kayobe/overcloud-dib.yml @@ -10,6 +10,10 @@ # is {{ os_distribution == 'rocky' }}. This will change in a future release. #overcloud_dib_build_host_images: +# List of additional host packages to install. +overcloud_dib_host_packages_extra: + - "{% if 'ubuntu-minimal' in overcloud_dib_host_images | map(attribute='elements') | flatten | list %}debootstrap{% endif %}" + # List of overcloud host disk images to build. Each element is a dict defining # an image in a format accepted by the stackhpc.os-images role. Default is to # build an image named "deployment_image" configured with the overcloud_dib_* @@ -64,7 +68,11 @@ # List of additional git repositories containing Diskimage Builder (DIB) # elements. See stackhpc.os-images role for usage. Default is empty. -#overcloud_dib_git_elements_extra: +overcloud_dib_git_elements_extra: + - repo: "https://github.com/stackhpc/stackhpc-image-elements" + local: "{{ source_checkout_path }}/stackhpc-image-elememts" + version: "v1.4.0" + elements_path: "elements" # List of git repositories containing Diskimage Builder (DIB) elements. See # stackhpc.os-images role for usage. Default is a combination of diff --git a/etc/kayobe/stackhpc-overcloud-dib.yml b/etc/kayobe/stackhpc-overcloud-dib.yml new file mode 100644 index 000000000..0482cae0e --- /dev/null +++ b/etc/kayobe/stackhpc-overcloud-dib.yml @@ -0,0 +1,212 @@ +--- +# StackHPC overcloud host disk image configuration. + +############################################################################### +# Diskimage-builder configuration for overcloud host disk images. + +# StackHPC overcloud host disk image Diskimage Builder recipe. This may be used +# as an item in the overcloud_dib_host_images list when +# overcloud_dib_build_host_images is true. +# Example: +# overcloud_dib_host_images: +# - "{{ stackhpc_overcloud_dib_host_image }}" +stackhpc_overcloud_dib_host_image: + name: "{{ stackhpc_overcloud_dib_name }}" + elements: "{{ stackhpc_overcloud_dib_elements }}" + env: "{{ stackhpc_overcloud_dib_env_vars }}" + packages: "{{ stackhpc_overcloud_dib_packages }}" + +# StackHPC overcloud DIB image name. +stackhpc_overcloud_dib_name: "deployment_image" + +# StackHPC overcloud DIB image elements. +stackhpc_overcloud_dib_elements: + - "{{ os_distribution }}-{% if os_distribution == 'rocky' %}container{% else %}minimal{% endif %}" + - "cloud-init-datasources" + - "{% if os_distribution in ['centos', 'rocky'] %}disable-selinux{% endif %}" + - "enable-serial-console" + - "vm" + - "block-device-efi" + - "cloud-init" + - "{% if os_distribution in ['centos', 'rocky'] %}dracut-regenerate{% endif %}" + - "{% if os_distribution == 'ubuntu' %}lvm{% endif %}" + - "openssh-server" + - "{% if os_distribution == 'ubuntu' %}sudoers{% endif %}" + +# StackHPC overcloud DIB image environment variables. +stackhpc_overcloud_dib_env_vars: + DIB_BLOCK_DEVICE_CONFIG: "{{ stackhpc_overcloud_dib_block_device_config_uefi_lvm }}" + DIB_BOOTLOADER_DEFAULT_CMDLINE: "nofb nomodeset gfxpayload=text net.ifnames=1 rd.auto" + DIB_CLOUD_INIT_DATASOURCES: "ConfigDrive" + DIB_CONTAINERFILE_RUNTIME: "docker" + DIB_CONTAINERFILE_NETWORK_DRIVER: "host" + DIB_DISTRIBUTION_MIRROR: "{{ stackhpc_repo_ubuntu_focal_url if os_distribution == 'ubuntu' else '' }}" + DIB_DRACUT_ENABLED_MODULES_DEFAULT_CONFIG: "{{ stackhpc_overcloud_dib_dracut_enabled_modules_default_config }}" + DIB_RELEASE: "{{ overcloud_dib_os_release }}" + DIB_SUDOERS_FILENAME: "no-fqdn" + # Avoid DNS queries during sudo commands, since we might not always have working DNS. + DIB_SUDOERS_CONFIG: | + Defaults !fqdn + # FIXME: Support templating repo files. + # DIB_YUM_MINIMAL_BOOTSTRAP_REPOS: /path/to/dir/containing/dib-mirror-*.repo + YUM: dnf + +# StackHPC overcloud DIB image packages. +stackhpc_overcloud_dib_packages: + - "logrotate" + - "net-tools" + +# StackHPC overcloud DIB image block device configuration. +# This image layout conforms to the CIS partition benchmarks. +# This configuration builds a UEFI-compatible image with 3 partitions. +# * p0: EFI ESP bootloader +# * p1: EFI BSP +# * p2: LVM PV (rootpv) +# The rootpv PV is in the rootvg VG, and has the following LVs: +# * lv_root -> / +# * lv_tmp -> /tmp +# * lv_var -> /var +# * lv_var_tmp -> /var/tmp +# * lv_log -> /var/log +# * lv_audit -> /var/log/audit +# * lv_home -> /home +stackhpc_overcloud_dib_block_device_config_uefi_lvm: | + - local_loop: + name: image0 + size: 20GiB + - partitioning: + base: image0 + label: gpt + partitions: + - name: ESP + type: 'EF00' + size: 550MiB + mkfs: + type: vfat + mount: + mount_point: /boot/efi + fstab: + options: "defaults" + fsck-passno: 2 + - name: BSP + type: 'EF02' + size: 8MiB + - name: root + type: '8E00' + flags: [ boot ] + size: 100% + - lvm: + name: lvm + base: [ root ] + pvs: + - name: rootpv + base: root + options: [ "--force" ] + vgs: + - name: rootvg + base: [ "rootpv" ] + options: [ "--force" ] + lvs: + - name: lv_root + base: rootvg + size: 5G + - name: lv_tmp + base: rootvg + size: 1G + - name: lv_var + base: rootvg + size: 1G + - name: lv_var_tmp + base: rootvg + size: 1G + - name: lv_log + base: rootvg + size: 1G + - name: lv_audit + base: rootvg + size: 128M + - name: lv_home + base: rootvg + size: 128M + - mkfs: + name: fs_root + base: lv_root + type: ext4 + label: "rootfs" + mount: + mount_point: / + fstab: + options: "defaults" + fsck-passno: 1 + - mkfs: + name: fs_tmp + base: lv_tmp + type: ext4 + label: "tmpfs" + mount: + mount_point: /tmp + fstab: + options: "rw,noexec,nosuid,nodev" + fsck-passno: 2 + - mkfs: + name: fs_var + base: lv_var + type: ext4 + label: "varfs" + mount: + mount_point: /var + fstab: + options: "defaults" + fsck-passno: 2 + - mkfs: + name: fs_var_tmp + base: lv_var_tmp + type: ext4 + label: "vartmpfs" + mount: + mount_point: /var/tmp + fstab: + options: "rw,noexec,nosuid,nodev" + fsck-passno: 2 + - mkfs: + name: fs_log + base: lv_log + type: ext4 + label: "logfs" + mount: + mount_point: /var/log + fstab: + options: "defaults" + fsck-passno: 2 + - mkfs: + name: fs_audit + base: lv_audit + type: ext4 + label: "auditfs" + mount: + mount_point: /var/log/audit + fstab: + options: "defaults" + fsck-passno: 2 + - mkfs: + name: fs_home + base: lv_home + type: ext4 + label: "homefs" + mount: + mount_point: /home + fstab: + options: "rw,nodev" + fsck-passno: 2 + +# StackHPC overcloud DIB image Dracut module configuration. +stackhpc_overcloud_dib_dracut_enabled_modules_default_config: | + - name: crypt + packages: + - cryptsetup + - name: lvm + packages: + - lvm2 + - name: mdraid + packages: + - mdraid diff --git a/releasenotes/notes/overcloud-dib-938eac005662a2cb.yaml b/releasenotes/notes/overcloud-dib-938eac005662a2cb.yaml new file mode 100644 index 000000000..c8178302a --- /dev/null +++ b/releasenotes/notes/overcloud-dib-938eac005662a2cb.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Adds a standard overcloud Diskimage Builder (DIB) host image configuration. From 4102cdeac08926cc7392f83884ae3f12f92bbc81 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 28 Jul 2022 15:47:42 +0000 Subject: [PATCH 02/36] Add StackHPC LVM configuration --- .../inventory/group_vars/all/stackhpc-lvm | 150 ++++++++++++++++++ releasenotes/notes/lvm-3c785770722f768a.yaml | 5 + 2 files changed, 155 insertions(+) create mode 100644 etc/kayobe/inventory/group_vars/all/stackhpc-lvm create mode 100644 releasenotes/notes/lvm-3c785770722f768a.yaml diff --git a/etc/kayobe/inventory/group_vars/all/stackhpc-lvm b/etc/kayobe/inventory/group_vars/all/stackhpc-lvm new file mode 100644 index 000000000..8d710ba0d --- /dev/null +++ b/etc/kayobe/inventory/group_vars/all/stackhpc-lvm @@ -0,0 +1,150 @@ +--- +############################################################################### +# StackHPC LVM Volume Group (VG) configuration. + +# StackHPC rootvg VG. +# This VG contains a set of LVs that conform to the CIS partition benchmarks. +# This layout is compatible with the overcloud DIB host image configuration in +# stackhpc-overcloud-dib.yml. This may be used as an item in one of the +# *_lvm_groups variables. +# Example: +# controller_lvm_groups: +# - "{{ stackhpc_lvm_group_rootvg }}" +stackhpc_lvm_group_rootvg: + vgname: rootvg + disks: "{{ stackhpc_lvm_group_rootvg_disks }}" + create: true + lvnames: "{{ stackhpc_lvm_group_rootvg_lvs_default + stackhpc_lvm_group_rootvg_lvs_extra }}" + +# List of disks to include in the rootvg VG. +stackhpc_lvm_group_rootvg_disks: + - /dev/disk/by-partlabel/root + +# List of default LVs to include in the rootvg VG. +stackhpc_lvm_group_rootvg_lvs_default: + - "{{ stackhpc_lvm_lv_swap }}" + - "{{ stackhpc_lvm_lv_root }}" + - "{{ stackhpc_lvm_lv_tmp }}" + - "{{ stackhpc_lvm_lv_var }}" + - "{{ stackhpc_lvm_lv_var_tmp }}" + - "{{ stackhpc_lvm_lv_log }}" + - "{{ stackhpc_lvm_lv_audit }}" + - "{{ stackhpc_lvm_lv_home }}" + +# List of extra LVs to include in the rootvg VG. +stackhpc_lvm_group_rootvg_lvs_extra: [] + +############################################################################### +# StackHPC LVM Logical Volume (LV) configuration. + +# StackHPC LVM lv_swap LV size. +stackhpc_lvm_lv_swap_size: 16g + +# StackHPC LVM lv_root LV size. +stackhpc_lvm_lv_root_size: 50g + +# StackHPC LVM lv_tmp LV size. +stackhpc_lvm_lv_tmp_size: 10g + +# StackHPC LVM lv_var LV size. +stackhpc_lvm_lv_var_size: 20g + +# StackHPC LVM lv_var_tmp LV size. +stackhpc_lvm_lv_var_tmp_size: 2g + +# StackHPC LVM lv_log LV size. +stackhpc_lvm_lv_log_size: 20g + +# StackHPC LVM lv_audit LV size. +stackhpc_lvm_lv_audit_size: 10g + +# StackHPC LVM lv_home LV size. +stackhpc_lvm_lv_home_size: 10g + +# StackHPC LVM lv_docker LV size. +stackhpc_lvm_lv_docker_size: 100%FREE + +# StackHPC LVM lv_swap LV. +stackhpc_lvm_lv_swap: + lvname: lv_swap + size: "{{ stackhpc_lvm_lv_swap_size }}" + create: true + mount: false + +# StackHPC LVM lv_root LV. +stackhpc_lvm_lv_root: + lvname: lv_root + size: "{{ stackhpc_lvm_lv_root_size }}" + create: true + filesystem: ext4 + mount: true + mntp: / + +# StackHPC LVM lv_tmp LV. +stackhpc_lvm_lv_tmp: + lvname: lv_tmp + size: "{{ stackhpc_lvm_lv_tmp_size }}" + create: true + filesystem: ext4 + mount: true + mntp: /tmp + +# StackHPC LVM lv_var LV. +stackhpc_lvm_lv_var: + lvname: lv_var + size: "{{ stackhpc_lvm_lv_var_size }}" + create: true + filesystem: ext4 + mount: true + mntp: /var + +# StackHPC LVM lv_var_tmp LV. +stackhpc_lvm_lv_var_tmp: + lvname: lv_var_tmp + size: "{{ stackhpc_lvm_lv_var_tmp_size }}" + create: true + filesystem: ext4 + mount: true + mntp: /var/tmp + +# StackHPC LVM lv_log LV. +stackhpc_lvm_lv_log: + lvname: lv_log + size: "{{ stackhpc_lvm_lv_log_size }}" + create: true + filesystem: ext4 + mount: true + mntp: /var/log + +# StackHPC LVM lv_audit LV. +stackhpc_lvm_lv_audit: + lvname: lv_audit + size: "{{ stackhpc_lvm_lv_audit_size }}" + create: true + filesystem: ext4 + mount: true + mntp: /var/log/audit + +# StackHPC LVM lv_home LV. +stackhpc_lvm_lv_home: + lvname: lv_home + size: "{{ stackhpc_lvm_lv_home_size }}" + create: true + filesystem: ext4 + mount: true + mntp: /home + +# StackHPC LVM lv_docker LV. +# NOTE: This is not included in the rootvg by default, but may be added via +# stackhpc_lvm_group_rootvg_lvs_extra. +# Example: +# stackhpc_lvm_group_rootvg_lvs_extra: +# - "{{ stackhpc_lvm_lv_docker }}" +# Alternatively, this may reside in a separate VG. +stackhpc_lvm_lv_docker: + lvname: lv_docker + size: "{{ stackhpc_lvm_lv_docker_size }}" + create: true + filesystem: ext4 + mount: true + mntp: /var/lib/docker diff --git a/releasenotes/notes/lvm-3c785770722f768a.yaml b/releasenotes/notes/lvm-3c785770722f768a.yaml new file mode 100644 index 000000000..5818c3e23 --- /dev/null +++ b/releasenotes/notes/lvm-3c785770722f768a.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Adds a standard LVM configuration that is compatible with the new overcloud + host image. From f093d1f7a79d3a0afd47eb0819545956a13a9347 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 22 Jul 2022 11:22:36 +0100 Subject: [PATCH 03/36] Add swap.yml custom playbook Creates a swap filesystem on the lv_swap LV in the rootvg VG. May be added as a post hook to overcloud host configure. --- etc/kayobe/ansible/swap.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 etc/kayobe/ansible/swap.yml diff --git a/etc/kayobe/ansible/swap.yml b/etc/kayobe/ansible/swap.yml new file mode 100644 index 000000000..e47397212 --- /dev/null +++ b/etc/kayobe/ansible/swap.yml @@ -0,0 +1,35 @@ +--- +# Custom playbook to configure a swap device. This may be used as a +# post-overcloud host configure hook. +# +# Variables: +# * swap_group: Host pattern against which to target the playbook. Default is +# 'overcloud'. +# * swap_device: Name of the swap device to configure. Default is +# '/dev/rootvg/lv_swap'. + +- name: Configure swap + hosts: "{{ swap_group | default('overcloud') }}" + tags: + - swap + vars: + swap_device: "/dev/rootvg/lv_swap" + become: true + tasks: + - name: Ensure swap filesystem is present + filesystem: + fstype: "swap" + dev: "{{ swap_device }}" + + - name: Ensure swap device present in fstab + mount: + name: "none" + src: "{{ swap_device }}" + fstype: "swap" + state: "present" + + # It does no harm to run this when swap is already active + - name: Enable swap devices + command: "/sbin/swapon -a" + when: + - ansible_facts.swaptotal_mb == 0 From 5ef8de0e558a45583c1691a8322f4dbb0b76e4b8 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 27 Jul 2022 15:05:48 +0000 Subject: [PATCH 04/36] Add growroot.yml custom playbook Grows the root partition & rootpv LVM PV to fill the available space. This playbook may be added as a pre-hook for overcloud host configure. --- etc/kayobe/ansible/growroot.yml | 62 +++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 etc/kayobe/ansible/growroot.yml diff --git a/etc/kayobe/ansible/growroot.yml b/etc/kayobe/ansible/growroot.yml new file mode 100644 index 000000000..62c7eed1f --- /dev/null +++ b/etc/kayobe/ansible/growroot.yml @@ -0,0 +1,62 @@ +--- +# Custom playbook to grow the partition and LVM PV of the root VG. This allows +# for expansion of LVs in that VG. This may be used as a pre-overcloud host +# configure hook. +# +# Variables: +# * growroot_group: Host pattern against which to target the playbook. Default +# is 'overcloud'. +# * growroot_vg: Name of the VG containing the PV to grow. Default is 'rootvg'. + +- name: Grow root PV + hosts: "{{ growroot_group | default('overcloud') }}" + # Avoid using facts because this may be used as a pre overcloud host + # configure hook, and we don't want to populate the fact cache (if one is in + # use) with the bootstrap user's context. + gather_facts: false + tags: + - growroot + vars: + ansible_user: "{{ bootstrap_user }}" + # We can't assume that a virtualenv exists at this point, so use the system + # python interpreter. + ansible_python_interpreter: /usr/bin/python3 + # Work around no known_hosts entry on first boot. + ansible_ssh_common_args: "-o StrictHostKeyChecking=no" + # Name of the LVM VG containing the root PV. + growroot_vg: "rootvg" + + tasks: + - name: Ensure growpart is installed + package: + name: "{% if os_distribution in ['centos', 'rocky'] %}cloud-utils-growpart{% else %}cloud-guest-utils{% endif %}" + state: present + cache_valid_time: "{{ apt_cache_valid_time if ansible_facts.os_family == 'Debian' else omit }}" + update_cache: "{{ True if ansible_facts.os_family == 'Debian' else omit }}" + become: True + + - name: Get root PV device + command: "pvs --select vg_name={{ growroot_vg }} --reportformat json" + register: pvs + become: True + changed_when: False + + - name: Grow partition + command: "growpart {{ disk }} {{ part_num }}" + vars: + pv: "{{ pvs.stdout | from_json }}" + disk_tmp: "{{ pv.report[0].pv[0].pv_name[:-1] }}" + disk: "{{ disk_tmp[:-1] if disk_tmp[-1] == 'p' else disk_tmp }}" + part_num: "{{ pv.report[0].pv[0].pv_name[-1] }}" + become: True + failed_when: "growpart.rc != 0 and 'NOCHANGE' not in growpart.stdout" + changed_when: "'NOCHANGE' not in growpart.stdout" + register: growpart + + - name: Grow LVM PV + command: "pvresize {{ disk }}" + vars: + pv: "{{ pvs.stdout | from_json }}" + disk: "{{ pv.report[0].pv[0].pv_name }}" + become: True + when: "'NOCHANGE' not in growpart.stdout" From 51148dbcf2fcfef1e1c5008a11175aa1fbf4b4b9 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 7 Jun 2022 10:31:15 +0100 Subject: [PATCH 05/36] docker: enable live restore --- etc/kayobe/docker.yml | 2 +- releasenotes/notes/docker-live-restore-6b6c55455202422c.yaml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/docker-live-restore-6b6c55455202422c.yaml diff --git a/etc/kayobe/docker.yml b/etc/kayobe/docker.yml index fcd7c8eb0..c4ac21c3b 100644 --- a/etc/kayobe/docker.yml +++ b/etc/kayobe/docker.yml @@ -31,7 +31,7 @@ docker_registry: "{{ stackhpc_docker_registry }}" #docker_registry_mirrors: # Enable live-restore on docker daemon -#docker_daemon_live_restore: +docker_daemon_live_restore: true ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/releasenotes/notes/docker-live-restore-6b6c55455202422c.yaml b/releasenotes/notes/docker-live-restore-6b6c55455202422c.yaml new file mode 100644 index 000000000..8c516d3d0 --- /dev/null +++ b/releasenotes/notes/docker-live-restore-6b6c55455202422c.yaml @@ -0,0 +1,5 @@ +--- +upgrade: + - | + Enables Docker live restore by default. This may be disabled by setting + ``docker_daemon_live_restore`` to ``false`` in ``docker.yml``. From e913d76110f4a23c7150a66dffb925e4e2ae9478 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 7 Jun 2022 15:01:57 +0100 Subject: [PATCH 06/36] Add basic Ansible tuning * Set forks to 20 * YAML-formatted output * Disable fact variable injection * Enable SSH pipelining --- etc/kayobe/ansible.cfg | 11 +++++++++++ .../notes/ansible-tuning-142bbcf4f646e6e0.yaml | 6 ++++++ 2 files changed, 17 insertions(+) create mode 100644 etc/kayobe/ansible.cfg create mode 100644 releasenotes/notes/ansible-tuning-142bbcf4f646e6e0.yaml diff --git a/etc/kayobe/ansible.cfg b/etc/kayobe/ansible.cfg new file mode 100644 index 000000000..696ed923a --- /dev/null +++ b/etc/kayobe/ansible.cfg @@ -0,0 +1,11 @@ +[defaults] +forks = 20 +# Use the YAML stdout callback plugin. +stdout_callback = yaml +# Use the stdout_callback when running ad-hoc commands. +bin_ansible_callbacks = True +# Disable fact variable injection to improve performance. +inject_facts_as_vars = False + +[ssh_connection] +pipelining = True diff --git a/releasenotes/notes/ansible-tuning-142bbcf4f646e6e0.yaml b/releasenotes/notes/ansible-tuning-142bbcf4f646e6e0.yaml new file mode 100644 index 000000000..13e4144e0 --- /dev/null +++ b/releasenotes/notes/ansible-tuning-142bbcf4f646e6e0.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Adds some basic tuning of Ansible, including use of 20 forks, enabling + SSH pipelining, YAML-formatted output, and disabling fact variable + injection. From d80d58ad59c1dc50eb11e429354be70ff436d4d6 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 8 Aug 2022 11:28:48 +0100 Subject: [PATCH 07/36] CI: Allow building Ubuntu Kolla container images --- .github/workflows/container-image-build.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/container-image-build.yml b/.github/workflows/container-image-build.yml index d04e1e001..306aa6f42 100644 --- a/.github/workflows/container-image-build.yml +++ b/.github/workflows/container-image-build.yml @@ -18,6 +18,14 @@ on: type: boolean required: false default: false + distro: + description: Container image OS distribution + type: choice + required: false + default: centos + options: + - centos + - ubuntu env: ANSIBLE_FORCE_COLOR: True @@ -85,7 +93,7 @@ jobs: run: | source venvs/kayobe/bin/activate && source src/kayobe-config/kayobe-env --environment ci-builder && - kayobe overcloud container image build ${{ github.event.inputs.regexes }} --push + kayobe overcloud container image build ${{ github.event.inputs.regexes }} --push -e kolla_base_distro=${{ inputs.distro }} env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: github.event.inputs.overcloud == 'true' @@ -112,7 +120,7 @@ jobs: run: | source venvs/kayobe/bin/activate && source src/kayobe-config/kayobe-env --environment ci-builder && - kayobe seed container image build --push + kayobe seed container image build --push -e kolla_base_distro=${{ inputs.distro }} env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: github.event.inputs.seed == 'true' From b3bb3a0545064f70c3e1a39056eb01ad8fc76d82 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 10 Aug 2022 11:32:29 +0100 Subject: [PATCH 08/36] CI: Pull Kayobe container image to ensure latest --- .github/workflows/stackhpc-all-in-one.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml index 20980bcb1..4c0a62f45 100644 --- a/.github/workflows/stackhpc-all-in-one.yml +++ b/.github/workflows/stackhpc-all-in-one.yml @@ -167,6 +167,11 @@ jobs: echo "::add-mask::$ssh_key" echo "::set-output name=ssh_key::$ssh_key" + # The same tag may be reused (e.g. pr-123), so ensure we have the latest image. + - name: Pull latest Kayobe image + run: | + sudo docker image pull $KAYOBE_IMAGE + - name: Host configure run: | sudo -E docker run -t --rm \ From 0d340cee835f48863e917d7d99b8e9dfc3ea24d9 Mon Sep 17 00:00:00 2001 From: Grzegorz Bialas Date: Wed, 10 Aug 2022 15:47:33 +0200 Subject: [PATCH 09/36] add skydive feature flag --- etc/kayobe/environments/ci-builder/stackhpc-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml index e61f78761..26d767d03 100644 --- a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml @@ -23,6 +23,7 @@ kolla_enable_octavia: true kolla_enable_ovn: true kolla_enable_prometheus: true kolla_enable_redis: true +kolla_enable_skydive: true ############################################################################### # Network configuration. From f3addef60cdcb42306e6e9467c0dbb898ba025a0 Mon Sep 17 00:00:00 2001 From: stackhpc-ci <22933334+stackhpc-ci@users.noreply.github.com> Date: Wed, 10 Aug 2022 15:03:29 +0000 Subject: [PATCH 10/36] feat: automatic update of workflows stackhpc/wallaby --- .github/workflows/tag-and-release.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/workflows/tag-and-release.yml diff --git a/.github/workflows/tag-and-release.yml b/.github/workflows/tag-and-release.yml new file mode 100644 index 000000000..154dbd51b --- /dev/null +++ b/.github/workflows/tag-and-release.yml @@ -0,0 +1,11 @@ +--- +name: Tag & Release +'on': + push: + branches: + - stackhpc/wallaby +permissions: + contents: write +jobs: + tag-and-release: + uses: stackhpc/.github/.github/workflows/tag-and-release.yml@main From c87747beca75ac1d83db657b5c3bb2f95cd84676 Mon Sep 17 00:00:00 2001 From: stackhpc-ci <22933334+stackhpc-ci@users.noreply.github.com> Date: Wed, 10 Aug 2022 15:04:30 +0000 Subject: [PATCH 11/36] feat: automatic update of community files stackhpc/wallaby --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..e9a948a0d --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @stackhpc/kayobe From 0dba1c6c4df333290b7079a2dfb1263594546dfd Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 10 Aug 2022 16:12:54 +0100 Subject: [PATCH 12/36] Update kayobe-automation to fix rsync issue --- .automation | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.automation b/.automation index d7cab138e..a5749321e 160000 --- a/.automation +++ b/.automation @@ -1 +1 @@ -Subproject commit d7cab138e827011b2574cd420d29c06fbd8dc56b +Subproject commit a5749321ebcaa9798f1d63d7dc15f1380540f0a3 From 6b62dab27f29d0a1d78b69eb73d24d77ea2ffab5 Mon Sep 17 00:00:00 2001 From: stackhpc-ci <22933334+stackhpc-ci@users.noreply.github.com> Date: Wed, 10 Aug 2022 15:52:47 +0000 Subject: [PATCH 13/36] feat: automatic update of workflows stackhpc/wallaby --- .github/workflows/tag-and-release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tag-and-release.yml b/.github/workflows/tag-and-release.yml index 154dbd51b..100525452 100644 --- a/.github/workflows/tag-and-release.yml +++ b/.github/workflows/tag-and-release.yml @@ -9,3 +9,5 @@ permissions: jobs: tag-and-release: uses: stackhpc/.github/.github/workflows/tag-and-release.yml@main + with: + upstream: https://github.com/openstack/kayobe-config From f3f95f1b3a2c51217090c2ec08df835bf503d936 Mon Sep 17 00:00:00 2001 From: Grzegorz Bialas Date: Thu, 11 Aug 2022 12:48:56 +0200 Subject: [PATCH 14/36] add skydive tag --- etc/kayobe/kolla/globals.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index a0c899f72..8a20a5145 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -27,4 +27,5 @@ prometheus_jiralert_tag: wallaby-20220119T122428 prometheus_libvirt_exporter_tag: wallaby-20220325T122042 prometheus_msteams_tag: wallaby-20220119T122428 prometheus_openstack_exporter_tag: wallaby-20220705T132206 +skydive_tag: wallaby-20220811T091848 {% endif %} From 7fc7e5b9a11c097b8eda866ad4504f18bee1ec86 Mon Sep 17 00:00:00 2001 From: Grzegorz Bialas Date: Thu, 11 Aug 2022 15:17:50 +0200 Subject: [PATCH 15/36] add skydive to pulled images --- etc/kayobe/pulp.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index 87b5e42cb..28939da9e 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -428,6 +428,9 @@ stackhpc_pulp_images: - redis - redis-base - redis-sentinel + - skydive-agent + - skydive-analyzer + - skydive-base # Common parameters for container image repositories. stackhpc_pulp_repository_container_repos_common: From e2e1650d7cc025ff57d3737c33956ad5bc38681c Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 12 Aug 2022 11:50:20 +0200 Subject: [PATCH 16/36] Remove sahara-dashboard fork from sources The fix on stackhpc/wallaby has been merged upstream. --- etc/kayobe/kolla.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 2ad38bb15..d2c037353 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -122,10 +122,6 @@ kolla_sources: type: git location: https://github.com/stackhpc/cloudkitty-dashboard.git reference: stackhpc/wallaby - horizon-plugin-sahara-dashboard: - type: git - location: https://github.com/stackhpc/sahara-dashboard.git - reference: stackhpc/wallaby ironic-inspector-additions-stackhpc-inspector-plugins: # Install our custom inspector plugins. type: git From d49cafa17050d12d3bf4c39dc0d2a006b481f861 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 17 Aug 2022 10:26:55 +0100 Subject: [PATCH 17/36] README: fix ci-aio and ci-builder setup instructions --- README.rst | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index c37c413ab..95476af75 100644 --- a/README.rst +++ b/README.rst @@ -238,6 +238,13 @@ Setup Access the host via SSH. +Install package dependencies. On Ubuntu: + +.. code-block:: console + + sudo apt update + sudo apt install python3-virtualenv + Clone the Kayobe and Kayobe configuration repositories (this one): .. code-block:: console @@ -246,7 +253,7 @@ Clone the Kayobe and Kayobe configuration repositories (this one): mkdir -p src pushd src git clone https://github.com/stackhpc/kayobe.git -b stackhpc/wallaby - git clone https://github.com/stackhpc/stackhpc-kayobe-config -b stackhpc/wallaby + git clone https://github.com/stackhpc/stackhpc-kayobe-config -b stackhpc/wallaby kayobe-config popd Create a virtual environment and install Kayobe: @@ -337,6 +344,13 @@ Setup Access the host via SSH. +Install package dependencies. On Ubuntu: + +.. code-block:: console + + sudo apt update + sudo apt install python3-virtualenv + Clone the Kayobe and Kayobe configuration repositories (this one): .. code-block:: console @@ -345,7 +359,7 @@ Clone the Kayobe and Kayobe configuration repositories (this one): mkdir -p src pushd src git clone https://github.com/stackhpc/kayobe.git -b stackhpc/wallaby - git clone https://github.com/stackhpc/stackhpc-kayobe-config -b stackhpc/wallaby + git clone https://github.com/stackhpc/stackhpc-kayobe-config -b stackhpc/wallaby kayobe-config popd Create a virtual environment and install Kayobe: From e1ce19ff6e2832358aff309bb17b6f957c19ccf0 Mon Sep 17 00:00:00 2001 From: Bartosz Bezak Date: Thu, 18 Aug 2022 12:56:22 +0200 Subject: [PATCH 18/36] add name to pulp container --- etc/kayobe/seed.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml index 3de3e3872..3dcfd138d 100644 --- a/etc/kayobe/seed.yml +++ b/etc/kayobe/seed.yml @@ -93,6 +93,7 @@ seed_pulp_container_enabled: true seed_pulp_container: pulp: + name: pulp image: pulp/pulp pre: "{{ kayobe_config_path }}/containers/pulp/pre.yml" post: "{{ kayobe_config_path }}/containers/pulp/post.yml" From b01b261e494db9be9d3ffaac227ab87298b688ea Mon Sep 17 00:00:00 2001 From: Bartosz Bezak Date: Mon, 22 Aug 2022 11:15:46 +0200 Subject: [PATCH 19/36] fix pulp post.yml instead of adding name to seed.yml --- etc/kayobe/containers/pulp/post.yml | 2 +- etc/kayobe/seed.yml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/etc/kayobe/containers/pulp/post.yml b/etc/kayobe/containers/pulp/post.yml index 0548b9926..fdb5506cc 100644 --- a/etc/kayobe/containers/pulp/post.yml +++ b/etc/kayobe/containers/pulp/post.yml @@ -10,7 +10,7 @@ - name: Set the Pulp admin password become: true command: >- - docker exec -u root {{ seed_containers.pulp.name }} + docker exec -u root pulp bash -c 'pulpcore-manager reset-admin-password -p {{ pulp_password }}' no_log: true diff --git a/etc/kayobe/seed.yml b/etc/kayobe/seed.yml index 3dcfd138d..3de3e3872 100644 --- a/etc/kayobe/seed.yml +++ b/etc/kayobe/seed.yml @@ -93,7 +93,6 @@ seed_pulp_container_enabled: true seed_pulp_container: pulp: - name: pulp image: pulp/pulp pre: "{{ kayobe_config_path }}/containers/pulp/pre.yml" post: "{{ kayobe_config_path }}/containers/pulp/post.yml" From 015c26ac09c37d62a9fbe3931474ac8046dff247 Mon Sep 17 00:00:00 2001 From: Grzegorz Bialas Date: Mon, 22 Aug 2022 12:49:41 +0200 Subject: [PATCH 20/36] add new skydive tags --- etc/kayobe/kolla/globals.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 8a20a5145..4dc615d49 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -27,5 +27,6 @@ prometheus_jiralert_tag: wallaby-20220119T122428 prometheus_libvirt_exporter_tag: wallaby-20220325T122042 prometheus_msteams_tag: wallaby-20220119T122428 prometheus_openstack_exporter_tag: wallaby-20220705T132206 -skydive_tag: wallaby-20220811T091848 +skydive_agent_tag: wallaby-20220817T151053 +skydive_analyzer_tag: wallaby-20220817T151053 {% endif %} From a14d4e4a35a68c61636feb313a50384a84d97bbb Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 19 Aug 2022 11:01:27 +0100 Subject: [PATCH 21/36] Revert "Ubuntu: add host Apt repository configuration" The Ubuntu mirrors in Ark aren't ready yet. This reverts commit 31bc7455823f788f1becc5e063914802d68ebbb5. --- etc/kayobe/apt.yml | 26 ++++---------------------- etc/kayobe/kolla/globals.yml | 2 -- 2 files changed, 4 insertions(+), 24 deletions(-) diff --git a/etc/kayobe/apt.yml b/etc/kayobe/apt.yml index 627951c17..0d5dd6bd8 100644 --- a/etc/kayobe/apt.yml +++ b/etc/kayobe/apt.yml @@ -17,21 +17,13 @@ # * filename: name of a file in /etc/apt/apt.conf.d/ in which to write the # configuration # Default is an empty list. -apt_config: - # NOTE: Currently the Pulp verbatim publisher does not sync translation - # files, which results in apt update failing. Disable translations until this - # is resolved. - - content: | - Acquire::Languages "none"; - filename: 99no-languages +#apt_config: # List of apt keys. Each item is a dict containing the following keys: # * url: URL of key # * filename: Name of a file in which to store the downloaded key # Default is an empty list. -apt_keys: - - url: "https://download.docker.com/linux/ubuntu/gpg" - filename: docker.asc +#apt_keys: # A list of Apt repositories. # * types: whitespace-separated list of repository types, e.g. deb or deb-src @@ -46,22 +38,12 @@ apt_keys: # * architecture: whitespace-separated list of architectures that will be used # (optional, default is unset) # Default is an empty list. -apt_repositories: - - url: "{{ stackhpc_repo_ubuntu_focal_url }}" - suites: "{{ ansible_facts.distribution_release }} {{ ansible_facts.distribution_release }}-updates {{ ansible_facts.distribution_release }}-backports" - components: main restricted universe multiverse - - url: "{{ stackhpc_repo_ubuntu_focal_security_url }}" - suites: "{{ ansible_facts.distribution_release }}-security" - components: main restricted universe multiverse - - url: "{{ stackhpc_repo_docker_ce_ubuntu_url }}" - suites: "{{ ansible_facts.distribution_release }}" - components: stable - signed_by: docker.asc +#apt_repositories: # Whether to disable repositories in /etc/apt/sources.list. This may be used # when replacing the distribution repositories via apt_repositories. # Default is false. -apt_disable_sources_list: true +#apt_disable_sources_list: ############################################################################### # Dummy variable to allow Ansible to accept this file. diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 8a20a5145..9901ec267 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -1,7 +1,5 @@ # yamllint disable-file --- -enable_docker_repo: "{% raw %}{{ ansible_facts.os_family == 'RedHat' }}{% endraw %}" - docker_yum_baseurl: "{{ stackhpc_repo_docker_url }}" docker_yum_gpgkey: "https://download.docker.com/linux/{% raw %}{{ ansible_facts.distribution | lower }}{% endraw %}/gpg" From 356da17ba66899c2c5f17b995986e4d774a90ee3 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 19 Aug 2022 11:09:23 +0100 Subject: [PATCH 22/36] Ubuntu: stop syncing Ark Ubuntu mirrors to local Pulp The Ubuntu mirrors in Ark aren't ready yet. --- etc/kayobe/pulp.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index 28939da9e..9c0545c39 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -32,7 +32,10 @@ stackhpc_release_pulp_content_url: "{{ stackhpc_release_pulp_url }}/pulp/content # Debs # Whether to sync Ubuntu packages. -stackhpc_pulp_sync_ubuntu_focal: "{{ os_distribution == 'ubuntu' }}" +# NOTE: Not currently syncing Ubuntu packages, since the on_demand mirror in +# Ark does not work if the upstream mirror pulls packages (which it does +# sometimes). +stackhpc_pulp_sync_ubuntu_focal: false stackhpc_pulp_repository_deb_repos: # Base Ubuntu Focal repositories From 1c7f77b48aa7dae07a4c7626664197b5a0ec2019 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 19 Aug 2022 11:27:39 +0100 Subject: [PATCH 23/36] Ubuntu: stop using Ark repo mirrors in container images The Ubuntu mirrors in Ark aren't ready yet. --- etc/kayobe/kolla.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index d2c037353..75f58a793 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -211,13 +211,16 @@ kolla_build_blocks: -e 's/^[# ]*\(baseurl *=.*\)/#\1/g' \ -e '/#baseurl.*/a baseurl={{ repo.url }}' /etc/yum.repos.d/{{ repo.file }}{% if not loop.last %} &&{% endif %} \ {% endfor %} - base_ubuntu_package_sources_list: | - RUN \ - rm -f /etc/apt/sources.list && \ - {% for repo in stackhpc_ubuntu_focal_repos %} - echo '{{ repo }}' >> /etc/apt/sources.list {% if not loop.last %} && \ - {% endif %} - {% endfor %} + # NOTE: Not currently syncing Ubuntu packages, since the on_demand mirror in + # Ark does not work if the upstream mirror pulls packages (which it does + # sometimes). + # base_ubuntu_package_sources_list: | + # RUN \ + # rm -f /etc/apt/sources.list && \ + # {% for repo in stackhpc_ubuntu_focal_repos %} + # echo '{{ repo }}' >> /etc/apt/sources.list {% if not loop.last %} && \ + # {% endif %} + # {% endfor %} grafana_plugins_install: | RUN grafana-cli plugins install vonage-status-panel ironic_inspector_header: | From e7ea0fa09d050853f7d97b2aef650e6d0ef8c371 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 23 Aug 2022 13:52:30 +0100 Subject: [PATCH 24/36] Ubuntu: use newly built Kolla images --- etc/kayobe/kolla.yml | 2 +- etc/kayobe/kolla/globals.yml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 75f58a793..2382158e4 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -91,7 +91,7 @@ kolla_docker_registry_password: "{{ stackhpc_docker_registry_password }}" # Kolla OpenStack release version. This should be a Docker image tag. # Default is {{ openstack_release }}. -kolla_openstack_release: wallaby-20211207T102718 +kolla_openstack_release: "{% if kolla_base_distro == 'centos' %}wallaby-20211207T102718{% else %}wallaby-20220819T112725{% endif %}" # Docker tag applied to built container images. Default is # {{ kolla_openstack_release }}. diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 9901ec267..3b9e46de4 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -3,7 +3,6 @@ docker_yum_baseurl: "{{ stackhpc_repo_docker_url }}" docker_yum_gpgkey: "https://download.docker.com/linux/{% raw %}{{ ansible_facts.distribution | lower }}{% endraw %}/gpg" -{# We are not currently providing container images for Ubuntu in Wallaby. #} {% if kolla_base_distro == 'centos' %} barbican_tag: wallaby-20220311T133847 cinder_tag: wallaby-20220309T083940 From 817bea96fc73097ec141a98ddc7c5da1ae39a1d4 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 24 Aug 2022 09:18:46 +0100 Subject: [PATCH 25/36] growpart.yml: Don't assume facts are present --- etc/kayobe/ansible/growroot.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/etc/kayobe/ansible/growroot.yml b/etc/kayobe/ansible/growroot.yml index 62c7eed1f..dd73e168a 100644 --- a/etc/kayobe/ansible/growroot.yml +++ b/etc/kayobe/ansible/growroot.yml @@ -25,14 +25,16 @@ ansible_ssh_common_args: "-o StrictHostKeyChecking=no" # Name of the LVM VG containing the root PV. growroot_vg: "rootvg" + # Don't assume facts are present. + os_family: "{{ ansible_facts.os_family | default('Debian' if os_distribution == 'ubuntu' else 'RedHat') }}" tasks: - name: Ensure growpart is installed package: - name: "{% if os_distribution in ['centos', 'rocky'] %}cloud-utils-growpart{% else %}cloud-guest-utils{% endif %}" + name: "{% if os_family == 'RedHat' %}cloud-utils-growpart{% else %}cloud-guest-utils{% endif %}" state: present - cache_valid_time: "{{ apt_cache_valid_time if ansible_facts.os_family == 'Debian' else omit }}" - update_cache: "{{ True if ansible_facts.os_family == 'Debian' else omit }}" + cache_valid_time: "{{ apt_cache_valid_time if os_family == 'Debian' else omit }}" + update_cache: "{{ True if os_family == 'Debian' else omit }}" become: True - name: Get root PV device From 04f1582dc16d1f0b96f1e69b2db3572da67222f5 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 24 Aug 2022 09:34:51 +0100 Subject: [PATCH 26/36] Overcloud DIB: stop using StackHPC package repos for Ubuntu image We are not currently syncing Ubuntu packages, since the on_demand mirror in Ark does not work if the upstream mirror pulls packages (which it does sometimes). --- etc/kayobe/stackhpc-overcloud-dib.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/stackhpc-overcloud-dib.yml b/etc/kayobe/stackhpc-overcloud-dib.yml index 0482cae0e..84fffbc0f 100644 --- a/etc/kayobe/stackhpc-overcloud-dib.yml +++ b/etc/kayobe/stackhpc-overcloud-dib.yml @@ -40,7 +40,10 @@ stackhpc_overcloud_dib_env_vars: DIB_CLOUD_INIT_DATASOURCES: "ConfigDrive" DIB_CONTAINERFILE_RUNTIME: "docker" DIB_CONTAINERFILE_NETWORK_DRIVER: "host" - DIB_DISTRIBUTION_MIRROR: "{{ stackhpc_repo_ubuntu_focal_url if os_distribution == 'ubuntu' else '' }}" + # NOTE: Not currently syncing Ubuntu packages, since the on_demand mirror in + # Ark does not work if the upstream mirror pulls packages (which it does + # sometimes). + # DIB_DISTRIBUTION_MIRROR: "{{ stackhpc_repo_ubuntu_focal_url if os_distribution == 'ubuntu' else '' }}" DIB_DRACUT_ENABLED_MODULES_DEFAULT_CONFIG: "{{ stackhpc_overcloud_dib_dracut_enabled_modules_default_config }}" DIB_RELEASE: "{{ overcloud_dib_os_release }}" DIB_SUDOERS_FILENAME: "no-fqdn" From b32e6f23b5b333eed3726460006264a3c196d45e Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 24 Aug 2022 10:07:38 +0100 Subject: [PATCH 27/36] Ubuntu: set Bifrost image tag --- etc/kayobe/kolla/globals.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index 3b9e46de4..2596006f3 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -25,4 +25,6 @@ prometheus_libvirt_exporter_tag: wallaby-20220325T122042 prometheus_msteams_tag: wallaby-20220119T122428 prometheus_openstack_exporter_tag: wallaby-20220705T132206 skydive_tag: wallaby-20220811T091848 +{% else %} +bifrost_tag: wallaby-20220819T150635 {% endif %} From 4bfb234077d294846091f600f1b5fc8ad00db847 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 24 Aug 2022 10:08:37 +0100 Subject: [PATCH 28/36] CI: add hook for seed container image build Without this, the seed tags artifact contains the overcloud images because ~/kolla_tag hasn't been updated. --- .../seed-container-image-build/post.d/50-write-kolla-tag.yml | 1 + 1 file changed, 1 insertion(+) create mode 120000 etc/kayobe/hooks/seed-container-image-build/post.d/50-write-kolla-tag.yml diff --git a/etc/kayobe/hooks/seed-container-image-build/post.d/50-write-kolla-tag.yml b/etc/kayobe/hooks/seed-container-image-build/post.d/50-write-kolla-tag.yml new file mode 120000 index 000000000..c685974a2 --- /dev/null +++ b/etc/kayobe/hooks/seed-container-image-build/post.d/50-write-kolla-tag.yml @@ -0,0 +1 @@ +../../../ansible/write-kolla-tag.yml \ No newline at end of file From 3931b6eebb58f15737d6735b11c17b39d0a471b7 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 24 Aug 2022 12:17:18 +0200 Subject: [PATCH 29/36] Support filtering Kolla container images to sync/publish It can take a long time to sync and publish all Kolla container images to the local Pulp service. Sometimes we know we just need to update a few images. This change adds support for filtering the image list, using regexes in a similar way to kolla-build. Example: kayobe playbook run ... -e stackhpc_pulp_images_kolla_filter='"^glance nova-compute$"' --- etc/kayobe/pulp.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/pulp.yml b/etc/kayobe/pulp.yml index e1a8e1526..bdc91053e 100644 --- a/etc/kayobe/pulp.yml +++ b/etc/kayobe/pulp.yml @@ -428,6 +428,15 @@ stackhpc_pulp_images_kolla: - redis-base - redis-sentinel +# Whitespace-separated list of regular expressions matching Kolla image names. +# Usage is similar to kolla-build CLI arguments. +# Example: +# kayobe playbook run ... -e stackhpc_pulp_images_kolla_filter='"^glance nova-compute$"' +stackhpc_pulp_images_kolla_filter: ".*" + +# Filtered list of Kolla container image names. +stackhpc_pulp_images_kolla_filtered: "{{ stackhpc_pulp_images_kolla | select('search', '(' ~ stackhpc_pulp_images_kolla_filter.split() | join('|') ~ ')') | list }}" + # Common parameters for container image repositories. stackhpc_pulp_repository_container_repos_kolla_common: url: "{{ stackhpc_release_pulp_registry_url }}" @@ -440,7 +449,7 @@ stackhpc_pulp_repository_container_repos_kolla_common: # List of Kolla container image repositories. stackhpc_pulp_repository_container_repos_kolla: >- {%- set repos = [] -%} - {%- for image in stackhpc_pulp_images_kolla -%} + {%- for image in stackhpc_pulp_images_kolla_filtered -%} {%- set image_repo = kolla_docker_namespace ~ "/" ~ kolla_base_distro ~ "-" ~ kolla_install_type ~ "-" ~ image -%} {%- set repo = {"name": image_repo} -%} {%- set _ = repos.append(stackhpc_pulp_repository_container_repos_kolla_common | combine(repo)) -%} @@ -455,7 +464,7 @@ stackhpc_pulp_distribution_container_kolla_common: # List of Kolla container image distributions. stackhpc_pulp_distribution_container_kolla: >- {%- set distributions = [] -%} - {%- for image in stackhpc_pulp_images_kolla -%} + {%- for image in stackhpc_pulp_images_kolla_filtered -%} {%- set image_repo = kolla_docker_namespace ~ "/" ~ kolla_base_distro ~ "-" ~ kolla_install_type ~ "-" ~ image -%} {%- set distribution = {"name": image_repo, "repository": image_repo, "base_path": image_repo} -%} {%- set _ = distributions.append(stackhpc_pulp_distribution_container_kolla_common | combine(distribution)) -%} From 0f61e4eaa170573d8144c66308616d15de13c6bb Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 25 Aug 2022 11:54:14 +0100 Subject: [PATCH 30/36] growroot.yml: Fail if the expected volume group doesn't exist --- etc/kayobe/ansible/growroot.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/etc/kayobe/ansible/growroot.yml b/etc/kayobe/ansible/growroot.yml index dd73e168a..40f3476fb 100644 --- a/etc/kayobe/ansible/growroot.yml +++ b/etc/kayobe/ansible/growroot.yml @@ -43,6 +43,12 @@ become: True changed_when: False + - name: Fail if root PV device not found + fail: + msg: > + Expected LVM physical volume devices not found in volume group {{ growroot_vg }} + when: (pvs.stdout | from_json).report[0].pv | length == 0 + - name: Grow partition command: "growpart {{ disk }} {{ part_num }}" vars: From b5347fc76dcf10b2a12732e9d08041cc677d7fe2 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 25 Aug 2022 10:17:34 +0100 Subject: [PATCH 31/36] bifrost: Add stackhpc-inspector-plugins to image --- etc/kayobe/kolla.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 2382158e4..2f19fb75d 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -114,6 +114,11 @@ kolla_sources: type: git location: "{{ kolla_bifrost_source_url }}" reference: "{{ kolla_bifrost_source_version }}" + bifrost-base-additions-stackhpc-inspector-plugins: + # Install our custom inspector plugins. + type: git + location: https://github.com/stackhpc/stackhpc-inspector-plugins.git + reference: 1.3.0 cloudkitty-base: type: git location: https://github.com/stackhpc/cloudkitty.git @@ -221,6 +226,8 @@ kolla_build_blocks: # echo '{{ repo }}' >> /etc/apt/sources.list {% if not loop.last %} && \ # {% endif %} # {% endfor %} + bifrost_base_header: | + ADD additions-archive / grafana_plugins_install: | RUN grafana-cli plugins install vonage-status-panel ironic_inspector_header: | @@ -244,6 +251,8 @@ kolla_build_blocks: ARG blackbox_exporter_version='0.19.0' kolla_build_customizations_common: + bifrost_base_pip_packages_append: + - /additions/* ironic_inspector_pip_packages_append: - /additions/* From f02e3bf8dd0bd79da2a6f69970cea1fa81c72bc8 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 25 Aug 2022 12:39:19 +0100 Subject: [PATCH 32/36] bifrost: update tags Includes stackhpc-inspector-plugins package. --- etc/kayobe/kolla/globals.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index db0992bf3..68c3951f7 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -5,6 +5,7 @@ docker_yum_gpgkey: "https://download.docker.com/linux/{% raw %}{{ ansible_facts. {% if kolla_base_distro == 'centos' %} barbican_tag: wallaby-20220311T133847 +bifrost_tag: wallaby-20220825T103439 cinder_tag: wallaby-20220309T083940 cloudkitty_tag: wallaby-20220119T122428 grafana_tag: wallaby-20220210T160332 @@ -27,5 +28,5 @@ prometheus_openstack_exporter_tag: wallaby-20220705T132206 skydive_agent_tag: wallaby-20220817T151053 skydive_analyzer_tag: wallaby-20220817T151053 {% else %} -bifrost_tag: wallaby-20220819T150635 +bifrost_tag: wallaby-20220825T112231 {% endif %} From 10b96b423f85fc73609e7e63f89b9d315788e529 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 26 Aug 2022 16:07:04 +0100 Subject: [PATCH 33/36] growroot: Avoid package installation when growpart is installed In a pre-bootstrapped environment which requires a proxy for package installation, we may not be able to perform package updates. --- etc/kayobe/ansible/growroot.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/etc/kayobe/ansible/growroot.yml b/etc/kayobe/ansible/growroot.yml index 40f3476fb..6bfe63e41 100644 --- a/etc/kayobe/ansible/growroot.yml +++ b/etc/kayobe/ansible/growroot.yml @@ -29,6 +29,14 @@ os_family: "{{ ansible_facts.os_family | default('Debian' if os_distribution == 'ubuntu' else 'RedHat') }}" tasks: + - name: Check if growpart is installed + shell: + cmd: type growpart + changed_when: false + failed_when: false + register: growpart_check + become: true + - name: Ensure growpart is installed package: name: "{% if os_family == 'RedHat' %}cloud-utils-growpart{% else %}cloud-guest-utils{% endif %}" @@ -36,6 +44,7 @@ cache_valid_time: "{{ apt_cache_valid_time if os_family == 'Debian' else omit }}" update_cache: "{{ True if os_family == 'Debian' else omit }}" become: True + when: growpart_check.rc != 0 - name: Get root PV device command: "pvs --select vg_name={{ growroot_vg }} --reportformat json" From 2de8b2de391175c88f7cc0faa67d781c2d0bdcff Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 26 Aug 2022 16:12:55 +0100 Subject: [PATCH 34/36] pulp: use pulp_url as advertised content origin This fixes an issue where a container image pull would fail if the seed's hostname is not resolvable from the host pulling the image. Typically it will not be. This hasn't always been an issue, and it's unclear what has changed. Perhaps something in the Pulp container registry code. --- etc/kayobe/containers/pulp/settings.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/etc/kayobe/containers/pulp/settings.py b/etc/kayobe/containers/pulp/settings.py index c30454845..7f5636dc1 100644 --- a/etc/kayobe/containers/pulp/settings.py +++ b/etc/kayobe/containers/pulp/settings.py @@ -1,4 +1,4 @@ -CONTENT_ORIGIN='http://{{ ansible_facts.fqdn }}' -ANSIBLE_API_HOSTNAME='http://{{ ansible_facts.fqdn }}' -ANSIBLE_CONTENT_HOSTNAME='http://{{ ansible_facts.fqdn }}/pulp/content' +CONTENT_ORIGIN='{{ pulp_url }}' +ANSIBLE_API_HOSTNAME='{{ pulp_url }}' +ANSIBLE_CONTENT_HOSTNAME='{{ pulp_url }}/pulp/content' TOKEN_AUTH_DISABLED=True From f43c3eea3a18106d3b81ae15ce7bfd967f038854 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 30 Aug 2022 13:02:06 +0100 Subject: [PATCH 35/36] README: Install python3-virtualenv for ci-aio & ci-builder environments --- README.rst | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 95476af75..304a8edbc 100644 --- a/README.rst +++ b/README.rst @@ -238,12 +238,20 @@ Setup Access the host via SSH. -Install package dependencies. On Ubuntu: +Install package dependencies. + +On CentOS: + +.. code-block:: console + + sudo dnf install -y python3-virtualenv + +On Ubuntu: .. code-block:: console sudo apt update - sudo apt install python3-virtualenv + sudo apt install -y python3-virtualenv Clone the Kayobe and Kayobe configuration repositories (this one): @@ -344,12 +352,20 @@ Setup Access the host via SSH. -Install package dependencies. On Ubuntu: +Install package dependencies. + +On CentOS: + +.. code-block:: console + + sudo dnf install -y python3-virtualenv + +On Ubuntu: .. code-block:: console sudo apt update - sudo apt install python3-virtualenv + sudo apt install -y python3-virtualenv Clone the Kayobe and Kayobe configuration repositories (this one): From e9373343c4f2af8f2b72b75cacdd14d9b64cc2e5 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 31 Aug 2022 15:05:05 +0100 Subject: [PATCH 36/36] Add ceph group as a child of the storage group --- etc/kayobe/inventory/groups | 3 +++ 1 file changed, 3 insertions(+) diff --git a/etc/kayobe/inventory/groups b/etc/kayobe/inventory/groups index e51195309..570779a93 100644 --- a/etc/kayobe/inventory/groups +++ b/etc/kayobe/inventory/groups @@ -101,6 +101,9 @@ hs-switches ############################################################################### # Ceph groups. +[storage:children] +ceph + [ceph:children] mons mgrs