yoga xena merge

.github/workflows/stackhpc-all-in-one.yml

@@ -167,6 +167,11 @@ jobs:
           echo "::add-mask::$ssh_key"
           echo "::set-output name=ssh_key::$ssh_key"
 
+      # The same tag may be reused (e.g. pr-123), so ensure we have the latest image.
+      - name: Pull latest Kayobe image
+        run: |
+          sudo docker image pull $KAYOBE_IMAGE
+
       - name: Host configure
         run: |
           sudo -E docker run -t --rm \

.github/workflows/stackhpc-container-image-build.yml

@@ -18,6 +18,14 @@ on:
         type: boolean
         required: false
         default: false
+      distro:
+        description: Container image OS distribution
+        type: choice
+        required: false
+        default: centos
+        options:
+          - centos
+          - ubuntu
 
 env:
   ANSIBLE_FORCE_COLOR: True
@@ -85,7 +93,7 @@ jobs:
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe overcloud container image build ${{ github.event.inputs.regexes }} --push
+          kayobe overcloud container image build ${{ github.event.inputs.regexes }} --push -e kolla_base_distro=${{ inputs.distro }}
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: github.event.inputs.overcloud == 'true'
@@ -112,7 +120,7 @@ jobs:
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe seed container image build --push
+          kayobe seed container image build --push -e kolla_base_distro=${{ inputs.distro }}
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: github.event.inputs.seed == 'true'

README.rst

@@ -238,6 +238,21 @@ Setup
 
 Access the host via SSH.
 
+Install package dependencies.
+
+On CentOS:
+
+.. code-block:: console
+
+   sudo dnf install -y python3-virtualenv
+
+On Ubuntu:
+
+.. code-block:: console
+
+   sudo apt update
+   sudo apt install -y python3-virtualenv
+
 Clone the Kayobe and Kayobe configuration repositories (this one):
 
 .. code-block:: console
@@ -246,7 +261,7 @@ Clone the Kayobe and Kayobe configuration repositories (this one):
    mkdir -p src
    pushd src
    git clone https://github.com/stackhpc/kayobe.git -b stackhpc/xena
-   git clone https://github.com/stackhpc/stackhpc-kayobe-config -b stackhpc/xena
+   git clone https://github.com/stackhpc/stackhpc-kayobe-config -b stackhpc/xena kayobe-config
    popd
 
 Create a virtual environment and install Kayobe:
@@ -337,6 +352,21 @@ Setup
 
 Access the host via SSH.
 
+Install package dependencies.
+
+On CentOS:
+
+.. code-block:: console
+
+   sudo dnf install -y python3-virtualenv
+
+On Ubuntu:
+
+.. code-block:: console
+
+   sudo apt update
+   sudo apt install -y python3-virtualenv
+
 Clone the Kayobe and Kayobe configuration repositories (this one):
 
 .. code-block:: console
@@ -345,7 +375,7 @@ Clone the Kayobe and Kayobe configuration repositories (this one):
    mkdir -p src
    pushd src
    git clone https://github.com/stackhpc/kayobe.git -b stackhpc/xena
-   git clone https://github.com/stackhpc/stackhpc-kayobe-config -b stackhpc/xena
+   git clone https://github.com/stackhpc/stackhpc-kayobe-config -b stackhpc/xena kayobe-config
    popd
 
 Create a virtual environment and install Kayobe:

etc/kayobe/ansible.cfg

@@ -0,0 +1,11 @@
+[defaults]
+forks = 20
+# Use the YAML stdout callback plugin.
+stdout_callback = yaml
+# Use the stdout_callback when running ad-hoc commands.
+bin_ansible_callbacks = True
+# Disable fact variable injection to improve performance.
+inject_facts_as_vars = False
+
+[ssh_connection]
+pipelining = True

etc/kayobe/ansible/growroot.yml

@@ -0,0 +1,79 @@
+---
+# Custom playbook to grow the partition and LVM PV of the root VG. This allows
+# for expansion of LVs in that VG. This may be used as a pre-overcloud host
+# configure hook.
+#
+# Variables:
+# * growroot_group: Host pattern against which to target the playbook. Default
+#                   is 'overcloud'.
+# * growroot_vg: Name of the VG containing the PV to grow. Default is 'rootvg'.
+
+- name: Grow root PV
+  hosts: "{{ growroot_group | default('overcloud') }}"
+  # Avoid using facts because this may be used as a pre overcloud host
+  # configure hook, and we don't want to populate the fact cache (if one is in
+  # use) with the bootstrap user's context.
+  gather_facts: false
+  tags:
+    - growroot
+  vars:
+    ansible_user: "{{ bootstrap_user }}"
+    # We can't assume that a virtualenv exists at this point, so use the system
+    # python interpreter.
+    ansible_python_interpreter: /usr/bin/python3
+    # Work around no known_hosts entry on first boot.
+    ansible_ssh_common_args: "-o StrictHostKeyChecking=no"
+    # Name of the LVM VG containing the root PV.
+    growroot_vg: "rootvg"
+    # Don't assume facts are present.
+    os_family: "{{ ansible_facts.os_family | default('Debian' if os_distribution == 'ubuntu' else 'RedHat') }}"
+
+  tasks:
+    - name: Check if growpart is installed
+      shell:
+        cmd: type growpart
+      changed_when: false
+      failed_when: false
+      register: growpart_check
+      become: true
+
+    - name: Ensure growpart is installed
+      package:
+        name: "{% if os_family == 'RedHat' %}cloud-utils-growpart{% else %}cloud-guest-utils{% endif %}"
+        state: present
+        cache_valid_time: "{{ apt_cache_valid_time if os_family == 'Debian' else omit }}"
+        update_cache: "{{ True if os_family == 'Debian' else omit }}"
+      become: True
+      when: growpart_check.rc != 0
+
+    - name: Get root PV device
+      command: "pvs --select vg_name={{ growroot_vg }} --reportformat json"
+      register: pvs
+      become: True
+      changed_when: False
+
+    - name: Fail if root PV device not found
+      fail:
+        msg: >
+          Expected LVM physical volume devices not found in volume group {{ growroot_vg }}
+      when: (pvs.stdout | from_json).report[0].pv | length == 0
+
+    - name: Grow partition
+      command: "growpart {{ disk }} {{ part_num }}"
+      vars:
+        pv: "{{ pvs.stdout | from_json }}"
+        disk_tmp: "{{ pv.report[0].pv[0].pv_name[:-1] }}"
+        disk: "{{ disk_tmp[:-1] if disk_tmp[-1] == 'p' else disk_tmp }}"
+        part_num: "{{ pv.report[0].pv[0].pv_name[-1] }}"
+      become: True
+      failed_when: "growpart.rc != 0 and 'NOCHANGE' not in growpart.stdout"
+      changed_when: "'NOCHANGE' not in growpart.stdout"
+      register: growpart
+
+    - name: Grow LVM PV
+      command: "pvresize {{ disk }}"
+      vars:
+        pv: "{{ pvs.stdout | from_json }}"
+        disk: "{{ pv.report[0].pv[0].pv_name }}"
+      become: True
+      when: "'NOCHANGE' not in growpart.stdout"

etc/kayobe/ansible/swap.yml

@@ -0,0 +1,35 @@
+---
+# Custom playbook to configure a swap device. This may be used as a
+# post-overcloud host configure hook.
+#
+# Variables:
+# * swap_group: Host pattern against which to target the playbook. Default is
+#               'overcloud'.
+# * swap_device: Name of the swap device to configure. Default is
+#                '/dev/rootvg/lv_swap'.
+
+- name: Configure swap
+  hosts: "{{ swap_group | default('overcloud') }}"
+  tags:
+    - swap
+  vars:
+    swap_device: "/dev/rootvg/lv_swap"
+  become: true
+  tasks:
+    - name: Ensure swap filesystem is present
+      filesystem:
+        fstype: "swap"
+        dev: "{{ swap_device }}"
+
+    - name: Ensure swap device present in fstab
+      mount:
+        name: "none"
+        src: "{{ swap_device }}"
+        fstype: "swap"
+        state: "present"
+
+    # It does no harm to run this when swap is already active
+    - name: Enable swap devices
+      command: "/sbin/swapon -a"
+      when:
+        - ansible_facts.swaptotal_mb == 0

etc/kayobe/apt.yml

@@ -17,21 +17,13 @@
 # * filename: name of a file in /etc/apt/apt.conf.d/ in which to write the
 #   configuration
 # Default is an empty list.
-apt_config:
-  # NOTE: Currently the Pulp verbatim publisher does not sync translation
-  # files, which results in apt update failing. Disable translations until this
-  # is resolved.
-  - content: |
-      Acquire::Languages "none";
-    filename: 99no-languages
+#apt_config:
 
 # List of apt keys. Each item is a dict containing the following keys:
 # * url: URL of key
 # * filename: Name of a file in which to store the downloaded key
 # Default is an empty list.
-apt_keys:
-  - url: "https://download.docker.com/linux/ubuntu/gpg"
-    filename: docker.asc
+#apt_keys:
 
 # A list of Apt repositories.
 # * types: whitespace-separated list of repository types, e.g. deb or deb-src
@@ -46,22 +38,12 @@ apt_keys:
 # * architecture: whitespace-separated list of architectures that will be used
 #   (optional, default is unset)
 # Default is an empty list.
-apt_repositories:
-  - url: "{{ stackhpc_repo_ubuntu_focal_url }}"
-    suites: "{{ ansible_facts.distribution_release }} {{ ansible_facts.distribution_release }}-updates {{ ansible_facts.distribution_release }}-backports"
-    components: main restricted universe multiverse
-  - url: "{{ stackhpc_repo_ubuntu_focal_security_url }}"
-    suites: "{{ ansible_facts.distribution_release }}-security"
-    components: main restricted universe multiverse
-  - url: "{{ stackhpc_repo_docker_ce_ubuntu_url }}"
-    suites: "{{ ansible_facts.distribution_release }}"
-    components: stable
-    signed_by: docker.asc
+#apt_repositories:
 
 # Whether to disable repositories in /etc/apt/sources.list. This may be used
 # when replacing the distribution repositories via apt_repositories.
 # Default is false.
-apt_disable_sources_list: true
+#apt_disable_sources_list:
 
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.

etc/kayobe/containers/pulp/post.yml

@@ -10,7 +10,7 @@
 - name: Set the Pulp admin password
   become: true
   command: >-
-    docker exec -u root {{ seed_containers.pulp.name }}
+    docker exec -u root pulp
     bash -c
       'pulpcore-manager reset-admin-password -p {{ pulp_password }}'
   no_log: true

etc/kayobe/containers/pulp/settings.py

@@ -1,4 +1,4 @@
-CONTENT_ORIGIN='http://{{ ansible_facts.fqdn }}'
-ANSIBLE_API_HOSTNAME='http://{{ ansible_facts.fqdn }}'
-ANSIBLE_CONTENT_HOSTNAME='http://{{ ansible_facts.fqdn }}/pulp/content'
+CONTENT_ORIGIN='{{ pulp_url }}'
+ANSIBLE_API_HOSTNAME='{{ pulp_url }}'
+ANSIBLE_CONTENT_HOSTNAME='{{ pulp_url }}/pulp/content'
 TOKEN_AUTH_DISABLED=True

etc/kayobe/docker.yml

@@ -31,7 +31,7 @@ docker_registry: "{{ stackhpc_docker_registry }}"
 #docker_registry_mirrors:
 
 # Enable live-restore on docker daemon
-#docker_daemon_live_restore:
+docker_daemon_live_restore: true
 
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.

etc/kayobe/environments/ci-builder/stackhpc-ci.yml

@@ -28,6 +28,7 @@ kolla_enable_octavia: true
 kolla_enable_ovn: true
 kolla_enable_prometheus: true
 kolla_enable_redis: true
+kolla_enable_skydive: true
 
 ###############################################################################
 # Network configuration.

etc/kayobe/hooks/seed-container-image-build/post.d/50-write-kolla-tag.yml

@@ -0,0 +1 @@
+../../../ansible/write-kolla-tag.yml