Merge wallaby into xena

.github/workflows/stackhpc-all-in-one.yml

@@ -167,6 +167,11 @@ jobs:
           echo "::add-mask::$ssh_key"
           echo "::set-output name=ssh_key::$ssh_key"
 
+      # The same tag may be reused (e.g. pr-123), so ensure we have the latest image.
+      - name: Pull latest Kayobe image
+        run: |
+          sudo docker image pull $KAYOBE_IMAGE
+
       - name: Host configure
         run: |
           sudo -E docker run -t --rm \

.github/workflows/stackhpc-container-image-build.yml

@@ -18,6 +18,14 @@ on:
         type: boolean
         required: false
         default: false
+      distro:
+        description: Container image OS distribution
+        type: choice
+        required: false
+        default: centos
+        options:
+          - centos
+          - ubuntu
 
 env:
   ANSIBLE_FORCE_COLOR: True
@@ -85,7 +93,7 @@ jobs:
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe overcloud container image build ${{ github.event.inputs.regexes }} --push
+          kayobe overcloud container image build ${{ github.event.inputs.regexes }} --push -e kolla_base_distro=${{ inputs.distro }}
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: github.event.inputs.overcloud == 'true'
@@ -112,7 +120,7 @@ jobs:
         run: |
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
-          kayobe seed container image build --push
+          kayobe seed container image build --push -e kolla_base_distro=${{ inputs.distro }}
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
         if: github.event.inputs.seed == 'true'

README.rst

@@ -238,6 +238,21 @@ Setup
 
 Access the host via SSH.
 
+Install package dependencies.
+
+On CentOS:
+
+.. code-block:: console
+
+   sudo dnf install -y python3-virtualenv
+
+On Ubuntu:
+
+.. code-block:: console
+
+   sudo apt update
+   sudo apt install -y python3-virtualenv
+
 Clone the Kayobe and Kayobe configuration repositories (this one):
 
 .. code-block:: console
@@ -246,7 +261,7 @@ Clone the Kayobe and Kayobe configuration repositories (this one):
    mkdir -p src
    pushd src
    git clone https://github.com/stackhpc/kayobe.git -b stackhpc/xena
-   git clone https://github.com/stackhpc/stackhpc-kayobe-config -b stackhpc/xena
+   git clone https://github.com/stackhpc/stackhpc-kayobe-config -b stackhpc/xena kayobe-config
    popd
 
 Create a virtual environment and install Kayobe:
@@ -337,6 +352,21 @@ Setup
 
 Access the host via SSH.
 
+Install package dependencies.
+
+On CentOS:
+
+.. code-block:: console
+
+   sudo dnf install -y python3-virtualenv
+
+On Ubuntu:
+
+.. code-block:: console
+
+   sudo apt update
+   sudo apt install -y python3-virtualenv
+
 Clone the Kayobe and Kayobe configuration repositories (this one):
 
 .. code-block:: console
@@ -345,7 +375,7 @@ Clone the Kayobe and Kayobe configuration repositories (this one):
    mkdir -p src
    pushd src
    git clone https://github.com/stackhpc/kayobe.git -b stackhpc/xena
-   git clone https://github.com/stackhpc/stackhpc-kayobe-config -b stackhpc/xena
+   git clone https://github.com/stackhpc/stackhpc-kayobe-config -b stackhpc/xena kayobe-config
    popd
 
 Create a virtual environment and install Kayobe:

etc/kayobe/apt.yml

@@ -17,21 +17,13 @@
 # * filename: name of a file in /etc/apt/apt.conf.d/ in which to write the
 #   configuration
 # Default is an empty list.
-apt_config:
-  # NOTE: Currently the Pulp verbatim publisher does not sync translation
-  # files, which results in apt update failing. Disable translations until this
-  # is resolved.
-  - content: |
-      Acquire::Languages "none";
-    filename: 99no-languages
+#apt_config:
 
 # List of apt keys. Each item is a dict containing the following keys:
 # * url: URL of key
 # * filename: Name of a file in which to store the downloaded key
 # Default is an empty list.
-apt_keys:
-  - url: "https://download.docker.com/linux/ubuntu/gpg"
-    filename: docker.asc
+#apt_keys:
 
 # A list of Apt repositories.
 # * types: whitespace-separated list of repository types, e.g. deb or deb-src
@@ -46,22 +38,12 @@ apt_keys:
 # * architecture: whitespace-separated list of architectures that will be used
 #   (optional, default is unset)
 # Default is an empty list.
-apt_repositories:
-  - url: "{{ stackhpc_repo_ubuntu_focal_url }}"
-    suites: "{{ ansible_facts.distribution_release }} {{ ansible_facts.distribution_release }}-updates {{ ansible_facts.distribution_release }}-backports"
-    components: main restricted universe multiverse
-  - url: "{{ stackhpc_repo_ubuntu_focal_security_url }}"
-    suites: "{{ ansible_facts.distribution_release }}-security"
-    components: main restricted universe multiverse
-  - url: "{{ stackhpc_repo_docker_ce_ubuntu_url }}"
-    suites: "{{ ansible_facts.distribution_release }}"
-    components: stable
-    signed_by: docker.asc
+#apt_repositories:
 
 # Whether to disable repositories in /etc/apt/sources.list. This may be used
 # when replacing the distribution repositories via apt_repositories.
 # Default is false.
-apt_disable_sources_list: true
+#apt_disable_sources_list:
 
 ###############################################################################
 # Dummy variable to allow Ansible to accept this file.

etc/kayobe/containers/pulp/post.yml

@@ -10,7 +10,7 @@
 - name: Set the Pulp admin password
   become: true
   command: >-
-    docker exec -u root {{ seed_containers.pulp.name }}
+    docker exec -u root pulp
     bash -c
       'pulpcore-manager reset-admin-password -p {{ pulp_password }}'
   no_log: true

etc/kayobe/containers/pulp/settings.py

@@ -1,4 +1,4 @@
-CONTENT_ORIGIN='http://{{ ansible_facts.fqdn }}'
-ANSIBLE_API_HOSTNAME='http://{{ ansible_facts.fqdn }}'
-ANSIBLE_CONTENT_HOSTNAME='http://{{ ansible_facts.fqdn }}/pulp/content'
+CONTENT_ORIGIN='{{ pulp_url }}'
+ANSIBLE_API_HOSTNAME='{{ pulp_url }}'
+ANSIBLE_CONTENT_HOSTNAME='{{ pulp_url }}/pulp/content'
 TOKEN_AUTH_DISABLED=True

etc/kayobe/environments/ci-builder/stackhpc-ci.yml

@@ -28,6 +28,7 @@ kolla_enable_octavia: true
 kolla_enable_ovn: true
 kolla_enable_prometheus: true
 kolla_enable_redis: true
+kolla_enable_skydive: true
 
 ###############################################################################
 # Network configuration.

etc/kayobe/hooks/seed-container-image-build/post.d/50-write-kolla-tag.yml

@@ -0,0 +1 @@
+../../../ansible/write-kolla-tag.yml

etc/kayobe/kolla.yml

@@ -115,6 +115,11 @@ kolla_sources:
     type: git
     location: "{{ kolla_bifrost_source_url }}"
     reference: "{{ kolla_bifrost_source_version }}"
+  bifrost-base-additions-stackhpc-inspector-plugins:
+    # Install our custom inspector plugins.
+    type: git
+    location: https://github.com/stackhpc/stackhpc-inspector-plugins.git
+    reference: 1.3.0
   cloudkitty-base:
     type: git
     location: https://github.com/stackhpc/cloudkitty.git
@@ -216,13 +221,18 @@ kolla_build_blocks:
                -e 's/^[# ]*\(baseurl *=.*\)/#\1/g' \
                -e '/#baseurl.*/a baseurl={{ repo.url }}' /etc/yum.repos.d/{{ repo.file }}{% if not loop.last %} &&{% endif %} \
     {% endfor %}
-  base_ubuntu_package_sources_list: |
-    RUN \
-        rm -f /etc/apt/sources.list && \
-    {% for repo in stackhpc_ubuntu_focal_repos %}
-        echo '{{ repo }}' >> /etc/apt/sources.list {% if not loop.last %} && \
-    {% endif %}
-    {% endfor %}
+  # NOTE: Not currently syncing Ubuntu packages, since the on_demand mirror in
+  # Ark does not work if the upstream mirror pulls packages (which it does
+  # sometimes).
+  # base_ubuntu_package_sources_list: |
+  #   RUN \
+  #       rm -f /etc/apt/sources.list && \
+  #   {% for repo in stackhpc_ubuntu_focal_repos %}
+  #       echo '{{ repo }}' >> /etc/apt/sources.list {% if not loop.last %} && \
+  #   {% endif %}
+  #   {% endfor %}
+  bifrost_base_header: |
+    ADD additions-archive /
   grafana_plugins_install: |
     RUN grafana-cli plugins install vonage-status-panel
   ironic_inspector_header: |
@@ -235,6 +245,8 @@ kolla_build_blocks:
 # customization is most commonly packages. The operation should be one of
 # override, append or remove. The value should be a list.
 kolla_build_customizations_common:
+  bifrost_base_pip_packages_append:
+    - /additions/*
   ironic_inspector_pip_packages_append:
     - /additions/*
 

etc/kayobe/kolla/globals.yml

@@ -1,9 +1,9 @@
 # yamllint disable-file
 ---
-enable_docker_repo: "{% raw %}{{ ansible_facts.os_family == 'RedHat' }}{% endraw %}"
-
 docker_yum_baseurl: "{{ stackhpc_repo_docker_url }}"
 docker_yum_gpgkey: "https://download.docker.com/linux/{% raw %}{{ ansible_facts.distribution | lower }}{% endraw %}/gpg"
 
+{% if kolla_base_distro == 'centos' %}
 barbican_tag: xena-20220728T143658
 prometheus_tag: xena-20220728T143658
+{% endif %}

etc/kayobe/pulp.yml

@@ -32,7 +32,10 @@ stackhpc_release_pulp_content_url: "{{ stackhpc_release_pulp_url }}/pulp/content
 # Debs
 
 # Whether to sync Ubuntu packages.
-stackhpc_pulp_sync_ubuntu_focal: "{{ os_distribution == 'ubuntu' }}"
+# NOTE: Not currently syncing Ubuntu packages, since the on_demand mirror in
+# Ark does not work if the upstream mirror pulls packages (which it does
+# sometimes).
+stackhpc_pulp_sync_ubuntu_focal: false
 
 stackhpc_pulp_repository_deb_repos:
   # Base Ubuntu Focal repositories
@@ -427,6 +430,9 @@ stackhpc_pulp_images_kolla:
   - redis
   - redis-base
   - redis-sentinel
+  - skydive-agent
+  - skydive-analyzer
+  - skydive-base
 
 # Common parameters for container image repositories.
 stackhpc_pulp_repository_container_repos_kolla_common: