CI: add Ubuntu overcloud deploy job

* Use source images
* Need to specify bash for &> syntax

Issues worked around:

* Manually configuring bridge via ip commands makes ifup fail to bring
  up the link. Adds a kayobe-network-bootstrap Zuul CI role that adds
  persistent configuration for the all-in-one network.

* bridge not active after interfaces role bounce. Added a pause, similar
  to michaelrigart/ansible-role-interfaces#31

* fails installing docker python module for kolla user. WARNING: The
  repository located at mirror-int.ord.rax.opendev.org is not a trusted
  or secure host and is being ignored ERROR: No matching distribution
  found for docker===4.4.0 Adding trusted host for PyPI mirror.

* Tenks fails to create block devices - missing qemu-img (in qemu-utils)

* Tenks qemu emulator is different on Ubuntu

Remaining issues:

* Bare metal testing is unreliable on Ubuntu - some jobs see IPMI
  failures such as the following:

    ipmitool chassis bootdev pxe

    Error setting Chassis Boot Parameter 5\nError setting Chassis Boot
    Parameter 0\n

  Bare metal testing is disabled on Ubuntu for now.

Depends-On: https://review.opendev.org/766984
Depends-On: https://review.opendev.org/766958

Story: 2004960
Task: 29393

Change-Id: I1985efae7c18f55c3ff7c27c17d6242523904f3e

dev/functions

@@ -144,7 +144,8 @@ function install_dependencies {
         echo "CentOS 7 is no longer supported"
         exit 1
     else
-        sudo apt install -y python-dev python3-virtualenv gcc git libffi-dev
+        sudo apt update
+        sudo apt install -y python3-dev python3-virtualenv gcc git libffi-dev
     fi
 }
 

dev/tenks-deploy-config-compute.yml

@@ -40,7 +40,7 @@ libvirt_host_install_daemon: false
 
 # QEMU may not be installed on the host, so set the path and avoid
 # autodetection.
-libvirt_vm_emulator: "/usr/libexec/qemu-kvm"
+libvirt_vm_emulator: "{% if ansible_os_family == 'RedHat' %}/usr/libexec/qemu-kvm{% else %}/usr/bin/qemu-system-x86_64{% endif %}"
 
 # Specify a log path in the kolla_logs Docker volume. It is accessible on the
 # host at the same path.

playbooks/kayobe-overcloud-base/overrides.yml.j2

@@ -3,10 +3,18 @@
 # Ansible is run directly on the controller.
 disable_selinux_do_reboot: false
 
+{% if ansible_os_family == 'Debian' %}
+# On Ubuntu, use 5 seconds delay to allow for interfaces to come up after ifup
+# exits.
+interfaces_pause_time: 5
+{% endif %}
+
 # Use the OpenStack infra's Dockerhub mirror.
 docker_registry_mirrors:
   - "http://{{ zuul_site_mirror_fqdn }}:8082/"
 
+kolla_base_distro: "{{ ansible_distribution | lower }}"
+kolla_install_type: "{{ 'source' if ansible_distribution == 'Ubuntu' else 'binary' }}"
 kolla_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kolla'].src_dir }}"
 kolla_source_version: "{{ zuul.projects['opendev.org/openstack/kolla'].checkout }}"
 kolla_ansible_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['opendev.org/openstack/kolla-ansible'].src_dir }}"
@@ -17,6 +25,8 @@ pip_upper_constraints_file: "/tmp/upper-constraints.txt"
 # Use the CI infra's PyPI mirror.
 pip_local_mirror: true
 pip_index_url: "http://{{ zuul_site_mirror_fqdn }}/pypi/simple"
+pip_trusted_hosts:
+  - "{{ zuul_site_mirror_fqdn }}"
 
 # NOTE(mgoddard): CentOS 8 removes interfaces from their bridge during ifdown,
 # and removes the bridge if there are no interfaces left. When Kayobe bounces

playbooks/kayobe-overcloud-base/pre.yml

@@ -3,6 +3,16 @@
   environment:
     KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}"
   tasks:
+    # NOTE(mgoddard): The kayobe dev config by default expects a bridge -
+    # breth1 - to exist with an IP address of 192.168.33.3.
+    - import_role:
+        name: kayobe-network-bootstrap
+      vars:
+        bridge_interface: breth1
+        bridge_ip: 192.168.33.3
+        bridge_prefix: 24
+        bridge_port_interface: dummy1
+
     # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
     # precedence over the standard config files.
     - name: Ensure kayobe-config override config file exists
@@ -16,30 +26,14 @@
         dest: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/globals.yml"
       when: tls_enabled
 
-    # NOTE(mgoddard): The kayobe dev config by default expects a bridge -
-    # breth1 - to exist with an IP address of 192.168.33.3.
-    - name: Ensure all-in-one network bridge interface exists
-      command: "{{ item }}"
-      become: true
-      with_items:
-        - "ip l add breth1 type bridge"
-        - "ip l set breth1 up"
-        - "ip a add 192.168.33.3/24 dev breth1"
-        # NOTE(mgoddard): CentOS 8 removes interfaces from their bridge during
-        # ifdown, and removes the bridge if there are no interfaces left. When
-        # Kayobe bounces veth links plugged into the bridge, it causes the
-        # bridge which has the IP we are using for SSH to be removed. Use a
-        # dummy interface.
-        - "ip l add dummy1 type dummy"
-        - "ip l set dummy1 up"
-        - "ip l set dummy1 master breth1"
-
     - name: Ensure kayobe is installed
       shell:
         cmd: dev/install.sh &> {{ logs_dir }}/ansible/install
         chdir: "{{ kayobe_src_dir }}"
+        executable: /bin/bash
 
     - name: Configure the firewall
       shell:
         cmd: dev/configure-firewall.sh
         chdir: "{{ kayobe_src_dir }}"
+        executable: /bin/bash

playbooks/kayobe-overcloud-base/run.yml

@@ -7,22 +7,37 @@
     - name: Ensure overcloud is deployed
       shell:
         cmd: "{{ kayobe_src_dir }}/dev/overcloud-deploy.sh &> {{ logs_dir }}/ansible/overcloud-deploy"
+        executable: /bin/bash
+
+    - name: Ensure qemu-img is available for Tenks
+      command: docker exec -u root nova_libvirt bash -c 'apt update && apt -y install qemu-utils'
+      become: true
+      when: ansible_os_family == 'Debian'
 
     - name: Ensure test Tenks cluster is deployed
       shell:
         # Pass absolute source directory, since otherwise the `chdir` will
         # cause this to fail.
         cmd: dev/tenks-deploy-compute.sh '{{ tenks_src_dir }}' &> {{ logs_dir }}/ansible/tenks-deploy
         chdir: "{{ kayobe_src_dir }}"
+        executable: /bin/bash
 
     - name: Perform testing of the virtualized machines
       # We must do this before tenks-deploy as that will stop the nova_libvirt
       # container
       shell:
         cmd: dev/overcloud-test-vm.sh &> {{ logs_dir }}/ansible/overcloud-test-vm
         chdir: "{{ kayobe_src_dir }}"
+        executable: /bin/bash
 
     - name: Perform testing of the baremetal machines
       shell:
         cmd: dev/overcloud-test-baremetal.sh &> {{ logs_dir }}/ansible/overcloud-test-baremetal
         chdir: "{{ kayobe_src_dir }}"
+        executable: /bin/bash
+      # FIXME(mgoddard): Bare metal testing is unreliable on Ubuntu - some jobs
+      # see IPMI failures such as the following:
+      # ipmitool chassis bootdev pxe
+      # Error setting Chassis Boot Parameter 5\nError setting Chassis Boot
+      # Parameter 0\n
+      when: ansible_os_family != 'Debian'

playbooks/kayobe-overcloud-upgrade-base/pre.yml

@@ -1,6 +1,16 @@
 ---
 - hosts: primary
   tasks:
+    # NOTE(mgoddard): The kayobe dev config by default expects a bridge -
+    # breth1 - to exist with an IP address of 192.168.33.3.
+    - import_role:
+        name: kayobe-network-bootstrap
+      vars:
+        bridge_interface: breth1
+        bridge_ip: 192.168.33.3
+        bridge_prefix: 24
+        bridge_port_interface: dummy1
+
     # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
     # precedence over the standard config files.
     - name: Ensure kayobe-config override config file exists
@@ -9,21 +19,3 @@
         dest: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
       vars:
         is_previous_release: true
-
-    # NOTE(mgoddard): The kayobe dev config by default expects a bridge -
-    # breth1 - to exist with an IP address of 192.168.33.3.
-    - name: Ensure all-in-one network bridge interface exists
-      command: "{{ item }}"
-      become: true
-      with_items:
-        - "ip l add breth1 type bridge"
-        - "ip l set breth1 up"
-        - "ip a add 192.168.33.3/24 dev breth1"
-        # NOTE(mgoddard): CentOS 8 removes interfaces from their bridge during
-        # ifdown, and removes the bridge if there are no interfaces left. When
-        # Kayobe bounces veth links plugged into the bridge, it causes the
-        # bridge which has the IP we are using for SSH to be removed. Use a
-        # dummy interface.
-        - "ip l add dummy1 type dummy"
-        - "ip l set dummy1 up"
-        - "ip l set dummy1 master breth1"

playbooks/kayobe-seed-base/pre.yml

@@ -3,6 +3,16 @@
   environment:
     KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}"
   tasks:
+    # NOTE(mgoddard): The kayobe dev config by default expects a bridge -
+    # breth1 - to exist with an IP address of 192.168.33.5.
+    - import_role:
+        name: kayobe-network-bootstrap
+      vars:
+        bridge_interface: breth1
+        bridge_ip: 192.168.33.5
+        bridge_prefix: 24
+        bridge_port_interface: dummy1
+
     # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
     # precedence over the standard config files.
     - name: Ensure kayobe-config override config file exists
@@ -20,24 +30,6 @@
         src: bifrost-overrides.yml.j2
         dest: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost/bifrost.yml"
 
-    # NOTE(mgoddard): The kayobe dev config by default expects a bridge -
-    # breth1 - to exist on the seed with an IP address of 192.168.33.5.
-    - name: Ensure all-in-one network bridge interface exists
-      command: "{{ item }}"
-      become: true
-      with_items:
-        - "ip l add breth1 type bridge"
-        - "ip l set breth1 up"
-        - "ip a add 192.168.33.5/24 dev breth1"
-        # NOTE(mgoddard): CentOS 8 removes interfaces from their bridge during
-        # ifdown, and removes the bridge if there are no interfaces left. When
-        # Kayobe bounces veth links plugged into the bridge, it causes the
-        # bridge which has the IP we are using for SSH to be removed. Use a
-        # dummy interface.
-        - "ip l add dummy1 type dummy"
-        - "ip l set dummy1 up"
-        - "ip l set dummy1 master breth1"
-
     - name: Ensure kayobe is installed
       shell:
         cmd: dev/install.sh &> {{ logs_dir }}/ansible/install

playbooks/kayobe-seed-upgrade-base/pre.yml

@@ -1,6 +1,16 @@
 ---
 - hosts: primary
   tasks:
+    # NOTE(mgoddard): The kayobe dev config by default expects a bridge -
+    # breth1 - to exist with an IP address of 192.168.33.5.
+    - import_role:
+        name: kayobe-network-bootstrap
+      vars:
+        bridge_interface: breth1
+        bridge_ip: 192.168.33.5
+        bridge_prefix: 24
+        bridge_port_interface: dummy1
+
     # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
     # precedence over the standard config files.
     - name: Ensure kayobe-config override config file exists
@@ -19,21 +29,3 @@
       template:
         src: bifrost-overrides.yml.j2
         dest: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost/bifrost.yml"
-
-    # NOTE(mgoddard): The kayobe dev config by default expects a bridge -
-    # breth1 - to exist on the seed with an IP address of 192.168.33.5.
-    - name: Ensure all-in-one network bridge interface exists
-      command: "{{ item }}"
-      become: true
-      with_items:
-        - "ip l add breth1 type bridge"
-        - "ip l set breth1 up"
-        - "ip a add 192.168.33.5/24 dev breth1"
-        # NOTE(mgoddard): CentOS 8 removes interfaces from their bridge during
-        # ifdown, and removes the bridge if there are no interfaces left. When
-        # Kayobe bounces veth links plugged into the bridge, it causes the
-        # bridge which has the IP we are using for SSH to be removed. Use a
-        # dummy interface.
-        - "ip l add dummy1 type dummy"
-        - "ip l set dummy1 up"
-        - "ip l set dummy1 master breth1"

playbooks/kayobe-seed-vm-base/pre.yml

@@ -3,35 +3,15 @@
   environment:
     KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}"
   tasks:
-    # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
-    # precedence over the standard config files.
-    - name: Ensure kayobe-config override config file exists
-      template:
-        src: overrides.yml.j2
-        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
-
-    - name: Ensure seed group variables exist
-      template:
-        src: seed-group-vars.j2
-        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/inventory/group_vars/seed/network-interfaces"
-
     # NOTE(mgoddard): The kayobe dev config by default expects a bridge -
-    # braio - to exist with an IP address of 192.168.33.4.
-    - name: Ensure all-in-one network bridge interface exists
-      command: "{{ item }}"
-      become: true
-      with_items:
-        - "ip l add braio type bridge"
-        - "ip l set braio up"
-        - "ip a add 192.168.33.4/24 dev braio"
-        # NOTE(mgoddard): CentOS 8 removes interfaces from their bridge during
-        # ifdown, and removes the bridge if there are no interfaces left. When
-        # Kayobe bounces veth links plugged into the bridge, it causes the
-        # bridge which has the IP we are using for SSH to be removed. Use a
-        # dummy interface.
-        - "ip l add dummy1 type dummy"
-        - "ip l set dummy1 up"
-        - "ip l set dummy1 master braio"
+    # braio - to exist with an IP address of 192.168.33.5.
+    - import_role:
+        name: kayobe-network-bootstrap
+      vars:
+        bridge_interface: braio
+        bridge_ip: 192.168.33.4
+        bridge_prefix: 24
+        bridge_port_interface: dummy1
 
     # NOTE(mgoddard): Configure IP forwarding and NAT to allow communication
     # from the seed VM to the outside world.
@@ -57,6 +37,18 @@
         state: disabled
       become: True
 
+    # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
+    # precedence over the standard config files.
+    - name: Ensure kayobe-config override config file exists
+      template:
+        src: overrides.yml.j2
+        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
+
+    - name: Ensure seed group variables exist
+      template:
+        src: seed-group-vars.j2
+        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/inventory/group_vars/seed/network-interfaces"
+
     - name: Ensure kayobe is installed
       shell:
         cmd: dev/install.sh &> {{ logs_dir }}/ansible/install

requirements.yml

@@ -5,7 +5,7 @@
   # There are no versioned releases of this role.
   version: 8438592c84585c86e62ae07e526d3da53629b377
 - src: MichaelRigart.interfaces
-  version: v1.9.2
+  version: v1.11.0
 - src: mrlesmithjr.manage-lvm
   version: v0.1.4
 - src: mrlesmithjr.mdadm

roles/kayobe-network-bootstrap/README.md

@@ -0,0 +1,13 @@
+# Kayobe network bootstrap
+
+Ansible role to bootstrap network configuration in CI.
+
+The role creates a bridge interface and a dummy interface, and adds the dummy
+interface as a port in the bridge. The bridge is assigned an IP address.
+
+## Role variables
+
+* `bridge_interface`: name of the bridge interface
+* `bridge_ip`: IP address to assign to the bridge
+* `bridge_prefix`: CIDR prefix to assign to the bridge
+* `bridge_port_interface`: name of the bridge port dummy interface

roles/kayobe-network-bootstrap/defaults/main.yml

@@ -0,0 +1,5 @@
+---
+bridge_interface:
+bridge_ip:
+bridge_prefix:
+bridge_port_interface:

roles/kayobe-network-bootstrap/tasks/Debian.yml

@@ -0,0 +1,38 @@
+---
+- name: Ensure interfaces.d directory exists
+  file:
+    path: /etc/network/interfaces.d
+    state: directory
+  become: true
+
+- name: Ensure interfaces.d directory is sourced
+  lineinfile:
+    path: /etc/network/interfaces
+    line: source /etc/network/interfaces.d/*
+  become: true
+
+- name: Ensure all-in-one network dummy interface exists
+  become: true
+  copy:
+    content: |
+      auto {{ bridge_port_interface }}
+      iface {{ bridge_port_interface }}  inet manual
+    dest: /etc/network/interfaces.d/ifcfg-{{ bridge_port_interface }}
+
+- name: Ensure all-in-one network bridge interface exists
+  become: true
+  copy:
+    content: |
+      auto {{ bridge_interface }}
+      iface {{ bridge_interface }} inet static
+      address {{ bridge_ip }}
+      netmask {{ (bridge_ip ~ '/' ~ bridge_prefix) | ipaddr('netmask') }}
+      bridge_ports {{ bridge_port_interface }}
+    dest: /etc/network/interfaces.d/ifcfg-{{ bridge_interface }}
+
+- name: Ensure all-in-one network bridge interfaces are up
+  become: true
+  command: "{{ item }}"
+  with_items:
+    - "ifup {{ bridge_interface }}"
+    - "ifup {{ bridge_port_interface }}"