Playbooks & scripts for building, registering and rotating octavia amphora images

doc/source/operations/index.rst

@@ -8,3 +8,4 @@ This guide is for operators of the StackHPC Kayobe configuration project.
    :maxdepth: 1
 
    rabbitmq
+   octavia

doc/source/operations/octavia.rst

@@ -0,0 +1,40 @@
+=======
+Octavia
+=======
+
+Building and rotating amphora images
+====================================
+
+StackHPC kayobe config contains utility playbooks to build and rotate the amphora images.
+With your kayobe environment activated, you can build a new amphora image with:
+
+.. code-block:: console
+
+  kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/octavia-amphora-image-build.yml
+
+The resultant image is based on Ubuntu. By default the image will be built on the
+seed, but it is possible to change the group in the ansible inventory using the
+``amphora_builder_group`` variable.
+
+To rotate the image, first activate an openrc file containing the credentials
+for the octavia service account, e.g:
+
+.. code-block:: console
+
+  . $KOLLA_CONFIG_PATH/octavia-openrc.sh
+
+You can then run the playbook to upload the image:
+
+.. code-block:: console
+
+  kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/octavia-amphora-image-register.yml
+
+This will rename the old image by adding a timestamp suffix, before uploading a
+new image with the name, ``amphora-x64-haproxy``. Octavia should be configured
+to discover the image by tag using the ``amp_image_tag`` config option. The
+images are tagged with ``amphora`` to match the kolla-ansible default for
+``octavia_amp_image_tag``. This prevents you needing to reconfigure octavia
+when building new images.
+
+To rollback an image update, simply delete the old image. The next newest image with
+a tag matching ``amp_image_tag`` will be selected.

etc/kayobe/ansible/octavia-amphora-image-build.yml

@@ -0,0 +1,95 @@
+---
+- name: Build an Octavia Amphora image
+  hosts: "{{ amphora_builder_group | default('seed') }}"
+  vars:
+    amphora_dib_upper_constraints_file: "{{ pip_upper_constraints_file }}"
+  tasks:
+    - name: Ensure packages are installed
+      become: true
+      vars:
+        packages_for_os_family:
+          RedHat:
+            - debootstrap
+            - qemu-img
+            - git
+            - e2fsprogs
+            - policycoreutils-python-utils
+            - yum-utils
+          Debian:
+            - debootstrap
+            - qemu-utils
+            - git
+            - kpartx
+      package:
+        name: "{{ packages_for_os_family[ansible_facts.os_family] }}"
+
+    - name: Create a temporary directory
+      tempfile:
+        state: directory
+      register: tempfile_result
+
+    - block:
+        - name: Check whether the image cache directory exists
+          stat:
+            path: "{{ image_cache_path }}"
+            get_md5: False
+            get_checksum: False
+            mime: False
+          register: image_cache_stat
+
+        - name: Ensure the image cache directory exists
+          file:
+            path: "{{ image_cache_path }}"
+            state: directory
+            owner: "{{ ansible_facts.user_uid }}"
+            group: "{{ ansible_facts.user_gid }}"
+          become: true
+          when: >-
+            not image_cache_stat.stat.exists or
+            not image_cache_stat.stat.writeable
+
+        - name: Set path facts
+          vars:
+            work_path: "{{ tempfile_result.path }}"
+          set_fact:
+            src_path: "{{ work_path }}/octavia"
+            venv_path: "{{ work_path }}/venv"
+            work_path: "{{ work_path }}"
+
+        - name: Clone Octavia source code
+          git:
+            depth: 1
+            dest: "{{ src_path }}"
+            repo: "https://opendev.org/openstack/octavia"
+            version: "{{ openstack_branch }}"
+
+        - name: Install diskimage-builder in a virtual environment
+          pip:
+            name: diskimage-builder
+            extra_args: "{% if amphora_dib_upper_constraints_file %}-c {{ amphora_dib_upper_constraints_file }}{% endif %}"
+            virtualenv: "{{ venv_path }}"
+
+        - name: Create build log file (/var/log/octavia-amphora-image-build.log)
+          file:
+            path: /var/log/octavia-amphora-image-build.log
+            state: touch
+            owner: "{{ ansible_facts.user_uid }}"
+            group: "{{ ansible_facts.user_gid }}"
+          become: true
+
+        - name: Create the Amphora image
+          shell:
+            cmd: "source {{ venv_path }}/bin/activate && ./diskimage-create.sh -i ubuntu-minimal -s 3 -g {{ openstack_branch }} >> /var/log/octavia-amphora-image-build.log 2>&1"
+            chdir: "{{ src_path }}/diskimage-create"
+          changed_when: true
+
+        - name: Copy image to image store
+          copy:
+            src: "{{ src_path }}/diskimage-create/amphora-x64-haproxy.qcow2"
+            dest: "{{ image_cache_path }}/amphora-x64-haproxy-{{ openstack_release }}.qcow2"
+            remote_src: true
+      always:
+        - name: Remove temporary files
+          file:
+            path: "{{ work_path }}"
+            state: absent

etc/kayobe/ansible/octavia-amphora-image-register.yml

@@ -0,0 +1,79 @@
+---
+- name: Register an Octavia Amphora image in Glance
+  gather_facts: yes
+  hosts: "{{ amphora_builder_group | default('seed') }}"
+  vars:
+    venv: "{{ virtualenv_path }}/octavia-amphora"
+  tasks:
+    - name: Fail if not using octavia user and service project
+      fail:
+        msg: >-
+          Source the octavia-openrc.sh file before executing this playbook
+      when: >-
+        lookup('env', 'OS_USERNAME') != 'octavia' or
+        lookup('env', 'OS_PROJECT_NAME') != 'service'
+
+    - name: Set up openstack virtualenv
+      pip:
+        virtualenv: "{{ venv }}"
+        name:
+          - openstacksdk
+          - python-openstackclient
+        state: latest
+        extra_args: "{% if pip_upper_constraints_file %}-c {{ pip_upper_constraints_file }}{% endif %}"
+
+    - name: Query Octavia Amphora image
+      vars:
+        ansible_python_interpreter: "{{ venv }}/bin/python"
+      os_image_info:
+        auth_type: password
+        auth: "{{ openstack_auth }}"
+        ca_cert: "{{ openstack_cacert }}"
+        interface: "{{ openstack_interface }}"
+        image: amphora-x64-haproxy
+      register: image_info
+
+    - name: Get image checksum
+      stat:
+        path: "{{ image_cache_path }}/amphora-x64-haproxy-{{ openstack_release }}.qcow2"
+        checksum_algorithm: md5
+      changed_when: false
+      register: image_checksum
+      when: image_info.openstack_image
+
+    - name: Ensure Octavia Amphora image is renamed
+      vars:
+        ansible_python_interpreter: "{{ venv }}/bin/python"
+      shell:
+        cmd: >-
+          {{ venv }}/bin/openstack image set amphora-x64-haproxy --name amphora-x64-haproxy-{{ ansible_facts.date_time.iso8601_basic_short }}
+      when:
+        - image_info.openstack_image
+        - image_info.openstack_image.checksum != image_checksum.stat.checksum
+      changed_when: true
+      environment: "{{ openstack_auth_env }}"
+
+    - name: Ensure Octavia Amphora image is registered
+      vars:
+        ansible_python_interpreter: "{{ venv }}/bin/python"
+      os_image:
+        auth_type: password
+        auth: "{{ openstack_auth }}"
+        ca_cert: "{{ openstack_cacert }}"
+        interface: "{{ openstack_interface }}"
+        name: amphora-x64-haproxy
+        container_format: bare
+        disk_format: qcow2
+        is_public: no
+        filename: "{{ image_cache_path }}/amphora-x64-haproxy-{{ openstack_release }}.qcow2"
+        properties:
+          hw_architecture: x86_64
+          hw_rng_model: virtio
+
+    # FIXME: Use 'tags' parameter of os_image module available from
+    # openstack.cloud.image 1.5.0.
+    - name: Ensure Octavia Amphora image is tagged
+      shell:
+        cmd: >-
+          {{ venv }}/bin/openstack image set amphora-x64-haproxy --tag amphora
+      environment: "{{ openstack_auth_env }}"

releasenotes/notes/adds-amphora-utility-playbooks-21f7c523f4960f89.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Adds utility playbooks to build and rotate amphora images. For more details
+    check out the Octavia section of the Operator Guide included in the
+    documentation.

tools/loadbalancer-failover.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Fail over octavia loadbalancers to the latest amphora image.
+
+set -ex
+
+expected_image=$(openstack image show amphora-x64-haproxy.qcow2 -f value -c id)
+
+openstack loadbalancer amphora list --status ALLOCATED -f value -c id | while read a; do
+  image=$(openstack loadbalancer amphora show $a -f value -c image_id)
+  if [[ $image != "None" ]] && [[ $image != $expected_image ]]; then
+    lb_id=$(openstack loadbalancer amphora show $a -f value -c loadbalancer_id)
+    echo "Failing over loadbalancer $lb_id (amphora $a)"
+    if ! openstack loadbalancer failover $lb_id --wait; then
+      echo "Failed failing over loadbalancer $lb_id"
+    fi
+  fi
+done