diff --git a/.github/workflows/stackhpc-promote.yml b/.github/workflows/stackhpc-promote.yml index b53d03070..b5b0e2c3c 100644 --- a/.github/workflows/stackhpc-promote.yml +++ b/.github/workflows/stackhpc-promote.yml @@ -3,7 +3,7 @@ name: Promote package repositories on: push: branches: - # NOTE(mgoddard): Reference only the current release branch here. + # NOTE(upgrade): Reference only the current release branch here. - stackhpc/2023.1 jobs: promote: diff --git a/doc/source/operations/octavia.rst b/doc/source/operations/octavia.rst index 57f3f6233..80d7bdd25 100644 --- a/doc/source/operations/octavia.rst +++ b/doc/source/operations/octavia.rst @@ -38,3 +38,55 @@ when building new images. To rollback an image update, simply delete the old image. The next newest image with a tag matching ``amp_image_tag`` will be selected. + +Manually deleting broken load balancers +======================================= + +Sometimes, a load balancer will get stuck in a broken state of ``PENDING_CREATE`` or ``PENDING_UPDATE``. +When in this state, the load balancer cannot be deleted; you will see the error ``Invalid state PENDING_CREATE of loadbalancer resource``. +To delete a load balancer in this state, you will need to manually update its provisioning status in the database. + +Find the database password: + +.. code-block:: console + + ansible-vault view --vault-password-file $KOLLA_CONFIG_PATH/passwords.yml + + # Search for database_password with: + /^database + +Access the database from a controller: + +.. code-block:: console + + docker exec -it mariadb bash + mysql -u root -p octavia + # Enter the database password when promted. + +List the load balancers to find the ID of the broken one(s): + +.. code-block:: console + + SELECT * FROM load_balancer; + +Set the provisioning status to ERROR for any broken load balancer: + +.. code-block:: console + + UPDATE load_balancer SET provisioning_status='ERROR' WHERE id=''; + +Delete the load balancer from the OpenStack CLI, cascading if any stray +Amphorae are hanging around: + +.. code-block:: console + + openstack loadbalancer delete --cascade + + +Sometimes, Amphora may also fail to delete if they are stuck in state +``BOOTING``. These can be resolved entirely from the OpenStack CLI: + +.. code-block:: console + + openstack loadbalancer amphora configure + openstack loadbalancer amphora delete diff --git a/doc/source/operations/upgrading.rst b/doc/source/operations/upgrading.rst index 10827fcb3..94fc6000a 100644 --- a/doc/source/operations/upgrading.rst +++ b/doc/source/operations/upgrading.rst @@ -71,6 +71,31 @@ to:: ark.stackhpc.com/stackhpc/etcd:zed-rocky-9-20230821T155947 +OVN enabled by default +---------------------- + +OVN is now enabled by default in StackHPC Kayobe Configuration. This change +was made to align with our standard deployment configuration. + +There is currently not a tested migration path from OVS to OVN on a running +system. If you are using a Neutron plugin other than ML2/OVN, set +``kolla_enable_ovn`` to ``false`` in ``etc/kayobe/kolla.yml``. + +For new deployments using OVN, see +:kolla-ansible-doc:`reference/networking/neutron.html#ovn-ml2-ovn`. + +Known issues +============ + +* Rebuilds of servers with volumes are broken if there are any Nova compute + services running an older release, including any that are down. Old compute + services should be removed using ``openstack compute service delete``, then + remaining compute services restarted. See `LP#2040264 + `__. + +* The OVN sync repair tool removes metadata ports, breaking OVN load balancers. + See `LP#2038091 `__. + Security baseline ================= diff --git a/etc/kayobe/ansible/growroot.yml b/etc/kayobe/ansible/growroot.yml index 54819b4c4..cdd7293d9 100644 --- a/etc/kayobe/ansible/growroot.yml +++ b/etc/kayobe/ansible/growroot.yml @@ -77,7 +77,7 @@ vars: pv: "{{ pvs.stdout | from_json }}" disk_tmp: "{{ pv.report[0].pv[0].pv_name[:-1] }}" - disk: "{{ disk_tmp[:-1] if disk_tmp[-1] == 'p' else disk_tmp }}" + disk: "{{ disk_tmp[:-1] if disk_tmp[-1] == 'p' and disk_tmp[:4] == 'nvme' else disk_tmp }}" part_num: "{{ pv.report[0].pv[0].pv_name[-1] }}" become: true failed_when: "growpart.rc != 0 and 'NOCHANGE' not in growpart.stdout" diff --git a/etc/kayobe/cephadm.yml b/etc/kayobe/cephadm.yml index edcc15c10..d8f7e54f7 100644 --- a/etc/kayobe/cephadm.yml +++ b/etc/kayobe/cephadm.yml @@ -12,7 +12,7 @@ cephadm_ceph_release: "quincy" cephadm_image: "{{ stackhpc_docker_registry if stackhpc_sync_ceph_images | bool else 'quay.io' }}/ceph/ceph:{{ cephadm_image_tag }}" # Ceph container image tag. -cephadm_image_tag: "v17.2.6" +cephadm_image_tag: "v17.2.7" # Ceph custom repo workaround for Ubuntu Jammy as there are no official ceph repos for jammy. cephadm_custom_repos: "{{ ansible_facts['distribution_release'] == 'jammy' }}" @@ -92,7 +92,7 @@ cephadm_commands_post: "{{ cephadm_commands_post_default + cephadm_commands_post cephadm_commands_pre_default: [] cephadm_commands_pre_extra: [] -cephadm_commands_post_default: "{{ ['mgr module enable prometheus'] if kolla_enable_prometheus_ceph_mgr_exporter | bool else [] }}" +cephadm_commands_post_default: "{{ ['mgr module enable prometheus'] if kolla_enable_prometheus_ceph_mgr_exporter | default(False) | bool else [] }}" cephadm_commands_post_extra: [] ############################################################################### diff --git a/etc/kayobe/inventory/group_vars/all/vault b/etc/kayobe/inventory/group_vars/all/vault index 3f004baff..dae698b54 100644 --- a/etc/kayobe/inventory/group_vars/all/vault +++ b/etc/kayobe/inventory/group_vars/all/vault @@ -10,13 +10,13 @@ seed_hashicorp_registry_password: "{{ stackhpc_docker_registry_password if stack seed_consul_docker_image: "{{ stackhpc_docker_registry ~ '/' if stackhpc_sync_hashicorp_images | bool else '' }}hashicorp/consul" # Seed Consul container image tag. -seed_consul_docker_tag: "1.16.1" +seed_consul_docker_tag: "1.16.3" # Seed Vault container image. seed_vault_docker_image: "{{ stackhpc_docker_registry ~ '/' if stackhpc_sync_hashicorp_images | bool else '' }}hashicorp/vault" # Seed Vault container image tag. -seed_vault_docker_tag: "1.14.1" +seed_vault_docker_tag: "1.14.6" # Seed Vault PKI Role name seed_vault_pki_role_name: "ServerCert" diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml index 77a0dcdea..0296253ff 100644 --- a/etc/kayobe/kolla.yml +++ b/etc/kayobe/kolla.yml @@ -106,12 +106,9 @@ kolla_tag: "{{ openstack_release }}-{{ kolla_base_distro }}-{{ kolla_base_distro # type: git # location: https://github.com/openstack/ironic # reference: master -# NOTE (Alex-Welsh): The reference for many of these entries could be -# 'stackhpc/{{ openstack_release }}' which would keep the branch up to date -# for the current release. This is nice in theory but in practice, the stackhpc -# forks change with every release and the elements in this list change with -# them. Explicitly using /2021.3 makes it more intuitive to find and edit these -# entries. +# NOTE(upgrade): These sources should be checked with each release. StackHPC +# branches are only required when we have custom backports. For a new release, +# we may have caught up with upstream. kolla_sources: bifrost-base-additions-stackhpc-inspector-plugins: # Install our custom inspector plugins. @@ -121,11 +118,11 @@ kolla_sources: cloudkitty-base: type: git location: https://github.com/stackhpc/cloudkitty.git - reference: stackhpc/2023.1 + reference: stackhpc/{{ openstack_release }} horizon-plugin-cloudkitty-dashboard: type: git location: https://github.com/stackhpc/cloudkitty-dashboard.git - reference: stackhpc/2023.1 + reference: stackhpc/{{ openstack_release }} ironic-inspector-additions-stackhpc-inspector-plugins: # Install our custom inspector plugins. type: git @@ -135,11 +132,15 @@ kolla_sources: magnum-base: type: git location: https://github.com/stackhpc/magnum.git - reference: stackhpc/2023.1 + reference: stackhpc/{{ openstack_release }} + neutron-base: + type: git + location: https://github.com/stackhpc/neutron.git + reference: stackhpc/{{ openstack_release }} neutron-base-plugin-networking-generic-switch: type: git location: https://github.com/stackhpc/networking-generic-switch.git - reference: stackhpc/2023.1 + reference: stackhpc/{{ openstack_release }} ############################################################################### # Kolla image build configuration. @@ -284,18 +285,22 @@ kolla_build_blocks: {% if stackhpc_kolla_clean_up_repo_mirrors | bool %} {% if kolla_base_distro == 'rocky' %} RUN \ - tar -xzf /etc/yum.repos.d.backup/repos.tar.gz -C /etc/yum.repos.d && \ - if grep -r '{{ stackhpc_repo_mirror_url }}' /etc/yum.repos.d; then \ - echo "Found repository mirror in Yum repositories"; \ - exit 1; \ + if [ -f /etc/yum.repos.d.backup/repos.tar.gz ]; then \ + tar -xzf /etc/yum.repos.d.backup/repos.tar.gz -C /etc/yum.repos.d && \ + rm -rf /etc/yum.repos.d.backup/; \ fi && \ - rm -rf /etc/yum.repos.d.backup/ + if grep -r '{{ stackhpc_repo_mirror_url }}' /etc/yum.repos.d; then \ + echo "Found repository mirror in Yum repositories"; \ + exit 1; \ + fi {% else %} RUN \ - mv /etc/apt/sources.list.backup /etc/apt/sources.list && \ + if [ -f /etc/apt/sources.list.backup ]; then \ + mv /etc/apt/sources.list.backup /etc/apt/sources.list; \ + fi && \ if grep -r '{{ stackhpc_repo_mirror_url }}' /etc/apt/sources.list; then \ - echo "Found repository mirror in APT repositories"; \ - exit 1; \ + echo "Found repository mirror in APT repositories"; \ + exit 1; \ fi {% endif %} {% endif %} diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml index a40648743..024efe4c8 100644 --- a/etc/kayobe/kolla/globals.yml +++ b/etc/kayobe/kolla/globals.yml @@ -12,9 +12,17 @@ kayobe_image_tags: bifrost: rocky: 2023.1-rocky-9-20231013T151957 ubuntu: 2023.1-ubuntu-jammy-20231013T151957 + cloudkitty: + rocky: 2023.1-rocky-9-20231115T110235 + ubuntu: 2023.1-ubuntu-jammy-20231115T110235 + neutron: + rocky: 2023.1-rocky-9-20231115T110249 + ubuntu: 2023.1-ubuntu-jammy-20231115T110249 openstack_tag: "{% raw %}{{ kayobe_image_tags['openstack'][kolla_base_distro] }}{% endraw %}" bifrost_tag: "{% raw %}{{ kayobe_image_tags['bifrost'][kolla_base_distro] }}{% endraw %}" +cloudkitty_tag: "{% raw %}{{ kayobe_image_tags['cloudkitty'][kolla_base_distro] }}{% endraw %}" +neutron_tag: "{% raw %}{{ kayobe_image_tags['neutron'][kolla_base_distro] }}{% endraw %}" om_enable_rabbitmq_high_availability: true @@ -40,6 +48,7 @@ prometheus_ceph_mgr_exporter_endpoints: # Use inventory hostnames as labels prometheus_instance_label: "{% raw %}{{ ansible_facts.hostname }}{% endraw %}" -############################################################################# - -neutron_ovn_distributed_fip: true +# Make openstack-exporter use Nova API version 2.1 to keep metrics the same as +# in Yoga. This is required to include a valid value for the flavor_id label on +# openstack_nova_server_status metrics. +prometheus_openstack_exporter_compute_api_version: "2.1" diff --git a/releasenotes/notes/bump-ceph-images-f9d4721aa1d37729.yaml b/releasenotes/notes/bump-ceph-images-f9d4721aa1d37729.yaml new file mode 100644 index 000000000..439f8b842 --- /dev/null +++ b/releasenotes/notes/bump-ceph-images-f9d4721aa1d37729.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + Updates default Ceph images to v17.2.7 for Quincy. diff --git a/releasenotes/notes/bump-hashicorp-images-7ef66e1328983bbe.yaml b/releasenotes/notes/bump-hashicorp-images-7ef66e1328983bbe.yaml new file mode 100644 index 000000000..6e25cb43b --- /dev/null +++ b/releasenotes/notes/bump-hashicorp-images-7ef66e1328983bbe.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + Updates Consul to 1.16.3 and Vault to 1.14.6. diff --git a/releasenotes/notes/cloudkitty-fix-es-to-os-migration-b0efd2626f59d977.yaml b/releasenotes/notes/cloudkitty-fix-es-to-os-migration-b0efd2626f59d977.yaml new file mode 100644 index 000000000..818f415a4 --- /dev/null +++ b/releasenotes/notes/cloudkitty-fix-es-to-os-migration-b0efd2626f59d977.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes the bulk API of CloudKitty so that it now supports the migration + from Elasticsearch to OpenSearch. diff --git a/releasenotes/notes/enable_ovn-b84974f398242dcb.yaml b/releasenotes/notes/enable_ovn-b84974f398242dcb.yaml index 21e253eda..3267a0447 100644 --- a/releasenotes/notes/enable_ovn-b84974f398242dcb.yaml +++ b/releasenotes/notes/enable_ovn-b84974f398242dcb.yaml @@ -1,8 +1,10 @@ --- upgrade: - | - Enabled ML2/OVN by default as checks preventing accidental migration - from ML2/OVS were added in kolla-ansible. If you are using a Neutron - plugin other than ML2/OVN, set `kolla_enable_ovn` to `false`. - OVN distributed FIP is also enabled, to disable it set - `neutron_ovn_distributed_fip` to `false` in `etc/kayobe/kolla/globals.yml`. \ No newline at end of file + Enabled ML2/OVN by default. Checks preventing accidental migration + from ML2/OVS were added in Kolla Ansible. If you are using a Neutron + plugin other than ML2/OVN, set ``kolla_enable_ovn`` to ``false``. + + OVN distributed FIP is disabled, to enable it set + ``neutron_ovn_distributed_fip`` to ``true`` in + ``etc/kayobe/kolla/globals.yml``. diff --git a/releasenotes/notes/fix-growroot-playbook-6a8ee02d7c0fbcb3.yaml b/releasenotes/notes/fix-growroot-playbook-6a8ee02d7c0fbcb3.yaml new file mode 100644 index 000000000..f3e4b84d2 --- /dev/null +++ b/releasenotes/notes/fix-growroot-playbook-6a8ee02d7c0fbcb3.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixes an issue with the growroot playbook where disks such as 'sdp' would + become 'sd' due to the removal of the trailing 'p' when dealing with nvme + devices. diff --git a/releasenotes/notes/fix-lb-fip-issue-when-neutron-restarts-6aafd03d115a1ee4.yaml b/releasenotes/notes/fix-lb-fip-issue-when-neutron-restarts-6aafd03d115a1ee4.yaml new file mode 100644 index 000000000..dc5f3c2bf --- /dev/null +++ b/releasenotes/notes/fix-lb-fip-issue-when-neutron-restarts-6aafd03d115a1ee4.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixes Neutron so that load balancer FIPs are not broken on Neutron restart. + See `Neutron bug report + `__. diff --git a/releasenotes/notes/fix-regression-plugging-802-3ad-port-group-cd48d2e6f43b363f.yaml b/releasenotes/notes/fix-regression-plugging-802-3ad-port-group-cd48d2e6f43b363f.yaml new file mode 100644 index 000000000..bec2cafb8 --- /dev/null +++ b/releasenotes/notes/fix-regression-plugging-802-3ad-port-group-cd48d2e6f43b363f.yaml @@ -0,0 +1,9 @@ +--- +fixes: + - | + Fixes issue where Netmiko devices were sending no commands to the switch + since plug_bond_to_network is overridden in + networking_generic_switch/devices/netmiko_devices/init.py and + PLUG_BOND_TO_NETWORK to set to None. + See `NGS bug report + `__. diff --git a/releasenotes/notes/neutron-containers-built-from-fork-e9f3c287b694fa8b.yaml b/releasenotes/notes/neutron-containers-built-from-fork-e9f3c287b694fa8b.yaml new file mode 100644 index 000000000..cf0a68153 --- /dev/null +++ b/releasenotes/notes/neutron-containers-built-from-fork-e9f3c287b694fa8b.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Neutron containers are now built from our StackHPC fork. diff --git a/releasenotes/notes/openstack-exporter-nova-api-fa5c2a9663bc97e2.yaml b/releasenotes/notes/openstack-exporter-nova-api-fa5c2a9663bc97e2.yaml new file mode 100644 index 000000000..e4e2f43ac --- /dev/null +++ b/releasenotes/notes/openstack-exporter-nova-api-fa5c2a9663bc97e2.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Restores valid value for the ``flavor_id`` label on + ``openstack_nova_server_status`` Prometheus metrics.