diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml index 67e9eda0c..adeb7ffe7 100644 --- a/.github/workflows/overcloud-host-image-build.yml +++ b/.github/workflows/overcloud-host-image-build.yml @@ -23,17 +23,34 @@ on: env: ANSIBLE_FORCE_COLOR: True + KAYOBE_ENVIRONMENT: ci-builder + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} jobs: overcloud-host-image-build: name: Build overcloud host images if: github.repository == 'stackhpc/stackhpc-kayobe-config' - runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder-rl9] + runs-on: arc-skc-host-image-builder-runner permissions: {} steps: - - uses: actions/checkout@v4 + - name: Install Package + uses: ConorMacBride/install-package@main + with: + apt: git unzip nodejs python3-pip python3-venv openssh-server openssh-client jq + + - name: Start the SSH service + run: | + sudo /etc/init.d/ssh start + + - name: Checkout + uses: actions/checkout@v4 with: path: src/kayobe-config + - name: Output image tag of the builder + id: builder_image_tag + run: | + echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + - name: Determine OpenStack release id: openstack_release run: | @@ -50,10 +67,6 @@ jobs: run: | echo "${{ steps.host_image_tag.outputs.host_image_tag }}" - - name: Clean any previous build artifact - run: | - rm -f /tmp/updated_images.txt - - name: Clone StackHPC Kayobe repository uses: actions/checkout@v4 with: @@ -61,25 +74,6 @@ jobs: ref: refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }} path: src/kayobe - - name: Install dependencies - run: | - sudo dnf -y install zstd debootstrap - - - name: Setup networking - run: | - if ! ip l show breth1 >/dev/null 2>&1; then - sudo ip l add breth1 type bridge - fi - sudo ip l set breth1 up - if ! ip a show breth1 | grep 192.168.33.3/24; then - sudo ip a add 192.168.33.3/24 dev breth1 - fi - if ! ip l show dummy1 >/dev/null 2>&1; then - sudo ip l add dummy1 type dummy - fi - sudo ip l set dummy1 up - sudo ip l set dummy1 master breth1 - - name: Install Kayobe run: | mkdir -p venvs && @@ -89,36 +83,132 @@ jobs: pip install -U pip && pip install ../src/kayobe + - name: Install terraform + uses: hashicorp/setup-terraform@v2 + + - name: Initialise terraform + run: terraform init + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate SSH keypair + run: ssh-keygen -f id_rsa -N '' + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate clouds.yaml + run: | + cat << EOF > clouds.yaml + ${{ secrets.CLOUDS_YAML }} + EOF + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Generate terraform.tfvars + run: | + cat << EOF > terraform.tfvars + ssh_public_key = "id_rsa.pub" + ssh_username = "rocky" + aio_vm_name = "skc-host-image-builder" + # Must be a Rocky Linux 9 host to successfully build all images + # This MUST NOT be an LVM image. It can cause confusing conficts with the built image. + aio_vm_image = "Rocky-9-GenericCloud-Base-9.3-20231113.0.x86_64.qcow2" + aio_vm_flavor = "en1.medium" + aio_vm_network = "stackhpc-ci" + aio_vm_subnet = "stackhpc-ci" + aio_vm_interface = "eth0" + EOF + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Terraform Plan + run: terraform plan + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: "openstack" + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Terraform Apply + run: | + for attempt in $(seq 5); do + if terraform apply -auto-approve; then + echo "Created infrastructure on attempt $attempt" + exit 0 + fi + echo "Failed to create infrastructure on attempt $attempt" + sleep 10 + terraform destroy -auto-approve + sleep 60 + done + echo "Failed to create infrastructure after $attempt attempts" + exit 1 + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: "openstack" + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Get Terraform outputs + id: tf_outputs + run: | + terraform output -json + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + + - name: Write Terraform outputs + run: | + cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml + ${{ steps.tf_outputs.outputs.stdout }} + EOF + + - name: Write Terraform network config + run: | + cat << EOF > src/kayobe-config/etc/kayobe/environments/ci-builder/tf-network-allocation.yml + --- + aio_ips: + builder: "{{ access_ip_v4.value }}" + EOF + + - name: Write Terraform network interface config + run: | + mkdir -p src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed + rm -f src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces + cat << EOF > src/kayobe-config/etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/seed/network-interfaces + admin_interface: "{{ access_interface.value }}" + aio_interface: "{{ access_interface.value }}" + EOF + + - name: Manage SSH keys + run: | + mkdir -p ~/.ssh + touch ~/.ssh/authorized_keys + cat src/kayobe-config/terraform/aio/id_rsa.pub >> ~/.ssh/authorized_keys + cp src/kayobe-config/terraform/aio/id_rsa* ~/.ssh/ + - name: Bootstrap the control host run: | source venvs/kayobe/bin/activate && source src/kayobe-config/kayobe-env --environment ci-builder && kayobe control host bootstrap - - name: Configure the seed host + - name: Configure the seed host (Builder VM) + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host configure -e seed_bootstrap_user=rocky --skip-tags network + + - name: Install dependencies run: | source venvs/kayobe/bin/activate && source src/kayobe-config/kayobe-env --environment ci-builder && - kayobe seed host configure + kayobe seed host command run \ + --command "sudo dnf config-manager --set-enabled crb && sudo dnf -y install epel-release && sudo dnf -y install zstd debootstrap kpartx cloud-init" --show-output env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - name: Create bifrost_httpboot Docker volume - run: | - if [[ $(sudo docker volume ls -f Name=bifrost_httpboot -q | wc -l) = 0 ]]; then - sudo docker volume create bifrost_httpboot - fi - - - name: Generate clouds.yaml - run: | - cat << EOF > clouds.yaml - ${{ secrets.CLOUDS_YAML }} - EOF - - - name: Install OpenStack client run: | source venvs/kayobe/bin/activate && - pip install python-openstackclient -c https://releases.openstack.org/constraints/upper/${{ steps.openstack_release.outputs.openstack_release }} + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "sudo mkdir -p /var/lib/docker/volumes/bifrost_httpboot/_data" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} - name: Build a Rocky Linux 9 overcloud host image id: build_rocky_9 @@ -134,6 +224,16 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.rocky9 + - name: Show last error logs + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.stdout" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: steps.build_rocky_9.outcome == 'failure' + - name: Upload Rocky Linux 9 overcloud host image to Ark run: | source venvs/kayobe/bin/activate && @@ -147,19 +247,17 @@ jobs: env: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.rocky9 && steps.build_rocky_9.outcome == 'success' - - - name: Upload Rocky Linux 9 overcloud host image to SMS + + - name: Upload Rocky Linux 9 overcloud host image to Dev Cloud run: | source venvs/kayobe/bin/activate && - openstack image create \ - overcloud-rocky-9-${{ steps.host_image_tag.outputs.host_image_tag }} \ - --container-format bare \ - --disk-format qcow2 \ - --file /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.qcow2 \ - --private \ - --os-cloud sms-lab-release \ - --progress + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \ + -e local_image_path="/opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.qcow2" \ + -e image_name=overcloud-rocky-9-${{ steps.host_image_tag.outputs.host_image_tag }} env: + CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} if: inputs.rocky9 && steps.build_rocky_9.outcome == 'success' @@ -178,6 +276,16 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.ubuntu-jammy + - name: Show last error logs + continue-on-error: true + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe seed host command run --command "tail -200 /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.stdout" --show-output + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: steps.build_ubuntu_jammy.outcome == 'failure' + - name: Upload Ubuntu Jammy 22.04 overcloud host image to Ark run: | source venvs/kayobe/bin/activate && @@ -192,50 +300,27 @@ jobs: KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy.outcome == 'success' - - name: Upload Ubuntu Jammy 22.04 overcloud host image to SMS + - name: Upload Ubuntu Jammy overcloud host image to Dev Cloud run: | source venvs/kayobe/bin/activate && - openstack image create \ - overcloud-ubuntu-jammy-${{ steps.host_image_tag.outputs.host_image_tag }} \ - --container-format bare \ - --disk-format qcow2 \ - --file /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.qcow2 \ - --private \ - --os-cloud sms-lab-release \ - --progress + source src/kayobe-config/kayobe-env --environment ci-builder && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/openstack-host-image-upload.yml \ + -e local_image_path="/opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.qcow2" \ + -e image_name=overcloud-ubuntu-jammy-${{ steps.host_image_tag.outputs.host_image_tag }} env: + CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} if: inputs.ubuntu-jammy && steps.build_ubuntu_jammy.outcome == 'success' - - name: Upload updated images artifact - uses: actions/upload-artifact@v4 - with: - name: Updated images list - path: /tmp/updated_images.txt - retention-days: 7 - if: steps.build_rocky_9.outcome == 'success' || - steps.build_ubuntu_jammy.outcome == 'success' - - - name: Upload Rocky 9 build logs if build failed - uses: actions/upload-artifact@v4 - with: - name: Rocky 9 build logs - path: | - /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.stdout - /opt/kayobe/images/overcloud-rocky-9/overcloud-rocky-9.stderr - retention-days: 7 - if: steps.build_rocky_9.outcome == 'failure' - - - name: Upload Ubuntu Jammy 22.04 build logs if build failed - uses: actions/upload-artifact@v4 - with: - name: Ubuntu Jammy 22.04 build logs - path: | - /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.stdout - /opt/kayobe/images/overcloud-ubuntu-jammy/overcloud-ubuntu-jammy.stderr - retention-days: 7 - if: steps.build_ubuntu_jammy.outcome == 'failure' + - name: Copy logs back + continue-on-error: true + run: | + mkdir logs + scp -r rocky@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/opt/kayobe/images/*/*.std* ./logs/ + scp -r rocky@$(jq -r .access_ip_v4.value src/kayobe-config/etc/kayobe/environments/ci-builder/tf-outputs.yml):/tmp/updated_images.txt ./logs/ || true + if: always() - name: Fail if any overcloud host image builds failed run: | @@ -244,7 +329,18 @@ jobs: if: steps.build_rocky_9.outcome == 'failure' || steps.build_ubuntu_jammy.outcome == 'failure' - - name: Clean up build artifacts - run: | - sudo rm -rf /opt/kayobe/images/ + - name: Upload logs artifact + uses: actions/upload-artifact@v4 + with: + name: Build logs + path: ./logs + if: always() + + - name: Destroy + run: terraform destroy -auto-approve + working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} if: always() diff --git a/.github/workflows/stackhpc-ci-cleanup.yml b/.github/workflows/stackhpc-ci-cleanup.yml index a769aa718..ed9ec327c 100644 --- a/.github/workflows/stackhpc-ci-cleanup.yml +++ b/.github/workflows/stackhpc-ci-cleanup.yml @@ -55,3 +55,23 @@ jobs: OS_CLOUD: openstack OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + + - name: Clean up host image builder instances over 5 hours old + run: | + result=0 + changes_before=$(date -Imin -d -5hours) + for status in ACTIVE BUILD ERROR SHUTOFF; do + for instance in $(openstack server list --tags skc-host-image-build --os-compute-api-version 2.66 --format value --column ID --changes-before $changes_before --status $status); do + echo "Cleaning up $status instance $instance" + openstack server show $instance + if ! openstack server delete $instance; then + echo "Failed to delete $status instance $instance" + result=1 + fi + done + done + exit $result + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} diff --git a/doc/source/configuration/monitoring.rst b/doc/source/configuration/monitoring.rst index 33d86c632..822a3c02f 100644 --- a/doc/source/configuration/monitoring.rst +++ b/doc/source/configuration/monitoring.rst @@ -141,36 +141,26 @@ OpenStack Capacity ================== OpenStack Capacity allows you to see how much space you have available -in your cloud. StackHPC Kayobe Config includes a playbook for manual -deployment, and it's necessary that some variables are set before -running this playbook. +in your cloud. StackHPC Kayobe Config will deploy OpenStack Capacity +by default on a service deploy, this can be disabled by setting +``stackhpc_enable_os_capacity`` to false. -To successfully deploy OpenStack Capacity, you are required to specify -the OpenStack application credentials in ``kayobe/secrets.yml`` as: +OpenStack Capacity is deployed automatically using a service deploy hook +with the generated kolla-ansible admin credentials, you can override these +by setting the authentication url, username, password, project name and +project domain name in ``stackhpc-monitoring.yml``: .. code-block:: yaml - secrets_os_capacity_credential_id: - secrets_os_capacity_credential_secret: + stackhpc_os_capacity_auth_url: + stackhpc_os_capacity_username: + stackhpc_os_capacity_password: + stackhpc_os_capacity_project_name: + stackhpc_os_capacity_domain_name: + stackhpc_os_capacity_openstack_region_name: -The Keystone authentication URL and OpenStack region can be changed -from their defaults in ``stackhpc-monitoring.yml`` should you need to -set a different OpenStack region for your cloud. The authentication -URL is set to use ``kolla_internal_fqdn`` by default: - -.. code-block:: yaml - - stackhpc_os_capacity_auth_url: - stackhpc_os_capacity_openstack_region_name: - -Additionally, you are required to enable a conditional flag to allow -HAProxy and Prometheus configuration to be templated during deployment. - -.. code-block:: yaml - - stackhpc_enable_os_capacity: true - -If you are deploying in a cloud with internal TLS, you may be required +Additionally, you should ensure these credentials have the correct permissions +for the exporter. If you are deploying in a cloud with internal TLS, you may be required to disable certificate verification for the OpenStack Capacity exporter if your certificate is not signed by a trusted CA. @@ -178,21 +168,14 @@ if your certificate is not signed by a trusted CA. stackhpc_os_capacity_openstack_verify: false -After defining your credentials, you may deploy OpenStack Capacity -using the ``ansible/deploy-os-capacity-exporter.yml`` Ansible playbook +If you've modified your credentials, you will need to re-deploy OpenStack Capacity +using the ``deploy-os-capacity-exporter.yml`` Ansible playbook via Kayobe. .. code-block:: console kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/deploy-os-capacity-exporter.yml -It is required that you re-configure the Prometheus, Grafana and HAProxy -services following deployment, to do this run the following Kayobe command. - -.. code-block:: console - - kayobe overcloud service reconfigure -kt grafana,prometheus,loadbalancer - If you notice ``HaproxyServerDown`` or ``HaproxyBackendDown`` prometheus alerts after deployment it's likely the os_exporter secrets have not been set correctly, double check you have entered the correct authentication diff --git a/doc/source/configuration/vault.rst b/doc/source/configuration/vault.rst index b60901a45..d598a63a5 100644 --- a/doc/source/configuration/vault.rst +++ b/doc/source/configuration/vault.rst @@ -111,7 +111,7 @@ Setup HAProxy config for Vault .. code-block:: - kayobe overcloud service deploy -kt haproxy + kayobe overcloud service deploy --skip-tags os_capacity -kt haproxy Setup Vault HA on the overcloud hosts ------------------------------------- diff --git a/etc/kayobe/ansible.cfg b/etc/kayobe/ansible.cfg index 901001ad8..b38cb8239 100644 --- a/etc/kayobe/ansible.cfg +++ b/etc/kayobe/ansible.cfg @@ -11,5 +11,9 @@ callbacks_enabled = ansible.posix.profile_tasks # Silence warning about invalid characters found in group names force_valid_group_names = ignore +[inventory] +# Fail when any inventory source cannot be parsed. +any_unparsed_is_failed = True + [ssh_connection] pipelining = True diff --git a/etc/kayobe/ansible/deploy-os-capacity-exporter.yml b/etc/kayobe/ansible/deploy-os-capacity-exporter.yml index 8cff6a89e..cc3afa7b0 100644 --- a/etc/kayobe/ansible/deploy-os-capacity-exporter.yml +++ b/etc/kayobe/ansible/deploy-os-capacity-exporter.yml @@ -1,6 +1,7 @@ --- - name: Remove legacy os_exporter.cfg file hosts: network + tags: os_capacity gather_facts: false tasks: - name: Ensure legacy os_exporter.cfg config file is deleted @@ -11,20 +12,40 @@ - name: Deploy os-capacity exporter hosts: monitoring + tags: os_capacity gather_facts: false tasks: - name: Create os-capacity directory ansible.builtin.file: path: /opt/kayobe/os-capacity/ state: directory + when: stackhpc_enable_os_capacity + + - name: Read admin-openrc credential file + ansible.builtin.command: + cmd: "cat {{ lookup('ansible.builtin.env', 'KOLLA_CONFIG_PATH') }}/admin-openrc.sh" + delegate_to: localhost + register: credential + when: stackhpc_enable_os_capacity + + - name: Set facts for admin credentials + ansible.builtin.set_fact: + stackhpc_os_capacity_auth_url: "{{ credential.stdout_lines | select('match', '.*OS_AUTH_URL*.') | first | split('=') | last | replace(\"'\",'') }}" + stackhpc_os_capacity_project_name: "{{ credential.stdout_lines | select('match', '.*OS_PROJECT_NAME*.') | first | split('=') | last | replace(\"'\",'') }}" + stackhpc_os_capacity_domain_name: "{{ credential.stdout_lines | select('match', '.*OS_PROJECT_DOMAIN_NAME*.') | first | split('=') | last | replace(\"'\",'') }}" + stackhpc_os_capacity_openstack_region_name: "{{ credential.stdout_lines | select('match', '.*OS_REGION_NAME*.') | first | split('=') | last | replace(\"'\",'') }}" + stackhpc_os_capacity_username: "{{ credential.stdout_lines | select('match', '.*OS_USERNAME*.') | first | split('=') | last | replace(\"'\",'') }}" + stackhpc_os_capacity_password: "{{ credential.stdout_lines | select('match', '.*OS_PASSWORD*.') | first | split('=') | last | replace(\"'\",'') }}" + when: stackhpc_enable_os_capacity - name: Template clouds.yml ansible.builtin.template: src: templates/os_capacity-clouds.yml.j2 dest: /opt/kayobe/os-capacity/clouds.yaml + when: stackhpc_enable_os_capacity - name: Ensure os_capacity container is running - docker_container: + community.docker.docker_container: name: os_capacity image: ghcr.io/stackhpc/os-capacity:master env: @@ -37,3 +58,4 @@ network_mode: host restart_policy: unless-stopped become: true + when: stackhpc_enable_os_capacity diff --git a/etc/kayobe/ansible/fix-networking.yml b/etc/kayobe/ansible/fix-networking.yml index 0b14f9ddf..01a833264 100644 --- a/etc/kayobe/ansible/fix-networking.yml +++ b/etc/kayobe/ansible/fix-networking.yml @@ -10,11 +10,16 @@ # Work around no known_hosts entry on first boot. ansible_ssh_common_args: "-o StrictHostKeyChecking=no" tasks: - - name: Ensure `hosts` file contains pulp entries + - name: Ensure hosts are reachable + ansible.builtin.wait_for_connection: + + - name: Ensure `hosts` file contains pulp and API entries blockinfile: path: /etc/hosts - marker: "# {mark} Kayobe Pulp entries" + marker: "# {mark} Kayobe entries" block: | 10.0.0.34 pelican pelican.service.compute.sms-lab.cloud 10.205.3.187 pulp-server pulp-server.internal.sms-cloud + 192.168.37.2 internal.infra.mos.{{ root_domain }} + 192.168.39.2 public.infra.mos.{{ root_domain }} become: true diff --git a/etc/kayobe/ansible/hotfix-containers.yml b/etc/kayobe/ansible/hotfix-containers.yml index b6a811801..677105f3e 100644 --- a/etc/kayobe/ansible/hotfix-containers.yml +++ b/etc/kayobe/ansible/hotfix-containers.yml @@ -30,13 +30,13 @@ - name: Set fact for containers list set_fact: - containers_list: host_containers.stdout + containers_list: "{{ host_containers.stdout }}" - name: Fail if no containers match given regex vars: hotfix_containers: "{{ containers_list | split('\n') | regex_search(container_hotfix_container_regex) }}" fail: - msg: "No containers matched. Please check your regex. Containers running on host: {{ host_containers | split('\n') }}" + msg: "No containers matched. Please check your regex. Containers running on host: {{ host_containers.stdout_lines }}" when: hotfix_containers == "" - name: Ensure hotfix-files directory exists on the remote host diff --git a/etc/kayobe/ansible/openstack-host-image-upload.yml b/etc/kayobe/ansible/openstack-host-image-upload.yml new file mode 100644 index 000000000..2c92d2446 --- /dev/null +++ b/etc/kayobe/ansible/openstack-host-image-upload.yml @@ -0,0 +1,54 @@ +--- +# This playbook is designed to be used by the overcloud-host-image-build.yml +# GitHub workflow to upload newly-built images to a development cloud for +# testing and use in CI. +- name: Upload an OS image to Glance + hosts: seed + vars: + local_image_path: "/opt/kayobe/images/overcloud-{{ os_distribution }}-{{ os_release }}/overcloud-{{ os_distribution }}-{{ os_release }}.qcow2" + image_name: "overcloud-{{ os_distribution }}-{{ os_release }}" + tasks: + - block: + - name: Write out clouds.yaml + copy: + content: "{{ lookup('ansible.builtin.env', 'CLOUDS_YAML') }}" + dest: clouds.yaml + mode: 0600 + + - name: Write out secure.yaml + no_log: true + vars: + os_secrets: + clouds: + openstack: + auth: + application_credential_id: "{{ lookup('ansible.builtin.env', 'OS_APPLICATION_CREDENTIAL_ID') }}" + application_credential_secret: "{{ lookup('ansible.builtin.env', 'OS_APPLICATION_CREDENTIAL_SECRET') }}" + copy: + content: "{{ os_secrets | to_nice_yaml }}" + dest: secure.yaml + mode: 0600 + + - name: Ensure dependencies are installed + pip: + name: openstacksdk + + - name: Upload an image to Glance + openstack.cloud.image: + cloud: openstack + name: "{{ image_name }}" + container_format: bare + disk_format: qcow2 + state: present + filename: "{{ local_image_path }}" + + always: + - name: Remove clouds.yaml + file: + path: clouds.yaml + state: absent + + - name: Remove secure.yaml + file: + path: secure.yaml + state: absent diff --git a/etc/kayobe/ansible/pulp-host-image-upload.yml b/etc/kayobe/ansible/pulp-host-image-upload.yml index a06897d90..d3a44f133 100644 --- a/etc/kayobe/ansible/pulp-host-image-upload.yml +++ b/etc/kayobe/ansible/pulp-host-image-upload.yml @@ -1,12 +1,12 @@ --- - name: Upload and create a distribution for an image - hosts: localhost + hosts: seed vars: remote_pulp_url: "{{ stackhpc_release_pulp_url }}" remote_pulp_username: "{{ stackhpc_image_repository_username }}" remote_pulp_password: "{{ stackhpc_image_repository_password }}" repository_name: "kayobe-images-{{ openstack_release }}-{{ os_distribution }}-{{ os_release }}" - base_path: "kayobe-images/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}" + pulp_base_path: "kayobe-images/{{ openstack_release }}/{{ os_distribution }}/{{ os_release }}" tasks: - name: Print image tag debug: @@ -74,7 +74,7 @@ username: "{{ remote_pulp_username }}" password: "{{ remote_pulp_password }}" name: "{{ repository_name }}_latest" - base_path: "{{ base_path }}/latest" + base_path: "{{ pulp_base_path }}/latest" publication: "{{ publication_details.publication.pulp_href }}" content_guard: development state: present @@ -86,7 +86,7 @@ username: "{{ remote_pulp_username }}" password: "{{ remote_pulp_password }}" name: "{{ repository_name }}_{{ host_image_tag }}" - base_path: "{{ base_path }}/{{ host_image_tag }}" + base_path: "{{ pulp_base_path }}/{{ host_image_tag }}" publication: "{{ publication_details.publication.pulp_href }}" content_guard: development state: present @@ -95,26 +95,26 @@ - name: Update new images file with versioned path lineinfile: path: /tmp/updated_images.txt - line: "{{ remote_pulp_url }}/pulp/content/{{ base_path }}/\ + line: "{{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\ {{ host_image_tag }}/{{ found_files.files[0].path | basename }}" create: true - name: Update new images file with latest path lineinfile: path: /tmp/updated_images.txt - line: "{{ remote_pulp_url }}/pulp/content/{{ base_path }}/\ + line: "{{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\ latest/{{ found_files.files[0].path | basename }}" when: latest_distribution_details.changed - name: Print versioned path debug: - msg: "New versioned path: {{ remote_pulp_url }}/pulp/content/{{ base_path }}/\ + msg: "New versioned path: {{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\ {{ host_image_tag }}/{{ found_files.files[0].path | basename }}" when: latest_distribution_details.changed - name: Print latest path debug: - msg: "New latest path: {{ remote_pulp_url }}/pulp/content/{{ base_path }}/\ + msg: "New latest path: {{ remote_pulp_url }}/pulp/content/{{ pulp_base_path }}/\ latest/{{ found_files.files[0].path | basename }}" when: latest_distribution_details.changed diff --git a/etc/kayobe/ansible/requirements.yml b/etc/kayobe/ansible/requirements.yml index e09ed5b85..f1ae3bb63 100644 --- a/etc/kayobe/ansible/requirements.yml +++ b/etc/kayobe/ansible/requirements.yml @@ -1,7 +1,7 @@ --- collections: - name: stackhpc.cephadm - version: 1.14.0 + version: 1.15.1 # NOTE: Pinning pulp.squeezer to 0.0.13 because 0.0.14+ depends on the # pulp_glue Python library being installed. - name: pulp.squeezer diff --git a/etc/kayobe/ansible/run-container-hotfix.yml b/etc/kayobe/ansible/run-container-hotfix.yml index 582ade5da..de652e451 100644 --- a/etc/kayobe/ansible/run-container-hotfix.yml +++ b/etc/kayobe/ansible/run-container-hotfix.yml @@ -20,3 +20,4 @@ - name: Run container_hotfix_command command: "{{ kolla_container_engine | default('docker')}} exec {{ '-u 0' if container_hotfix_become else '' }} {{ hotfix_container }} {{ container_hotfix_command }}" + when: container_hotfix_command diff --git a/etc/kayobe/ansible/templates/os_capacity-clouds.yml.j2 b/etc/kayobe/ansible/templates/os_capacity-clouds.yml.j2 index a821d6dcb..ef3c8d7a5 100644 --- a/etc/kayobe/ansible/templates/os_capacity-clouds.yml.j2 +++ b/etc/kayobe/ansible/templates/os_capacity-clouds.yml.j2 @@ -2,12 +2,14 @@ clouds: openstack: auth: auth_url: "{{ stackhpc_os_capacity_auth_url }}" - application_credential_id: "{{ secrets_os_capacity_credential_id }}" - application_credential_secret: "{{ secrets_os_capacity_credential_secret }}" + project_name: "{{ stackhpc_os_capacity_project_name }}" + domain_name: "{{ stackhpc_os_capacity_domain_name }}" + username: "{{ stackhpc_os_capacity_username }}" + password: "{{ stackhpc_os_capacity_password }}" region_name: "{{ stackhpc_os_capacity_openstack_region_name }}" interface: "internal" identity_api_version: 3 - auth_type: "v3applicationcredential" + auth_type: "password" {% if not stackhpc_os_capacity_openstack_verify | bool %} verify: False {% endif %} diff --git a/etc/kayobe/environments/ci-aio/automated-setup.sh b/etc/kayobe/environments/ci-aio/automated-setup.sh index 7d00d2302..d0f9a390d 100644 --- a/etc/kayobe/environments/ci-aio/automated-setup.sh +++ b/etc/kayobe/environments/ci-aio/automated-setup.sh @@ -72,6 +72,10 @@ fi sudo ip l set dummy1 up sudo ip l set dummy1 master breth1 +if type apt; then + sudo cp /run/systemd/network/* /etc/systemd/network +fi + export KAYOBE_VAULT_PASSWORD=$(cat $BASE_PATH/vault-pw) pushd $BASE_PATH/src/kayobe-config source kayobe-env --environment ci-aio diff --git a/etc/kayobe/environments/ci-multinode/cephadm.yml b/etc/kayobe/environments/ci-multinode/cephadm.yml index 7885a5735..4a9d3f448 100644 --- a/etc/kayobe/environments/ci-multinode/cephadm.yml +++ b/etc/kayobe/environments/ci-multinode/cephadm.yml @@ -2,6 +2,12 @@ ############################################################################### # Cephadm deployment configuration. +# Ceph release name. +cephadm_ceph_release: "{{ 'quincy' if (ansible_facts['distribution_release'] == 'jammy' or ansible_facts.distribution_major_version == '9') else 'pacific' }}" + +# Ceph container image tag. +cephadm_image_tag: "{{ 'v17.2.7' if cephadm_ceph_release == 'quincy' else 'v16.2.14' }}" + # Ceph OSD specification. cephadm_osd_spec: service_type: osd diff --git a/etc/kayobe/environments/ci-multinode/kolla/globals.yml b/etc/kayobe/environments/ci-multinode/kolla/globals.yml index f7f7ca77d..0f9dfe6f0 100644 --- a/etc/kayobe/environments/ci-multinode/kolla/globals.yml +++ b/etc/kayobe/environments/ci-multinode/kolla/globals.yml @@ -1,4 +1,9 @@ --- +# Most development environments will use nested virtualisation, and we can't +# guarantee that nested KVM support is available. Use QEMU as a lowest common +# denominator. +nova_compute_virt_type: qemu + # Reduce the control plane's memory footprint by limiting the number of worker # processes to two per-service when running in a VM. openstack_service_workers: "{% raw %}{{ [ansible_facts.processor_vcpus, 2 if ansible_facts.virtualization_role == 'guest' else 5] | min }}{% endraw %}" diff --git a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml index 84c528207..d7cc442d6 100644 --- a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml @@ -10,7 +10,16 @@ kolla_docker_namespace: stackhpc-dev # Host and port of a package repository mirror. # Build and deploy the development Pulp service repositories. -stackhpc_repo_mirror_url: "http://pulp-server.internal.sms-cloud:8080" +# Use Ark's package repositories to install packages. +stackhpc_repo_mirror_url: "{{ stackhpc_release_pulp_url }}" +stackhpc_repo_mirror_username: "skc-ci-aio" +stackhpc_repo_mirror_password: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 36373536303261313239613761653261663437356566343865383563346334396136653666383765 + 6634396534653865633936653038383132396532386665370a366562383166353966663838316266 + 65333133636330623936623438666632316238376264313234346333346461623765633163353635 + 6565326136313564320a303231383438333062643533333335663034613439393665656162626137 + 65356232656164663831316530333136336362393636656566353635306565626636 # Build and deploy released Pulp repository versions. stackhpc_repo_grafana_version: "{{ stackhpc_pulp_repo_grafana_version }}" @@ -46,12 +55,5 @@ stackhpc_include_os_minor_version_in_repo_url: true # Push built images to the development Pulp service registry. stackhpc_docker_registry: "{{ stackhpc_repo_mirror_url | regex_replace('^https?://', '') }}" -# Username and password of container registry. -stackhpc_docker_registry_username: "stackhpc-kayobe-ci" -stackhpc_docker_registry_password: !vault | - $ANSIBLE_VAULT;1.1;AES256 - 33356166343730633865363431306535613736663764373034396132356131343066636530393534 - 3262646436663034633131316438633230383330633533350a386365313239303464383636376338 - 61656662333939333063343131633963636431663136643137636664633233633133396339613861 - 3038613063626138610a333566393937643630366564653163613364323965396130613433316537 - 39653335393831633362343934363866346262613166393561666336623062393935 +stackhpc_docker_registry_username: "{{ stackhpc_repo_mirror_username }}" +stackhpc_docker_registry_password: "{{ stackhpc_repo_mirror_password }}" diff --git a/etc/kayobe/hooks/overcloud-service-deploy/post.d/deploy-os-capacity-exporter.yml b/etc/kayobe/hooks/overcloud-service-deploy/post.d/deploy-os-capacity-exporter.yml new file mode 120000 index 000000000..0cc70aace --- /dev/null +++ b/etc/kayobe/hooks/overcloud-service-deploy/post.d/deploy-os-capacity-exporter.yml @@ -0,0 +1 @@ +../../../ansible/deploy-os-capacity-exporter.yml \ No newline at end of file diff --git a/etc/kayobe/overcloud-dib.yml b/etc/kayobe/overcloud-dib.yml index 638fdfbfb..73ae57302 100644 --- a/etc/kayobe/overcloud-dib.yml +++ b/etc/kayobe/overcloud-dib.yml @@ -71,7 +71,7 @@ overcloud_dib_host_packages_extra: overcloud_dib_git_elements_extra: - repo: "https://github.com/stackhpc/stackhpc-image-elements" local: "{{ source_checkout_path }}/stackhpc-image-elements" - version: "v1.6.0" + version: "v1.6.1" elements_path: "elements" # List of git repositories containing Diskimage Builder (DIB) elements. See diff --git a/etc/kayobe/stackhpc-monitoring.yml b/etc/kayobe/stackhpc-monitoring.yml index 13bf6ba0f..f08e552c3 100644 --- a/etc/kayobe/stackhpc-monitoring.yml +++ b/etc/kayobe/stackhpc-monitoring.yml @@ -14,13 +14,7 @@ alertmanager_low_memory_threshold_gib: 5 # Whether the OpenStack Capacity exporter is enabled. # Enabling this flag will result in HAProxy configuration and Prometheus scrape # targets being templated during deployment. -stackhpc_enable_os_capacity: false - -# Keystone authentication URL for OpenStack Capacity -stackhpc_os_capacity_auth_url: "http{% if kolla_enable_tls_internal | bool %}s{% endif %}://{{ kolla_internal_fqdn }}:5000" - -# OpenStack region for OpenStack Capacity -stackhpc_os_capacity_openstack_region_name: "RegionOne" +stackhpc_enable_os_capacity: true # Whether TLS certificate verification is enabled for the OpenStack Capacity # exporter during Keystone authentication. diff --git a/releasenotes/notes/bump-magnum-capi-helm-6723d89456e6a590.yaml b/releasenotes/notes/bump-magnum-capi-helm-6723d89456e6a590.yaml new file mode 100644 index 000000000..7fc3cca1a --- /dev/null +++ b/releasenotes/notes/bump-magnum-capi-helm-6723d89456e6a590.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Updates Magnum CAPI Helm driver version to v0.11.0 diff --git a/releasenotes/notes/fail-unparsed-inventory-c3b4e2ffcb620a6b.yaml b/releasenotes/notes/fail-unparsed-inventory-c3b4e2ffcb620a6b.yaml new file mode 100644 index 000000000..335691c30 --- /dev/null +++ b/releasenotes/notes/fail-unparsed-inventory-c3b4e2ffcb620a6b.yaml @@ -0,0 +1,7 @@ +--- +upgrade: + - | + Updates the Ansible configuration to `fail on any unparsed inventory source + `__. + If you are using a separate Ansible configuration for Kolla Ansible, you + may wish to add this setting in ``etc/kayobe/kolla/ansible.cfg``. diff --git a/releasenotes/notes/os-capacity-deploy-hook-b52e87c0819df6fd.yaml b/releasenotes/notes/os-capacity-deploy-hook-b52e87c0819df6fd.yaml new file mode 100644 index 000000000..547939199 --- /dev/null +++ b/releasenotes/notes/os-capacity-deploy-hook-b52e87c0819df6fd.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + Automatic deployment for OpenStack Capacity via a Kayobe service + deploy hook using kolla admin credentials. +upgrade: + - | + OpenStack Capacity no longer uses application credentials. Please + delete any previously generated application credentials. \ No newline at end of file