diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml index e8584ef7e..7f7b6fe5b 100644 --- a/.github/workflows/overcloud-host-image-build.yml +++ b/.github/workflows/overcloud-host-image-build.yml @@ -55,7 +55,7 @@ jobs: rm -f /tmp/updated_images.txt - name: Clone StackHPC Kayobe repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: stackhpc/kayobe ref: refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }} diff --git a/.github/workflows/overcloud-host-image-promote.yml b/.github/workflows/overcloud-host-image-promote.yml index 52674db04..449068f6f 100644 --- a/.github/workflows/overcloud-host-image-promote.yml +++ b/.github/workflows/overcloud-host-image-promote.yml @@ -21,9 +21,9 @@ jobs: overcloud-host-image-promote: name: Promote overcloud host image if: github.repository == 'stackhpc/stackhpc-kayobe-config' - runs-on: [self-hosted, stackhpc-kayobe-config-kolla-builder-rl9] + runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: path: src/kayobe-config @@ -32,9 +32,10 @@ jobs: run: | BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview) echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT + working-directory: src/kayobe-config - name: Clone StackHPC Kayobe repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: stackhpc/kayobe ref: refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }} diff --git a/.github/workflows/overcloud-host-image-upload.yml b/.github/workflows/overcloud-host-image-upload.yml new file mode 100644 index 000000000..e1468df87 --- /dev/null +++ b/.github/workflows/overcloud-host-image-upload.yml @@ -0,0 +1,173 @@ +--- +name: Upload overcloud host images +on: + workflow_dispatch: + inputs: + rocky9: + description: Upload Rocky Linux 9 + type: boolean + default: true + ubuntu-jammy: + description: Upload Ubuntu 22.04 Jammy + type: boolean + default: true + kayobe-environment: + description: Kayobe environment to use + type: string + default: "ci-builder" + secrets: + KAYOBE_VAULT_PASSWORD: + required: true + CLOUDS_YAML: + required: true + OS_APPLICATION_CREDENTIAL_ID: + required: true + OS_APPLICATION_CREDENTIAL_SECRET: + required: true + +env: + ANSIBLE_FORCE_COLOR: True +jobs: + overcloud-host-image-upload: + name: Upload overcloud host images + if: github.repository == 'stackhpc/stackhpc-kayobe-config' + runs-on: arc-skc-host-image-builder-runner + permissions: {} + steps: + - name: Install package dependencies + run: | + sudo apt update + sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv + + - uses: actions/checkout@v4 + with: + path: src/kayobe-config + + - name: Determine OpenStack release + id: openstack_release + run: | + BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' src/kayobe-config/.gitreview) + echo "openstack_release=${BRANCH}" | sed "s|stable/||" >> $GITHUB_OUTPUT + + - name: Clone StackHPC Kayobe repository + uses: actions/checkout@v4 + with: + repository: stackhpc/kayobe + ref: refs/heads/stackhpc/${{ steps.openstack_release.outputs.openstack_release }} + path: src/kayobe + + - name: Install Kayobe + run: | + mkdir -p venvs && + pushd venvs && + python3 -m venv kayobe && + source kayobe/bin/activate && + pip install -U pip && + pip install ../src/kayobe + + - name: Bootstrap the control host + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ${{ inputs.kayobe-environment }} && + kayobe control host bootstrap --skip-tags bootstrap + + - name: Generate clouds.yaml + run: | + cat << EOF > clouds.yaml + ${{ secrets.CLOUDS_YAML }} + EOF + + - name: Install OpenStack client + run: | + source venvs/kayobe/bin/activate && + pip install python-openstackclient -c https://opendev.org/openstack/requirements/raw/branch/stable/zed/upper-constraints.txt + + - name: Output Rocky Linux 9 image tag + id: rocky_9_image_tag + run: | + echo image_tag=$(grep stackhpc_rocky_9_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + + - name: Check if image exists already + id: rocky_9_image_exists + run: | + source venvs/kayobe/bin/activate && + openstack image show \ + overcloud-rocky-9-${{ steps.rocky_9_image_tag.outputs.image_tag }} + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + continue-on-error: true + + - name: Download Rocky Linux 9 overcloud host image from Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ${{ inputs.kayobe-environment }} && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-host-image-download.yml \ + -e os_distribution="rocky" \ + -e os_release="9" + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: inputs.rocky9 && steps.rocky_9_image_exists.outcome == 'failure' + + - name: Upload Rocky Linux 9 overcloud host image to Cloud + run: | + source venvs/kayobe/bin/activate && + openstack image create \ + overcloud-rocky-9-${{ steps.rocky_9_image_tag.outputs.image_tag }} \ + --container-format bare \ + --disk-format qcow2 \ + --file /tmp/rocky-9.qcow2 \ + --private \ + --progress + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: inputs.rocky9 && steps.rocky_9_image_exists.outcome == 'failure' + + - name: Output Ubuntu Jammy image tag + id: ubuntu_jammy_image_tag + run: | + echo image_tag=$(grep stackhpc_ubuntu_jammy_overcloud_host_image_version: src/kayobe-config/etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT + + - name: Check if image exists already + id: ubuntu_jammy_image_exists + run: | + source venvs/kayobe/bin/activate && + openstack image show \ + overcloud-ubuntu-jammy-${{ steps.ubuntu_jammy_image_tag.outputs.image_tag }} + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + continue-on-error: true + + - name: Download Ubuntu Jammy 22.04 overcloud host image from Ark + run: | + source venvs/kayobe/bin/activate && + source src/kayobe-config/kayobe-env --environment ${{ inputs.kayobe-environment }} && + kayobe playbook run \ + src/kayobe-config/etc/kayobe/ansible/pulp-host-image-download.yml \ + -e os_distribution="ubuntu" \ + -e os_release="jammy" + env: + KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} + if: inputs.ubuntu-jammy && steps.ubuntu_jammy_image_exists.outcome == 'failure' + + - name: Upload Ubuntu Jammy 22.04 overcloud host image to Cloud + run: | + source venvs/kayobe/bin/activate && + openstack image create \ + overcloud-ubuntu-jammy-${{ steps.ubuntu_jammy_image_tag.outputs.image_tag }} \ + --container-format bare \ + --disk-format qcow2 \ + --file /tmp/ubuntu-jammy.qcow2 \ + --private \ + --progress + env: + OS_CLOUD: openstack + OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} + OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} + if: inputs.ubuntu-jammy && steps.ubuntu_jammy_image_exists.outcome == 'failure' diff --git a/.github/workflows/stackhpc-build-kayobe-image.yml b/.github/workflows/stackhpc-build-kayobe-image.yml index ab67cb406..2c39e5006 100644 --- a/.github/workflows/stackhpc-build-kayobe-image.yml +++ b/.github/workflows/stackhpc-build-kayobe-image.yml @@ -51,7 +51,7 @@ jobs: steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - name: Checkout kayobe config - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true diff --git a/.github/workflows/stackhpc-container-image-build.yml b/.github/workflows/stackhpc-container-image-build.yml index 7cd09b585..63f649398 100644 --- a/.github/workflows/stackhpc-container-image-build.yml +++ b/.github/workflows/stackhpc-container-image-build.yml @@ -48,7 +48,7 @@ jobs: openstack_release: ${{ steps.openstack_release.outputs.openstack_release }} steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Determine OpenStack release id: openstack_release @@ -97,12 +97,12 @@ jobs: needs: - generate-tag steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: path: src/kayobe-config - name: Clone StackHPC Kayobe repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: stackhpc/kayobe ref: refs/heads/stackhpc/${{ needs.generate-tag.outputs.openstack_release }} diff --git a/.github/workflows/stackhpc-pull-request.yml b/.github/workflows/stackhpc-pull-request.yml index 99350e829..79405720c 100644 --- a/.github/workflows/stackhpc-pull-request.yml +++ b/.github/workflows/stackhpc-pull-request.yml @@ -22,7 +22,7 @@ jobs: aio: ${{ steps.changes.outputs.aio }} steps: - name: GitHub Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Check changed files uses: dorny/paths-filter@v2 @@ -47,7 +47,7 @@ jobs: if: github.repository == 'stackhpc/stackhpc-kayobe-config' steps: - name: GitHub Checkout 🛎 - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - name: Setup Python ${{ matrix.python-version }} 🐍 diff --git a/doc/source/configuration/wazuh.rst b/doc/source/configuration/wazuh.rst index 50717d88a..1b23361ef 100644 --- a/doc/source/configuration/wazuh.rst +++ b/doc/source/configuration/wazuh.rst @@ -226,11 +226,12 @@ You may need to modify some of the variables, including: .. note:: - NOTE: If you are using multiple environments, and you need to customise Wazuh in each environment, create override files in an appropriate directory, - for example `etc/kayobe/environments/production/inventory/group_vars/` + for example ``etc/kayobe/environments/production/inventory/group_vars/``. + Files which values can be overridden (in the context of Wazuh): + - etc/kayobe/inventory/group_vars/wazuh/wazuh-manager/wazuh-manager - etc/kayobe/wazuh-manager.yml - etc/kayobe/inventory/group_vars/wazuh/wazuh-agent/wazuh-agent @@ -316,6 +317,9 @@ If you are using the wazuh generated certificates, this will result in the creation of some certificates and keys (in case of custom certs adjust path to it). Encrypt the keys (and remember to commit to git): +``ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments//wazuh/wazuh-certificates/*.key`` + +If using the kayobe environments feature, otherwise: ``ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/ansible/wazuh/certificates/certs/*.key`` diff --git a/etc/kayobe/ansible/pulp-host-image-download.yml b/etc/kayobe/ansible/pulp-host-image-download.yml new file mode 100644 index 000000000..6b3494dfe --- /dev/null +++ b/etc/kayobe/ansible/pulp-host-image-download.yml @@ -0,0 +1,38 @@ +--- +- name: Download an overcloud host image from Ark + hosts: localhost + vars: + # This var is an edited version of stackhpc_overcloud_host_image_url + # without the auth credentials in it. Auth is handled by username and + # password in the get_url task of this playbook + stackhpc_overcloud_host_image_url_no_auth: "{{ stackhpc_release_pulp_content_url }}/kayobe-images/\ + {{ openstack_release }}/{{ os_distribution }}/{{ os_release }}/\ + {{ 'ofed/' if stackhpc_overcloud_host_image_is_ofed else '' }}\ + {{ stackhpc_overcloud_host_image_version }}/\ + overcloud-{{ os_distribution }}-{{ os_release }}\ + {{ '-ofed' if stackhpc_overcloud_host_image_is_ofed else '' }}.qcow2" + + tasks: + - name: Print image information + debug: + msg: | + OS Distribution: {{ os_distribution }} + OS Release: {{ os_release }} + Image tag: {{ stackhpc_overcloud_host_image_version }} + OFED: {{ stackhpc_overcloud_host_image_is_ofed }} + + # TODO: Add checksum support + - name: Download image artifact + get_url: + url: "{{ stackhpc_overcloud_host_image_url_no_auth }}" + username: "{{ stackhpc_image_repository_username }}" + password: "{{ stackhpc_image_repository_password }}" + force_basic_auth: true + unredirected_headers: + - "Authorization" + dest: "/tmp/{{ os_distribution }}-{{ os_release }}.qcow2" + mode: "0644" + register: image_download_result + until: image_download_result.status_code == 200 + retries: 3 + delay: 60