From c808cfadaee62d4209aae3dd7ae94a805e4b061a Mon Sep 17 00:00:00 2001 From: hanwenli Date: Fri, 6 Jun 2025 06:19:56 -0700 Subject: [PATCH 1/3] Remove pinning of kernel-abi-stablelists According to https://access.redhat.com/solutions/444773 With Red Hat Enterprise Linux 7 and 8, the stablelist is valid for the particular major release. This means that once a symbol has been introduced into kABI for a particular major release, it will not be removed, nor will its meaning be changed during that kernel major release complete life cycle. With Red Hat Enterprise Linux 9, each minor release will have a unique stablelist that is valid throughout the minor release lifecycle. For more information on this, please refer to the following knowledgebase article; We are pinning the minor release for the duration of the build. So the stablelist for the minor version will be installed --- .../resources/imagebuilder/parallelcluster.yaml | 12 ------------ .../resources/imagebuilder/update_and_reboot.yaml | 12 ------------ 2 files changed, 24 deletions(-) diff --git a/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml b/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml index 364a0785e6..2286e0c323 100644 --- a/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml +++ b/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml @@ -198,12 +198,6 @@ phases: # listing all the packages because wildcard does not work as expected yum versionlock kernel kernel-core kernel-modules - if [[ ${!OS} == "alinux2" ]] || [[ ${!OS} == "alinux2023" ]] ; then - yum versionlock kernel-abi-whitelists - else - yum versionlock kernel-abi-stablelists - fi - if [[ ${!OS} == "rocky8" ]] || [[ ${!OS} == "rocky9" ]] ; then yum versionlock rocky-release rocky-repos elif [[ ${!OS} == "rhel8" ]] || [[ ${!OS} == "rhel9" ]] ; then @@ -350,12 +344,6 @@ phases: if [[ ${!PLATFORM} == RHEL ]]; then yum versionlock delete kernel kernel-core kernel-modules - if [[ ${!OS} == "alinux2" ]] || [[ ${!OS} == "alinux2023" ]] ; then - yum versionlock delete kernel-abi-whitelists - else - yum versionlock delete kernel-abi-stablelists - fi - if [[ ${!OS} == "rocky8" ]] || [[ ${!OS} == "rocky9" ]] ; then yum versionlock delete rocky-release rocky-repos elif [[ ${!OS} == "rhel8" ]] || [[ ${!OS} == "rhel9" ]] ; then diff --git a/cli/src/pcluster/resources/imagebuilder/update_and_reboot.yaml b/cli/src/pcluster/resources/imagebuilder/update_and_reboot.yaml index 2b7607b1ea..29554560bb 100644 --- a/cli/src/pcluster/resources/imagebuilder/update_and_reboot.yaml +++ b/cli/src/pcluster/resources/imagebuilder/update_and_reboot.yaml @@ -202,12 +202,6 @@ phases: # listing all the packages because wildcard does not work as expected yum versionlock kernel kernel-core kernel-modules - if [[ ${!OS} == "alinux2" ]] || [[ ${!OS} == "alinux2023" ]] ; then - yum versionlock kernel-abi-whitelists - else - yum versionlock kernel-abi-stablelists - fi - if [[ ${!OS} == "rocky8" ]] || [[ ${!OS} == "rocky9" ]] ; then yum versionlock rocky-release rocky-repos elif [[ ${!OS} == "rhel8" ]] || [[ ${!OS} == "rhel9" ]] ; then @@ -303,12 +297,6 @@ phases: if [[ ${!DISABLE_KERNEL_UPDATE} == true ]] && [[ ${!PLATFORM} == RHEL ]]; then yum versionlock delete kernel kernel-core kernel-modules - if [[ ${!OS} == "alinux2" ]] || [[ ${!OS} == "alinux2023" ]] ; then - yum versionlock delete kernel-abi-whitelists - else - yum versionlock delete kernel-abi-stablelists - fi - if [[ ${!OS} == "rocky8" ]] || [[ ${!OS} == "rocky9" ]] ; then yum versionlock delete rocky-release elif [[ ${!OS} == "rhel8" ]] || [[ ${!OS} == "rhel9" ]] ; then From cc79bf7ebdbcdda39ea076f244d327938868d62a Mon Sep 17 00:00:00 2001 From: hanwenli Date: Mon, 16 Jun 2025 12:10:03 -0700 Subject: [PATCH 2/3] Install kernel packages before pinning 1. Move DisableNouveau, InstallAdditionalKernelPackages from `update_and_reboot.yaml` to `parallelcluster.yaml`. This improves consistency of build between with and without `UpdateOsPackages` 2. Remove InstallEfiBootManager because the bug it was trying to work around already got resolved. 3. Move RHEL minor version pinning from `InstallPrerequisite` to the beginning of `PinVersion` for the following reasons 3.1 Even if we make mistakes pinning packages without installing, with minor version pinned, we will pin to the latest within the minor 3.2 `yum install -y yum-plugin-versionlock` will install the version within the minor 3.3 In terms of code style, RHEL minor version pinning fits in `PinVersion` better than `InstallPrerequisite` 4. For RHEL/Rocky, install kernel-headers, kernel-devel, kernel-devel-matched(RHEL/Rocky >=9) before pinning the version. For Ubuntu, install `linux-headers` before pinning the version 5. Add a Reboot after installing and pinning kernel packages --- CHANGELOG.md | 1 + .../imagebuilder/parallelcluster.yaml | 72 ++++++++++++++++--- .../imagebuilder/update_and_reboot.yaml | 64 ----------------- 3 files changed, 63 insertions(+), 74 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e417bbed5..f0ac7f39aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ CHANGELOG **BUG FIXES** - Fix a bug that update-cluster, update-compute-fleet may fail when compute resources use an expired Capacity Reservation. +- Fix build image failures on Rocky 9 non-lastest versions. 3.13.1 ------ diff --git a/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml b/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml index 2286e0c323..f20e57c9f6 100644 --- a/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml +++ b/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml @@ -184,7 +184,7 @@ phases: fi fi - - name: PinKernelVersion + - name: PinVersion action: ExecuteBash inputs: commands: @@ -193,7 +193,45 @@ phases: OS='{{ build.OperatingSystemName.outputs.stdout }}' PLATFORM='{{ build.PlatformName.outputs.stdout }}' KERNEL_VERSION=$(uname -a) + RELEASE_VERSION='{{ build.OperatingSystemVersion.outputs.stdout }}' if [[ ${!PLATFORM} == RHEL ]]; then + if [[ ${!OS} == rhel9 ]] || [[ ${!OS} == rocky9 ]]; then + if [[ ! -f /etc/yum/vars/releasever ]]; then + echo "yes" > /opt/parallelcluster/pin_releasesever + echo ${!RELEASE_VERSION} > /etc/yum/vars/releasever + yum clean all + fi + fi + PACKAGE_LIST="kernel-headers-$(uname -r) kernel-devel-$(uname -r)" + if [[ ${!OS} != "rocky8" ]] && [[ ${!OS} != "rhel8" ]]; then + PACKAGE_LIST+=" kernel-devel-matched-$(uname -r)" + fi + + if [[ ${!OS} == "rocky8" ]] || [[ ${!OS} == "rocky9" ]] ; then + if [[ ${!OS} == "rocky8" ]] ; then + REPOSITORY="BaseOS" + else + REPOSITORY="AppStream" + fi + for PACKAGE in ${!PACKAGE_LIST} + do + # try to install kernel source for a specific release version + yum install -y ${!PACKAGE} + if [ $? -ne 0 ]; then + yum install -y wget + # Previous releases are moved into a vault area once a new minor release version is available for at least a week. + # https://wiki.rockylinux.org/rocky/repo/#notes-on-devel + wget https://dl.rockylinux.org/vault/rocky/${!RELEASE_VERSION}/${!REPOSITORY}/$(uname -m)/os/Packages/k/${!PACKAGE}.rpm + yum install -y ./${!PACKAGE}.rpm + fi + done + else + for PACKAGE in ${!PACKAGE_LIST} + do + yum -y install ${!PACKAGE} + done + fi + yum install -y yum-plugin-versionlock # listing all the packages because wildcard does not work as expected yum versionlock kernel kernel-core kernel-modules @@ -204,9 +242,24 @@ phases: yum versionlock redhat-release fi else + apt-get -y install linux-headers-$(uname -r) apt-mark hold linux-aws* linux-base* linux-headers* linux-image* fi - echo "Kernel version is ${!KERNEL_VERSION}" + echo "Kernel version is ${!KERNEL_VERSION}" + + - name: DisableNouveau + action: ExecuteBash + inputs: + commands: + - | + set -v + PLATFORM='{{ build.PlatformName.outputs.stdout }}' + /bin/sed -r -i -e 's/GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 rd.driver.blacklist=nouveau nouveau.modeset=0"/' /etc/default/grub + if [[ ${!PLATFORM} == RHEL ]]; then + grub2-mkconfig -o /boot/grub2/grub.cfg + elif [[ ${!PLATFORM} == DEBIAN ]]; then + update-grub + fi # Install prerequisite OS packages - name: InstallPrerequisite @@ -217,16 +270,8 @@ phases: set -v OS='{{ build.OperatingSystemName.outputs.stdout }}' PLATFORM='{{ build.PlatformName.outputs.stdout }}' - VERSION='{{ build.OperatingSystemVersion.outputs.stdout }}' if [[ ${!PLATFORM} == RHEL ]]; then - if [[ ${!OS} == rhel9 ]] || [[ ${!OS} == rocky9 ]]; then - if [[ ! -f /etc/yum/vars/releasever ]]; then - echo "yes" > /opt/parallelcluster/pin_releasesever - echo ${!VERSION} > /etc/yum/vars/releasever - yum clean all - fi - fi yum -y update krb5-libs yum -y groupinstall development && sudo yum -y install wget jq if [[ ${!OS} != alinux2023 ]]; then @@ -246,6 +291,13 @@ phases: apt-get -y install build-essential curl wget jq fi + - name: RebootStep + action: Reboot + onFailure: Abort + maxAttempts: 2 + inputs: + delaySeconds: 10 + # Install Cinc - name: InstallCinc action: ExecuteBash diff --git a/cli/src/pcluster/resources/imagebuilder/update_and_reboot.yaml b/cli/src/pcluster/resources/imagebuilder/update_and_reboot.yaml index 29554560bb..b7a5157215 100644 --- a/cli/src/pcluster/resources/imagebuilder/update_and_reboot.yaml +++ b/cli/src/pcluster/resources/imagebuilder/update_and_reboot.yaml @@ -99,20 +99,6 @@ phases: fi fi - - name: DisableNouveau - action: ExecuteBash - inputs: - commands: - - | - set -v - PLATFORM='{{ build.PlatformName.outputs.stdout }}' - /bin/sed -r -i -e 's/GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 rd.driver.blacklist=nouveau nouveau.modeset=0"/' /etc/default/grub - if [[ ${!PLATFORM} == RHEL ]]; then - grub2-mkconfig -o /boot/grub2/grub.cfg - elif [[ ${!PLATFORM} == DEBIAN ]]; then - update-grub - fi - - name: DisableUnattendedUpgrades action: ExecuteBash inputs: @@ -131,20 +117,6 @@ phases: # update package index DEBIAN_FRONTEND=noninteractive apt-get -y update fi - - - name: InstallEfiBootManager - action: ExecuteBash - inputs: - commands: - - | - set -v - PLATFORM='{{ build.PlatformName.outputs.stdout }}' - ARCH=$(uname -m) - if [[ `echo ${!ARCH}` == 'aarch64' ]] && [[ ${!PLATFORM} == DEBIAN ]]; then - # temporary workaround to solve https://bugs.launchpad.net/ubuntu/+source/grub2-signed/+bug/1936857 - apt-get -y install efibootmgr - fi - - name: InstallPrerequisites action: ExecuteBash inputs: @@ -246,42 +218,6 @@ phases: maxAttempts: 2 inputs: delaySeconds: 10 - - name: InstallAdditionalKernelPackages - action: ExecuteBash - inputs: - commands: - - | - set -v - OS='{{ build.OperatingSystemName.outputs.stdout }}' - PLATFORM='{{ build.PlatformName.outputs.stdout }}' - DISABLE_KERNEL_UPDATE='{{ build.DisableKernelUpdate.outputs.stdout }}' - - if [[ ${!PLATFORM} == RHEL ]]; then - # Install kernel-devel during OS update, so that headers are aligned with new kernel. - # The same is done for Debian through `apt-get -y install linux-aws` - if [[ ${!OS} == "rocky8" ]] ; then - PACKAGE="kernel-devel-$(uname -r)" - RELEASE_VERSION=$(source /etc/os-release && echo ${!VERSION_ID}) - - # try to install kernel source for a specific release version - yum install -y ${!PACKAGE} --releasever ${!RELEASE_VERSION} - if [ $? -ne 0 ]; then - yum install -y wget - # Previous releases are moved into a vault area once a new minor release version is available for at least a week. - # https://wiki.rockylinux.org/rocky/repo/#notes-on-devel - wget https://dl.rockylinux.org/vault/rocky/${!RELEASE_VERSION}/BaseOS/$(uname -m)/os/Packages/k/${!PACKAGE}.rpm - yum install -y ./${!PACKAGE}.rpm - fi - else - yum -y install kernel-headers-$(uname -r) - yum -y install kernel-devel-$(uname -r) - fi - - elif [[ ${!PLATFORM} == DEBIAN ]]; then - if [[ ${!DISABLE_KERNEL_UPDATE} != true ]]; then - apt-get -y install linux-aws linux-headers-aws linux-image-aws - fi - fi - name: RemoveKernelPin action: ExecuteBash From 6a820d74aa6076b23e490ec69d54b3e75a9ac712 Mon Sep 17 00:00:00 2001 From: hanwenli Date: Tue, 17 Jun 2025 08:39:50 -0700 Subject: [PATCH 3/3] Enable vault repository for Rocky when mirrors don't work This commit improves the Rocky9 vault repository usage. Instead of using `wget` from the URL of the vault, this commit enables vault repository, so all subsequent `yum` commands will be able to use the vault. This reduces code duplication and improves robustness. Example of repo configuration in Rocky: ``` cat /etc/yum.repos.d/Rocky-AppStream.repo # Rocky-AppStream.repo # # The mirrorlist system uses the connecting IP address of the client and the # update status of each mirror to pick current mirrors that are geographically # close to the client. You should use this for Rocky updates unless you are # manually picking other mirrors. # # If the mirrorlist does not work for you, you can try the commented out # baseurl line instead. [appstream] name=Rocky Linux $releasever - AppStream mirrorlist=https://mirrors.rockylinux.org/mirrorlist?arch=$basearch&repo=AppStream-$releasever #baseurl=http://dl.rockylinux.org/$contentdir/$releasever/AppStream/$basearch/os/ gpgcheck=1 enabled=1 countme=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-rockyofficial ``` --- .../resources/imagebuilder/parallelcluster.yaml | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml b/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml index f20e57c9f6..326a8c25f9 100644 --- a/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml +++ b/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml @@ -208,21 +208,14 @@ phases: fi if [[ ${!OS} == "rocky8" ]] || [[ ${!OS} == "rocky9" ]] ; then - if [[ ${!OS} == "rocky8" ]] ; then - REPOSITORY="BaseOS" - else - REPOSITORY="AppStream" - fi for PACKAGE in ${!PACKAGE_LIST} do - # try to install kernel source for a specific release version yum install -y ${!PACKAGE} if [ $? -ne 0 ]; then - yum install -y wget - # Previous releases are moved into a vault area once a new minor release version is available for at least a week. - # https://wiki.rockylinux.org/rocky/repo/#notes-on-devel - wget https://dl.rockylinux.org/vault/rocky/${!RELEASE_VERSION}/${!REPOSITORY}/$(uname -m)/os/Packages/k/${!PACKAGE}.rpm - yum install -y ./${!PACKAGE}.rpm + # Enable vault repository + sed -i 's|^#baseurl=http://dl.rockylinux.org/$contentdir|baseurl=http://dl.rockylinux.org/vault/rocky|g' /etc/yum.repos.d/*.repo + sed -i 's|^#baseurl=https://dl.rockylinux.org/$contentdir|baseurl=https://dl.rockylinux.org/vault/rocky|g' /etc/yum.repos.d/*.repo + yum install -y ${!PACKAGE} fi done else